1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using HtmlAgilityPack;
using Pathoschild.Http.Client;
using StardewModdingAPI.Toolkit;
namespace StardewModdingAPI.Web.Framework.Clients.Nexus
{
/// <summary>An HTTP client for fetching mod metadata from the Nexus website.</summary>
internal class NexusWebScrapeClient : INexusClient
{
/*********
** Fields
*********/
/// <summary>The URL for a Nexus mod page for the user, excluding the base URL, where {0} is the mod ID.</summary>
private readonly string ModUrlFormat;
/// <summary>The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID.</summary>
public string ModScrapeUrlFormat { get; set; }
/// <summary>The underlying HTTP client.</summary>
private readonly IClient Client;
/*********
** Public methods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="userAgent">The user agent for the Nexus Mods API client.</param>
/// <param name="baseUrl">The base URL for the Nexus Mods site.</param>
/// <param name="modUrlFormat">The URL for a Nexus Mods mod page for the user, excluding the <paramref name="baseUrl"/>, where {0} is the mod ID.</param>
/// <param name="modScrapeUrlFormat">The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID.</param>
public NexusWebScrapeClient(string userAgent, string baseUrl, string modUrlFormat, string modScrapeUrlFormat)
{
this.ModUrlFormat = modUrlFormat;
this.ModScrapeUrlFormat = modScrapeUrlFormat;
this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent);
}
/// <summary>Get metadata about a mod.</summary>
/// <param name="id">The Nexus mod ID.</param>
/// <returns>Returns the mod info if found, else <c>null</c>.</returns>
public async Task<NexusMod> GetModAsync(uint id)
{
// fetch HTML
string html;
try
{
html = await this.Client
.GetAsync(string.Format(this.ModScrapeUrlFormat, id))
.AsString();
}
catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound)
{
return null;
}
// parse HTML
var doc = new HtmlDocument();
doc.LoadHtml(html);
// handle Nexus error message
HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'site-notice')][contains(@class, 'warning')]");
if (node != null)
{
string[] errorParts = node.InnerText.Trim().Split(new[] { '\n' }, 2, System.StringSplitOptions.RemoveEmptyEntries);
string errorCode = errorParts[0];
string errorText = errorParts.Length > 1 ? errorParts[1] : null;
switch (errorCode.Trim().ToLower())
{
case "not found":
return null;
case "hidden mod":
return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Hidden };
case "not published":
return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.NotPublished };
default:
return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Other };
}
}
// extract mod info
string url = this.GetModUrl(id);
string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim();
string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim();
SemanticVersion.TryParse(version, out ISemanticVersion parsedVersion);
// extract file versions
List<string> rawVersions = new List<string>();
foreach (var fileSection in doc.DocumentNode.SelectNodes("//div[contains(@class, 'files-tabs')]"))
{
string sectionName = fileSection.Descendants("h2").First().InnerText;
if (sectionName != "Main files" && sectionName != "Optional files")
continue;
rawVersions.AddRange(
from statBox in fileSection.Descendants().Where(p => p.HasClass("stat-version"))
from versionStat in statBox.Descendants().Where(p => p.HasClass("stat"))
select versionStat.InnerText.Trim()
);
}
// choose latest file version
ISemanticVersion latestFileVersion = null;
foreach (string rawVersion in rawVersions)
{
if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur))
continue;
if (parsedVersion != null && !cur.IsNewerThan(parsedVersion))
continue;
if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion))
continue;
latestFileVersion = cur;
}
// yield info
return new NexusMod
{
Name = name,
Version = parsedVersion?.ToString() ?? version,
LatestFileVersion = latestFileVersion,
Url = url
};
}
/// <summary>Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.</summary>
public void Dispose()
{
this.Client?.Dispose();
}
/*********
** Private methods
*********/
/// <summary>Get the full mod page URL for a given ID.</summary>
/// <param name="id">The mod ID.</param>
private string GetModUrl(uint id)
{
UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress);
builder.Path += string.Format(this.ModUrlFormat, id);
return builder.Uri.ToString();
}
}
}
|
