From 890c6b3ea7c4aedf4a9130aceff8d80c78bd6e0f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 24 Jul 2019 22:08:15 -0400 Subject: rename Nexus API client for upcoming API usage (#651) --- .../Framework/Clients/Nexus/NexusClient.cs | 152 +++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs (limited to 'src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs') diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs new file mode 100644 index 00000000..87393367 --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using HtmlAgilityPack; +using Pathoschild.Http.Client; +using StardewModdingAPI.Toolkit; + +namespace StardewModdingAPI.Web.Framework.Clients.Nexus +{ + /// An HTTP client for fetching mod metadata from the Nexus website. + internal class NexusClient : INexusClient + { + /********* + ** Fields + *********/ + /// The URL for a Nexus mod page for the user, excluding the base URL, where {0} is the mod ID. + private readonly string ModUrlFormat; + + /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. + public string ModScrapeUrlFormat { get; set; } + + /// The underlying HTTP client. + private readonly IClient Client; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The user agent for the Nexus Mods API client. + /// The base URL for the Nexus Mods site. + /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. + /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. + public NexusClient(string userAgent, string baseUrl, string modUrlFormat, string modScrapeUrlFormat) + { + this.ModUrlFormat = modUrlFormat; + this.ModScrapeUrlFormat = modScrapeUrlFormat; + this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); + } + + /// Get metadata about a mod. + /// The Nexus mod ID. + /// Returns the mod info if found, else null. + public async Task GetModAsync(uint id) + { + // fetch HTML + string html; + try + { + html = await this.Client + .GetAsync(string.Format(this.ModScrapeUrlFormat, id)) + .AsString(); + } + catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + return null; + } + + // parse HTML + var doc = new HtmlDocument(); + doc.LoadHtml(html); + + // handle Nexus error message + HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'site-notice')][contains(@class, 'warning')]"); + if (node != null) + { + string[] errorParts = node.InnerText.Trim().Split(new[] { '\n' }, 2, System.StringSplitOptions.RemoveEmptyEntries); + string errorCode = errorParts[0]; + string errorText = errorParts.Length > 1 ? errorParts[1] : null; + switch (errorCode.Trim().ToLower()) + { + case "not found": + return null; + + case "hidden mod": + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Hidden }; + + case "not published": + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.NotPublished }; + + default: + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Other }; + } + } + + // extract mod info + string url = this.GetModUrl(id); + string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); + string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); + SemanticVersion.TryParse(version, out ISemanticVersion parsedVersion); + + // extract file versions + List rawVersions = new List(); + foreach (var fileSection in doc.DocumentNode.SelectNodes("//div[contains(@class, 'files-tabs')]")) + { + string sectionName = fileSection.Descendants("h2").First().InnerText; + if (sectionName != "Main files" && sectionName != "Optional files") + continue; + + rawVersions.AddRange( + from statBox in fileSection.Descendants().Where(p => p.HasClass("stat-version")) + from versionStat in statBox.Descendants().Where(p => p.HasClass("stat")) + select versionStat.InnerText.Trim() + ); + } + + // choose latest file version + ISemanticVersion latestFileVersion = null; + foreach (string rawVersion in rawVersions) + { + if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur)) + continue; + if (parsedVersion != null && !cur.IsNewerThan(parsedVersion)) + continue; + if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion)) + continue; + + latestFileVersion = cur; + } + + // yield info + return new NexusMod + { + Name = name, + Version = parsedVersion?.ToString() ?? version, + LatestFileVersion = latestFileVersion, + Url = url + }; + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.Client?.Dispose(); + } + + + /********* + ** Private methods + *********/ + /// Get the full mod page URL for a given ID. + /// The mod ID. + private string GetModUrl(uint id) + { + UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); + builder.Path += string.Format(this.ModUrlFormat, id); + return builder.Uri.ToString(); + } + } +} -- cgit From 95f261b1f30d8c5ad6c179cd75a220dcca3c6395 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 24 Jul 2019 22:11:51 -0400 Subject: fetch mod info from Nexus API if the web page is hidden due to adult content (#651) --- docs/release-notes.md | 1 + .../Framework/Clients/Nexus/NexusClient.cs | 133 ++++++++++++++++----- .../Framework/Clients/Nexus/NexusModStatus.cs | 3 + .../Framework/ConfigModels/ApiClientsConfig.cs | 3 + src/SMAPI.Web/SMAPI.Web.csproj | 1 + src/SMAPI.Web/Startup.cs | 10 +- src/SMAPI.Web/appsettings.Development.json | 2 + src/SMAPI.Web/appsettings.json | 1 + 8 files changed, 120 insertions(+), 34 deletions(-) (limited to 'src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs') diff --git a/docs/release-notes.md b/docs/release-notes.md index 4b380564..2b7d1545 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -24,6 +24,7 @@ These changes have not been released yet. * Fixed map reloads resetting tilesheet seasons. * Fixed map reloads not updating door warps. * Fixed outdoor tilesheets being seasonalised when added to an indoor location. + * Fixed update checks failing for Nexus mods marked as adult content. * For the mod compatibility list: * Now loads faster (since data is fetched in a background service). diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs index 87393367..753d3b4f 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs @@ -4,8 +4,10 @@ using System.Linq; using System.Net; using System.Threading.Tasks; using HtmlAgilityPack; +using Pathoschild.FluentNexus.Models; using Pathoschild.Http.Client; using StardewModdingAPI.Toolkit; +using FluentNexusClient = Pathoschild.FluentNexus.NexusClient; namespace StardewModdingAPI.Web.Framework.Clients.Nexus { @@ -16,41 +18,73 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus ** Fields *********/ /// The URL for a Nexus mod page for the user, excluding the base URL, where {0} is the mod ID. - private readonly string ModUrlFormat; + private readonly string WebModUrlFormat; /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. - public string ModScrapeUrlFormat { get; set; } + public string WebModScrapeUrlFormat { get; set; } - /// The underlying HTTP client. - private readonly IClient Client; + /// The underlying HTTP client for the Nexus Mods website. + private readonly IClient WebClient; + + /// The underlying HTTP client for the Nexus API. + private readonly FluentNexusClient ApiClient; /********* ** Public methods *********/ /// Construct an instance. - /// The user agent for the Nexus Mods API client. - /// The base URL for the Nexus Mods site. - /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. - /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. - public NexusClient(string userAgent, string baseUrl, string modUrlFormat, string modScrapeUrlFormat) + /// The user agent for the Nexus Mods web client. + /// The base URL for the Nexus Mods site. + /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. + /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. + /// The app version to show in API user agents. + /// The Nexus API authentication key. + public NexusClient(string webUserAgent, string webBaseUrl, string webModUrlFormat, string webModScrapeUrlFormat, string apiAppVersion, string apiKey) { - this.ModUrlFormat = modUrlFormat; - this.ModScrapeUrlFormat = modScrapeUrlFormat; - this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); + this.WebModUrlFormat = webModUrlFormat; + this.WebModScrapeUrlFormat = webModScrapeUrlFormat; + this.WebClient = new FluentClient(webBaseUrl).SetUserAgent(webUserAgent); + this.ApiClient = new FluentNexusClient(apiKey, "SMAPI", apiAppVersion); } /// Get metadata about a mod. /// The Nexus mod ID. /// Returns the mod info if found, else null. public async Task GetModAsync(uint id) + { + // Fetch from the Nexus website when possible, since it has no rate limits. Mods with + // adult content are hidden for anonymous users, so fall back to the API in that case. + // Note that the API has very restrictive rate limits which means we can't just use it + // for all cases. + NexusMod mod = await this.GetModFromWebsiteAsync(id); + if (mod?.Status == NexusModStatus.AdultContentForbidden) + mod = await this.GetModFromApiAsync(id); + + return mod; + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.WebClient?.Dispose(); + } + + + /********* + ** Private methods + *********/ + /// Get metadata about a mod by scraping the Nexus website. + /// The Nexus mod ID. + /// Returns the mod info if found, else null. + private async Task GetModFromWebsiteAsync(uint id) { // fetch HTML string html; try { - html = await this.Client - .GetAsync(string.Format(this.ModScrapeUrlFormat, id)) + html = await this.WebClient + .GetAsync(string.Format(this.WebModScrapeUrlFormat, id)) .AsString(); } catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) @@ -74,14 +108,8 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus case "not found": return null; - case "hidden mod": - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Hidden }; - - case "not published": - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.NotPublished }; - default: - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Other }; + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = this.GetWebStatus(errorCode) }; } } @@ -130,23 +158,68 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus }; } - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - public void Dispose() + /// Get metadata about a mod from the Nexus API. + /// The Nexus mod ID. + /// Returns the mod info if found, else null. + private async Task GetModFromApiAsync(uint id) { - this.Client?.Dispose(); - } + // fetch mod + Mod mod = await this.ApiClient.Mods.GetMod("stardewvalley", (int)id); + ModFileList files = await this.ApiClient.ModFiles.GetModFiles("stardewvalley", (int)id, FileCategory.Main, FileCategory.Optional); + + // get versions + if (!SemanticVersion.TryParse(mod.Version, out ISemanticVersion mainVersion)) + mainVersion = null; + ISemanticVersion latestFileVersion = null; + foreach (string rawVersion in files.Files.Select(p => p.FileVersion)) + { + if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur)) + continue; + if (mainVersion != null && !cur.IsNewerThan(mainVersion)) + continue; + if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion)) + continue; + latestFileVersion = cur; + } + + // yield info + return new NexusMod + { + Name = mod.Name, + Version = SemanticVersion.TryParse(mod.Version, out ISemanticVersion version) ? version?.ToString() : mod.Version, + LatestFileVersion = latestFileVersion, + Url = this.GetModUrl(id) + }; + } - /********* - ** Private methods - *********/ /// Get the full mod page URL for a given ID. /// The mod ID. private string GetModUrl(uint id) { - UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); - builder.Path += string.Format(this.ModUrlFormat, id); + UriBuilder builder = new UriBuilder(this.WebClient.BaseClient.BaseAddress); + builder.Path += string.Format(this.WebModUrlFormat, id); return builder.Uri.ToString(); } + + /// Get the mod status for a web error code. + /// The Nexus error code. + private NexusModStatus GetWebStatus(string errorCode) + { + switch (errorCode.Trim().ToLower()) + { + case "adult content": + return NexusModStatus.AdultContentForbidden; + + case "hidden mod": + return NexusModStatus.Hidden; + + case "not published": + return NexusModStatus.NotPublished; + + default: + return NexusModStatus.Other; + } + } } } diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusModStatus.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusModStatus.cs index c5723093..9ef314cd 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusModStatus.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusModStatus.cs @@ -12,6 +12,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus /// The mod hasn't been published yet. NotPublished, + /// The mod contains adult content which is hidden for anonymous web users. + AdultContentForbidden, + /// The Nexus API returned an unhandled error. Other } diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index c27cadab..d2e9a2fe 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -65,6 +65,9 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The URL for a Nexus mod page to scrape for versions, excluding the , where {0} is the mod ID. public string NexusModScrapeUrlFormat { get; set; } + /// The Nexus API authentication key. + public string NexusApiKey { get; set; } + /**** ** Pastebin ****/ diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index f4e99e0c..26b29fce 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -26,6 +26,7 @@ + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 66e1f00d..4d23fe65 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -136,10 +136,12 @@ namespace StardewModdingAPI.Web )); services.AddSingleton(new NexusClient( - userAgent: userAgent, - baseUrl: api.NexusBaseUrl, - modUrlFormat: api.NexusModUrlFormat, - modScrapeUrlFormat: api.NexusModScrapeUrlFormat + webUserAgent: userAgent, + webBaseUrl: api.NexusBaseUrl, + webModUrlFormat: api.NexusModUrlFormat, + webModScrapeUrlFormat: api.NexusModScrapeUrlFormat, + apiAppVersion: version, + apiKey: api.NexusApiKey )); services.AddSingleton(new PastebinClient( diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 5856b96c..e6b4a1b1 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -20,6 +20,8 @@ "GitHubUsername": null, "GitHubPassword": null, + "NexusApiKey": null, + "PastebinUserKey": null, "PastebinDevKey": null }, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 77d13924..3ea37dea 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -39,6 +39,7 @@ "ModDropApiUrl": "https://www.moddrop.com/api/mods/data", "ModDropModPageUrl": "https://www.moddrop.com/sdv/mod/{0}", + "NexusApiKey": null, // see top note "NexusBaseUrl": "https://www.nexusmods.com/stardewvalley/", "NexusModUrlFormat": "mods/{0}", "NexusModScrapeUrlFormat": "mods/{0}?tab=files", -- cgit