From 08d83aa039dd57efcc6ab50595fe5c0ea1003527 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 24 Jul 2019 21:28:33 -0400 Subject: treat hidden/unpublished Nexus mods as not found (#651) --- src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs') diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs index e83a6041..061de5de 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs @@ -74,8 +74,14 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus case "not found": return null; + case "hidden mod": + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Hidden }; + + case "not published": + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.NotPublished }; + default: - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText})." }; + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Other }; } } -- cgit From 890c6b3ea7c4aedf4a9130aceff8d80c78bd6e0f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 24 Jul 2019 22:08:15 -0400 Subject: rename Nexus API client for upcoming API usage (#651) --- .../Framework/Clients/Nexus/NexusClient.cs | 152 +++++++++++++++++++++ .../Clients/Nexus/NexusWebScrapeClient.cs | 152 --------------------- src/SMAPI.Web/Startup.cs | 2 +- 3 files changed, 153 insertions(+), 153 deletions(-) create mode 100644 src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs delete mode 100644 src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs (limited to 'src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs') diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs new file mode 100644 index 00000000..87393367 --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using HtmlAgilityPack; +using Pathoschild.Http.Client; +using StardewModdingAPI.Toolkit; + +namespace StardewModdingAPI.Web.Framework.Clients.Nexus +{ + /// An HTTP client for fetching mod metadata from the Nexus website. + internal class NexusClient : INexusClient + { + /********* + ** Fields + *********/ + /// The URL for a Nexus mod page for the user, excluding the base URL, where {0} is the mod ID. + private readonly string ModUrlFormat; + + /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. + public string ModScrapeUrlFormat { get; set; } + + /// The underlying HTTP client. + private readonly IClient Client; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The user agent for the Nexus Mods API client. + /// The base URL for the Nexus Mods site. + /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. + /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. + public NexusClient(string userAgent, string baseUrl, string modUrlFormat, string modScrapeUrlFormat) + { + this.ModUrlFormat = modUrlFormat; + this.ModScrapeUrlFormat = modScrapeUrlFormat; + this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); + } + + /// Get metadata about a mod. + /// The Nexus mod ID. + /// Returns the mod info if found, else null. + public async Task GetModAsync(uint id) + { + // fetch HTML + string html; + try + { + html = await this.Client + .GetAsync(string.Format(this.ModScrapeUrlFormat, id)) + .AsString(); + } + catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + return null; + } + + // parse HTML + var doc = new HtmlDocument(); + doc.LoadHtml(html); + + // handle Nexus error message + HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'site-notice')][contains(@class, 'warning')]"); + if (node != null) + { + string[] errorParts = node.InnerText.Trim().Split(new[] { '\n' }, 2, System.StringSplitOptions.RemoveEmptyEntries); + string errorCode = errorParts[0]; + string errorText = errorParts.Length > 1 ? errorParts[1] : null; + switch (errorCode.Trim().ToLower()) + { + case "not found": + return null; + + case "hidden mod": + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Hidden }; + + case "not published": + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.NotPublished }; + + default: + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Other }; + } + } + + // extract mod info + string url = this.GetModUrl(id); + string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); + string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); + SemanticVersion.TryParse(version, out ISemanticVersion parsedVersion); + + // extract file versions + List rawVersions = new List(); + foreach (var fileSection in doc.DocumentNode.SelectNodes("//div[contains(@class, 'files-tabs')]")) + { + string sectionName = fileSection.Descendants("h2").First().InnerText; + if (sectionName != "Main files" && sectionName != "Optional files") + continue; + + rawVersions.AddRange( + from statBox in fileSection.Descendants().Where(p => p.HasClass("stat-version")) + from versionStat in statBox.Descendants().Where(p => p.HasClass("stat")) + select versionStat.InnerText.Trim() + ); + } + + // choose latest file version + ISemanticVersion latestFileVersion = null; + foreach (string rawVersion in rawVersions) + { + if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur)) + continue; + if (parsedVersion != null && !cur.IsNewerThan(parsedVersion)) + continue; + if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion)) + continue; + + latestFileVersion = cur; + } + + // yield info + return new NexusMod + { + Name = name, + Version = parsedVersion?.ToString() ?? version, + LatestFileVersion = latestFileVersion, + Url = url + }; + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.Client?.Dispose(); + } + + + /********* + ** Private methods + *********/ + /// Get the full mod page URL for a given ID. + /// The mod ID. + private string GetModUrl(uint id) + { + UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); + builder.Path += string.Format(this.ModUrlFormat, id); + return builder.Uri.ToString(); + } + } +} diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs deleted file mode 100644 index 061de5de..00000000 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs +++ /dev/null @@ -1,152 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Threading.Tasks; -using HtmlAgilityPack; -using Pathoschild.Http.Client; -using StardewModdingAPI.Toolkit; - -namespace StardewModdingAPI.Web.Framework.Clients.Nexus -{ - /// An HTTP client for fetching mod metadata from the Nexus website. - internal class NexusWebScrapeClient : INexusClient - { - /********* - ** Fields - *********/ - /// The URL for a Nexus mod page for the user, excluding the base URL, where {0} is the mod ID. - private readonly string ModUrlFormat; - - /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. - public string ModScrapeUrlFormat { get; set; } - - /// The underlying HTTP client. - private readonly IClient Client; - - - /********* - ** Public methods - *********/ - /// Construct an instance. - /// The user agent for the Nexus Mods API client. - /// The base URL for the Nexus Mods site. - /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. - /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. - public NexusWebScrapeClient(string userAgent, string baseUrl, string modUrlFormat, string modScrapeUrlFormat) - { - this.ModUrlFormat = modUrlFormat; - this.ModScrapeUrlFormat = modScrapeUrlFormat; - this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); - } - - /// Get metadata about a mod. - /// The Nexus mod ID. - /// Returns the mod info if found, else null. - public async Task GetModAsync(uint id) - { - // fetch HTML - string html; - try - { - html = await this.Client - .GetAsync(string.Format(this.ModScrapeUrlFormat, id)) - .AsString(); - } - catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) - { - return null; - } - - // parse HTML - var doc = new HtmlDocument(); - doc.LoadHtml(html); - - // handle Nexus error message - HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'site-notice')][contains(@class, 'warning')]"); - if (node != null) - { - string[] errorParts = node.InnerText.Trim().Split(new[] { '\n' }, 2, System.StringSplitOptions.RemoveEmptyEntries); - string errorCode = errorParts[0]; - string errorText = errorParts.Length > 1 ? errorParts[1] : null; - switch (errorCode.Trim().ToLower()) - { - case "not found": - return null; - - case "hidden mod": - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Hidden }; - - case "not published": - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.NotPublished }; - - default: - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Other }; - } - } - - // extract mod info - string url = this.GetModUrl(id); - string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); - string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); - SemanticVersion.TryParse(version, out ISemanticVersion parsedVersion); - - // extract file versions - List rawVersions = new List(); - foreach (var fileSection in doc.DocumentNode.SelectNodes("//div[contains(@class, 'files-tabs')]")) - { - string sectionName = fileSection.Descendants("h2").First().InnerText; - if (sectionName != "Main files" && sectionName != "Optional files") - continue; - - rawVersions.AddRange( - from statBox in fileSection.Descendants().Where(p => p.HasClass("stat-version")) - from versionStat in statBox.Descendants().Where(p => p.HasClass("stat")) - select versionStat.InnerText.Trim() - ); - } - - // choose latest file version - ISemanticVersion latestFileVersion = null; - foreach (string rawVersion in rawVersions) - { - if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur)) - continue; - if (parsedVersion != null && !cur.IsNewerThan(parsedVersion)) - continue; - if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion)) - continue; - - latestFileVersion = cur; - } - - // yield info - return new NexusMod - { - Name = name, - Version = parsedVersion?.ToString() ?? version, - LatestFileVersion = latestFileVersion, - Url = url - }; - } - - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - public void Dispose() - { - this.Client?.Dispose(); - } - - - /********* - ** Private methods - *********/ - /// Get the full mod page URL for a given ID. - /// The mod ID. - private string GetModUrl(uint id) - { - UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); - builder.Path += string.Format(this.ModUrlFormat, id); - return builder.Uri.ToString(); - } - } -} diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index fd229b5e..66e1f00d 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -135,7 +135,7 @@ namespace StardewModdingAPI.Web modUrlFormat: api.ModDropModPageUrl )); - services.AddSingleton(new NexusWebScrapeClient( + services.AddSingleton(new NexusClient( userAgent: userAgent, baseUrl: api.NexusBaseUrl, modUrlFormat: api.NexusModUrlFormat, -- cgit