From cea113eb2908105c5b5d5ed86a8221e95ea92fb4 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 4 Mar 2018 17:32:29 -0500 Subject: fetch mod info from Nexus website until we can use their API again (#454) --- .../Framework/Clients/Nexus/NexusClient.cs | 2 +- src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs | 4 + .../Clients/Nexus/NexusWebScrapeClient.cs | 93 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs (limited to 'src/SMAPI.Web/Framework/Clients/Nexus') diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs index 1a7011e2..adec41be 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs @@ -3,7 +3,7 @@ using Pathoschild.Http.Client; namespace StardewModdingAPI.Web.Framework.Clients.Nexus { - /// An HTTP client for fetching mod metadata from Nexus Mods. + /// An HTTP client for fetching mod metadata from the Nexus Mods API. internal class NexusClient : INexusClient { /********* diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs index 2b04104f..cd52c72b 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs @@ -17,5 +17,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus /// The mod's web URL. [JsonProperty("mod_page_uri")] public string Url { get; set; } + + /// A user-friendly error which indicates why fetching the mod info failed (if applicable). + [JsonIgnore] + public string Error { get; set; } } } diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs new file mode 100644 index 00000000..1e941e3d --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs @@ -0,0 +1,93 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using HtmlAgilityPack; +using Pathoschild.Http.Client; + +namespace StardewModdingAPI.Web.Framework.Clients.Nexus +{ + /// An HTTP client for fetching mod metadata from the Nexus website. + internal class NexusWebScrapeClient : INexusClient + { + /********* + ** Properties + *********/ + /// The URL for a Nexus web page excluding the base URL, where {0} is the mod ID. + private readonly string ModUrlFormat; + + /// The underlying HTTP client. + private readonly IClient Client; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The user agent for the Nexus Mods API client. + /// The base URL for the Nexus Mods site. + /// The URL for a Nexus Mods web page excluding the , where {0} is the mod ID. + public NexusWebScrapeClient(string userAgent, string baseUrl, string modUrlFormat) + { + this.ModUrlFormat = modUrlFormat; + this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); + } + + /// Get metadata about a mod. + /// The Nexus mod ID. + /// Returns the mod info if found, else null. + public async Task GetModAsync(uint id) + { + // fetch HTML + string html; + try + { + html = await this.Client + .GetAsync(string.Format(this.ModUrlFormat, id)) + .AsString(); + } + catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + return null; + } + + // parse HTML + var doc = new HtmlDocument(); + doc.LoadHtml(html); + + // handle Nexus error message + HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'site-notice')][contains(@class, 'warning')]"); + if (node != null) + { + string[] errorParts = node.InnerText.Trim().Split(new[] { '\n' }, 2, System.StringSplitOptions.RemoveEmptyEntries); + string errorCode = errorParts[0]; + string errorText = errorParts.Length > 1 ? errorParts[1] : null; + switch (errorCode.Trim().ToLower()) + { + case "not found": + return null; + + default: + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText})." }; + } + } + + // extract mod info + string url = new UriBuilder(this.Client.BaseClient.BaseAddress) { Path = string.Format(this.ModUrlFormat, id) }.Uri.ToString(); + string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); + string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); + + return new NexusMod + { + Name = name, + Version = version, + Url = url + }; + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.Client?.Dispose(); + } + } +} -- cgit From 0a71285c631a6ef52ea381c35224f353a618e3fa Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 4 Mar 2018 21:05:50 -0500 Subject: fix broken Nexus mod URLs --- .../Clients/Chucklefish/ChucklefishClient.cs | 19 ++++++++++++++----- .../Framework/Clients/Nexus/NexusWebScrapeClient.cs | 15 ++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) (limited to 'src/SMAPI.Web/Framework/Clients/Nexus') diff --git a/src/SMAPI.Web/Framework/Clients/Chucklefish/ChucklefishClient.cs b/src/SMAPI.Web/Framework/Clients/Chucklefish/ChucklefishClient.cs index 6772672c..029553ce 100644 --- a/src/SMAPI.Web/Framework/Clients/Chucklefish/ChucklefishClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Chucklefish/ChucklefishClient.cs @@ -12,9 +12,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Chucklefish /********* ** Properties *********/ - /// The base URL for the Chucklefish mod site. - private readonly string BaseUrl; - /// The URL for a mod page excluding the base URL, where {0} is the mod ID. private readonly string ModPageUrlFormat; @@ -31,7 +28,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Chucklefish /// The URL for a mod page excluding the , where {0} is the mod ID. public ChucklefishClient(string userAgent, string baseUrl, string modPageUrlFormat) { - this.BaseUrl = baseUrl; this.ModPageUrlFormat = modPageUrlFormat; this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); } @@ -59,7 +55,7 @@ namespace StardewModdingAPI.Web.Framework.Clients.Chucklefish doc.LoadHtml(html); // extract mod info - string url = new UriBuilder(new Uri(this.BaseUrl)) { Path = string.Format(this.ModPageUrlFormat, id) }.Uri.ToString(); + string url = this.GetModUrl(id); string name = doc.DocumentNode.SelectSingleNode("//meta[@name='twitter:title']").Attributes["content"].Value; if (name.StartsWith("[SMAPI] ")) name = name.Substring("[SMAPI] ".Length); @@ -79,5 +75,18 @@ namespace StardewModdingAPI.Web.Framework.Clients.Chucklefish { this.Client?.Dispose(); } + + + /********* + ** Private methods + *********/ + /// Get the full mod page URL for a given ID. + /// The mod ID. + private string GetModUrl(uint id) + { + UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); + builder.Path += string.Format(this.ModPageUrlFormat, id); + return builder.Uri.ToString(); + } } } diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs index 1e941e3d..d0597965 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs @@ -72,7 +72,7 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus } // extract mod info - string url = new UriBuilder(this.Client.BaseClient.BaseAddress) { Path = string.Format(this.ModUrlFormat, id) }.Uri.ToString(); + string url = this.GetModUrl(id); string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); @@ -89,5 +89,18 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus { this.Client?.Dispose(); } + + + /********* + ** Private methods + *********/ + /// Get the full mod page URL for a given ID. + /// The mod ID. + private string GetModUrl(uint id) + { + UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); + builder.Path += string.Format(this.ModUrlFormat, id); + return builder.Uri.ToString(); + } } } -- cgit