using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Threading.Tasks; using HtmlAgilityPack; using Pathoschild.FluentNexus.Models; using Pathoschild.Http.Client; using StardewModdingAPI.Toolkit; using FluentNexusClient = Pathoschild.FluentNexus.NexusClient; namespace StardewModdingAPI.Web.Framework.Clients.Nexus { /// An HTTP client for fetching mod metadata from the Nexus website. internal class NexusClient : INexusClient { /********* ** Fields *********/ /// The URL for a Nexus mod page for the user, excluding the base URL, where {0} is the mod ID. private readonly string WebModUrlFormat; /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. public string WebModScrapeUrlFormat { get; set; } /// The underlying HTTP client for the Nexus Mods website. private readonly IClient WebClient; /// The underlying HTTP client for the Nexus API. private readonly FluentNexusClient ApiClient; /********* ** Public methods *********/ /// Construct an instance. /// The user agent for the Nexus Mods web client. /// The base URL for the Nexus Mods site. /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. /// The app version to show in API user agents. /// The Nexus API authentication key. public NexusClient(string webUserAgent, string webBaseUrl, string webModUrlFormat, string webModScrapeUrlFormat, string apiAppVersion, string apiKey) { this.WebModUrlFormat = webModUrlFormat; this.WebModScrapeUrlFormat = webModScrapeUrlFormat; this.WebClient = new FluentClient(webBaseUrl).SetUserAgent(webUserAgent); this.ApiClient = new FluentNexusClient(apiKey, "SMAPI", apiAppVersion); } /// Get metadata about a mod. /// The Nexus mod ID. /// Returns the mod info if found, else null. public async Task GetModAsync(uint id) { // Fetch from the Nexus website when possible, since it has no rate limits. Mods with // adult content are hidden for anonymous users, so fall back to the API in that case. // Note that the API has very restrictive rate limits which means we can't just use it // for all cases. NexusMod mod = await this.GetModFromWebsiteAsync(id); if (mod?.Status == NexusModStatus.AdultContentForbidden) mod = await this.GetModFromApiAsync(id); return mod; } /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. public void Dispose() { this.WebClient?.Dispose(); } /********* ** Private methods *********/ /// Get metadata about a mod by scraping the Nexus website. /// The Nexus mod ID. /// Returns the mod info if found, else null. private async Task GetModFromWebsiteAsync(uint id) { // fetch HTML string html; try { html = await this.WebClient .GetAsync(string.Format(this.WebModScrapeUrlFormat, id)) .AsString(); } catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) { return null; } // parse HTML var doc = new HtmlDocument(); doc.LoadHtml(html); // handle Nexus error message HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'site-notice')][contains(@class, 'warning')]"); if (node != null) { string[] errorParts = node.InnerText.Trim().Split(new[] { '\n' }, 2, System.StringSplitOptions.RemoveEmptyEntries); string errorCode = errorParts[0]; string errorText = errorParts.Length > 1 ? errorParts[1] : null; switch (errorCode.Trim().ToLower()) { case "not found": return null; default: return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = this.GetWebStatus(errorCode) }; } } // extract mod info string url = this.GetModUrl(id); string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); SemanticVersion.TryParse(version, out ISemanticVersion parsedVersion); // extract file versions List rawVersions = new List(); foreach (var fileSection in doc.DocumentNode.SelectNodes("//div[contains(@class, 'files-tabs')]")) { string sectionName = fileSection.Descendants("h2").First().InnerText; if (sectionName != "Main files" && sectionName != "Optional files") continue; rawVersions.AddRange( from statBox in fileSection.Descendants().Where(p => p.HasClass("stat-version")) from versionStat in statBox.Descendants().Where(p => p.HasClass("stat")) select versionStat.InnerText.Trim() ); } // choose latest file version ISemanticVersion latestFileVersion = null; foreach (string rawVersion in rawVersions) { if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur)) continue; if (parsedVersion != null && !cur.IsNewerThan(parsedVersion)) continue; if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion)) continue; latestFileVersion = cur; } // yield info return new NexusMod { Name = name, Version = parsedVersion?.ToString() ?? version, LatestFileVersion = latestFileVersion, Url = url }; } /// Get metadata about a mod from the Nexus API. /// The Nexus mod ID. /// Returns the mod info if found, else null. private async Task GetModFromApiAsync(uint id) { // fetch mod Mod mod = await this.ApiClient.Mods.GetMod("stardewvalley", (int)id); ModFileList files = await this.ApiClient.ModFiles.GetModFiles("stardewvalley", (int)id, FileCategory.Main, FileCategory.Optional); // get versions if (!SemanticVersion.TryParse(mod.Version, out ISemanticVersion mainVersion)) mainVersion = null; ISemanticVersion latestFileVersion = null; foreach (string rawVersion in files.Files.Select(p => p.FileVersion)) { if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur)) continue; if (mainVersion != null && !cur.IsNewerThan(mainVersion)) continue; if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion)) continue; latestFileVersion = cur; } // yield info return new NexusMod { Name = mod.Name, Version = SemanticVersion.TryParse(mod.Version, out ISemanticVersion version) ? version?.ToString() : mod.Version, LatestFileVersion = latestFileVersion, Url = this.GetModUrl(id) }; } /// Get the full mod page URL for a given ID. /// The mod ID. private string GetModUrl(uint id) { UriBuilder builder = new UriBuilder(this.WebClient.BaseClient.BaseAddress); builder.Path += string.Format(this.WebModUrlFormat, id); return builder.Uri.ToString(); } /// Get the mod status for a web error code. /// The Nexus error code. private NexusModStatus GetWebStatus(string errorCode) { switch (errorCode.Trim().ToLower()) { case "adult content": return NexusModStatus.AdultContentForbidden; case "hidden mod": return NexusModStatus.Hidden; case "not published": return NexusModStatus.NotPublished; default: return NexusModStatus.Other; } } } }