using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Threading.Tasks; using HtmlAgilityPack; using Pathoschild.Http.Client; using StardewModdingAPI.Toolkit; namespace StardewModdingAPI.Web.Framework.Clients.Nexus { /// An HTTP client for fetching mod metadata from the Nexus website. internal class NexusWebScrapeClient : INexusClient { /********* ** Fields *********/ /// The URL for a Nexus mod page for the user, excluding the base URL, where {0} is the mod ID. private readonly string ModUrlFormat; /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. public string ModScrapeUrlFormat { get; set; } /// The underlying HTTP client. private readonly IClient Client; /********* ** Public methods *********/ /// Construct an instance. /// The user agent for the Nexus Mods API client. /// The base URL for the Nexus Mods site. /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. public NexusWebScrapeClient(string userAgent, string baseUrl, string modUrlFormat, string modScrapeUrlFormat) { this.ModUrlFormat = modUrlFormat; this.ModScrapeUrlFormat = modScrapeUrlFormat; this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); } /// Get metadata about a mod. /// The Nexus mod ID. /// Returns the mod info if found, else null. public async Task GetModAsync(uint id) { // fetch HTML string html; try { html = await this.Client .GetAsync(string.Format(this.ModScrapeUrlFormat, id)) .AsString(); } catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) { return null; } // parse HTML var doc = new HtmlDocument(); doc.LoadHtml(html); // handle Nexus error message HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'site-notice')][contains(@class, 'warning')]"); if (node != null) { string[] errorParts = node.InnerText.Trim().Split(new[] { '\n' }, 2, System.StringSplitOptions.RemoveEmptyEntries); string errorCode = errorParts[0]; string errorText = errorParts.Length > 1 ? errorParts[1] : null; switch (errorCode.Trim().ToLower()) { case "not found": return null; case "hidden mod": return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Hidden }; case "not published": return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.NotPublished }; default: return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Other }; } } // extract mod info string url = this.GetModUrl(id); string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); SemanticVersion.TryParse(version, out ISemanticVersion parsedVersion); // extract file versions List rawVersions = new List(); foreach (var fileSection in doc.DocumentNode.SelectNodes("//div[contains(@class, 'files-tabs')]")) { string sectionName = fileSection.Descendants("h2").First().InnerText; if (sectionName != "Main files" && sectionName != "Optional files") continue; rawVersions.AddRange( from statBox in fileSection.Descendants().Where(p => p.HasClass("stat-version")) from versionStat in statBox.Descendants().Where(p => p.HasClass("stat")) select versionStat.InnerText.Trim() ); } // choose latest file version ISemanticVersion latestFileVersion = null; foreach (string rawVersion in rawVersions) { if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur)) continue; if (parsedVersion != null && !cur.IsNewerThan(parsedVersion)) continue; if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion)) continue; latestFileVersion = cur; } // yield info return new NexusMod { Name = name, Version = parsedVersion?.ToString() ?? version, LatestFileVersion = latestFileVersion, Url = url }; } /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. public void Dispose() { this.Client?.Dispose(); } /********* ** Private methods *********/ /// Get the full mod page URL for a given ID. /// The mod ID. private string GetModUrl(uint id) { UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); builder.Path += string.Format(this.ModUrlFormat, id); return builder.Uri.ToString(); } } }