From 03860cca868aceb6af67d9a6ad171c8ceb9be7eb Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Fri, 25 May 2018 19:26:09 -0400 Subject: add wiki compatibility list parsing to toolkit (#532) --- .../Clients/Wiki/WikiCompatibilityClient.cs | 142 +++++++++++++++++++++ .../Clients/Wiki/WikiCompatibilityEntry.cs | 24 ++++ .../Clients/Wiki/WikiCompatibilityStatus.cs | 24 ++++ 3 files changed, 190 insertions(+) create mode 100644 src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs create mode 100644 src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs create mode 100644 src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityStatus.cs (limited to 'src/StardewModdingAPI.Toolkit/Framework/Clients') diff --git a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs new file mode 100644 index 00000000..be03a120 --- /dev/null +++ b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading.Tasks; +using HtmlAgilityPack; +using Pathoschild.Http.Client; + +namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki +{ + /// An HTTP client for fetching mod metadata from the wiki compatibility list. + public class WikiCompatibilityClient : IDisposable + { + /********* + ** Properties + *********/ + /// The underlying HTTP client. + private readonly IClient Client; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The user agent for the wiki API. + /// The base URL for the wiki API. + public WikiCompatibilityClient(string userAgent, string baseUrl = "https://stardewvalleywiki.com/mediawiki/api.php") + { + this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); + } + + /// Fetch mod compatibility entries. + public async Task FetchAsync() + { + // fetch HTML + ResponseModel response = await this.Client + .GetAsync("") + .WithArguments(new + { + action = "parse", + page = "Modding:SMAPI_compatibility", + format = "json" + }) + .As(); + string html = response.Parse.Text["*"]; + + // parse HTML + var doc = new HtmlDocument(); + doc.LoadHtml(html); + + // find mod entries + HtmlNodeCollection modNodes = doc.DocumentNode.SelectNodes("table[@id='mod-list']//tr[@class='mod']"); + if (modNodes == null) + throw new InvalidOperationException("Can't parse wiki compatibility list, no mods found."); + + // parse + return this.ParseEntries(modNodes).ToArray(); + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.Client?.Dispose(); + } + + + /********* + ** Private methods + *********/ + /// Parse valid mod compatibility entries. + /// The HTML compatibility entries. + private IEnumerable ParseEntries(IEnumerable nodes) + { + foreach (HtmlNode node in nodes) + { + // parse status + WikiCompatibilityStatus status; + { + string rawStatus = node.GetAttributeValue("data-status", null); + if (rawStatus == null) + continue; // not a mod node? + if (!Enum.TryParse(rawStatus, true, out status)) + throw new InvalidOperationException($"Unknown status '{rawStatus}' when parsing compatibility list."); + } + + // parse unofficial version + ISemanticVersion unofficialVersion = null; + { + string rawUnofficialVersion = node.GetAttributeValue("data-unofficial-version", null); + SemanticVersion.TryParse(rawUnofficialVersion, out unofficialVersion); + } + + // parse other fields + int? nexusID = this.GetNullableIntAttribute(node, "data-nexus-id"); + int? chucklefishID = this.GetNullableIntAttribute(node, "data-chucklefish-id"); + string githubRepo = node.GetAttributeValue("data-github", null); + string customSourceUrl = node.GetAttributeValue("data-custom-source", null); + + // yield model + yield return new WikiCompatibilityEntry + { + Status = status, + NexusID = nexusID, + ChucklefishID = chucklefishID, + GitHubRepo = githubRepo, + CustomSourceUrl = customSourceUrl, + UnofficialVersion = unofficialVersion + }; + } + } + + /// Get a nullable integer attribute value. + /// The HTML node. + /// The attribute name. + private int? GetNullableIntAttribute(HtmlNode node, string attributeName) + { + string raw = node.GetAttributeValue(attributeName, null); + if (raw != null && int.TryParse(raw, out int value)) + return value; + return null; + } + + /// The response model for the MediaWiki parse API. + [SuppressMessage("ReSharper", "ClassNeverInstantiated.Local")] + [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Local")] + private class ResponseModel + { + /// The parse API results. + public ResponseParseModel Parse { get; set; } + } + + /// The inner response model for the MediaWiki parse API. + [SuppressMessage("ReSharper", "ClassNeverInstantiated.Local")] + [SuppressMessage("ReSharper", "CollectionNeverUpdated.Local")] + [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Local")] + private class ResponseParseModel + { + /// The parsed text. + public IDictionary Text { get; set; } + } + } +} diff --git a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs new file mode 100644 index 00000000..ceeb2de5 --- /dev/null +++ b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs @@ -0,0 +1,24 @@ +namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki +{ + /// An entry in the mod compatibility list. + public class WikiCompatibilityEntry + { + /// The mod ID on Nexus. + public int? NexusID { get; set; } + + /// The mod ID in the Chucklefish mod repo. + public int? ChucklefishID { get; set; } + + /// The GitHub repository in the form 'owner/repo'. + public string GitHubRepo { get; set; } + + /// The URL to a non-GitHub source repo. + public string CustomSourceUrl { get; set; } + + /// The version of the latest unofficial update, if applicable. + public ISemanticVersion UnofficialVersion { get; set; } + + /// The compatibility status. + public WikiCompatibilityStatus Status { get; set; } + } +} diff --git a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityStatus.cs b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityStatus.cs new file mode 100644 index 00000000..06252b2f --- /dev/null +++ b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityStatus.cs @@ -0,0 +1,24 @@ +namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki +{ + /// The compatibility status for a mod. + public enum WikiCompatibilityStatus + { + /// The mod is compatible. + Ok = 0, + + /// The mod is compatible if you use an optional official download. + Optional = 1, + + /// The mod isn't compatible, but the player can fix it or there's a good alternative. + Workaround = 2, + + /// The mod isn't compatible. + Broken = 3, + + /// The mod is no longer maintained by the author, and an unofficial update or continuation is unlikely. + Abandoned = 4, + + /// The mod is no longer needed and should be removed. + Obsolete = 5 + } +} -- cgit From 738c0ce386590f55c185297065aeda62b1f5d06f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 26 May 2018 15:21:07 -0400 Subject: improve wiki parsing (#532) --- .../Clients/Wiki/WikiCompatibilityClient.cs | 23 +++++++++++++++++++--- .../Clients/Wiki/WikiCompatibilityEntry.cs | 9 +++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) (limited to 'src/StardewModdingAPI.Toolkit/Framework/Clients') diff --git a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs index be03a120..a68a0b4e 100644 --- a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs +++ b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs @@ -91,19 +91,25 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki } // parse other fields + string name = node.Descendants("td").FirstOrDefault()?.InnerText?.Trim(); + string[] ids = this.GetAttribute(node, "data-id")?.Split(new [] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToArray() ?? new string[0]; int? nexusID = this.GetNullableIntAttribute(node, "data-nexus-id"); int? chucklefishID = this.GetNullableIntAttribute(node, "data-chucklefish-id"); - string githubRepo = node.GetAttributeValue("data-github", null); - string customSourceUrl = node.GetAttributeValue("data-custom-source", null); + string githubRepo = this.GetAttribute(node, "data-github"); + string customSourceUrl = this.GetAttribute(node, "data-custom-source"); + string customUrl = this.GetAttribute(node, "data-custom-url"); // yield model yield return new WikiCompatibilityEntry { + ID = ids, + Name = name, Status = status, NexusID = nexusID, ChucklefishID = chucklefishID, GitHubRepo = githubRepo, CustomSourceUrl = customSourceUrl, + CustomUrl = customUrl, UnofficialVersion = unofficialVersion }; } @@ -114,12 +120,23 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki /// The attribute name. private int? GetNullableIntAttribute(HtmlNode node, string attributeName) { - string raw = node.GetAttributeValue(attributeName, null); + string raw = this.GetAttribute(node, attributeName); if (raw != null && int.TryParse(raw, out int value)) return value; return null; } + /// Get a strings attribute value. + /// The HTML node. + /// The attribute name. + private string GetAttribute(HtmlNode node, string attributeName) + { + string raw = node.GetAttributeValue(attributeName, null); + if (raw != null) + raw = HtmlEntity.DeEntitize(raw); + return raw; + } + /// The response model for the MediaWiki parse API. [SuppressMessage("ReSharper", "ClassNeverInstantiated.Local")] [SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Local")] diff --git a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs index ceeb2de5..9aabdcba 100644 --- a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs +++ b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs @@ -3,6 +3,12 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki /// An entry in the mod compatibility list. public class WikiCompatibilityEntry { + /// The mod's unique ID. A mod may have multiple current IDs in rare cases (e.g. due to parallel releases or unofficial updates). + public string[] ID { get; set; } + + /// The mod's display name. + public string Name { get; set; } + /// The mod ID on Nexus. public int? NexusID { get; set; } @@ -15,6 +21,9 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki /// The URL to a non-GitHub source repo. public string CustomSourceUrl { get; set; } + /// The custom mod page URL (if applicable). + public string CustomUrl { get; set; } + /// The version of the latest unofficial update, if applicable. public ISemanticVersion UnofficialVersion { get; set; } -- cgit From 9945408aa4ea2f6e07c2820f6aa1c160c68423d5 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 30 May 2018 21:21:33 -0400 Subject: add summary and 'unofficial' status to wiki client (#532) --- .../Framework/Clients/Wiki/WikiCompatibilityClient.cs | 6 ++++-- .../Framework/Clients/Wiki/WikiCompatibilityEntry.cs | 3 +++ .../Framework/Clients/Wiki/WikiCompatibilityStatus.cs | 11 +++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) (limited to 'src/StardewModdingAPI.Toolkit/Framework/Clients') diff --git a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs index a68a0b4e..d0da42df 100644 --- a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs +++ b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityClient.cs @@ -92,7 +92,8 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki // parse other fields string name = node.Descendants("td").FirstOrDefault()?.InnerText?.Trim(); - string[] ids = this.GetAttribute(node, "data-id")?.Split(new [] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToArray() ?? new string[0]; + string summary = node.Descendants("td").FirstOrDefault(p => p.GetAttributeValue("class", null) == "summary")?.InnerText.Trim(); + string[] ids = this.GetAttribute(node, "data-id")?.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToArray() ?? new string[0]; int? nexusID = this.GetNullableIntAttribute(node, "data-nexus-id"); int? chucklefishID = this.GetNullableIntAttribute(node, "data-chucklefish-id"); string githubRepo = this.GetAttribute(node, "data-github"); @@ -110,7 +111,8 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki GitHubRepo = githubRepo, CustomSourceUrl = customSourceUrl, CustomUrl = customUrl, - UnofficialVersion = unofficialVersion + UnofficialVersion = unofficialVersion, + Summary = summary }; } } diff --git a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs index 9aabdcba..8bc66e20 100644 --- a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs +++ b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityEntry.cs @@ -29,5 +29,8 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki /// The compatibility status. public WikiCompatibilityStatus Status { get; set; } + + /// The human-readable summary of the compatibility status or workaround, without HTML formatitng. + public string Summary { get; set; } } } diff --git a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityStatus.cs b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityStatus.cs index 06252b2f..a1d2dfae 100644 --- a/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityStatus.cs +++ b/src/StardewModdingAPI.Toolkit/Framework/Clients/Wiki/WikiCompatibilityStatus.cs @@ -9,16 +9,19 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki /// The mod is compatible if you use an optional official download. Optional = 1, + /// The mod is compatible if you use an unofficial update. + Unofficial = 2, + /// The mod isn't compatible, but the player can fix it or there's a good alternative. - Workaround = 2, + Workaround = 3, /// The mod isn't compatible. - Broken = 3, + Broken = 4, /// The mod is no longer maintained by the author, and an unofficial update or continuation is unlikely. - Abandoned = 4, + Abandoned = 5, /// The mod is no longer needed and should be removed. - Obsolete = 5 + Obsolete = 6 } } -- cgit