From 212e85489a456acca6889faff5da835cbb707080 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 25 Feb 2018 23:27:44 -0500 Subject: fix log parser not correctly parsing mod list if a mod has no author name --- src/SMAPI.Web/Framework/LogParsing/LogParser.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/LogParsing/LogParser.cs b/src/SMAPI.Web/Framework/LogParsing/LogParser.cs index 23a1baa4..385accf0 100644 --- a/src/SMAPI.Web/Framework/LogParsing/LogParser.cs +++ b/src/SMAPI.Web/Framework/LogParsing/LogParser.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; +using StardewModdingAPI.Common; using StardewModdingAPI.Web.Framework.LogParsing.Models; namespace StardewModdingAPI.Web.Framework.LogParsing @@ -29,7 +30,8 @@ namespace StardewModdingAPI.Web.Framework.LogParsing private readonly Regex ModListStartPattern = new Regex(@"^Loaded \d+ mods:$", RegexOptions.Compiled | RegexOptions.IgnoreCase); /// A regex pattern matching an entry in SMAPI's mod list. - private readonly Regex ModListEntryPattern = new Regex(@"^ (?.+) (?.+) by (?.+) \| (?.+)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + /// The author name and description are optional. + private readonly Regex ModListEntryPattern = new Regex(@"^ (?.+?) (?" + SemanticVersionImpl.UnboundedVersionPattern + @")(?: by (?[^\|]+))?(?: \| (?.+))?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); /// A regex pattern matching the start of SMAPI's content pack list. private readonly Regex ContentPackListStartPattern = new Regex(@"^Loaded \d+ content packs:$", RegexOptions.Compiled | RegexOptions.IgnoreCase); -- cgit From 5c1318431b716e13feb214abe8116cb1498047a5 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 25 Feb 2018 23:35:14 -0500 Subject: always include raw taxt in model --- src/SMAPI.Web/Framework/LogParsing/LogParser.cs | 8 +++++--- src/SMAPI.Web/Framework/LogParsing/Models/ParsedLog.cs | 4 ++-- src/SMAPI.Web/Views/LogParser/Index.cshtml | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/LogParsing/LogParser.cs b/src/SMAPI.Web/Framework/LogParsing/LogParser.cs index 385accf0..9e44f163 100644 --- a/src/SMAPI.Web/Framework/LogParsing/LogParser.cs +++ b/src/SMAPI.Web/Framework/LogParsing/LogParser.cs @@ -55,6 +55,7 @@ namespace StardewModdingAPI.Web.Framework.LogParsing return new ParsedLog { IsValid = false, + RawText = logText, Error = "The log is empty." }; } @@ -63,7 +64,8 @@ namespace StardewModdingAPI.Web.Framework.LogParsing ParsedLog log = new ParsedLog { IsValid = true, - Messages = this.CollapseRepeats(this.GetMessages(logText)).ToArray(), + RawText = logText, + Messages = this.CollapseRepeats(this.GetMessages(logText)).ToArray() }; // parse log messages @@ -154,7 +156,7 @@ namespace StardewModdingAPI.Web.Framework.LogParsing { IsValid = false, Error = ex.Message, - RawTextIfError = logText + RawText = logText }; } catch (Exception ex) @@ -163,7 +165,7 @@ namespace StardewModdingAPI.Web.Framework.LogParsing { IsValid = false, Error = $"Parsing the log file failed. Technical details:\n{ex}", - RawTextIfError = logText + RawText = logText }; } } diff --git a/src/SMAPI.Web/Framework/LogParsing/Models/ParsedLog.cs b/src/SMAPI.Web/Framework/LogParsing/Models/ParsedLog.cs index 31ef2fe1..a82b6a1b 100644 --- a/src/SMAPI.Web/Framework/LogParsing/Models/ParsedLog.cs +++ b/src/SMAPI.Web/Framework/LogParsing/Models/ParsedLog.cs @@ -17,8 +17,8 @@ namespace StardewModdingAPI.Web.Framework.LogParsing.Models /// An error message indicating why the log file is invalid. public string Error { get; set; } - /// The raw text if is false. - public string RawTextIfError { get; set; } + /// The raw log text. + public string RawText { get; set; } /**** ** Log data diff --git a/src/SMAPI.Web/Views/LogParser/Index.cshtml b/src/SMAPI.Web/Views/LogParser/Index.cshtml index 39557d50..310277eb 100644 --- a/src/SMAPI.Web/Views/LogParser/Index.cshtml +++ b/src/SMAPI.Web/Views/LogParser/Index.cshtml @@ -155,7 +155,7 @@ else if (Model.ParsedLog?.IsValid == false)

Raw log

- 
@Model.ParsedLog.RawTextIfError
+ 
@Model.ParsedLog.RawText
}
-- cgit From cea113eb2908105c5b5d5ed86a8221e95ea92fb4 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 4 Mar 2018 17:32:29 -0500 Subject: fetch mod info from Nexus website until we can use their API again (#454) --- .../Framework/Clients/Nexus/NexusClient.cs | 2 +- src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs | 4 + .../Clients/Nexus/NexusWebScrapeClient.cs | 93 ++++++++++++++++++++++ .../Framework/ModRepositories/NexusRepository.cs | 8 +- src/SMAPI.Web/Startup.cs | 9 ++- 5 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs index 1a7011e2..adec41be 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs @@ -3,7 +3,7 @@ using Pathoschild.Http.Client; namespace StardewModdingAPI.Web.Framework.Clients.Nexus { - /// An HTTP client for fetching mod metadata from Nexus Mods. + /// An HTTP client for fetching mod metadata from the Nexus Mods API. internal class NexusClient : INexusClient { /********* diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs index 2b04104f..cd52c72b 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusMod.cs @@ -17,5 +17,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus /// The mod's web URL. [JsonProperty("mod_page_uri")] public string Url { get; set; } + + /// A user-friendly error which indicates why fetching the mod info failed (if applicable). + [JsonIgnore] + public string Error { get; set; } } } diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs new file mode 100644 index 00000000..1e941e3d --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs @@ -0,0 +1,93 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using HtmlAgilityPack; +using Pathoschild.Http.Client; + +namespace StardewModdingAPI.Web.Framework.Clients.Nexus +{ + /// An HTTP client for fetching mod metadata from the Nexus website. + internal class NexusWebScrapeClient : INexusClient + { + /********* + ** Properties + *********/ + /// The URL for a Nexus web page excluding the base URL, where {0} is the mod ID. + private readonly string ModUrlFormat; + + /// The underlying HTTP client. + private readonly IClient Client; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The user agent for the Nexus Mods API client. + /// The base URL for the Nexus Mods site. + /// The URL for a Nexus Mods web page excluding the , where {0} is the mod ID. + public NexusWebScrapeClient(string userAgent, string baseUrl, string modUrlFormat) + { + this.ModUrlFormat = modUrlFormat; + this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); + } + + /// Get metadata about a mod. + /// The Nexus mod ID. + /// Returns the mod info if found, else null. + public async Task GetModAsync(uint id) + { + // fetch HTML + string html; + try + { + html = await this.Client + .GetAsync(string.Format(this.ModUrlFormat, id)) + .AsString(); + } + catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + return null; + } + + // parse HTML + var doc = new HtmlDocument(); + doc.LoadHtml(html); + + // handle Nexus error message + HtmlNode node = doc.DocumentNode.SelectSingleNode("//div[contains(@class, 'site-notice')][contains(@class, 'warning')]"); + if (node != null) + { + string[] errorParts = node.InnerText.Trim().Split(new[] { '\n' }, 2, System.StringSplitOptions.RemoveEmptyEntries); + string errorCode = errorParts[0]; + string errorText = errorParts.Length > 1 ? errorParts[1] : null; + switch (errorCode.Trim().ToLower()) + { + case "not found": + return null; + + default: + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText})." }; + } + } + + // extract mod info + string url = new UriBuilder(this.Client.BaseClient.BaseAddress) { Path = string.Format(this.ModUrlFormat, id) }.Uri.ToString(); + string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); + string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); + + return new NexusMod + { + Name = name, + Version = version, + Url = url + }; + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.Client?.Dispose(); + } + } +} diff --git a/src/SMAPI.Web/Framework/ModRepositories/NexusRepository.cs b/src/SMAPI.Web/Framework/ModRepositories/NexusRepository.cs index cfa757ab..e1dc0fcc 100644 --- a/src/SMAPI.Web/Framework/ModRepositories/NexusRepository.cs +++ b/src/SMAPI.Web/Framework/ModRepositories/NexusRepository.cs @@ -39,9 +39,11 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories try { NexusMod mod = await this.Client.GetModAsync(nexusID); - return mod != null - ? new ModInfoModel(mod.Name, this.NormaliseVersion(mod.Version), mod.Url) - : new ModInfoModel("Found no mod with this ID."); + if (mod == null) + return new ModInfoModel("Found no mod with this ID."); + if (mod.Error != null) + return new ModInfoModel(mod.Error); + return new ModInfoModel(mod.Name, this.NormaliseVersion(mod.Version), mod.Url); } catch (Exception ex) { diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 93135239..d7d4d074 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -81,8 +81,13 @@ namespace StardewModdingAPI.Web password: api.GitHubPassword )); - services.AddSingleton(new NexusClient( - userAgent: api.NexusUserAgent, + //services.AddSingleton(new NexusClient( + // userAgent: api.NexusUserAgent, + // baseUrl: api.NexusBaseUrl, + // modUrlFormat: api.NexusModUrlFormat + //)); + services.AddSingleton(new NexusWebScrapeClient( + userAgent: userAgent, baseUrl: api.NexusBaseUrl, modUrlFormat: api.NexusModUrlFormat )); -- cgit From 0a71285c631a6ef52ea381c35224f353a618e3fa Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 4 Mar 2018 21:05:50 -0500 Subject: fix broken Nexus mod URLs --- .../Clients/Chucklefish/ChucklefishClient.cs | 19 ++++++++++++++----- .../Framework/Clients/Nexus/NexusWebScrapeClient.cs | 15 ++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/Clients/Chucklefish/ChucklefishClient.cs b/src/SMAPI.Web/Framework/Clients/Chucklefish/ChucklefishClient.cs index 6772672c..029553ce 100644 --- a/src/SMAPI.Web/Framework/Clients/Chucklefish/ChucklefishClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Chucklefish/ChucklefishClient.cs @@ -12,9 +12,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Chucklefish /********* ** Properties *********/ - /// The base URL for the Chucklefish mod site. - private readonly string BaseUrl; - /// The URL for a mod page excluding the base URL, where {0} is the mod ID. private readonly string ModPageUrlFormat; @@ -31,7 +28,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Chucklefish /// The URL for a mod page excluding the , where {0} is the mod ID. public ChucklefishClient(string userAgent, string baseUrl, string modPageUrlFormat) { - this.BaseUrl = baseUrl; this.ModPageUrlFormat = modPageUrlFormat; this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); } @@ -59,7 +55,7 @@ namespace StardewModdingAPI.Web.Framework.Clients.Chucklefish doc.LoadHtml(html); // extract mod info - string url = new UriBuilder(new Uri(this.BaseUrl)) { Path = string.Format(this.ModPageUrlFormat, id) }.Uri.ToString(); + string url = this.GetModUrl(id); string name = doc.DocumentNode.SelectSingleNode("//meta[@name='twitter:title']").Attributes["content"].Value; if (name.StartsWith("[SMAPI] ")) name = name.Substring("[SMAPI] ".Length); @@ -79,5 +75,18 @@ namespace StardewModdingAPI.Web.Framework.Clients.Chucklefish { this.Client?.Dispose(); } + + + /********* + ** Private methods + *********/ + /// Get the full mod page URL for a given ID. + /// The mod ID. + private string GetModUrl(uint id) + { + UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); + builder.Path += string.Format(this.ModPageUrlFormat, id); + return builder.Uri.ToString(); + } } } diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs index 1e941e3d..d0597965 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusWebScrapeClient.cs @@ -72,7 +72,7 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus } // extract mod info - string url = new UriBuilder(this.Client.BaseClient.BaseAddress) { Path = string.Format(this.ModUrlFormat, id) }.Uri.ToString(); + string url = this.GetModUrl(id); string name = doc.DocumentNode.SelectSingleNode("//h1")?.InnerText.Trim(); string version = doc.DocumentNode.SelectSingleNode("//ul[contains(@class, 'stats')]//li[@class='stat-version']//div[@class='stat']")?.InnerText.Trim(); @@ -89,5 +89,18 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus { this.Client?.Dispose(); } + + + /********* + ** Private methods + *********/ + /// Get the full mod page URL for a given ID. + /// The mod ID. + private string GetModUrl(uint id) + { + UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); + builder.Path += string.Format(this.ModUrlFormat, id); + return builder.Uri.ToString(); + } } } -- cgit