diff options
author | Jesse Plamondon-Willard <Pathoschild@users.noreply.github.com> | 2019-11-24 13:49:30 -0500 |
---|---|---|
committer | Jesse Plamondon-Willard <Pathoschild@users.noreply.github.com> | 2019-11-24 13:49:30 -0500 |
commit | a3f21685049cabf2d824c8060dc0b1de47e9449e (patch) | |
tree | ad9add30e9da2a50e0ea0245f1546b7378f0d282 /src/SMAPI.Web/Controllers | |
parent | 6521df7b131924835eb797251c1e956fae0d6e13 (diff) | |
parent | 277bf082675b98b95bf6184fe3c7a45b969c7ac2 (diff) | |
download | SMAPI-a3f21685049cabf2d824c8060dc0b1de47e9449e.tar.gz SMAPI-a3f21685049cabf2d824c8060dc0b1de47e9449e.tar.bz2 SMAPI-a3f21685049cabf2d824c8060dc0b1de47e9449e.zip |
Merge branch 'develop' into stable
Diffstat (limited to 'src/SMAPI.Web/Controllers')
-rw-r--r-- | src/SMAPI.Web/Controllers/JsonValidatorController.cs | 349 | ||||
-rw-r--r-- | src/SMAPI.Web/Controllers/LogParserController.cs | 109 | ||||
-rw-r--r-- | src/SMAPI.Web/Controllers/ModsApiController.cs | 279 | ||||
-rw-r--r-- | src/SMAPI.Web/Controllers/ModsController.cs | 50 |
4 files changed, 603 insertions, 184 deletions
diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs new file mode 100644 index 00000000..b2eb9a87 --- /dev/null +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -0,0 +1,349 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Schema; +using StardewModdingAPI.Web.Framework; +using StardewModdingAPI.Web.Framework.Clients.Pastebin; +using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.ConfigModels; +using StardewModdingAPI.Web.ViewModels.JsonValidator; + +namespace StardewModdingAPI.Web.Controllers +{ + /// <summary>Provides a web UI for validating JSON schemas.</summary> + internal class JsonValidatorController : Controller + { + /********* + ** Fields + *********/ + /// <summary>The site config settings.</summary> + private readonly SiteConfig Config; + + /// <summary>The underlying Pastebin client.</summary> + private readonly IPastebinClient Pastebin; + + /// <summary>The underlying text compression helper.</summary> + private readonly IGzipHelper GzipHelper; + + /// <summary>The section URL for the schema validator.</summary> + private string SectionUrl => this.Config.JsonValidatorUrl; + + /// <summary>The supported JSON schemas (names indexed by ID).</summary> + private readonly IDictionary<string, string> SchemaFormats = new Dictionary<string, string> + { + ["none"] = "None", + ["manifest"] = "Manifest", + ["content-patcher"] = "Content Patcher" + }; + + /// <summary>The schema ID to use if none was specified.</summary> + private string DefaultSchemaID = "manifest"; + + /// <summary>A token in an error message which indicates that the child errors should be displayed instead.</summary> + private readonly string TransparentToken = "$transparent"; + + + /********* + ** Public methods + *********/ + /*** + ** Constructor + ***/ + /// <summary>Construct an instance.</summary> + /// <param name="siteConfig">The context config settings.</param> + /// <param name="pastebin">The Pastebin API client.</param> + /// <param name="gzipHelper">The underlying text compression helper.</param> + public JsonValidatorController(IOptions<SiteConfig> siteConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + { + this.Config = siteConfig.Value; + this.Pastebin = pastebin; + this.GzipHelper = gzipHelper; + } + + /*** + ** Web UI + ***/ + /// <summary>Render the schema validator UI.</summary> + /// <param name="schemaName">The schema name with which to validate the JSON.</param> + /// <param name="id">The paste ID.</param> + [HttpGet] + [Route("json")] + [Route("json/{schemaName}")] + [Route("json/{schemaName}/{id}")] + public async Task<ViewResult> Index(string schemaName = null, string id = null) + { + schemaName = this.NormalizeSchemaName(schemaName); + + var result = new JsonValidatorModel(this.SectionUrl, id, schemaName, this.SchemaFormats); + if (string.IsNullOrWhiteSpace(id)) + return this.View("Index", result); + + // fetch raw JSON + PasteInfo paste = await this.GetAsync(id); + if (string.IsNullOrWhiteSpace(paste.Content)) + return this.View("Index", result.SetUploadError("The JSON file seems to be empty.")); + result.SetContent(paste.Content); + + // parse JSON + JToken parsed; + try + { + parsed = JToken.Parse(paste.Content, new JsonLoadSettings + { + DuplicatePropertyNameHandling = DuplicatePropertyNameHandling.Error, + CommentHandling = CommentHandling.Load + }); + } + catch (JsonReaderException ex) + { + return this.View("Index", result.AddErrors(new JsonValidatorErrorModel(ex.LineNumber, ex.Path, ex.Message, ErrorType.None))); + } + + // format JSON + result.SetContent(parsed.ToString(Formatting.Indented)); + + // skip if no schema selected + if (schemaName == "none") + return this.View("Index", result); + + // load schema + JSchema schema; + { + FileInfo schemaFile = this.FindSchemaFile(schemaName); + if (schemaFile == null) + return this.View("Index", result.SetParseError($"Invalid schema '{schemaName}'.")); + schema = JSchema.Parse(System.IO.File.ReadAllText(schemaFile.FullName)); + } + + // get format doc URL + result.FormatUrl = this.GetExtensionField<string>(schema, "@documentationUrl"); + + // validate JSON + parsed.IsValid(schema, out IList<ValidationError> rawErrors); + var errors = rawErrors + .SelectMany(this.GetErrorModels) + .ToArray(); + return this.View("Index", result.AddErrors(errors)); + } + + /*** + ** JSON + ***/ + /// <summary>Save raw JSON data.</summary> + [HttpPost, AllowLargePosts] + [Route("json")] + public async Task<ActionResult> PostAsync(JsonValidatorRequestModel request) + { + if (request == null) + return this.View("Index", new JsonValidatorModel(this.SectionUrl, null, null, this.SchemaFormats).SetUploadError("The request seems to be invalid.")); + + // normalize schema name + string schemaName = this.NormalizeSchemaName(request.SchemaName); + + // get raw log text + string input = request.Content; + if (string.IsNullOrWhiteSpace(input)) + return this.View("Index", new JsonValidatorModel(this.SectionUrl, null, schemaName, this.SchemaFormats).SetUploadError("The JSON file seems to be empty.")); + + // upload log + input = this.GzipHelper.CompressString(input); + SavePasteResult result = await this.Pastebin.PostAsync($"JSON validator {DateTime.UtcNow:s}", input); + + // handle errors + if (!result.Success) + return this.View("Index", new JsonValidatorModel(this.SectionUrl, result.ID, schemaName, this.SchemaFormats).SetUploadError($"Pastebin error: {result.Error ?? "unknown error"}")); + + // redirect to view + UriBuilder uri = new UriBuilder(new Uri(this.SectionUrl)); + uri.Path = $"{uri.Path.TrimEnd('/')}/{schemaName}/{result.ID}"; + return this.Redirect(uri.Uri.ToString()); + } + + + /********* + ** Private methods + *********/ + /// <summary>Fetch raw text from Pastebin.</summary> + /// <param name="id">The Pastebin paste ID.</param> + private async Task<PasteInfo> GetAsync(string id) + { + PasteInfo response = await this.Pastebin.GetAsync(id); + response.Content = this.GzipHelper.DecompressString(response.Content); + return response; + } + + /// <summary>Get a normalized schema name, or the <see cref="DefaultSchemaID"/> if blank.</summary> + /// <param name="schemaName">The raw schema name to normalize.</param> + private string NormalizeSchemaName(string schemaName) + { + schemaName = schemaName?.Trim().ToLower(); + return !string.IsNullOrWhiteSpace(schemaName) + ? schemaName + : this.DefaultSchemaID; + } + + /// <summary>Get the schema file given its unique ID.</summary> + /// <param name="id">The schema ID.</param> + private FileInfo FindSchemaFile(string id) + { + // normalize ID + id = id?.Trim().ToLower(); + if (string.IsNullOrWhiteSpace(id)) + return null; + + // get matching file + DirectoryInfo schemaDir = new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "schemas")); + foreach (FileInfo file in schemaDir.EnumerateFiles("*.json")) + { + if (file.Name.Equals($"{id}.json")) + return file; + } + + return null; + } + + /// <summary>Get view models representing a schema validation error and any child errors.</summary> + /// <param name="error">The error to represent.</param> + private IEnumerable<JsonValidatorErrorModel> GetErrorModels(ValidationError error) + { + // skip through transparent errors + if (this.IsTransparentError(error)) + { + foreach (var model in error.ChildErrors.SelectMany(this.GetErrorModels)) + yield return model; + yield break; + } + + // get message + string message = this.GetOverrideError(error); + if (message == null || message == this.TransparentToken) + message = this.FlattenErrorMessage(error); + + // build model + yield return new JsonValidatorErrorModel(error.LineNumber, error.Path, message, error.ErrorType); + } + + /// <summary>Get a flattened, human-readable message for a schema validation error and any child errors.</summary> + /// <param name="error">The error to represent.</param> + /// <param name="indent">The indentation level to apply for inner errors.</param> + private string FlattenErrorMessage(ValidationError error, int indent = 0) + { + // get override + string message = this.GetOverrideError(error); + if (message != null && message != this.TransparentToken) + return message; + + // skip through transparent errors + if (this.IsTransparentError(error)) + error = error.ChildErrors[0]; + + // get friendly representation of main error + message = error.Message; + switch (error.ErrorType) + { + case ErrorType.Const: + message = $"Invalid value. Found '{error.Value}', but expected '{error.Schema.Const}'."; + break; + + case ErrorType.Enum: + message = $"Invalid value. Found '{error.Value}', but expected one of '{string.Join("', '", error.Schema.Enum)}'."; + break; + + case ErrorType.Required: + message = $"Missing required fields: {string.Join(", ", (List<string>)error.Value)}."; + break; + } + + // add inner errors + foreach (ValidationError childError in error.ChildErrors) + message += "\n" + "".PadLeft(indent * 2, ' ') + $"==> {childError.Path}: " + this.FlattenErrorMessage(childError, indent + 1); + return message; + } + + /// <summary>Get whether a validation error should be omitted in favor of its child errors in user-facing error messages.</summary> + /// <param name="error">The error to check.</param> + private bool IsTransparentError(ValidationError error) + { + if (!error.ChildErrors.Any()) + return false; + + string @override = this.GetOverrideError(error); + return + @override == this.TransparentToken + || (error.ErrorType == ErrorType.Then && @override == null); + } + + /// <summary>Get an override error from the JSON schema, if any.</summary> + /// <param name="error">The schema validation error.</param> + private string GetOverrideError(ValidationError error) + { + string GetRawOverrideError() + { + // get override errors + IDictionary<string, string> errors = this.GetExtensionField<Dictionary<string, string>>(error.Schema, "@errorMessages"); + if (errors == null) + return null; + errors = new Dictionary<string, string>(errors, StringComparer.InvariantCultureIgnoreCase); + + // match error by type and message + foreach (var pair in errors) + { + if (!pair.Key.Contains(":")) + continue; + + string[] parts = pair.Key.Split(':', 2); + if (parts[0].Equals(error.ErrorType.ToString(), StringComparison.InvariantCultureIgnoreCase) && Regex.IsMatch(error.Message, parts[1])) + return pair.Value?.Trim(); + } + + // match by type + if (errors.TryGetValue(error.ErrorType.ToString(), out string message)) + return message?.Trim(); + + return null; + } + + return GetRawOverrideError() + ?.Replace("@value", this.FormatValue(error.Value)); + } + + /// <summary>Get an extension field from a JSON schema.</summary> + /// <typeparam name="T">The field type.</typeparam> + /// <param name="schema">The schema whose extension fields to search.</param> + /// <param name="key">The case-insensitive field key.</param> + private T GetExtensionField<T>(JSchema schema, string key) + { + if (schema.ExtensionData != null) + { + foreach (var pair in schema.ExtensionData) + { + if (pair.Key.Equals(key, StringComparison.InvariantCultureIgnoreCase)) + return pair.Value.ToObject<T>(); + } + } + + return default; + } + + /// <summary>Format a schema value for display.</summary> + /// <param name="value">The value to format.</param> + private string FormatValue(object value) + { + switch (value) + { + case List<string> list: + return string.Join(", ", list); + + default: + return value?.ToString() ?? "null"; + } + } + } +} diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index 21e4a56f..f7f19cd8 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,13 +1,12 @@ using System; -using System.IO; -using System.IO.Compression; using System.Linq; -using System.Text; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; +using StardewModdingAPI.Toolkit.Utilities; using StardewModdingAPI.Web.Framework; using StardewModdingAPI.Web.Framework.Clients.Pastebin; +using StardewModdingAPI.Web.Framework.Compression; using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.Framework.LogParsing; using StardewModdingAPI.Web.Framework.LogParsing.Models; @@ -27,9 +26,8 @@ namespace StardewModdingAPI.Web.Controllers /// <summary>The underlying Pastebin client.</summary> private readonly IPastebinClient Pastebin; - /// <summary>The first bytes in a valid zip file.</summary> - /// <remarks>See <a href="https://en.wikipedia.org/wiki/Zip_(file_format)#File_headers"/>.</remarks> - private const uint GzipLeadBytes = 0x8b1f; + /// <summary>The underlying text compression helper.</summary> + private readonly IGzipHelper GzipHelper; /********* @@ -41,10 +39,12 @@ namespace StardewModdingAPI.Web.Controllers /// <summary>Construct an instance.</summary> /// <param name="siteConfig">The context config settings.</param> /// <param name="pastebin">The Pastebin API client.</param> - public LogParserController(IOptions<SiteConfig> siteConfig, IPastebinClient pastebin) + /// <param name="gzipHelper">The underlying text compression helper.</param> + public LogParserController(IOptions<SiteConfig> siteConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) { this.Config = siteConfig.Value; this.Pastebin = pastebin; + this.GzipHelper = gzipHelper; } /*** @@ -60,14 +60,14 @@ namespace StardewModdingAPI.Web.Controllers { // fresh page if (string.IsNullOrWhiteSpace(id)) - return this.View("Index", new LogParserModel(this.Config.LogParserUrl, id)); + return this.View("Index", this.GetModel(id)); // log page PasteInfo paste = await this.GetAsync(id); ParsedLog log = paste.Success ? new LogParser().Parse(paste.Content) : new ParsedLog { IsValid = false, Error = "Pastebin error: " + paste.Error }; - return this.View("Index", new LogParserModel(this.Config.LogParserUrl, id, log, raw)); + return this.View("Index", this.GetModel(id).SetResult(log, raw)); } /*** @@ -81,15 +81,15 @@ namespace StardewModdingAPI.Web.Controllers // get raw log text string input = this.Request.Form["input"].FirstOrDefault(); if (string.IsNullOrWhiteSpace(input)) - return this.View("Index", new LogParserModel(this.Config.LogParserUrl, null) { UploadError = "The log file seems to be empty." }); + return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty.")); // upload log - input = this.CompressString(input); - SavePasteResult result = await this.Pastebin.PostAsync(input); + input = this.GzipHelper.CompressString(input); + SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", input); // handle errors if (!result.Success) - return this.View("Index", new LogParserModel(this.Config.LogParserUrl, result.ID) { UploadError = $"Pastebin error: {result.Error ?? "unknown error"}" }); + return this.View("Index", this.GetModel(result.ID, uploadError: $"Pastebin error: {result.Error ?? "unknown error"}")); // redirect to view UriBuilder uri = new UriBuilder(new Uri(this.Config.LogParserUrl)); @@ -106,74 +106,41 @@ namespace StardewModdingAPI.Web.Controllers private async Task<PasteInfo> GetAsync(string id) { PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.DecompressString(response.Content); + response.Content = this.GzipHelper.DecompressString(response.Content); return response; } - /// <summary>Compress a string.</summary> - /// <param name="text">The text to compress.</param> - /// <remarks>Derived from <a href="https://stackoverflow.com/a/17993002/262123"/>.</remarks> - private string CompressString(string text) + /// <summary>Build a log parser model.</summary> + /// <param name="pasteID">The paste ID.</param> + /// <param name="uploadError">An error which occurred while uploading the log to Pastebin.</param> + private LogParserModel GetModel(string pasteID, string uploadError = null) { - // get raw bytes - byte[] buffer = Encoding.UTF8.GetBytes(text); - - // compressed - byte[] compressedData; - using (MemoryStream stream = new MemoryStream()) - { - using (GZipStream zipStream = new GZipStream(stream, CompressionLevel.Optimal, leaveOpen: true)) - zipStream.Write(buffer, 0, buffer.Length); - - stream.Position = 0; - compressedData = new byte[stream.Length]; - stream.Read(compressedData, 0, compressedData.Length); - } - - // prefix length - byte[] zipBuffer = new byte[compressedData.Length + 4]; - Buffer.BlockCopy(compressedData, 0, zipBuffer, 4, compressedData.Length); - Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, zipBuffer, 0, 4); - - // return string representation - return Convert.ToBase64String(zipBuffer); + string sectionUrl = this.Config.LogParserUrl; + Platform? platform = this.DetectClientPlatform(); + return new LogParserModel(sectionUrl, pasteID, platform) { UploadError = uploadError }; } - /// <summary>Decompress a string.</summary> - /// <param name="rawText">The compressed text.</param> - /// <remarks>Derived from <a href="https://stackoverflow.com/a/17993002/262123"/>.</remarks> - private string DecompressString(string rawText) + /// <summary>Detect the viewer's OS.</summary> + /// <returns>Returns the viewer OS if known, else null.</returns> + private Platform? DetectClientPlatform() { - // get raw bytes - byte[] zipBuffer; - try + string userAgent = this.Request.Headers["User-Agent"]; + switch (userAgent) { - zipBuffer = Convert.FromBase64String(rawText); - } - catch - { - return rawText; // not valid base64, wasn't compressed by the log parser - } + case string ua when ua.Contains("Windows"): + return Platform.Windows; - // skip if not gzip - if (BitConverter.ToUInt16(zipBuffer, 4) != LogParserController.GzipLeadBytes) - return rawText; + case string ua when ua.Contains("Android"): // check for Android before Linux because Android user agents also contain Linux + return Platform.Android; - // decompress - using (MemoryStream memoryStream = new MemoryStream()) - { - // read length prefix - int dataLength = BitConverter.ToInt32(zipBuffer, 0); - memoryStream.Write(zipBuffer, 4, zipBuffer.Length - 4); - - // read data - byte[] buffer = new byte[dataLength]; - memoryStream.Position = 0; - using (GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) - gZipStream.Read(buffer, 0, buffer.Length); - - // return original string - return Encoding.UTF8.GetString(buffer); + case string ua when ua.Contains("Linux"): + return Platform.Linux; + + case string ua when ua.Contains("Mac"): + return Platform.Mac; + + default: + return null; } } } diff --git a/src/SMAPI.Web/Controllers/ModsApiController.cs b/src/SMAPI.Web/Controllers/ModsApiController.cs index 7e6f592c..fe220eb5 100644 --- a/src/SMAPI.Web/Controllers/ModsApiController.cs +++ b/src/SMAPI.Web/Controllers/ModsApiController.cs @@ -2,18 +2,19 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit; using StardewModdingAPI.Toolkit.Framework.Clients.WebApi; using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; using StardewModdingAPI.Toolkit.Framework.ModData; using StardewModdingAPI.Toolkit.Framework.UpdateData; +using StardewModdingAPI.Web.Framework.Caching.Mods; +using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.Clients.Chucklefish; +using StardewModdingAPI.Web.Framework.Clients.CurseForge; using StardewModdingAPI.Web.Framework.Clients.GitHub; using StardewModdingAPI.Web.Framework.Clients.ModDrop; using StardewModdingAPI.Web.Framework.Clients.Nexus; @@ -33,8 +34,11 @@ namespace StardewModdingAPI.Web.Controllers /// <summary>The mod repositories which provide mod metadata.</summary> private readonly IDictionary<ModRepositoryKey, IModRepository> Repositories; - /// <summary>The cache in which to store mod metadata.</summary> - private readonly IMemoryCache Cache; + /// <summary>The cache in which to store wiki data.</summary> + private readonly IWikiCacheRepository WikiCache; + + /// <summary>The cache in which to store mod data.</summary> + private readonly IModCacheRepository ModCache; /// <summary>The number of minutes successful update checks should be cached before refetching them.</summary> private readonly int SuccessCacheMinutes; @@ -42,9 +46,6 @@ namespace StardewModdingAPI.Web.Controllers /// <summary>The number of minutes failed update checks should be cached before refetching them.</summary> private readonly int ErrorCacheMinutes; - /// <summary>A regex which matches SMAPI-style semantic version.</summary> - private readonly string VersionRegex; - /// <summary>The internal mod metadata list.</summary> private readonly ModDatabase ModDatabase; @@ -57,26 +58,29 @@ namespace StardewModdingAPI.Web.Controllers *********/ /// <summary>Construct an instance.</summary> /// <param name="environment">The web hosting environment.</param> - /// <param name="cache">The cache in which to store mod metadata.</param> + /// <param name="wikiCache">The cache in which to store wiki data.</param> + /// <param name="modCache">The cache in which to store mod metadata.</param> /// <param name="configProvider">The config settings for mod update checks.</param> /// <param name="chucklefish">The Chucklefish API client.</param> + /// <param name="curseForge">The CurseForge API client.</param> /// <param name="github">The GitHub API client.</param> /// <param name="modDrop">The ModDrop API client.</param> /// <param name="nexus">The Nexus API client.</param> - public ModsApiController(IHostingEnvironment environment, IMemoryCache cache, IOptions<ModUpdateCheckConfig> configProvider, IChucklefishClient chucklefish, IGitHubClient github, IModDropClient modDrop, INexusClient nexus) + public ModsApiController(IHostingEnvironment environment, IWikiCacheRepository wikiCache, IModCacheRepository modCache, IOptions<ModUpdateCheckConfig> configProvider, IChucklefishClient chucklefish, ICurseForgeClient curseForge, IGitHubClient github, IModDropClient modDrop, INexusClient nexus) { - this.ModDatabase = new ModToolkit().GetModDatabase(Path.Combine(environment.WebRootPath, "StardewModdingAPI.metadata.json")); + this.ModDatabase = new ModToolkit().GetModDatabase(Path.Combine(environment.WebRootPath, "SMAPI.metadata.json")); ModUpdateCheckConfig config = configProvider.Value; this.CompatibilityPageUrl = config.CompatibilityPageUrl; - this.Cache = cache; + this.WikiCache = wikiCache; + this.ModCache = modCache; this.SuccessCacheMinutes = config.SuccessCacheMinutes; this.ErrorCacheMinutes = config.ErrorCacheMinutes; - this.VersionRegex = config.SemanticVersionRegex; this.Repositories = new IModRepository[] { new ChucklefishRepository(chucklefish), + new CurseForgeRepository(curseForge), new GitHubRepository(github), new ModDropRepository(modDrop), new NexusRepository(nexus) @@ -86,21 +90,42 @@ namespace StardewModdingAPI.Web.Controllers /// <summary>Fetch version metadata for the given mods.</summary> /// <param name="model">The mod search criteria.</param> + /// <param name="version">The requested API version.</param> [HttpPost] - public async Task<IEnumerable<ModEntryModel>> PostAsync([FromBody] ModSearchModel model) + public async Task<IEnumerable<ModEntryModel>> PostAsync([FromBody] ModSearchModel model, [FromRoute] string version) { if (model?.Mods == null) return new ModEntryModel[0]; + bool legacyMode = SemanticVersion.TryParse(version, out ISemanticVersion parsedVersion) && parsedVersion.IsOlderThan("3.0.0-beta.20191109"); + // fetch wiki data - WikiModEntry[] wikiData = await this.GetWikiDataAsync(); + WikiModEntry[] wikiData = this.WikiCache.GetWikiMods().Select(p => p.GetModel()).ToArray(); IDictionary<string, ModEntryModel> mods = new Dictionary<string, ModEntryModel>(StringComparer.CurrentCultureIgnoreCase); foreach (ModSearchEntryModel mod in model.Mods) { if (string.IsNullOrWhiteSpace(mod.ID)) continue; - ModEntryModel result = await this.GetModData(mod, wikiData, model.IncludeExtendedMetadata); + ModEntryModel result = await this.GetModData(mod, wikiData, model.IncludeExtendedMetadata || legacyMode, model.ApiVersion); + if (legacyMode) + { + result.Main = result.Metadata.Main; + result.Optional = result.Metadata.Optional; + result.Unofficial = result.Metadata.Unofficial; + result.UnofficialForBeta = result.Metadata.UnofficialForBeta; + result.HasBetaInfo = result.Metadata.BetaCompatibilityStatus != null; + result.SuggestedUpdate = null; + if (!model.IncludeExtendedMetadata) + result.Metadata = null; + } + else if (!model.IncludeExtendedMetadata && (model.ApiVersion == null || mod.InstalledVersion == null)) + { + var errors = new List<string>(result.Errors); + errors.Add($"This API can't suggest an update because {nameof(model.ApiVersion)} or {nameof(mod.InstalledVersion)} are null, and you didn't specify {nameof(model.IncludeExtendedMetadata)} to get other info. See the SMAPI technical docs for usage."); + result.Errors = errors.ToArray(); + } + mods[mod.ID] = result; } @@ -116,19 +141,31 @@ namespace StardewModdingAPI.Web.Controllers /// <param name="search">The mod data to match.</param> /// <param name="wikiData">The wiki data.</param> /// <param name="includeExtendedMetadata">Whether to include extended metadata for each mod.</param> + /// <param name="apiVersion">The SMAPI version installed by the player.</param> /// <returns>Returns the mod data if found, else <c>null</c>.</returns> - private async Task<ModEntryModel> GetModData(ModSearchEntryModel search, WikiModEntry[] wikiData, bool includeExtendedMetadata) + private async Task<ModEntryModel> GetModData(ModSearchEntryModel search, WikiModEntry[] wikiData, bool includeExtendedMetadata, ISemanticVersion apiVersion) { - // crossreference data + // cross-reference data ModDataRecord record = this.ModDatabase.Get(search.ID); WikiModEntry wikiEntry = wikiData.FirstOrDefault(entry => entry.ID.Contains(search.ID.Trim(), StringComparer.InvariantCultureIgnoreCase)); - string[] updateKeys = this.GetUpdateKeys(search.UpdateKeys, record, wikiEntry).ToArray(); + UpdateKey[] updateKeys = this.GetUpdateKeys(search.UpdateKeys, record, wikiEntry).ToArray(); // get latest versions ModEntryModel result = new ModEntryModel { ID = search.ID }; IList<string> errors = new List<string>(); - foreach (string updateKey in updateKeys) + ModEntryVersionModel main = null; + ModEntryVersionModel optional = null; + ModEntryVersionModel unofficial = null; + ModEntryVersionModel unofficialForBeta = null; + foreach (UpdateKey updateKey in updateKeys) { + // validate update key + if (!updateKey.LooksValid) + { + errors.Add($"The update key '{updateKey}' isn't in a valid format. It should contain the site key and mod ID like 'Nexus:541'."); + continue; + } + // fetch data ModInfoModel data = await this.GetInfoForUpdateKeyAsync(updateKey); if (data.Error != null) @@ -140,76 +177,118 @@ namespace StardewModdingAPI.Web.Controllers // handle main version if (data.Version != null) { - if (!SemanticVersion.TryParse(data.Version, out ISemanticVersion version)) + ISemanticVersion version = this.GetMappedVersion(data.Version, wikiEntry?.MapRemoteVersions); + if (version == null) { errors.Add($"The update key '{updateKey}' matches a mod with invalid semantic version '{data.Version}'."); continue; } - if (this.IsNewer(version, result.Main?.Version)) - result.Main = new ModEntryVersionModel(version, data.Url); + if (this.IsNewer(version, main?.Version)) + main = new ModEntryVersionModel(version, data.Url); } // handle optional version if (data.PreviewVersion != null) { - if (!SemanticVersion.TryParse(data.PreviewVersion, out ISemanticVersion version)) + ISemanticVersion version = this.GetMappedVersion(data.PreviewVersion, wikiEntry?.MapRemoteVersions); + if (version == null) { errors.Add($"The update key '{updateKey}' matches a mod with invalid optional semantic version '{data.PreviewVersion}'."); continue; } - if (this.IsNewer(version, result.Optional?.Version)) - result.Optional = new ModEntryVersionModel(version, data.Url); + if (this.IsNewer(version, optional?.Version)) + optional = new ModEntryVersionModel(version, data.Url); } } // get unofficial version - if (wikiEntry?.Compatibility.UnofficialVersion != null && this.IsNewer(wikiEntry.Compatibility.UnofficialVersion, result.Main?.Version) && this.IsNewer(wikiEntry.Compatibility.UnofficialVersion, result.Optional?.Version)) - result.Unofficial = new ModEntryVersionModel(wikiEntry.Compatibility.UnofficialVersion, $"{this.CompatibilityPageUrl}/#{wikiEntry.Anchor}"); + if (wikiEntry?.Compatibility.UnofficialVersion != null && this.IsNewer(wikiEntry.Compatibility.UnofficialVersion, main?.Version) && this.IsNewer(wikiEntry.Compatibility.UnofficialVersion, optional?.Version)) + unofficial = new ModEntryVersionModel(wikiEntry.Compatibility.UnofficialVersion, $"{this.CompatibilityPageUrl}/#{wikiEntry.Anchor}"); // get unofficial version for beta if (wikiEntry?.HasBetaInfo == true) { - result.HasBetaInfo = true; if (wikiEntry.BetaCompatibility.Status == WikiCompatibilityStatus.Unofficial) { if (wikiEntry.BetaCompatibility.UnofficialVersion != null) { - result.UnofficialForBeta = (wikiEntry.BetaCompatibility.UnofficialVersion != null && this.IsNewer(wikiEntry.BetaCompatibility.UnofficialVersion, result.Main?.Version) && this.IsNewer(wikiEntry.BetaCompatibility.UnofficialVersion, result.Optional?.Version)) + unofficialForBeta = (wikiEntry.BetaCompatibility.UnofficialVersion != null && this.IsNewer(wikiEntry.BetaCompatibility.UnofficialVersion, main?.Version) && this.IsNewer(wikiEntry.BetaCompatibility.UnofficialVersion, optional?.Version)) ? new ModEntryVersionModel(wikiEntry.BetaCompatibility.UnofficialVersion, $"{this.CompatibilityPageUrl}/#{wikiEntry.Anchor}") : null; } else - result.UnofficialForBeta = result.Unofficial; + unofficialForBeta = unofficial; } } // fallback to preview if latest is invalid - if (result.Main == null && result.Optional != null) + if (main == null && optional != null) { - result.Main = result.Optional; - result.Optional = null; + main = optional; + optional = null; } // special cases if (result.ID == "Pathoschild.SMAPI") { - if (result.Main != null) - result.Main.Url = "https://smapi.io/"; - if (result.Optional != null) - result.Optional.Url = "https://smapi.io/"; + if (main != null) + main.Url = "https://smapi.io/"; + if (optional != null) + optional.Url = "https://smapi.io/"; + } + + // get recommended update (if any) + ISemanticVersion installedVersion = this.GetMappedVersion(search.InstalledVersion?.ToString(), wikiEntry?.MapLocalVersions); + if (apiVersion != null && installedVersion != null) + { + // get newer versions + List<ModEntryVersionModel> updates = new List<ModEntryVersionModel>(); + if (this.IsRecommendedUpdate(installedVersion, main?.Version, useBetaChannel: true)) + updates.Add(main); + if (this.IsRecommendedUpdate(installedVersion, optional?.Version, useBetaChannel: installedVersion.IsPrerelease())) + updates.Add(optional); + if (this.IsRecommendedUpdate(installedVersion, unofficial?.Version, useBetaChannel: search.IsBroken)) + updates.Add(unofficial); + if (this.IsRecommendedUpdate(installedVersion, unofficialForBeta?.Version, useBetaChannel: apiVersion.IsPrerelease())) + updates.Add(unofficialForBeta); + + // get newest version + ModEntryVersionModel newest = null; + foreach (ModEntryVersionModel update in updates) + { + if (newest == null || update.Version.IsNewerThan(newest.Version)) + newest = update; + } + + // set field + result.SuggestedUpdate = newest != null + ? new ModEntryVersionModel(newest.Version, newest.Url) + : null; } // add extended metadata - if (includeExtendedMetadata && (wikiEntry != null || record != null)) - result.Metadata = new ModExtendedMetadataModel(wikiEntry, record); + if (includeExtendedMetadata) + result.Metadata = new ModExtendedMetadataModel(wikiEntry, record, main: main, optional: optional, unofficial: unofficial, unofficialForBeta: unofficialForBeta); // add result result.Errors = errors.ToArray(); return result; } + /// <summary>Get whether a given version should be offered to the user as an update.</summary> + /// <param name="currentVersion">The current semantic version.</param> + /// <param name="newVersion">The target semantic version.</param> + /// <param name="useBetaChannel">Whether the user enabled the beta channel and should be offered prerelease updates.</param> + private bool IsRecommendedUpdate(ISemanticVersion currentVersion, ISemanticVersion newVersion, bool useBetaChannel) + { + return + newVersion != null + && newVersion.IsNewerThan(currentVersion) + && (useBetaChannel || !newVersion.IsPrerelease()); + } + /// <summary>Get whether a <paramref name="current"/> version is newer than an <paramref name="other"/> version.</summary> /// <param name="current">The current version.</param> /// <param name="other">The other version.</param> @@ -218,60 +297,38 @@ namespace StardewModdingAPI.Web.Controllers return current != null && (other == null || other.IsOlderThan(current)); } - /// <summary>Get mod data from the wiki compatibility list.</summary> - private async Task<WikiModEntry[]> GetWikiDataAsync() - { - ModToolkit toolkit = new ModToolkit(); - return await this.Cache.GetOrCreateAsync("_wiki", async entry => - { - try - { - WikiModEntry[] entries = (await toolkit.GetWikiCompatibilityListAsync()).Mods; - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.SuccessCacheMinutes); - return entries; - } - catch - { - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.ErrorCacheMinutes); - return new WikiModEntry[0]; - } - }); - } - /// <summary>Get the mod info for an update key.</summary> /// <param name="updateKey">The namespaced update key.</param> - private async Task<ModInfoModel> GetInfoForUpdateKeyAsync(string updateKey) + private async Task<ModInfoModel> GetInfoForUpdateKeyAsync(UpdateKey updateKey) { - // parse update key - UpdateKey parsed = UpdateKey.Parse(updateKey); - if (!parsed.LooksValid) - return new ModInfoModel($"The update key '{updateKey}' isn't in a valid format. It should contain the site key and mod ID like 'Nexus:541'."); - - // get matching repository - if (!this.Repositories.TryGetValue(parsed.Repository, out IModRepository repository)) - return new ModInfoModel($"There's no mod site with key '{parsed.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}]."); - - // fetch mod info - return await this.Cache.GetOrCreateAsync($"{repository.VendorKey}:{parsed.ID}".ToLower(), async entry => + // get mod + if (!this.ModCache.TryGetMod(updateKey.Repository, updateKey.ID, out CachedMod mod) || this.ModCache.IsStale(mod.LastUpdated, mod.FetchStatus == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes)) { - ModInfoModel result = await repository.GetModInfoAsync(parsed.ID); + // get site + if (!this.Repositories.TryGetValue(updateKey.Repository, out IModRepository repository)) + return new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{updateKey.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}]."); + + // fetch mod + ModInfoModel result = await repository.GetModInfoAsync(updateKey.ID); if (result.Error == null) { if (result.Version == null) - result.Error = $"The update key '{updateKey}' matches a mod with no version number."; - else if (!Regex.IsMatch(result.Version, this.VersionRegex, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)) - result.Error = $"The update key '{updateKey}' matches a mod with invalid semantic version '{result.Version}'."; + result.SetError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with no version number."); + else if (!SemanticVersion.TryParse(result.Version, out _)) + result.SetError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with invalid semantic version '{result.Version}'."); } - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(result.Error == null ? this.SuccessCacheMinutes : this.ErrorCacheMinutes); - return result; - }); + + // cache mod + this.ModCache.SaveMod(repository.VendorKey, updateKey.ID, result, out mod); + } + return mod.GetModel(); } /// <summary>Get update keys based on the available mod metadata, while maintaining the precedence order.</summary> /// <param name="specifiedKeys">The specified update keys.</param> /// <param name="record">The mod's entry in SMAPI's internal database.</param> /// <param name="entry">The mod's entry in the wiki list.</param> - public IEnumerable<string> GetUpdateKeys(string[] specifiedKeys, ModDataRecord record, WikiModEntry entry) + private IEnumerable<UpdateKey> GetUpdateKeys(string[] specifiedKeys, ModDataRecord record, WikiModEntry entry) { IEnumerable<string> GetRaw() { @@ -291,20 +348,70 @@ namespace StardewModdingAPI.Web.Controllers if (entry != null) { if (entry.NexusID.HasValue) - yield return $"Nexus:{entry.NexusID}"; - if (entry.ChucklefishID.HasValue) - yield return $"Chucklefish:{entry.ChucklefishID}"; + yield return $"{ModRepositoryKey.Nexus}:{entry.NexusID}"; if (entry.ModDropID.HasValue) - yield return $"ModDrop:{entry.ModDropID}"; + yield return $"{ModRepositoryKey.ModDrop}:{entry.ModDropID}"; + if (entry.CurseForgeID.HasValue) + yield return $"{ModRepositoryKey.CurseForge}:{entry.CurseForgeID}"; + if (entry.ChucklefishID.HasValue) + yield return $"{ModRepositoryKey.Chucklefish}:{entry.ChucklefishID}"; } } - HashSet<string> seen = new HashSet<string>(StringComparer.InvariantCulture); - foreach (string key in GetRaw()) + HashSet<UpdateKey> seen = new HashSet<UpdateKey>(); + foreach (string rawKey in GetRaw()) { - if (!string.IsNullOrWhiteSpace(key) && seen.Add(key)) + if (string.IsNullOrWhiteSpace(rawKey)) + continue; + + UpdateKey key = UpdateKey.Parse(rawKey); + if (seen.Add(key)) yield return key; } } + + /// <summary>Get a semantic local version for update checks.</summary> + /// <param name="version">The version to parse.</param> + /// <param name="map">A map of version replacements.</param> + private ISemanticVersion GetMappedVersion(string version, IDictionary<string, string> map) + { + // try mapped version + string rawNewVersion = this.GetRawMappedVersion(version, map); + if (SemanticVersion.TryParse(rawNewVersion, out ISemanticVersion parsedNew)) + return parsedNew; + + // return original version + return SemanticVersion.TryParse(version, out ISemanticVersion parsedOld) + ? parsedOld + : null; + } + + /// <summary>Get a semantic local version for update checks.</summary> + /// <param name="version">The version to map.</param> + /// <param name="map">A map of version replacements.</param> + private string GetRawMappedVersion(string version, IDictionary<string, string> map) + { + if (version == null || map == null || !map.Any()) + return version; + + // match exact raw version + if (map.ContainsKey(version)) + return map[version]; + + // match parsed version + if (SemanticVersion.TryParse(version, out ISemanticVersion parsed)) + { + if (map.ContainsKey(parsed.ToString())) + return map[parsed.ToString()]; + + foreach (var pair in map) + { + if (SemanticVersion.TryParse(pair.Key, out ISemanticVersion target) && parsed.Equals(target) && SemanticVersion.TryParse(pair.Value, out ISemanticVersion newVersion)) + return newVersion.ToString(); + } + } + + return version; + } } } diff --git a/src/SMAPI.Web/Controllers/ModsController.cs b/src/SMAPI.Web/Controllers/ModsController.cs index ca866a8d..b621ded0 100644 --- a/src/SMAPI.Web/Controllers/ModsController.cs +++ b/src/SMAPI.Web/Controllers/ModsController.cs @@ -1,12 +1,8 @@ -using System; using System.Linq; using System.Text.RegularExpressions; -using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; -using StardewModdingAPI.Toolkit; -using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; +using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.ViewModels; @@ -19,10 +15,10 @@ namespace StardewModdingAPI.Web.Controllers ** Fields *********/ /// <summary>The cache in which to store mod metadata.</summary> - private readonly IMemoryCache Cache; + private readonly IWikiCacheRepository Cache; - /// <summary>The number of minutes successful update checks should be cached before refetching them.</summary> - private readonly int CacheMinutes; + /// <summary>The number of minutes before which wiki data should be considered old.</summary> + private readonly int StaleMinutes; /********* @@ -31,20 +27,20 @@ namespace StardewModdingAPI.Web.Controllers /// <summary>Construct an instance.</summary> /// <param name="cache">The cache in which to store mod metadata.</param> /// <param name="configProvider">The config settings for mod update checks.</param> - public ModsController(IMemoryCache cache, IOptions<ModCompatibilityListConfig> configProvider) + public ModsController(IWikiCacheRepository cache, IOptions<ModCompatibilityListConfig> configProvider) { ModCompatibilityListConfig config = configProvider.Value; this.Cache = cache; - this.CacheMinutes = config.CacheMinutes; + this.StaleMinutes = config.StaleMinutes; } /// <summary>Display information for all mods.</summary> [HttpGet] [Route("mods")] - public async Task<ViewResult> Index() + public ViewResult Index() { - return this.View("Index", await this.FetchDataAsync()); + return this.View("Index", this.FetchData()); } @@ -52,23 +48,23 @@ namespace StardewModdingAPI.Web.Controllers ** Private methods *********/ /// <summary>Asynchronously fetch mod metadata from the wiki.</summary> - public async Task<ModListModel> FetchDataAsync() + public ModListModel FetchData() { - return await this.Cache.GetOrCreateAsync($"{nameof(ModsController)}_mod_list", async entry => - { - WikiModList data = await new ModToolkit().GetWikiCompatibilityListAsync(); - ModListModel model = new ModListModel( - stableVersion: data.StableVersion, - betaVersion: data.BetaVersion, - mods: data - .Mods - .Select(mod => new ModModel(mod)) - .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")) // ignore case, spaces, and special characters when sorting - ); + // fetch cached data + if (!this.Cache.TryGetWikiMetadata(out CachedWikiMetadata metadata)) + return new ModListModel(); - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.CacheMinutes); - return model; - }); + // build model + return new ModListModel( + stableVersion: metadata.StableVersion, + betaVersion: metadata.BetaVersion, + mods: this.Cache + .GetWikiMods() + .Select(mod => new ModModel(mod.GetModel())) + .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")), // ignore case, spaces, and special characters when sorting + lastUpdated: metadata.LastUpdated, + isStale: this.Cache.IsStale(metadata.LastUpdated, this.StaleMinutes) + ); } } } |