From e00fb85ee7822bc7fed2d6bd5a2e4c207a799418 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 7 Jul 2019 00:29:22 -0400 Subject: migrate compatibility list's wiki data to MongoDB cache (#651) --- .../Framework/ConfigModels/MongoDbConfig.cs | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs (limited to 'src/SMAPI.Web/Framework/ConfigModels') diff --git a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs new file mode 100644 index 00000000..352eb960 --- /dev/null +++ b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs @@ -0,0 +1,36 @@ +using System; + +namespace StardewModdingAPI.Web.Framework.ConfigModels +{ + /// The config settings for mod compatibility list. + internal class MongoDbConfig + { + /********* + ** Accessors + *********/ + /// The MongoDB hostname. + public string Host { get; set; } + + /// The MongoDB username (if any). + public string Username { get; set; } + + /// The MongoDB password (if any). + public string Password { get; set; } + + + /********* + ** Public method + *********/ + /// Get the MongoDB connection string. + /// The initial database for which to authenticate. + public string GetConnectionString(string authDatabase) + { + bool isLocal = this.Host == "localhost"; + bool hasLogin = !string.IsNullOrWhiteSpace(this.Username) && !string.IsNullOrWhiteSpace(this.Password); + + return $"mongodb{(isLocal ? "" : "+srv")}://" + + (hasLogin ? $"{Uri.EscapeDataString(this.Username)}:{Uri.EscapeDataString(this.Password)}@" : "") + + $"{this.Host}/{authDatabase}retryWrites=true&w=majority"; + } + } +} -- cgit From 2b3f0e740bc09630e881c9967e5ad53d66384094 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 7 Jul 2019 00:39:52 -0400 Subject: make MongoDB database name configurable (#651) --- docs/technical/web.md | 1 + src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs | 8 +++++--- src/SMAPI.Web/Startup.cs | 4 ++-- src/SMAPI.Web/appsettings.Development.json | 3 ++- src/SMAPI.Web/appsettings.json | 3 ++- 5 files changed, 12 insertions(+), 7 deletions(-) (limited to 'src/SMAPI.Web/Framework/ConfigModels') diff --git a/docs/technical/web.md b/docs/technical/web.md index db54a87a..f1cb755b 100644 --- a/docs/technical/web.md +++ b/docs/technical/web.md @@ -123,6 +123,7 @@ Initial setup: `MongoDB:Host` | The hostname for the MongoDB instance. `MongoDB:Username` | The login username for the MongoDB instance. `MongoDB:Password` | The login password for the MongoDB instance. + `MongoDB:Database` | The database name (e.g. `smapi` in production or `smapi-edge` in testing environments). To deploy updates: 1. Deploy the web project using [AWS Toolkit for Visual Studio](https://aws.amazon.com/visualstudio/). diff --git a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs index 352eb960..3c508300 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs @@ -17,20 +17,22 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The MongoDB password (if any). public string Password { get; set; } + /// The database name. + public string Database { get; set; } + /********* ** Public method *********/ /// Get the MongoDB connection string. - /// The initial database for which to authenticate. - public string GetConnectionString(string authDatabase) + public string GetConnectionString() { bool isLocal = this.Host == "localhost"; bool hasLogin = !string.IsNullOrWhiteSpace(this.Username) && !string.IsNullOrWhiteSpace(this.Password); return $"mongodb{(isLocal ? "" : "+srv")}://" + (hasLogin ? $"{Uri.EscapeDataString(this.Username)}:{Uri.EscapeDataString(this.Password)}@" : "") - + $"{this.Host}/{authDatabase}retryWrites=true&w=majority"; + + $"{this.Host}/{this.Database}?retryWrites=true&w=majority"; } } } diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 0a8d23a8..315f5f88 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -115,9 +115,9 @@ namespace StardewModdingAPI.Web // init MongoDB { MongoDbConfig mongoConfig = this.Configuration.GetSection("MongoDB").Get(); - string connectionString = mongoConfig.GetConnectionString("smapi"); + string connectionString = mongoConfig.GetConnectionString(); - services.AddSingleton(serv => new MongoClient(connectionString).GetDatabase("smapi")); + services.AddSingleton(serv => new MongoClient(connectionString).GetDatabase(mongoConfig.Database)); services.AddSingleton(serv => new WikiCacheRepository(serv.GetService())); } } diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 9b0ec535..3b7ed8bd 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -36,6 +36,7 @@ "MongoDB": { "Host": "localhost", "Username": null, - "Password": null + "Password": null, + "Database": "smapi-edge" } } diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 65ccea75..532ea017 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -50,7 +50,8 @@ "MongoDB": { "Host": null, // see top note "Username": null, // see top note - "Password": null // see top note + "Password": null, // see top note + "Database": null // see top note }, "ModCompatibilityList": { -- cgit From ce6cedaf4be53d52f2e558055b91e515b92e4c83 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Fri, 19 Jul 2019 13:15:45 -0400 Subject: add background fetch for mod compatibility list (#651) --- docs/release-notes.md | 2 + src/SMAPI.Web/BackgroundService.cs | 92 +++++++++++ src/SMAPI.Web/Controllers/ModsController.cs | 33 ++-- .../Framework/Caching/BaseCacheRepository.cs | 6 +- .../Framework/Caching/Wiki/CachedWikiMetadata.cs | 2 +- .../Framework/Caching/Wiki/IWikiCacheRepository.cs | 4 +- .../ConfigModels/BackgroundServicesConfig.cs | 12 ++ .../ConfigModels/ModCompatibilityListConfig.cs | 6 +- src/SMAPI.Web/SMAPI.Web.csproj | 2 + src/SMAPI.Web/Startup.cs | 48 ++++-- src/SMAPI.Web/ViewModels/ModListModel.cs | 19 ++- src/SMAPI.Web/Views/Mods/Index.cshtml | 172 +++++++++++---------- src/SMAPI.Web/appsettings.Development.json | 7 - src/SMAPI.Web/appsettings.json | 9 +- src/SMAPI.Web/wwwroot/Content/css/mods.css | 57 ++++--- 15 files changed, 317 insertions(+), 154 deletions(-) create mode 100644 src/SMAPI.Web/BackgroundService.cs create mode 100644 src/SMAPI.Web/Framework/ConfigModels/BackgroundServicesConfig.cs (limited to 'src/SMAPI.Web/Framework/ConfigModels') diff --git a/docs/release-notes.md b/docs/release-notes.md index ea988fe5..9c3e3d28 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -25,6 +25,8 @@ These changes have not been released yet. * Fixed outdoor tilesheets being seasonalised when added to an indoor location. * For the mod compatibility list: + * Now loads faster (since data is fetched in a background service). + * Now continues working with cached data when the wiki is offline. * Clicking a mod link now automatically adds it to the visible mods when the list is filtered. * Added metadata links and dev notes (if any) to advanced info. diff --git a/src/SMAPI.Web/BackgroundService.cs b/src/SMAPI.Web/BackgroundService.cs new file mode 100644 index 00000000..2ccfd5f7 --- /dev/null +++ b/src/SMAPI.Web/BackgroundService.cs @@ -0,0 +1,92 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Hangfire; +using Microsoft.Extensions.Hosting; +using StardewModdingAPI.Toolkit; +using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; +using StardewModdingAPI.Web.Framework.Caching.Wiki; + +namespace StardewModdingAPI.Web +{ + /// A hosted service which runs background data updates. + /// Task methods need to be static, since otherwise Hangfire will try to serialise the entire instance. + internal class BackgroundService : IHostedService, IDisposable + { + /********* + ** Fields + *********/ + /// The background task server. + private static BackgroundJobServer JobServer; + + /// The cache in which to store mod metadata. + private static IWikiCacheRepository WikiCache; + + + /********* + ** Public methods + *********/ + /**** + ** Hosted service + ****/ + /// Construct an instance. + /// The cache in which to store mod metadata. + public BackgroundService(IWikiCacheRepository wikiCache) + { + BackgroundService.WikiCache = wikiCache; + } + + /// Start the service. + /// Tracks whether the start process has been aborted. + public Task StartAsync(CancellationToken cancellationToken) + { + this.TryInit(); + + // set startup tasks + BackgroundJob.Enqueue(() => BackgroundService.UpdateWikiAsync()); + + // set recurring tasks + RecurringJob.AddOrUpdate(() => BackgroundService.UpdateWikiAsync(), "*/10 * * * *"); + + return Task.CompletedTask; + } + + /// Triggered when the application host is performing a graceful shutdown. + /// Tracks whether the shutdown process should no longer be graceful. + public async Task StopAsync(CancellationToken cancellationToken) + { + if (BackgroundService.JobServer != null) + await BackgroundService.JobServer.WaitForShutdownAsync(cancellationToken); + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + BackgroundService.JobServer?.Dispose(); + } + + /**** + ** Tasks + ****/ + /// Update the cached wiki metadata. + public static async Task UpdateWikiAsync() + { + WikiModList wikiCompatList = await new ModToolkit().GetWikiCompatibilityListAsync(); + BackgroundService.WikiCache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods, out _, out _); + } + + + /********* + ** Private method + *********/ + /// Initialise the background service if it's not already initialised. + /// The background service is already initialised. + private void TryInit() + { + if (BackgroundService.JobServer != null) + throw new InvalidOperationException("The scheduler service is already started."); + + BackgroundService.JobServer = new BackgroundJobServer(); + } + } +} diff --git a/src/SMAPI.Web/Controllers/ModsController.cs b/src/SMAPI.Web/Controllers/ModsController.cs index b6040e06..b621ded0 100644 --- a/src/SMAPI.Web/Controllers/ModsController.cs +++ b/src/SMAPI.Web/Controllers/ModsController.cs @@ -1,9 +1,7 @@ using System.Linq; using System.Text.RegularExpressions; -using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; -using StardewModdingAPI.Toolkit; using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.ViewModels; @@ -19,8 +17,8 @@ namespace StardewModdingAPI.Web.Controllers /// The cache in which to store mod metadata. private readonly IWikiCacheRepository Cache; - /// The number of minutes successful update checks should be cached before refetching them. - private readonly int CacheMinutes; + /// The number of minutes before which wiki data should be considered old. + private readonly int StaleMinutes; /********* @@ -34,15 +32,15 @@ namespace StardewModdingAPI.Web.Controllers ModCompatibilityListConfig config = configProvider.Value; this.Cache = cache; - this.CacheMinutes = config.CacheMinutes; + this.StaleMinutes = config.StaleMinutes; } /// Display information for all mods. [HttpGet] [Route("mods")] - public async Task Index() + public ViewResult Index() { - return this.View("Index", await this.FetchDataAsync()); + return this.View("Index", this.FetchData()); } @@ -50,25 +48,22 @@ namespace StardewModdingAPI.Web.Controllers ** Private methods *********/ /// Asynchronously fetch mod metadata from the wiki. - public async Task FetchDataAsync() + public ModListModel FetchData() { - // refresh cache - CachedWikiMod[] mods; - if (!this.Cache.TryGetWikiMetadata(out CachedWikiMetadata metadata) || this.Cache.IsStale(metadata.LastUpdated, this.CacheMinutes)) - { - var wikiCompatList = await new ModToolkit().GetWikiCompatibilityListAsync(); - this.Cache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods, out metadata, out mods); - } - else - mods = this.Cache.GetWikiMods().ToArray(); + // fetch cached data + if (!this.Cache.TryGetWikiMetadata(out CachedWikiMetadata metadata)) + return new ModListModel(); // build model return new ModListModel( stableVersion: metadata.StableVersion, betaVersion: metadata.BetaVersion, - mods: mods + mods: this.Cache + .GetWikiMods() .Select(mod => new ModModel(mod.GetModel())) - .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")) // ignore case, spaces, and special characters when sorting + .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")), // ignore case, spaces, and special characters when sorting + lastUpdated: metadata.LastUpdated, + isStale: this.Cache.IsStale(metadata.LastUpdated, this.StaleMinutes) ); } } diff --git a/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs index 904455c5..f5354b93 100644 --- a/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs @@ -10,10 +10,10 @@ namespace StardewModdingAPI.Web.Framework.Caching *********/ /// Whether cached data is stale. /// The date when the data was updated. - /// The age in minutes before data is considered stale. - public bool IsStale(DateTimeOffset lastUpdated, int cacheMinutes) + /// The age in minutes before data is considered stale. + public bool IsStale(DateTimeOffset lastUpdated, int staleMinutes) { - return lastUpdated < DateTimeOffset.UtcNow.AddMinutes(-cacheMinutes); + return lastUpdated < DateTimeOffset.UtcNow.AddMinutes(-staleMinutes); } } } diff --git a/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs b/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs index de1ea9db..4d6b4b10 100644 --- a/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs +++ b/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs @@ -37,7 +37,7 @@ namespace StardewModdingAPI.Web.Framework.Caching.Wiki { this.StableVersion = stableVersion; this.BetaVersion = betaVersion; - this.LastUpdated = DateTimeOffset.UtcNow;; + this.LastUpdated = DateTimeOffset.UtcNow; } } } diff --git a/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs index d319db69..6031123d 100644 --- a/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs @@ -17,8 +17,8 @@ namespace StardewModdingAPI.Web.Framework.Caching.Wiki /// Whether cached data is stale. /// The date when the data was updated. - /// The age in minutes before data is considered stale. - bool IsStale(DateTimeOffset lastUpdated, int cacheMinutes); + /// The age in minutes before data is considered stale. + bool IsStale(DateTimeOffset lastUpdated, int staleMinutes); /// Get the cached wiki mods. /// A filter to apply, if any. diff --git a/src/SMAPI.Web/Framework/ConfigModels/BackgroundServicesConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/BackgroundServicesConfig.cs new file mode 100644 index 00000000..de871c9a --- /dev/null +++ b/src/SMAPI.Web/Framework/ConfigModels/BackgroundServicesConfig.cs @@ -0,0 +1,12 @@ +namespace StardewModdingAPI.Web.Framework.ConfigModels +{ + /// The config settings for background services. + internal class BackgroundServicesConfig + { + /********* + ** Accessors + *********/ + /// Whether to enable background update services. + public bool Enabled { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/ConfigModels/ModCompatibilityListConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ModCompatibilityListConfig.cs index d9ac9f02..24b540cd 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ModCompatibilityListConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ModCompatibilityListConfig.cs @@ -1,12 +1,12 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels { - /// The config settings for mod compatibility list. + /// The config settings for the mod compatibility list. internal class ModCompatibilityListConfig { /********* ** Accessors *********/ - /// The number of minutes data from the wiki should be cached before refetching it. - public int CacheMinutes { get; set; } + /// The number of minutes before which wiki data should be considered old. + public int StaleMinutes { get; set; } } } diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index 2f90389b..d53914d5 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -16,6 +16,8 @@ + + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 7beb0bcc..bdfa5ed9 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -1,4 +1,6 @@ using System.Collections.Generic; +using Hangfire; +using Hangfire.Mongo; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Rewrite; @@ -49,12 +51,13 @@ namespace StardewModdingAPI.Web /// The service injection container. public void ConfigureServices(IServiceCollection services) { - // init configuration + // init basic services services + .Configure(this.Configuration.GetSection("BackgroundServices")) .Configure(this.Configuration.GetSection("ModCompatibilityList")) .Configure(this.Configuration.GetSection("ModUpdateCheck")) - .Configure(this.Configuration.GetSection("Site")) .Configure(this.Configuration.GetSection("MongoDB")) + .Configure(this.Configuration.GetSection("Site")) .Configure(options => options.ConstraintMap.Add("semanticVersion", typeof(VersionConstraint))) .AddLogging() .AddMemoryCache() @@ -69,6 +72,33 @@ namespace StardewModdingAPI.Web options.SerializerSettings.NullValueHandling = NullValueHandling.Ignore; }); + // init background service + { + BackgroundServicesConfig config = this.Configuration.GetSection("BackgroundServices").Get(); + if (config.Enabled) + services.AddHostedService(); + } + + // init MongoDB + MongoDbConfig mongoConfig = this.Configuration.GetSection("MongoDB").Get(); + string mongoConnectionStr = mongoConfig.GetConnectionString(); + services.AddSingleton(serv => new MongoClient(mongoConnectionStr).GetDatabase(mongoConfig.Database)); + services.AddSingleton(serv => new WikiCacheRepository(serv.GetService())); + + // init Hangfire (needs MongoDB) + services + .AddHangfire(config => + { + config + .SetDataCompatibilityLevel(CompatibilityLevel.Version_170) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMongoStorage(mongoConnectionStr, $"{mongoConfig.Database}-hangfire", new MongoStorageOptions + { + MigrationOptions = new MongoMigrationOptions(MongoMigrationStrategy.Drop) + }); + }); + // init API clients { ApiClientsConfig api = this.Configuration.GetSection("ApiClients").Get(); @@ -111,15 +141,6 @@ namespace StardewModdingAPI.Web devKey: api.PastebinDevKey )); } - - // init MongoDB - { - MongoDbConfig mongoConfig = this.Configuration.GetSection("MongoDB").Get(); - string connectionString = mongoConfig.GetConnectionString(); - - services.AddSingleton(serv => new MongoClient(connectionString).GetDatabase(mongoConfig.Database)); - services.AddSingleton(serv => new WikiCacheRepository(serv.GetService())); - } } /// The method called by the runtime to configure the HTTP request pipeline. @@ -127,9 +148,9 @@ namespace StardewModdingAPI.Web /// The hosting environment. public void Configure(IApplicationBuilder app, IHostingEnvironment env) { + // basic config if (env.IsDevelopment()) app.UseDeveloperExceptionPage(); - app .UseCors(policy => policy .AllowAnyHeader() @@ -140,6 +161,9 @@ namespace StardewModdingAPI.Web .UseRewriter(this.GetRedirectRules()) .UseStaticFiles() // wwwroot folder .UseMvc(); + + // config Hangfire + app.UseHangfireDashboard("/tasks"); } diff --git a/src/SMAPI.Web/ViewModels/ModListModel.cs b/src/SMAPI.Web/ViewModels/ModListModel.cs index 3b87d393..ff7513bc 100644 --- a/src/SMAPI.Web/ViewModels/ModListModel.cs +++ b/src/SMAPI.Web/ViewModels/ModListModel.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; @@ -18,19 +19,35 @@ namespace StardewModdingAPI.Web.ViewModels /// The mods to display. public ModModel[] Mods { get; set; } + /// When the data was last updated. + public DateTimeOffset LastUpdated { get; set; } + + /// Whether the data hasn't been updated in a while. + public bool IsStale { get; set; } + + /// Whether the mod metadata is available. + public bool HasData => this.Mods != null; + /********* ** Public methods *********/ + /// Construct an empty instance. + public ModListModel() { } + /// Construct an instance. /// The current stable version of the game. /// The current beta version of the game (if any). /// The mods to display. - public ModListModel(string stableVersion, string betaVersion, IEnumerable mods) + /// When the data was last updated. + /// Whether the data hasn't been updated in a while. + public ModListModel(string stableVersion, string betaVersion, IEnumerable mods, DateTimeOffset lastUpdated, bool isStale) { this.StableVersion = stableVersion; this.BetaVersion = betaVersion; this.Mods = mods.ToArray(); + this.LastUpdated = lastUpdated; + this.IsStale = isStale; } } } diff --git a/src/SMAPI.Web/Views/Mods/Index.cshtml b/src/SMAPI.Web/Views/Mods/Index.cshtml index 2d45a64d..aa7c0678 100644 --- a/src/SMAPI.Web/Views/Mods/Index.cshtml +++ b/src/SMAPI.Web/Views/Mods/Index.cshtml @@ -18,92 +18,104 @@ } -
-
-

This page shows all known SMAPI mods and (incompatible) content packs, whether they work with the latest versions of Stardew Valley and SMAPI, and how to fix them if not. If a mod doesn't work after following the instructions below, check the troubleshooting guide or ask for help.

+@if (!Model.HasData) +{ +
↻ The mod data hasn't been fetched yet; please try again in a few minutes.
+} +else +{ + @if (Model.IsStale) + { +
Showing data from @(Math.Round((DateTimeOffset.UtcNow - Model.LastUpdated).TotalMinutes)) minutes ago. (Couldn't fetch newer data; the wiki API may be offline.)
+ } -

The list is updated every few days (you can help update it!). It doesn't include XNB mods (see using XNB mods on the wiki instead) or compatible content packs.

+
+
+

This page shows all known SMAPI mods and (incompatible) content packs, whether they work with the latest versions of Stardew Valley and SMAPI, and how to fix them if not. If a mod doesn't work after following the instructions below, check the troubleshooting guide or ask for help.

- @if (Model.BetaVersion != null) - { -

Note: "SDV @Model.BetaVersion only" lines are for an unreleased version of the game, not the stable version most players have. If a mod doesn't have that line, the info applies to both versions of the game.

- } -
+

The list is updated every few days (you can help update it!). It doesn't include XNB mods (see using XNB mods on the wiki instead) or compatible content packs.

-
-
- - + @if (Model.BetaVersion != null) + { +

Note: "SDV @Model.BetaVersion only" lines are for an unreleased version of the game, not the stable version most players have. If a mod doesn't have that line, the info applies to both versions of the game.

+ }
-
- - -
-
- {{filterGroup.label}}: + +
+
+ + +
+
+ + +
+
+ {{filterGroup.label}}: +
-
-
-
- {{visibleStats.total}} mods shown ({{Math.round((visibleStats.compatible + visibleStats.workaround) / visibleStats.total * 100)}}% compatible or have a workaround, {{Math.round((visibleStats.soon + visibleStats.broken) / visibleStats.total * 100)}}% broken, {{Math.round(visibleStats.abandoned / visibleStats.total * 100)}}% obsolete). +
+
+ {{visibleStats.total}} mods shown ({{Math.round((visibleStats.compatible + visibleStats.workaround) / visibleStats.total * 100)}}% compatible or have a workaround, {{Math.round((visibleStats.soon + visibleStats.broken) / visibleStats.total * 100)}}% broken, {{Math.round(visibleStats.abandoned / visibleStats.total * 100)}}% obsolete). +
+ No matching mods found.
- No matching mods found. -
- - - - - - - - - - - - - - - - - - - - - + + + + + + + +
mod namelinksauthorcompatibilitybroke incode 
- {{mod.Name}} - (aka {{mod.AlternateNames}}) - - - {{link.Text}}{{i < mod.ModPages.length - 1 ? ', ' : ''}} - - - {{mod.Author}} - (aka {{mod.AlternateAuthors}}) - -
-
- SDV @Model.BetaVersion only: - -
-
⚠ {{warning}}
- 		- source - no source - - - # - - - - [dev note] + + + + + + + + + + + + + + + + - - -
mod namelinksauthorcompatibilitybroke incode 
+ {{mod.Name}} + (aka {{mod.AlternateNames}}) + + + {{link.Text}}{{i < mod.ModPages.length - 1 ? ', ' : ''}} - -
- + 		+ {{mod.Author}} + (aka {{mod.AlternateAuthors}}) + +
+
+ SDV @Model.BetaVersion only: + +
+
⚠ {{warning}}
+ 		+ source + no source + + + # + + + + [dev note] + + +
+
+} diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index c50557b5..5856b96c 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -8,13 +8,6 @@ */ { - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Warning" - } - }, - "Site": { "RootUrl": "http://localhost:59482/", "ModListUrl": "http://localhost:59482/mods/", diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 532ea017..ea7e9cd2 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -10,7 +10,8 @@ "Logging": { "IncludeScopes": false, "LogLevel": { - "Default": "Warning" + "Default": "Warning", + "Hangfire": "Information" } }, @@ -55,7 +56,11 @@ }, "ModCompatibilityList": { - "CacheMinutes": 10 + "StaleMinutes": 15 + }, + + "BackgroundServices": { + "Enabled": true }, "ModUpdateCheck": { diff --git a/src/SMAPI.Web/wwwroot/Content/css/mods.css b/src/SMAPI.Web/wwwroot/Content/css/mods.css index fc5fff47..1c2b8056 100644 --- a/src/SMAPI.Web/wwwroot/Content/css/mods.css +++ b/src/SMAPI.Web/wwwroot/Content/css/mods.css @@ -15,30 +15,6 @@ border: 3px solid darkgreen; } -table.wikitable { - background-color:#f8f9fa; - color:#222; - border:1px solid #a2a9b1; - border-collapse:collapse -} - -table.wikitable > tr > th, -table.wikitable > tr > td, -table.wikitable > * > tr > th, -table.wikitable > * > tr > td { - border:1px solid #a2a9b1; - padding:0.2em 0.4em -} - -table.wikitable > tr > th, -table.wikitable > * > tr > th { - background-color:#eaecf0; -} - -table.wikitable > caption { - font-weight:bold -} - #options { margin-bottom: 1em; } @@ -73,6 +49,39 @@ table.wikitable > caption { opacity: 0.5; } +div.error { + padding: 2em 0; + color: red; + font-weight: bold; +} + +/********* +** Mod list +*********/ +table.wikitable { + background-color:#f8f9fa; + color:#222; + border:1px solid #a2a9b1; + border-collapse:collapse +} + +table.wikitable > tr > th, +table.wikitable > tr > td, +table.wikitable > * > tr > th, +table.wikitable > * > tr > td { + border:1px solid #a2a9b1; + padding:0.2em 0.4em +} + +table.wikitable > tr > th, +table.wikitable > * > tr > th { + background-color:#eaecf0; +} + +table.wikitable > caption { + font-weight:bold +} + #mod-list { font-size: 0.9em; } -- cgit From 17c6ae7ed995344111513ca91b18ec6598ec2399 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 24 Jul 2019 18:33:26 -0400 Subject: migrate update check caching to MongoDB (#651) --- src/SMAPI.Web/Controllers/ModsApiController.cs | 68 ++++++--------- src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs | 97 ++++++++++++++++++++++ .../Framework/Caching/Mods/IModCacheRepository.cs | 26 ++++++ .../Framework/Caching/Mods/ModCacheRepository.cs | 96 +++++++++++++++++++++ .../Framework/ConfigModels/ModUpdateCheckConfig.cs | 4 - .../Framework/ModRepositories/ModInfoModel.cs | 4 +- src/SMAPI.Web/Startup.cs | 2 + src/SMAPI.Web/appsettings.json | 1 - 8 files changed, 249 insertions(+), 49 deletions(-) create mode 100644 src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs create mode 100644 src/SMAPI.Web/Framework/Caching/Mods/IModCacheRepository.cs create mode 100644 src/SMAPI.Web/Framework/Caching/Mods/ModCacheRepository.cs (limited to 'src/SMAPI.Web/Framework/ConfigModels') diff --git a/src/SMAPI.Web/Controllers/ModsApiController.cs b/src/SMAPI.Web/Controllers/ModsApiController.cs index a74d0d8a..195ee5bf 100644 --- a/src/SMAPI.Web/Controllers/ModsApiController.cs +++ b/src/SMAPI.Web/Controllers/ModsApiController.cs @@ -2,17 +2,17 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit; using StardewModdingAPI.Toolkit.Framework.Clients.WebApi; using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; using StardewModdingAPI.Toolkit.Framework.ModData; using StardewModdingAPI.Toolkit.Framework.UpdateData; +using StardewModdingAPI.Web.Framework.Caching.Mods; +using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.Clients.Chucklefish; using StardewModdingAPI.Web.Framework.Clients.GitHub; using StardewModdingAPI.Web.Framework.Clients.ModDrop; @@ -33,8 +33,11 @@ namespace StardewModdingAPI.Web.Controllers /// The mod repositories which provide mod metadata. private readonly IDictionary Repositories; - /// The cache in which to store mod metadata. - private readonly IMemoryCache Cache; + /// The cache in which to store wiki data. + private readonly IWikiCacheRepository WikiCache; + + /// The cache in which to store mod data. + private readonly IModCacheRepository ModCache; /// The number of minutes successful update checks should be cached before refetching them. private readonly int SuccessCacheMinutes; @@ -42,9 +45,6 @@ namespace StardewModdingAPI.Web.Controllers /// The number of minutes failed update checks should be cached before refetching them. private readonly int ErrorCacheMinutes; - /// A regex which matches SMAPI-style semantic version. - private readonly string VersionRegex; - /// The internal mod metadata list. private readonly ModDatabase ModDatabase; @@ -57,22 +57,23 @@ namespace StardewModdingAPI.Web.Controllers *********/ /// Construct an instance. /// The web hosting environment. - /// The cache in which to store mod metadata. + /// The cache in which to store wiki data. + /// The cache in which to store mod metadata. /// The config settings for mod update checks. /// The Chucklefish API client. /// The GitHub API client. /// The ModDrop API client. /// The Nexus API client. - public ModsApiController(IHostingEnvironment environment, IMemoryCache cache, IOptions configProvider, IChucklefishClient chucklefish, IGitHubClient github, IModDropClient modDrop, INexusClient nexus) + public ModsApiController(IHostingEnvironment environment, IWikiCacheRepository wikiCache, IModCacheRepository modCache, IOptions configProvider, IChucklefishClient chucklefish, IGitHubClient github, IModDropClient modDrop, INexusClient nexus) { this.ModDatabase = new ModToolkit().GetModDatabase(Path.Combine(environment.WebRootPath, "SMAPI.metadata.json")); ModUpdateCheckConfig config = configProvider.Value; this.CompatibilityPageUrl = config.CompatibilityPageUrl; - this.Cache = cache; + this.WikiCache = wikiCache; + this.ModCache = modCache; this.SuccessCacheMinutes = config.SuccessCacheMinutes; this.ErrorCacheMinutes = config.ErrorCacheMinutes; - this.VersionRegex = config.SemanticVersionRegex; this.Repositories = new IModRepository[] { @@ -93,7 +94,7 @@ namespace StardewModdingAPI.Web.Controllers return new ModEntryModel[0]; // fetch wiki data - WikiModEntry[] wikiData = await this.GetWikiDataAsync(); + WikiModEntry[] wikiData = this.WikiCache.GetWikiMods().Select(p => p.GetModel()).ToArray(); IDictionary mods = new Dictionary(StringComparer.CurrentCultureIgnoreCase); foreach (ModSearchEntryModel mod in model.Mods) { @@ -218,26 +219,6 @@ namespace StardewModdingAPI.Web.Controllers return current != null && (other == null || other.IsOlderThan(current)); } - /// Get mod data from the wiki compatibility list. - private async Task GetWikiDataAsync() - { - ModToolkit toolkit = new ModToolkit(); - return await this.Cache.GetOrCreateAsync("_wiki", async entry => - { - try - { - WikiModEntry[] entries = (await toolkit.GetWikiCompatibilityListAsync()).Mods; - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.SuccessCacheMinutes); - return entries; - } - catch - { - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.ErrorCacheMinutes); - return new WikiModEntry[0]; - } - }); - } - /// Get the mod info for an update key. /// The namespaced update key. private async Task GetInfoForUpdateKeyAsync(string updateKey) @@ -247,24 +228,27 @@ namespace StardewModdingAPI.Web.Controllers if (!parsed.LooksValid) return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"The update key '{updateKey}' isn't in a valid format. It should contain the site key and mod ID like 'Nexus:541'."); - // get matching repository - if (!this.Repositories.TryGetValue(parsed.Repository, out IModRepository repository)) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{parsed.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}]."); - - // fetch mod info - return await this.Cache.GetOrCreateAsync($"{repository.VendorKey}:{parsed.ID}".ToLower(), async entry => + // get mod + if (!this.ModCache.TryGetMod(parsed.Repository, parsed.ID, out CachedMod mod) || this.ModCache.IsStale(mod.LastUpdated, mod.FetchStatus == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes)) { + // get site + if (!this.Repositories.TryGetValue(parsed.Repository, out IModRepository repository)) + return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{parsed.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}]."); + + // fetch mod ModInfoModel result = await repository.GetModInfoAsync(parsed.ID); if (result.Error == null) { if (result.Version == null) result.WithError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with no version number."); - else if (!Regex.IsMatch(result.Version, this.VersionRegex, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)) + else if (!SemanticVersion.TryParse(result.Version, out _)) result.WithError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with invalid semantic version '{result.Version}'."); } - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(result.Status == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes); - return result; - }); + + // cache mod + this.ModCache.SaveMod(repository.VendorKey, parsed.ID, result, out mod); + } + return mod.GetModel(); } /// Get update keys based on the available mod metadata, while maintaining the precedence order. diff --git a/src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs b/src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs new file mode 100644 index 00000000..fe8a7a1f --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs @@ -0,0 +1,97 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using StardewModdingAPI.Toolkit.Framework.UpdateData; +using StardewModdingAPI.Web.Framework.ModRepositories; + +namespace StardewModdingAPI.Web.Framework.Caching.Mods +{ + /// The model for cached mod data. + internal class CachedMod + { + /********* + ** Accessors + *********/ + /**** + ** Tracking + ****/ + /// The internal MongoDB ID. + [SuppressMessage("ReSharper", "InconsistentNaming", Justification = "Named per MongoDB conventions.")] + [BsonIgnoreIfDefault] + public ObjectId _id { get; set; } + + /// When the data was last updated. + public DateTimeOffset LastUpdated { get; set; } + + /// When the data was last requested through the web API. + public DateTimeOffset LastRequested { get; set; } + + /**** + ** Metadata + ****/ + /// The mod site on which the mod is found. + public ModRepositoryKey Site { get; set; } + + /// The mod's unique ID within the . + public string ID { get; set; } + + /// The mod availability status on the remote site. + public RemoteModStatus FetchStatus { get; set; } + + /// The error message providing more info for the , if applicable. + public string FetchError { get; set; } + + + /**** + ** Mod info + ****/ + /// The mod's display name. + public string Name { get; set; } + + /// The mod's latest version. + public string MainVersion { get; set; } + + /// The mod's latest optional or prerelease version, if newer than . + public string PreviewVersion { get; set; } + + /// The URL for the mod page. + public string Url { get; set; } + + + /********* + ** Accessors + *********/ + /// Construct an instance. + public CachedMod() { } + + /// Construct an instance. + /// The mod site on which the mod is found. + /// The mod's unique ID within the . + /// The mod data. + public CachedMod(ModRepositoryKey site, string id, ModInfoModel mod) + { + // tracking + this.LastUpdated = DateTimeOffset.UtcNow; + this.LastRequested = DateTimeOffset.UtcNow; + + // metadata + this.Site = site; + this.ID = id; + this.FetchStatus = mod.Status; + this.FetchError = mod.Error; + + // mod info + this.Name = mod.Name; + this.MainVersion = mod.Version; + this.PreviewVersion = mod.PreviewVersion; + this.Url = mod.Url; + } + + /// Get the API model for the cached data. + public ModInfoModel GetModel() + { + return new ModInfoModel(name: this.Name, version: this.MainVersion, previewVersion: this.PreviewVersion, url: this.Url).WithError(this.FetchStatus, this.FetchError); + } + } +} diff --git a/src/SMAPI.Web/Framework/Caching/Mods/IModCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/Mods/IModCacheRepository.cs new file mode 100644 index 00000000..23929d1d --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/Mods/IModCacheRepository.cs @@ -0,0 +1,26 @@ +using StardewModdingAPI.Toolkit.Framework.UpdateData; +using StardewModdingAPI.Web.Framework.ModRepositories; + +namespace StardewModdingAPI.Web.Framework.Caching.Mods +{ + /// Encapsulates logic for accessing the mod data cache. + internal interface IModCacheRepository : ICacheRepository + { + /********* + ** Methods + *********/ + /// Get the cached mod data. + /// The mod site to search. + /// The mod's unique ID within the . + /// The fetched mod. + /// Whether to update the mod's 'last requested' date. + bool TryGetMod(ModRepositoryKey site, string id, out CachedMod mod, bool markRequested = true); + + /// Save data fetched for a mod. + /// The mod site on which the mod is found. + /// The mod's unique ID within the . + /// The mod data. + /// The stored mod record. + void SaveMod(ModRepositoryKey site, string id, ModInfoModel mod, out CachedMod cachedMod); + } +} diff --git a/src/SMAPI.Web/Framework/Caching/Mods/ModCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/Mods/ModCacheRepository.cs new file mode 100644 index 00000000..d8ad7d21 --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/Mods/ModCacheRepository.cs @@ -0,0 +1,96 @@ +using System; +using MongoDB.Driver; +using StardewModdingAPI.Toolkit.Framework.UpdateData; +using StardewModdingAPI.Web.Framework.ModRepositories; + +namespace StardewModdingAPI.Web.Framework.Caching.Mods +{ + /// Encapsulates logic for accessing the mod data cache. + internal class ModCacheRepository : BaseCacheRepository, IModCacheRepository + { + /********* + ** Fields + *********/ + /// The collection for cached mod data. + private readonly IMongoCollection Mods; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The authenticated MongoDB database. + public ModCacheRepository(IMongoDatabase database) + { + // get collections + this.Mods = database.GetCollection("mods"); + + // add indexes if needed + this.Mods.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys.Ascending(p => p.ID).Ascending(p => p.Site))); + } + + /********* + ** Public methods + *********/ + /// Get the cached mod data. + /// The mod site to search. + /// The mod's unique ID within the . + /// The fetched mod. + /// Whether to update the mod's 'last requested' date. + public bool TryGetMod(ModRepositoryKey site, string id, out CachedMod mod, bool markRequested = true) + { + // get mod + id = this.NormaliseId(id); + mod = this.Mods.Find(entry => entry.ID == id && entry.Site == site).FirstOrDefault(); + if (mod == null) + return false; + + // bump 'last requested' + if (markRequested) + { + mod.LastRequested = DateTimeOffset.UtcNow; + mod = this.SaveMod(mod); + } + + return true; + } + + /// Save data fetched for a mod. + /// The mod site on which the mod is found. + /// The mod's unique ID within the . + /// The mod data. + /// The stored mod record. + public void SaveMod(ModRepositoryKey site, string id, ModInfoModel mod, out CachedMod cachedMod) + { + id = this.NormaliseId(id); + + cachedMod = this.SaveMod(new CachedMod(site, id, mod)); + } + + + /********* + ** Private methods + *********/ + /// Save data fetched for a mod. + /// The mod data. + public CachedMod SaveMod(CachedMod mod) + { + string id = this.NormaliseId(mod.ID); + + this.Mods.ReplaceOne( + entry => entry.ID == id && entry.Site == mod.Site, + mod, + new UpdateOptions { IsUpsert = true } + ); + + return mod; + } + + /// Normalise a mod ID for case-insensitive search. + /// The mod ID. + public string NormaliseId(string id) + { + return id.Trim().ToLower(); + } + } +} diff --git a/src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs index bde566c0..ab935bb3 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs @@ -12,10 +12,6 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The number of minutes failed update checks should be cached before refetching them. public int ErrorCacheMinutes { get; set; } - /// A regex which matches SMAPI-style semantic version. - /// Derived from SMAPI's SemanticVersion implementation. - public string SemanticVersionRegex { get; set; } - /// The web URL for the wiki compatibility list. public string CompatibilityPageUrl { get; set; } } diff --git a/src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs b/src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs index 16885bfd..15e6c213 100644 --- a/src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs +++ b/src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs @@ -18,7 +18,7 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories /// The mod's web URL. public string Url { get; set; } - /// The mod availability status. + /// The mod availability status on the remote site. public RemoteModStatus Status { get; set; } = RemoteModStatus.Ok; /// The error message indicating why the mod is invalid (if applicable). @@ -45,7 +45,7 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories } /// Set a mod error. - /// The mod availability status. + /// The mod availability status on the remote site. /// The error message indicating why the mod is invalid (if applicable). public ModInfoModel WithError(RemoteModStatus status, string error) { diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index caa7b056..fd229b5e 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -13,6 +13,7 @@ using Newtonsoft.Json; using StardewModdingAPI.Toolkit.Serialisation; using StardewModdingAPI.Web.Framework; using StardewModdingAPI.Web.Framework.Caching; +using StardewModdingAPI.Web.Framework.Caching.Mods; using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.Clients.Chucklefish; using StardewModdingAPI.Web.Framework.Clients.GitHub; @@ -88,6 +89,7 @@ namespace StardewModdingAPI.Web BsonSerializer.RegisterSerializer(new UtcDateTimeOffsetSerializer()); return new MongoClient(mongoConfig.GetConnectionString()).GetDatabase(mongoConfig.Database); }); + services.AddSingleton(serv => new ModCacheRepository(serv.GetRequiredService())); services.AddSingleton(serv => new WikiCacheRepository(serv.GetRequiredService())); // init Hangfire diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index ea7e9cd2..77d13924 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -66,7 +66,6 @@ "ModUpdateCheck": { "SuccessCacheMinutes": 60, "ErrorCacheMinutes": 5, - "SemanticVersionRegex": "^(?>(?0|[1-9]\\d*))\\.(?>(?0|[1-9]\\d*))(?>(?:\\.(?0|[1-9]\\d*))?)(?:-(?(?>[a-z0-9]+[\\-\\.]?)+))?$", "CompatibilityPageUrl": "https://mods.smapi.io" } } -- cgit From 95f261b1f30d8c5ad6c179cd75a220dcca3c6395 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 24 Jul 2019 22:11:51 -0400 Subject: fetch mod info from Nexus API if the web page is hidden due to adult content (#651) --- docs/release-notes.md | 1 + .../Framework/Clients/Nexus/NexusClient.cs | 133 ++++++++++++++++----- .../Framework/Clients/Nexus/NexusModStatus.cs | 3 + .../Framework/ConfigModels/ApiClientsConfig.cs | 3 + src/SMAPI.Web/SMAPI.Web.csproj | 1 + src/SMAPI.Web/Startup.cs | 10 +- src/SMAPI.Web/appsettings.Development.json | 2 + src/SMAPI.Web/appsettings.json | 1 + 8 files changed, 120 insertions(+), 34 deletions(-) (limited to 'src/SMAPI.Web/Framework/ConfigModels') diff --git a/docs/release-notes.md b/docs/release-notes.md index 4b380564..2b7d1545 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -24,6 +24,7 @@ These changes have not been released yet. * Fixed map reloads resetting tilesheet seasons. * Fixed map reloads not updating door warps. * Fixed outdoor tilesheets being seasonalised when added to an indoor location. + * Fixed update checks failing for Nexus mods marked as adult content. * For the mod compatibility list: * Now loads faster (since data is fetched in a background service). diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs index 87393367..753d3b4f 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusClient.cs @@ -4,8 +4,10 @@ using System.Linq; using System.Net; using System.Threading.Tasks; using HtmlAgilityPack; +using Pathoschild.FluentNexus.Models; using Pathoschild.Http.Client; using StardewModdingAPI.Toolkit; +using FluentNexusClient = Pathoschild.FluentNexus.NexusClient; namespace StardewModdingAPI.Web.Framework.Clients.Nexus { @@ -16,41 +18,73 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus ** Fields *********/ /// The URL for a Nexus mod page for the user, excluding the base URL, where {0} is the mod ID. - private readonly string ModUrlFormat; + private readonly string WebModUrlFormat; /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. - public string ModScrapeUrlFormat { get; set; } + public string WebModScrapeUrlFormat { get; set; } - /// The underlying HTTP client. - private readonly IClient Client; + /// The underlying HTTP client for the Nexus Mods website. + private readonly IClient WebClient; + + /// The underlying HTTP client for the Nexus API. + private readonly FluentNexusClient ApiClient; /********* ** Public methods *********/ /// Construct an instance. - /// The user agent for the Nexus Mods API client. - /// The base URL for the Nexus Mods site. - /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. - /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. - public NexusClient(string userAgent, string baseUrl, string modUrlFormat, string modScrapeUrlFormat) + /// The user agent for the Nexus Mods web client. + /// The base URL for the Nexus Mods site. + /// The URL for a Nexus Mods mod page for the user, excluding the , where {0} is the mod ID. + /// The URL for a Nexus mod page to scrape for versions, excluding the base URL, where {0} is the mod ID. + /// The app version to show in API user agents. + /// The Nexus API authentication key. + public NexusClient(string webUserAgent, string webBaseUrl, string webModUrlFormat, string webModScrapeUrlFormat, string apiAppVersion, string apiKey) { - this.ModUrlFormat = modUrlFormat; - this.ModScrapeUrlFormat = modScrapeUrlFormat; - this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); + this.WebModUrlFormat = webModUrlFormat; + this.WebModScrapeUrlFormat = webModScrapeUrlFormat; + this.WebClient = new FluentClient(webBaseUrl).SetUserAgent(webUserAgent); + this.ApiClient = new FluentNexusClient(apiKey, "SMAPI", apiAppVersion); } /// Get metadata about a mod. /// The Nexus mod ID. /// Returns the mod info if found, else null. public async Task GetModAsync(uint id) + { + // Fetch from the Nexus website when possible, since it has no rate limits. Mods with + // adult content are hidden for anonymous users, so fall back to the API in that case. + // Note that the API has very restrictive rate limits which means we can't just use it + // for all cases. + NexusMod mod = await this.GetModFromWebsiteAsync(id); + if (mod?.Status == NexusModStatus.AdultContentForbidden) + mod = await this.GetModFromApiAsync(id); + + return mod; + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.WebClient?.Dispose(); + } + + + /********* + ** Private methods + *********/ + /// Get metadata about a mod by scraping the Nexus website. + /// The Nexus mod ID. + /// Returns the mod info if found, else null. + private async Task GetModFromWebsiteAsync(uint id) { // fetch HTML string html; try { - html = await this.Client - .GetAsync(string.Format(this.ModScrapeUrlFormat, id)) + html = await this.WebClient + .GetAsync(string.Format(this.WebModScrapeUrlFormat, id)) .AsString(); } catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) @@ -74,14 +108,8 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus case "not found": return null; - case "hidden mod": - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Hidden }; - - case "not published": - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.NotPublished }; - default: - return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = NexusModStatus.Other }; + return new NexusMod { Error = $"Nexus error: {errorCode} ({errorText}).", Status = this.GetWebStatus(errorCode) }; } } @@ -130,23 +158,68 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus }; } - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - public void Dispose() + /// Get metadata about a mod from the Nexus API. + /// The Nexus mod ID. + /// Returns the mod info if found, else null. + private async Task GetModFromApiAsync(uint id) { - this.Client?.Dispose(); - } + // fetch mod + Mod mod = await this.ApiClient.Mods.GetMod("stardewvalley", (int)id); + ModFileList files = await this.ApiClient.ModFiles.GetModFiles("stardewvalley", (int)id, FileCategory.Main, FileCategory.Optional); + + // get versions + if (!SemanticVersion.TryParse(mod.Version, out ISemanticVersion mainVersion)) + mainVersion = null; + ISemanticVersion latestFileVersion = null; + foreach (string rawVersion in files.Files.Select(p => p.FileVersion)) + { + if (!SemanticVersion.TryParse(rawVersion, out ISemanticVersion cur)) + continue; + if (mainVersion != null && !cur.IsNewerThan(mainVersion)) + continue; + if (latestFileVersion != null && !cur.IsNewerThan(latestFileVersion)) + continue; + latestFileVersion = cur; + } + + // yield info + return new NexusMod + { + Name = mod.Name, + Version = SemanticVersion.TryParse(mod.Version, out ISemanticVersion version) ? version?.ToString() : mod.Version, + LatestFileVersion = latestFileVersion, + Url = this.GetModUrl(id) + }; + } - /********* - ** Private methods - *********/ /// Get the full mod page URL for a given ID. /// The mod ID. private string GetModUrl(uint id) { - UriBuilder builder = new UriBuilder(this.Client.BaseClient.BaseAddress); - builder.Path += string.Format(this.ModUrlFormat, id); + UriBuilder builder = new UriBuilder(this.WebClient.BaseClient.BaseAddress); + builder.Path += string.Format(this.WebModUrlFormat, id); return builder.Uri.ToString(); } + + /// Get the mod status for a web error code. + /// The Nexus error code. + private NexusModStatus GetWebStatus(string errorCode) + { + switch (errorCode.Trim().ToLower()) + { + case "adult content": + return NexusModStatus.AdultContentForbidden; + + case "hidden mod": + return NexusModStatus.Hidden; + + case "not published": + return NexusModStatus.NotPublished; + + default: + return NexusModStatus.Other; + } + } } } diff --git a/src/SMAPI.Web/Framework/Clients/Nexus/NexusModStatus.cs b/src/SMAPI.Web/Framework/Clients/Nexus/NexusModStatus.cs index c5723093..9ef314cd 100644 --- a/src/SMAPI.Web/Framework/Clients/Nexus/NexusModStatus.cs +++ b/src/SMAPI.Web/Framework/Clients/Nexus/NexusModStatus.cs @@ -12,6 +12,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Nexus /// The mod hasn't been published yet. NotPublished, + /// The mod contains adult content which is hidden for anonymous web users. + AdultContentForbidden, + /// The Nexus API returned an unhandled error. Other } diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index c27cadab..d2e9a2fe 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -65,6 +65,9 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The URL for a Nexus mod page to scrape for versions, excluding the , where {0} is the mod ID. public string NexusModScrapeUrlFormat { get; set; } + /// The Nexus API authentication key. + public string NexusApiKey { get; set; } + /**** ** Pastebin ****/ diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index f4e99e0c..26b29fce 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -26,6 +26,7 @@ + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 66e1f00d..4d23fe65 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -136,10 +136,12 @@ namespace StardewModdingAPI.Web )); services.AddSingleton(new NexusClient( - userAgent: userAgent, - baseUrl: api.NexusBaseUrl, - modUrlFormat: api.NexusModUrlFormat, - modScrapeUrlFormat: api.NexusModScrapeUrlFormat + webUserAgent: userAgent, + webBaseUrl: api.NexusBaseUrl, + webModUrlFormat: api.NexusModUrlFormat, + webModScrapeUrlFormat: api.NexusModScrapeUrlFormat, + apiAppVersion: version, + apiKey: api.NexusApiKey )); services.AddSingleton(new PastebinClient( diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 5856b96c..e6b4a1b1 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -20,6 +20,8 @@ "GitHubUsername": null, "GitHubPassword": null, + "NexusApiKey": null, + "PastebinUserKey": null, "PastebinDevKey": null }, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 77d13924..3ea37dea 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -39,6 +39,7 @@ "ModDropApiUrl": "https://www.moddrop.com/api/mods/data", "ModDropModPageUrl": "https://www.moddrop.com/sdv/mod/{0}", + "NexusApiKey": null, // see top note "NexusBaseUrl": "https://www.nexusmods.com/stardewvalley/", "NexusModUrlFormat": "mods/{0}", "NexusModScrapeUrlFormat": "mods/{0}?tab=files", -- cgit From 1d085df5b796e02b3e9e6874bd4e5684e840cb92 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Mon, 29 Jul 2019 16:43:25 -0400 Subject: track license info for mod GitHub repos (#651) --- docs/release-notes.md | 1 + .../Framework/Clients/Wiki/WikiModEntry.cs | 1 - .../Framework/UpdateData/UpdateKey.cs | 32 ++++++++++++- src/SMAPI.Web/BackgroundService.cs | 7 +-- src/SMAPI.Web/Controllers/ModsApiController.cs | 54 ++++++++++++++-------- src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs | 12 ++++- .../Framework/Clients/GitHub/GitHubClient.cs | 39 +++++++++------- .../Framework/Clients/GitHub/GitLicense.cs | 20 ++++++++ src/SMAPI.Web/Framework/Clients/GitHub/GitRepo.cs | 20 ++++++++ .../Framework/Clients/GitHub/IGitHubClient.cs | 5 ++ .../Framework/ConfigModels/ApiClientsConfig.cs | 6 --- .../ModRepositories/ChucklefishRepository.cs | 12 ++--- .../Framework/ModRepositories/GitHubRepository.cs | 24 +++++++--- .../Framework/ModRepositories/ModDropRepository.cs | 10 ++-- .../Framework/ModRepositories/ModInfoModel.cs | 42 ++++++++++++++++- .../Framework/ModRepositories/NexusRepository.cs | 8 ++-- src/SMAPI.Web/Startup.cs | 2 - src/SMAPI.Web/appsettings.json | 2 - 18 files changed, 220 insertions(+), 77 deletions(-) create mode 100644 src/SMAPI.Web/Framework/Clients/GitHub/GitLicense.cs create mode 100644 src/SMAPI.Web/Framework/Clients/GitHub/GitRepo.cs (limited to 'src/SMAPI.Web/Framework/ConfigModels') diff --git a/docs/release-notes.md b/docs/release-notes.md index b425112d..fd0cda51 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -26,6 +26,7 @@ These changes have not been released yet. * Fixed map reloads not updating door warps. * Fixed outdoor tilesheets being seasonalised when added to an indoor location. * Fixed update checks failing for Nexus mods marked as adult content. + * Fixed update checks not recognising releases on GitHub if they're not explicitly listed as update keys. * For the mod compatibility list: * Now loads faster (since data is fetched in a background service). diff --git a/src/SMAPI.Toolkit/Framework/Clients/Wiki/WikiModEntry.cs b/src/SMAPI.Toolkit/Framework/Clients/Wiki/WikiModEntry.cs index fff86891..06c44308 100644 --- a/src/SMAPI.Toolkit/Framework/Clients/Wiki/WikiModEntry.cs +++ b/src/SMAPI.Toolkit/Framework/Clients/Wiki/WikiModEntry.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki { diff --git a/src/SMAPI.Toolkit/Framework/UpdateData/UpdateKey.cs b/src/SMAPI.Toolkit/Framework/UpdateData/UpdateKey.cs index 865ebcf7..3fc1759e 100644 --- a/src/SMAPI.Toolkit/Framework/UpdateData/UpdateKey.cs +++ b/src/SMAPI.Toolkit/Framework/UpdateData/UpdateKey.cs @@ -3,7 +3,7 @@ using System; namespace StardewModdingAPI.Toolkit.Framework.UpdateData { /// A namespaced mod ID which uniquely identifies a mod within a mod repository. - public class UpdateKey + public class UpdateKey : IEquatable { /********* ** Accessors @@ -38,6 +38,12 @@ namespace StardewModdingAPI.Toolkit.Framework.UpdateData && !string.IsNullOrWhiteSpace(id); } + /// Construct an instance. + /// The mod repository containing the mod. + /// The mod ID within the repository. + public UpdateKey(ModRepositoryKey repository, string id) + : this($"{repository}:{id}", repository, id) { } + /// Parse a raw update key. /// The raw update key to parse. public static UpdateKey Parse(string raw) @@ -69,5 +75,29 @@ namespace StardewModdingAPI.Toolkit.Framework.UpdateData ? $"{this.Repository}:{this.ID}" : this.RawText; } + + /// Indicates whether the current object is equal to another object of the same type. + /// An object to compare with this object. + public bool Equals(UpdateKey other) + { + return + other != null + && this.Repository == other.Repository + && string.Equals(this.ID, other.ID, StringComparison.InvariantCultureIgnoreCase); + } + + /// Determines whether the specified object is equal to the current object. + /// The object to compare with the current object. + public override bool Equals(object obj) + { + return obj is UpdateKey other && this.Equals(other); + } + + /// Serves as the default hash function. + /// A hash code for the current object. + public override int GetHashCode() + { + return $"{this.Repository}:{this.ID}".ToLower().GetHashCode(); + } } } diff --git a/src/SMAPI.Web/BackgroundService.cs b/src/SMAPI.Web/BackgroundService.cs index 2dd3b921..dfd2c1b9 100644 --- a/src/SMAPI.Web/BackgroundService.cs +++ b/src/SMAPI.Web/BackgroundService.cs @@ -50,11 +50,11 @@ namespace StardewModdingAPI.Web // set startup tasks BackgroundJob.Enqueue(() => BackgroundService.UpdateWikiAsync()); - BackgroundJob.Enqueue(() => BackgroundService.RemoveStaleMods()); + BackgroundJob.Enqueue(() => BackgroundService.RemoveStaleModsAsync()); // set recurring tasks RecurringJob.AddOrUpdate(() => BackgroundService.UpdateWikiAsync(), "*/10 * * * *"); // every 10 minutes - RecurringJob.AddOrUpdate(() => BackgroundService.RemoveStaleMods(), "0 * * * *"); // hourly + RecurringJob.AddOrUpdate(() => BackgroundService.RemoveStaleModsAsync(), "0 * * * *"); // hourly return Task.CompletedTask; } @@ -84,9 +84,10 @@ namespace StardewModdingAPI.Web } /// Remove mods which haven't been requested in over 48 hours. - public static async Task RemoveStaleMods() + public static Task RemoveStaleModsAsync() { BackgroundService.ModCache.RemoveStaleMods(TimeSpan.FromHours(48)); + return Task.CompletedTask; } diff --git a/src/SMAPI.Web/Controllers/ModsApiController.cs b/src/SMAPI.Web/Controllers/ModsApiController.cs index 195ee5bf..13dd5529 100644 --- a/src/SMAPI.Web/Controllers/ModsApiController.cs +++ b/src/SMAPI.Web/Controllers/ModsApiController.cs @@ -123,18 +123,33 @@ namespace StardewModdingAPI.Web.Controllers // crossreference data ModDataRecord record = this.ModDatabase.Get(search.ID); WikiModEntry wikiEntry = wikiData.FirstOrDefault(entry => entry.ID.Contains(search.ID.Trim(), StringComparer.InvariantCultureIgnoreCase)); - string[] updateKeys = this.GetUpdateKeys(search.UpdateKeys, record, wikiEntry).ToArray(); + UpdateKey[] updateKeys = this.GetUpdateKeys(search.UpdateKeys, record, wikiEntry).ToArray(); + + // add soft lookups (don't log errors if the target doesn't exist) + UpdateKey[] softUpdateKeys = updateKeys.All(key => key.Repository != ModRepositoryKey.GitHub) && !string.IsNullOrWhiteSpace(wikiEntry?.GitHubRepo) + ? new[] { new UpdateKey(ModRepositoryKey.GitHub, wikiEntry.GitHubRepo) } + : new UpdateKey[0]; // get latest versions ModEntryModel result = new ModEntryModel { ID = search.ID }; IList errors = new List(); - foreach (string updateKey in updateKeys) + foreach (UpdateKey updateKey in updateKeys.Concat(softUpdateKeys)) { + bool isSoftLookup = softUpdateKeys.Contains(updateKey); + + // validate update key + if (!updateKey.LooksValid) + { + errors.Add($"The update key '{updateKey}' isn't in a valid format. It should contain the site key and mod ID like 'Nexus:541'."); + continue; + } + // fetch data ModInfoModel data = await this.GetInfoForUpdateKeyAsync(updateKey); if (data.Error != null) { - errors.Add(data.Error); + if (!isSoftLookup || data.Status != RemoteModStatus.DoesNotExist) + errors.Add(data.Error); continue; } @@ -221,32 +236,27 @@ namespace StardewModdingAPI.Web.Controllers /// Get the mod info for an update key. /// The namespaced update key. - private async Task GetInfoForUpdateKeyAsync(string updateKey) + private async Task GetInfoForUpdateKeyAsync(UpdateKey updateKey) { - // parse update key - UpdateKey parsed = UpdateKey.Parse(updateKey); - if (!parsed.LooksValid) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"The update key '{updateKey}' isn't in a valid format. It should contain the site key and mod ID like 'Nexus:541'."); - // get mod - if (!this.ModCache.TryGetMod(parsed.Repository, parsed.ID, out CachedMod mod) || this.ModCache.IsStale(mod.LastUpdated, mod.FetchStatus == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes)) + if (!this.ModCache.TryGetMod(updateKey.Repository, updateKey.ID, out CachedMod mod) || this.ModCache.IsStale(mod.LastUpdated, mod.FetchStatus == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes)) { // get site - if (!this.Repositories.TryGetValue(parsed.Repository, out IModRepository repository)) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{parsed.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}]."); + if (!this.Repositories.TryGetValue(updateKey.Repository, out IModRepository repository)) + return new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{updateKey.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}]."); // fetch mod - ModInfoModel result = await repository.GetModInfoAsync(parsed.ID); + ModInfoModel result = await repository.GetModInfoAsync(updateKey.ID); if (result.Error == null) { if (result.Version == null) - result.WithError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with no version number."); + result.SetError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with no version number."); else if (!SemanticVersion.TryParse(result.Version, out _)) - result.WithError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with invalid semantic version '{result.Version}'."); + result.SetError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with invalid semantic version '{result.Version}'."); } // cache mod - this.ModCache.SaveMod(repository.VendorKey, parsed.ID, result, out mod); + this.ModCache.SaveMod(repository.VendorKey, updateKey.ID, result, out mod); } return mod.GetModel(); } @@ -255,7 +265,7 @@ namespace StardewModdingAPI.Web.Controllers /// The specified update keys. /// The mod's entry in SMAPI's internal database. /// The mod's entry in the wiki list. - public IEnumerable GetUpdateKeys(string[] specifiedKeys, ModDataRecord record, WikiModEntry entry) + public IEnumerable GetUpdateKeys(string[] specifiedKeys, ModDataRecord record, WikiModEntry entry) { IEnumerable GetRaw() { @@ -283,10 +293,14 @@ namespace StardewModdingAPI.Web.Controllers } } - HashSet seen = new HashSet(StringComparer.InvariantCulture); - foreach (string key in GetRaw()) + HashSet seen = new HashSet(); + foreach (string rawKey in GetRaw()) { - if (!string.IsNullOrWhiteSpace(key) && seen.Add(key)) + if (string.IsNullOrWhiteSpace(rawKey)) + continue; + + UpdateKey key = UpdateKey.Parse(rawKey); + if (seen.Add(key)) yield return key; } } diff --git a/src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs b/src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs index fe8a7a1f..96eca847 100644 --- a/src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs +++ b/src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs @@ -58,6 +58,12 @@ namespace StardewModdingAPI.Web.Framework.Caching.Mods /// The URL for the mod page. public string Url { get; set; } + /// The license URL, if available. + public string LicenseUrl { get; set; } + + /// The license name, if available. + public string LicenseName { get; set; } + /********* ** Accessors @@ -86,12 +92,16 @@ namespace StardewModdingAPI.Web.Framework.Caching.Mods this.MainVersion = mod.Version; this.PreviewVersion = mod.PreviewVersion; this.Url = mod.Url; + this.LicenseUrl = mod.LicenseUrl; + this.LicenseName = mod.LicenseName; } /// Get the API model for the cached data. public ModInfoModel GetModel() { - return new ModInfoModel(name: this.Name, version: this.MainVersion, previewVersion: this.PreviewVersion, url: this.Url).WithError(this.FetchStatus, this.FetchError); + return new ModInfoModel(name: this.Name, version: this.MainVersion, url: this.Url, previewVersion: this.PreviewVersion) + .SetLicense(this.LicenseUrl, this.LicenseName) + .SetError(this.FetchStatus, this.FetchError); } } } diff --git a/src/SMAPI.Web/Framework/Clients/GitHub/GitHubClient.cs b/src/SMAPI.Web/Framework/Clients/GitHub/GitHubClient.cs index 22950db9..84c20957 100644 --- a/src/SMAPI.Web/Framework/Clients/GitHub/GitHubClient.cs +++ b/src/SMAPI.Web/Framework/Clients/GitHub/GitHubClient.cs @@ -12,12 +12,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.GitHub /********* ** Fields *********/ - /// The URL for a GitHub API query for the latest stable release, excluding the base URL, where {0} is the organisation and project name. - private readonly string StableReleaseUrlFormat; - - /// The URL for a GitHub API query for the latest release (including prerelease), excluding the base URL, where {0} is the organisation and project name. - private readonly string AnyReleaseUrlFormat; - /// The underlying HTTP client. private readonly IClient Client; @@ -27,17 +21,12 @@ namespace StardewModdingAPI.Web.Framework.Clients.GitHub *********/ /// Construct an instance. /// The base URL for the GitHub API. - /// The URL for a GitHub API query for the latest stable release, excluding the , where {0} is the organisation and project name. - /// The URL for a GitHub API query for the latest release (including prerelease), excluding the , where {0} is the organisation and project name. /// The user agent for the API client. /// The Accept header value expected by the GitHub API. /// The username with which to authenticate to the GitHub API. /// The password with which to authenticate to the GitHub API. - public GitHubClient(string baseUrl, string stableReleaseUrlFormat, string anyReleaseUrlFormat, string userAgent, string acceptHeader, string username, string password) + public GitHubClient(string baseUrl, string userAgent, string acceptHeader, string username, string password) { - this.StableReleaseUrlFormat = stableReleaseUrlFormat; - this.AnyReleaseUrlFormat = anyReleaseUrlFormat; - this.Client = new FluentClient(baseUrl) .SetUserAgent(userAgent) .AddDefault(req => req.WithHeader("Accept", acceptHeader)); @@ -45,25 +34,43 @@ namespace StardewModdingAPI.Web.Framework.Clients.GitHub this.Client = this.Client.SetBasicAuthentication(username, password); } + /// Get basic metadata for a GitHub repository, if available. + /// The repository key (like Pathoschild/SMAPI). + /// Returns the repository info if it exists, else null. + public async Task GetRepositoryAsync(string repo) + { + this.AssertKeyFormat(repo); + try + { + return await this.Client + .GetAsync($"repos/{repo}") + .As(); + } + catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + return null; + } + } + /// Get the latest release for a GitHub repository. /// The repository key (like Pathoschild/SMAPI). /// Whether to return a prerelease version if it's latest. /// Returns the release if found, else null. public async Task GetLatestReleaseAsync(string repo, bool includePrerelease = false) { - this.AssetKeyFormat(repo); + this.AssertKeyFormat(repo); try { if (includePrerelease) { GitRelease[] results = await this.Client - .GetAsync(string.Format(this.AnyReleaseUrlFormat, repo)) + .GetAsync($"repos/{repo}/releases?per_page=2") // allow for draft release (only visible if GitHub repo is owned by same account as the update check credentials) .AsArray(); return results.FirstOrDefault(p => !p.IsDraft); } return await this.Client - .GetAsync(string.Format(this.StableReleaseUrlFormat, repo)) + .GetAsync($"repos/{repo}/releases/latest") .As(); } catch (ApiException ex) when (ex.Status == HttpStatusCode.NotFound) @@ -85,7 +92,7 @@ namespace StardewModdingAPI.Web.Framework.Clients.GitHub /// Assert that a repository key is formatted correctly. /// The repository key (like Pathoschild/SMAPI). /// The repository key is invalid. - private void AssetKeyFormat(string repo) + private void AssertKeyFormat(string repo) { if (repo == null || !repo.Contains("/") || repo.IndexOf("/", StringComparison.InvariantCultureIgnoreCase) != repo.LastIndexOf("/", StringComparison.InvariantCultureIgnoreCase)) throw new ArgumentException($"The value '{repo}' isn't a valid GitHub repository key, must be a username and project name like 'Pathoschild/SMAPI'.", nameof(repo)); diff --git a/src/SMAPI.Web/Framework/Clients/GitHub/GitLicense.cs b/src/SMAPI.Web/Framework/Clients/GitHub/GitLicense.cs new file mode 100644 index 00000000..736efbe6 --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/GitHub/GitLicense.cs @@ -0,0 +1,20 @@ +using Newtonsoft.Json; + +namespace StardewModdingAPI.Web.Framework.Clients.GitHub +{ + /// The license info for a GitHub project. + internal class GitLicense + { + /// The license display name. + [JsonProperty("name")] + public string Name { get; set; } + + /// The SPDX ID for the license. + [JsonProperty("spdx_id")] + public string SpdxId { get; set; } + + /// The URL for the license info. + [JsonProperty("url")] + public string Url { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/Clients/GitHub/GitRepo.cs b/src/SMAPI.Web/Framework/Clients/GitHub/GitRepo.cs new file mode 100644 index 00000000..7d80576e --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/GitHub/GitRepo.cs @@ -0,0 +1,20 @@ +using Newtonsoft.Json; + +namespace StardewModdingAPI.Web.Framework.Clients.GitHub +{ + /// Basic metadata about a GitHub project. + internal class GitRepo + { + /// The full repository name, including the owner. + [JsonProperty("full_name")] + public string FullName { get; set; } + + /// The URL to the repository web page, if any. + [JsonProperty("html_url")] + public string WebUrl { get; set; } + + /// The code license, if any. + [JsonProperty("license")] + public GitLicense License { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/Clients/GitHub/IGitHubClient.cs b/src/SMAPI.Web/Framework/Clients/GitHub/IGitHubClient.cs index 9519c26f..a34f03bd 100644 --- a/src/SMAPI.Web/Framework/Clients/GitHub/IGitHubClient.cs +++ b/src/SMAPI.Web/Framework/Clients/GitHub/IGitHubClient.cs @@ -9,6 +9,11 @@ namespace StardewModdingAPI.Web.Framework.Clients.GitHub /********* ** Methods *********/ + /// Get basic metadata for a GitHub repository, if available. + /// The repository key (like Pathoschild/SMAPI). + /// Returns the repository info if it exists, else null. + Task GetRepositoryAsync(string repo); + /// Get the latest release for a GitHub repository. /// The repository key (like Pathoschild/SMAPI). /// Whether to return a prerelease version if it's latest. diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index d2e9a2fe..a0a1f42a 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -29,12 +29,6 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The base URL for the GitHub API. public string GitHubBaseUrl { get; set; } - /// The URL for a GitHub API query for the latest stable release, excluding the , where {0} is the organisation and project name. - public string GitHubStableReleaseUrlFormat { get; set; } - - /// The URL for a GitHub API query for the latest release (including prerelease), excluding the , where {0} is the organisation and project name. - public string GitHubAnyReleaseUrlFormat { get; set; } - /// The Accept header value expected by the GitHub API. public string GitHubAcceptHeader { get; set; } diff --git a/src/SMAPI.Web/Framework/ModRepositories/ChucklefishRepository.cs b/src/SMAPI.Web/Framework/ModRepositories/ChucklefishRepository.cs index 04c80dd2..c14fb45d 100644 --- a/src/SMAPI.Web/Framework/ModRepositories/ChucklefishRepository.cs +++ b/src/SMAPI.Web/Framework/ModRepositories/ChucklefishRepository.cs @@ -32,21 +32,19 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories { // validate ID format if (!uint.TryParse(id, out uint realID)) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid Chucklefish mod ID, must be an integer ID."); + return new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid Chucklefish mod ID, must be an integer ID."); // fetch info try { var mod = await this.Client.GetModAsync(realID); - if (mod == null) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, "Found no Chucklefish mod with this ID."); - - // create model - return new ModInfoModel(name: mod.Name, version: this.NormaliseVersion(mod.Version), url: mod.Url); + return mod != null + ? new ModInfoModel(name: mod.Name, version: this.NormaliseVersion(mod.Version), url: mod.Url) + : new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, "Found no Chucklefish mod with this ID."); } catch (Exception ex) { - return new ModInfoModel().WithError(RemoteModStatus.TemporaryError, ex.ToString()); + return new ModInfoModel().SetError(RemoteModStatus.TemporaryError, ex.ToString()); } } diff --git a/src/SMAPI.Web/Framework/ModRepositories/GitHubRepository.cs b/src/SMAPI.Web/Framework/ModRepositories/GitHubRepository.cs index 614e00c2..0e254b39 100644 --- a/src/SMAPI.Web/Framework/ModRepositories/GitHubRepository.cs +++ b/src/SMAPI.Web/Framework/ModRepositories/GitHubRepository.cs @@ -30,36 +30,46 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories /// The mod ID in this repository. public override async Task GetModInfoAsync(string id) { + ModInfoModel result = new ModInfoModel().SetBasicInfo(id, $"https://github.com/{id}/releases"); + // validate ID format if (!id.Contains("/") || id.IndexOf("/", StringComparison.InvariantCultureIgnoreCase) != id.LastIndexOf("/", StringComparison.InvariantCultureIgnoreCase)) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid GitHub mod ID, must be a username and project name like 'Pathoschild/LookupAnything'."); + return result.SetError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid GitHub mod ID, must be a username and project name like 'Pathoschild/LookupAnything'."); // fetch info try { + // fetch repo info + GitRepo repository = await this.Client.GetRepositoryAsync(id); + if (repository == null) + return result.SetError(RemoteModStatus.DoesNotExist, "Found no GitHub repository for this ID."); + result + .SetBasicInfo(repository.FullName, $"{repository.WebUrl}/releases") + .SetLicense(url: repository.License?.Url, name: repository.License?.Name); + // get latest release (whether preview or stable) GitRelease latest = await this.Client.GetLatestReleaseAsync(id, includePrerelease: true); if (latest == null) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, "Found no GitHub release for this ID."); + return result.SetError(RemoteModStatus.DoesNotExist, "Found no GitHub release for this ID."); // split stable/prerelease if applicable GitRelease preview = null; if (latest.IsPrerelease) { - GitRelease result = await this.Client.GetLatestReleaseAsync(id, includePrerelease: false); - if (result != null) + GitRelease release = await this.Client.GetLatestReleaseAsync(id, includePrerelease: false); + if (release != null) { preview = latest; - latest = result; + latest = release; } } // return data - return new ModInfoModel(name: id, version: this.NormaliseVersion(latest.Tag), previewVersion: this.NormaliseVersion(preview?.Tag), url: $"https://github.com/{id}/releases"); + return result.SetVersions(version: this.NormaliseVersion(latest.Tag), previewVersion: this.NormaliseVersion(preview?.Tag)); } catch (Exception ex) { - return new ModInfoModel().WithError(RemoteModStatus.TemporaryError, ex.ToString()); + return result.SetError(RemoteModStatus.TemporaryError, ex.ToString()); } } diff --git a/src/SMAPI.Web/Framework/ModRepositories/ModDropRepository.cs b/src/SMAPI.Web/Framework/ModRepositories/ModDropRepository.cs index 5703b34e..62142668 100644 --- a/src/SMAPI.Web/Framework/ModRepositories/ModDropRepository.cs +++ b/src/SMAPI.Web/Framework/ModRepositories/ModDropRepository.cs @@ -32,19 +32,19 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories { // validate ID format if (!long.TryParse(id, out long modDropID)) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid ModDrop mod ID, must be an integer ID."); + return new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid ModDrop mod ID, must be an integer ID."); // fetch info try { ModDropMod mod = await this.Client.GetModAsync(modDropID); - if (mod == null) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, "Found no ModDrop mod with this ID."); - return new ModInfoModel(name: mod.Name, version: mod.LatestDefaultVersion?.ToString(), previewVersion: mod.LatestOptionalVersion?.ToString(), url: mod.Url); + return mod != null + ? new ModInfoModel(name: mod.Name, version: mod.LatestDefaultVersion?.ToString(), previewVersion: mod.LatestOptionalVersion?.ToString(), url: mod.Url) + : new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, "Found no ModDrop mod with this ID."); } catch (Exception ex) { - return new ModInfoModel().WithError(RemoteModStatus.TemporaryError, ex.ToString()); + return new ModInfoModel().SetError(RemoteModStatus.TemporaryError, ex.ToString()); } } diff --git a/src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs b/src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs index 15e6c213..46b98860 100644 --- a/src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs +++ b/src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs @@ -18,6 +18,12 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories /// The mod's web URL. public string Url { get; set; } + /// The license URL, if available. + public string LicenseUrl { get; set; } + + /// The license name, if available. + public string LicenseName { get; set; } + /// The mod availability status on the remote site. public RemoteModStatus Status { get; set; } = RemoteModStatus.Ok; @@ -37,17 +43,49 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories /// The semantic version for the mod's latest preview release, if available and different from . /// The mod's web URL. public ModInfoModel(string name, string version, string url, string previewVersion = null) + { + this + .SetBasicInfo(name, url) + .SetVersions(version, previewVersion); + } + + /// Set the basic mod info. + /// The mod name. + /// The mod's web URL. + public ModInfoModel SetBasicInfo(string name, string url) { this.Name = name; + this.Url = url; + + return this; + } + + /// Set the mod version info. + /// The semantic version for the mod's latest release. + /// The semantic version for the mod's latest preview release, if available and different from . + public ModInfoModel SetVersions(string version, string previewVersion = null) + { this.Version = version; this.PreviewVersion = previewVersion; - this.Url = url; + + return this; + } + + /// Set the license info, if available. + /// The license URL. + /// The license name. + public ModInfoModel SetLicense(string url, string name) + { + this.LicenseUrl = url; + this.LicenseName = name; + + return this; } /// Set a mod error. /// The mod availability status on the remote site. /// The error message indicating why the mod is invalid (if applicable). - public ModInfoModel WithError(RemoteModStatus status, string error) + public ModInfoModel SetError(RemoteModStatus status, string error) { this.Status = status; this.Error = error; diff --git a/src/SMAPI.Web/Framework/ModRepositories/NexusRepository.cs b/src/SMAPI.Web/Framework/ModRepositories/NexusRepository.cs index a4ae61eb..b4791f56 100644 --- a/src/SMAPI.Web/Framework/ModRepositories/NexusRepository.cs +++ b/src/SMAPI.Web/Framework/ModRepositories/NexusRepository.cs @@ -32,27 +32,27 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories { // validate ID format if (!uint.TryParse(id, out uint nexusID)) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid Nexus mod ID, must be an integer ID."); + return new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid Nexus mod ID, must be an integer ID."); // fetch info try { NexusMod mod = await this.Client.GetModAsync(nexusID); if (mod == null) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, "Found no Nexus mod with this ID."); + return new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, "Found no Nexus mod with this ID."); if (mod.Error != null) { RemoteModStatus remoteStatus = mod.Status == NexusModStatus.Hidden || mod.Status == NexusModStatus.NotPublished ? RemoteModStatus.DoesNotExist : RemoteModStatus.TemporaryError; - return new ModInfoModel().WithError(remoteStatus, mod.Error); + return new ModInfoModel().SetError(remoteStatus, mod.Error); } return new ModInfoModel(name: mod.Name, version: this.NormaliseVersion(mod.Version), previewVersion: mod.LatestFileVersion?.ToString(), url: mod.Url); } catch (Exception ex) { - return new ModInfoModel().WithError(RemoteModStatus.TemporaryError, ex.ToString()); + return new ModInfoModel().SetError(RemoteModStatus.TemporaryError, ex.ToString()); } } diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 4d23fe65..33737235 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -121,8 +121,6 @@ namespace StardewModdingAPI.Web services.AddSingleton(new GitHubClient( baseUrl: api.GitHubBaseUrl, - stableReleaseUrlFormat: api.GitHubStableReleaseUrlFormat, - anyReleaseUrlFormat: api.GitHubAnyReleaseUrlFormat, userAgent: userAgent, acceptHeader: api.GitHubAcceptHeader, username: api.GitHubUsername, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 3ea37dea..f9777f87 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -30,8 +30,6 @@ "ChucklefishModPageUrlFormat": "resources/{0}", "GitHubBaseUrl": "https://api.github.com", - "GitHubStableReleaseUrlFormat": "repos/{0}/releases/latest", - "GitHubAnyReleaseUrlFormat": "repos/{0}/releases?per_page=2", // allow for draft release (only visible if GitHub repo is owned by same account as the update check credentials) "GitHubAcceptHeader": "application/vnd.github.v3+json", "GitHubUsername": null, // see top note "GitHubPassword": null, // see top note -- cgit From 3ba567eaddeaa0bb2bdd749b56e0601d1cf65a25 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 4 Aug 2019 03:28:34 -0400 Subject: add JSON validator with initial support for manifest format (#654) --- .../Controllers/JsonValidatorController.cs | 217 +++++++++++++++++++++ src/SMAPI.Web/Controllers/LogParserController.cs | 4 +- .../Framework/Clients/Pastebin/IPastebinClient.cs | 3 +- .../Framework/Clients/Pastebin/PastebinClient.cs | 5 +- src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs | 3 + src/SMAPI.Web/SMAPI.Web.csproj | 1 + src/SMAPI.Web/Startup.cs | 2 +- .../JsonValidator/JsonValidatorErrorModel.cs | 36 ++++ .../ViewModels/JsonValidator/JsonValidatorModel.cs | 92 +++++++++ .../JsonValidator/JsonValidatorRequestModel.cs | 15 ++ src/SMAPI.Web/Views/JsonValidator/Index.cshtml | 126 ++++++++++++ src/SMAPI.Web/Views/Shared/_Layout.cshtml | 7 +- src/SMAPI.Web/appsettings.Development.json | 1 + src/SMAPI.Web/appsettings.json | 1 + .../wwwroot/Content/css/json-validator.css | 98 ++++++++++ src/SMAPI.Web/wwwroot/Content/css/main.css | 2 +- src/SMAPI.Web/wwwroot/Content/js/json-validator.js | 60 ++++++ src/SMAPI.Web/wwwroot/Content/js/log-parser.js | 17 +- src/SMAPI.Web/wwwroot/schemas/manifest.json | 120 ++++++++++++ 19 files changed, 792 insertions(+), 18 deletions(-) create mode 100644 src/SMAPI.Web/Controllers/JsonValidatorController.cs create mode 100644 src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorErrorModel.cs create mode 100644 src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs create mode 100644 src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorRequestModel.cs create mode 100644 src/SMAPI.Web/Views/JsonValidator/Index.cshtml create mode 100644 src/SMAPI.Web/wwwroot/Content/css/json-validator.css create mode 100644 src/SMAPI.Web/wwwroot/Content/js/json-validator.js create mode 100644 src/SMAPI.Web/wwwroot/schemas/manifest.json (limited to 'src/SMAPI.Web/Framework/ConfigModels') diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs new file mode 100644 index 00000000..9d1685ac --- /dev/null +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -0,0 +1,217 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Schema; +using StardewModdingAPI.Web.Framework; +using StardewModdingAPI.Web.Framework.Clients.Pastebin; +using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.ConfigModels; +using StardewModdingAPI.Web.ViewModels.JsonValidator; + +namespace StardewModdingAPI.Web.Controllers +{ + /// Provides a web UI for validating JSON schemas. + internal class JsonValidatorController : Controller + { + /********* + ** Fields + *********/ + /// The site config settings. + private readonly SiteConfig Config; + + /// The underlying Pastebin client. + private readonly IPastebinClient Pastebin; + + /// The underlying text compression helper. + private readonly IGzipHelper GzipHelper; + + /// The section URL for the schema validator. + private string SectionUrl => this.Config.JsonValidatorUrl; + + /// The supported JSON schemas (names indexed by ID). + private readonly IDictionary SchemaFormats = new Dictionary + { + ["none"] = "None", + ["manifest"] = "Manifest" + }; + + /// The schema ID to use if none was specified. + private string DefaultSchemaID = "manifest"; + + + /********* + ** Public methods + *********/ + /*** + ** Constructor + ***/ + /// Construct an instance. + /// The context config settings. + /// The Pastebin API client. + /// The underlying text compression helper. + public JsonValidatorController(IOptions siteConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + { + this.Config = siteConfig.Value; + this.Pastebin = pastebin; + this.GzipHelper = gzipHelper; + } + + /*** + ** Web UI + ***/ + /// Render the schema validator UI. + /// The schema name with which to validate the JSON. + /// The paste ID. + [HttpGet] + [Route("json")] + [Route("json/{schemaName}")] + [Route("json/{schemaName}/{id}")] + public async Task Index(string schemaName = null, string id = null) + { + schemaName = this.NormaliseSchemaName(schemaName); + + var result = new JsonValidatorModel(this.SectionUrl, id, schemaName, this.SchemaFormats); + if (string.IsNullOrWhiteSpace(id)) + return this.View("Index", result); + + // fetch raw JSON + PasteInfo paste = await this.GetAsync(id); + if (string.IsNullOrWhiteSpace(paste.Content)) + return this.View("Index", result.SetUploadError("The JSON file seems to be empty.")); + result.SetContent(paste.Content); + + // parse JSON + JToken parsed; + try + { + parsed = JToken.Parse(paste.Content); + } + catch (JsonReaderException ex) + { + return this.View("Index", result.AddErrors(new JsonValidatorErrorModel(ex.LineNumber, ex.Path, ex.Message))); + } + + // skip if no schema selected + if (schemaName == "none") + return this.View("Index", result); + + // load schema + JSchema schema; + { + FileInfo schemaFile = this.FindSchemaFile(schemaName); + if (schemaFile == null) + return this.View("Index", result.SetParseError($"Invalid schema '{schemaName}'.")); + schema = JSchema.Parse(System.IO.File.ReadAllText(schemaFile.FullName)); + } + + // validate JSON + parsed.IsValid(schema, out IList rawErrors); + var errors = rawErrors + .Select(error => new JsonValidatorErrorModel(error.LineNumber, error.Path, this.GetFlattenedError(error))) + .ToArray(); + return this.View("Index", result.AddErrors(errors)); + } + + /*** + ** JSON + ***/ + /// Save raw JSON data. + [HttpPost, AllowLargePosts] + [Route("json")] + public async Task PostAsync(JsonValidatorRequestModel request) + { + if (request == null) + return this.View("Index", new JsonValidatorModel(this.SectionUrl, null, null, this.SchemaFormats).SetUploadError("The request seems to be invalid.")); + + // normalise schema name + string schemaName = this.NormaliseSchemaName(request.SchemaName); + + // get raw log text + string input = request.Content; + if (string.IsNullOrWhiteSpace(input)) + return this.View("Index", new JsonValidatorModel(this.SectionUrl, null, schemaName, this.SchemaFormats).SetUploadError("The JSON file seems to be empty.")); + + // upload log + input = this.GzipHelper.CompressString(input); + SavePasteResult result = await this.Pastebin.PostAsync($"JSON validator {DateTime.UtcNow:s}", input); + + // handle errors + if (!result.Success) + return this.View("Index", new JsonValidatorModel(this.SectionUrl, result.ID, schemaName, this.SchemaFormats).SetUploadError($"Pastebin error: {result.Error ?? "unknown error"}")); + + // redirect to view + UriBuilder uri = new UriBuilder(new Uri(this.SectionUrl)); + uri.Path = $"{uri.Path.TrimEnd('/')}/{schemaName}/{result.ID}"; + return this.Redirect(uri.Uri.ToString()); + } + + + /********* + ** Private methods + *********/ + /// Fetch raw text from Pastebin. + /// The Pastebin paste ID. + private async Task GetAsync(string id) + { + PasteInfo response = await this.Pastebin.GetAsync(id); + response.Content = this.GzipHelper.DecompressString(response.Content); + return response; + } + + /// Get a flattened, human-readable message representing a schema validation error. + /// The error to represent. + /// The indentation level to apply for inner errors. + private string GetFlattenedError(ValidationError error, int indent = 0) + { + // get friendly representation of main error + string message = error.Message; + switch (error.ErrorType) + { + case ErrorType.Enum: + message = $"Invalid value. Found '{error.Value}', but expected one of '{string.Join("', '", error.Schema.Enum)}'."; + break; + } + + // add inner errors + foreach (ValidationError childError in error.ChildErrors) + message += "\n" + "".PadLeft(indent * 2, ' ') + $"==> {childError.Path}: " + this.GetFlattenedError(childError, indent + 1); + return message; + } + + /// Get a normalised schema name, or the if blank. + /// The raw schema name to normalise. + private string NormaliseSchemaName(string schemaName) + { + schemaName = schemaName?.Trim().ToLower(); + return !string.IsNullOrWhiteSpace(schemaName) + ? schemaName + : this.DefaultSchemaID; + } + + /// Get the schema file given its unique ID. + /// The schema ID. + private FileInfo FindSchemaFile(string id) + { + // normalise ID + id = id?.Trim().ToLower(); + if (string.IsNullOrWhiteSpace(id)) + return null; + + // get matching file + DirectoryInfo schemaDir = new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "schemas")); + foreach (FileInfo file in schemaDir.EnumerateFiles("*.json")) + { + if (file.Name.Equals($"{id}.json")) + return file; + } + + return null; + } + } +} diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index dc5895b0..0556a81e 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -84,7 +84,7 @@ namespace StardewModdingAPI.Web.Controllers // upload log input = this.GzipHelper.CompressString(input); - SavePasteResult result = await this.Pastebin.PostAsync(input); + SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", input); // handle errors if (!result.Success) @@ -108,7 +108,5 @@ namespace StardewModdingAPI.Web.Controllers response.Content = this.GzipHelper.DecompressString(response.Content); return response; } - - } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs index 630dfb76..a635abe3 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs @@ -11,7 +11,8 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin Task GetAsync(string id); /// Save a paste to Pastebin. + /// The paste name. /// The paste content. - Task PostAsync(string content); + Task PostAsync(string name, string content); } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs index 1e46f2dc..2e8a8c68 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs @@ -67,8 +67,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin } /// Save a paste to Pastebin. + /// The paste name. /// The paste content. - public async Task PostAsync(string content) + public async Task PostAsync(string name, string content) { try { @@ -85,7 +86,7 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin api_user_key = this.UserKey, api_dev_key = this.DevKey, api_paste_private = 1, // unlisted - api_paste_name = $"SMAPI log {DateTime.UtcNow:s}", + api_paste_name = name, api_paste_expire_date = "N", // never expire api_paste_code = content })) diff --git a/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs index d89a4260..bc6e868a 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs @@ -12,6 +12,9 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The root URL for the log parser. public string LogParserUrl { get; set; } + /// The root URL for the JSON validator. + public string JsonValidatorUrl { get; set; } + /// The root URL for the mod list. public string ModListUrl { get; set; } diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index 26b29fce..98517818 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -26,6 +26,7 @@ + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index bb43f5a5..de45b8a4 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -203,7 +203,7 @@ namespace StardewModdingAPI.Web redirects.Add(new ConditionalRewriteSubdomainRule( shouldRewrite: req => req.Host.Host != "localhost" - && (req.Host.Host.StartsWith("api.") || req.Host.Host.StartsWith("log.") || req.Host.Host.StartsWith("mods.")) + && (req.Host.Host.StartsWith("api.") || req.Host.Host.StartsWith("json.") || req.Host.Host.StartsWith("log.") || req.Host.Host.StartsWith("mods.")) && !req.Path.StartsWithSegments("/content") && !req.Path.StartsWithSegments("/favicon.ico") )); diff --git a/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorErrorModel.cs b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorErrorModel.cs new file mode 100644 index 00000000..f9497a38 --- /dev/null +++ b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorErrorModel.cs @@ -0,0 +1,36 @@ +namespace StardewModdingAPI.Web.ViewModels.JsonValidator +{ + /// The view model for a JSON validator error. + public class JsonValidatorErrorModel + { + /********* + ** Accessors + *********/ + /// The line number on which the error occurred. + public int Line { get; set; } + + /// The field path in the JSON file where the error occurred. + public string Path { get; set; } + + /// A human-readable description of the error. + public string Message { get; set; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + public JsonValidatorErrorModel() { } + + /// Construct an instance. + /// The line number on which the error occurred. + /// The field path in the JSON file where the error occurred. + /// A human-readable description of the error. + public JsonValidatorErrorModel(int line, string path, string message) + { + this.Line = line; + this.Path = path; + this.Message = message; + } + } +} diff --git a/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs new file mode 100644 index 00000000..4c122d4f --- /dev/null +++ b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; +using System.Linq; + +namespace StardewModdingAPI.Web.ViewModels.JsonValidator +{ + /// The view model for the JSON validator page. + public class JsonValidatorModel + { + /********* + ** Accessors + *********/ + /// The root URL for the log parser controller. + public string SectionUrl { get; set; } + + /// The paste ID. + public string PasteID { get; set; } + + /// The schema name with which the JSON was validated. + public string SchemaName { get; set; } + + /// The supported JSON schemas (names indexed by ID). + public readonly IDictionary SchemaFormats; + + /// The validated content. + public string Content { get; set; } + + /// The schema validation errors, if any. + public JsonValidatorErrorModel[] Errors { get; set; } = new JsonValidatorErrorModel[0]; + + /// An error which occurred while uploading the JSON to Pastebin. + public string UploadError { get; set; } + + /// An error which occurred while parsing the JSON. + public string ParseError { get; set; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + public JsonValidatorModel() { } + + /// Construct an instance. + /// The root URL for the log parser controller. + /// The paste ID. + /// The schema name with which the JSON was validated. + /// The supported JSON schemas (names indexed by ID). + public JsonValidatorModel(string sectionUrl, string pasteID, string schemaName, IDictionary schemaFormats) + { + this.SectionUrl = sectionUrl; + this.PasteID = pasteID; + this.SchemaName = schemaName; + this.SchemaFormats = schemaFormats; + } + + /// Set the validated content. + /// The validated content. + public JsonValidatorModel SetContent(string content) + { + this.Content = content; + + return this; + } + + /// Set the error which occurred while uploading the log to Pastebin. + /// The error message. + public JsonValidatorModel SetUploadError(string error) + { + this.UploadError = error; + + return this; + } + + /// Set the error which occurred while parsing the JSON. + /// The error message. + public JsonValidatorModel SetParseError(string error) + { + this.ParseError = error; + + return this; + } + + /// Add validation errors to the response. + /// The schema validation errors. + public JsonValidatorModel AddErrors(params JsonValidatorErrorModel[] errors) + { + this.Errors = this.Errors.Concat(errors).ToArray(); + + return this; + } + } +} diff --git a/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorRequestModel.cs b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorRequestModel.cs new file mode 100644 index 00000000..c8e851bf --- /dev/null +++ b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorRequestModel.cs @@ -0,0 +1,15 @@ +namespace StardewModdingAPI.Web.ViewModels.JsonValidator +{ + /// The view model for a JSON validation request. + public class JsonValidatorRequestModel + { + /********* + ** Accessors + *********/ + /// The schema name with which to validate the JSON. + public string SchemaName { get; set; } + + /// The raw content to validate. + public string Content { get; set; } + } +} diff --git a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml new file mode 100644 index 00000000..cd7ca912 --- /dev/null +++ b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml @@ -0,0 +1,126 @@ +@using StardewModdingAPI.Web.ViewModels.JsonValidator +@model JsonValidatorModel + +@{ + ViewData["Title"] = "JSON validator"; +} + +@section Head { + @if (Model.PasteID != null) + { + + } + + + + + + + + + +} + +@* upload result banner *@ +@if (Model.UploadError != null) +{ + +} +else if (Model.ParseError != null) +{ + +} +else if (Model.PasteID != null) +{ + +} + +@* upload new file *@ +@if (Model.Content == null) +{ +

Upload a JSON file

+
+
    +
  1. + Choose the JSON format:
    + +
    2. +
  2. + Drag the file onto this textbox (or paste the text in):
    + +
    3. +
  3. + Click this button:
    + +
    4. +
+
+} + +@* validation results *@ +@if (Model.Content != null) +{ +
+ @if (Model.UploadError == null) + { +
+ Change JSON format: + +
+ +

Validation errors

+ @if (Model.Errors.Any()) + { + + + + + + + + @foreach (JsonValidatorErrorModel error in Model.Errors) + { + + + + + + } +
LineFieldError
@error.Line@error.Path@error.Message
+ } + else + { +

No errors found.

+ } + } + +

Raw content

+ 
@Model.Content
+
+} diff --git a/src/SMAPI.Web/Views/Shared/_Layout.cshtml b/src/SMAPI.Web/Views/Shared/_Layout.cshtml index 4c602b29..9911ef0e 100644 --- a/src/SMAPI.Web/Views/Shared/_Layout.cshtml +++ b/src/SMAPI.Web/Views/Shared/_Layout.cshtml @@ -16,9 +16,14 @@

SMAPI

+ +

Tools

+
diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index e6b4a1b1..baf7efb7 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -12,6 +12,7 @@ "RootUrl": "http://localhost:59482/", "ModListUrl": "http://localhost:59482/mods/", "LogParserUrl": "http://localhost:59482/log/", + "JsonValidatorUrl": "http://localhost:59482/json/", "BetaEnabled": false, "BetaBlurb": null }, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index f9777f87..a440cf42 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -19,6 +19,7 @@ "RootUrl": null, // see top note "ModListUrl": null, // see top note "LogParserUrl": null, // see top note + "JsonValidatorUrl": null, // see top note "BetaEnabled": null, // see top note "BetaBlurb": null // see top note }, diff --git a/src/SMAPI.Web/wwwroot/Content/css/json-validator.css b/src/SMAPI.Web/wwwroot/Content/css/json-validator.css new file mode 100644 index 00000000..f9aeb18b --- /dev/null +++ b/src/SMAPI.Web/wwwroot/Content/css/json-validator.css @@ -0,0 +1,98 @@ +/********* +** Main layout +*********/ +#content { + max-width: 100%; +} + +#output { + padding: 10px; + overflow: auto; +} + +#output table td { + font-family: monospace; +} + +#output table tr th, +#output table tr td { + padding: 0 0.75rem; + white-space: pre-wrap; +} + + +/********* +** Result banner +*********/ +.banner { + border: 2px solid gray; + border-radius: 5px; + margin-top: 1em; + padding: 1em; +} + +.banner.success { + border-color: green; + background: #CFC; +} + +.banner.error { + border-color: red; + background: #FCC; +} + +/********* +** Validation results +*********/ +.table { + border-bottom: 1px dashed #888888; + margin-bottom: 5px; +} + +#metadata th, #metadata td { + text-align: left; + padding-right: 0.7em; +} + +.table { + border: 1px solid #000000; + background: #ffffff; + border-radius: 5px; + border-spacing: 1px; + overflow: hidden; + cursor: default; + box-shadow: 1px 1px 1px 1px #dddddd; +} + +.table tr { + background: #eee; +} + +.table tr:nth-child(even) { + background: #fff; +} + +/********* +** Upload form +*********/ +#input { + width: 100%; + height: 20em; + max-height: 70%; + margin: auto; + box-sizing: border-box; + border-radius: 5px; + border: 1px solid #000088; + outline: none; + box-shadow: inset 0px 0px 1px 1px rgba(0, 0, 192, .2); +} + +#submit { + font-size: 1.5em; + border-radius: 5px; + outline: none; + box-shadow: inset 0 0 1px 1px rgba(0, 0, 0, .2); + cursor: pointer; + border: 1px solid #008800; + background-color: #cfc; +} diff --git a/src/SMAPI.Web/wwwroot/Content/css/main.css b/src/SMAPI.Web/wwwroot/Content/css/main.css index 57eeee88..dcc7a798 100644 --- a/src/SMAPI.Web/wwwroot/Content/css/main.css +++ b/src/SMAPI.Web/wwwroot/Content/css/main.css @@ -73,7 +73,7 @@ a { } #sidebar h4 { - margin: 0 0 0.2em 0; + margin: 1.5em 0 0.2em 0; width: 10em; border-bottom: 1px solid #CCC; font-size: 0.8em; diff --git a/src/SMAPI.Web/wwwroot/Content/js/json-validator.js b/src/SMAPI.Web/wwwroot/Content/js/json-validator.js new file mode 100644 index 00000000..3f7a1775 --- /dev/null +++ b/src/SMAPI.Web/wwwroot/Content/js/json-validator.js @@ -0,0 +1,60 @@ +/* globals $ */ + +var smapi = smapi || {}; +smapi.jsonValidator = function (sectionUrl, pasteID) { + /** + * Rebuild the syntax-highlighted element. + */ + var formatCode = function () { + Sunlight.highlightAll(); + }; + + /** + * Initialise the JSON validator page. + */ + var init = function () { + // code formatting + formatCode(); + + // change format + $("#output #format").on("change", function() { + var schemaName = $(this).val(); + location.href = new URL(schemaName + "/" + pasteID, sectionUrl).toString(); + }); + + // upload form + var input = $("#input"); + if (input.length) { + // disable submit if it's empty + var toggleSubmit = function () { + var hasText = !!input.val().trim(); + submit.prop("disabled", !hasText); + }; + input.on("input", toggleSubmit); + toggleSubmit(); + + // drag & drop file + input.on({ + 'dragover dragenter': function (e) { + e.preventDefault(); + e.stopPropagation(); + }, + 'drop': function (e) { + var dataTransfer = e.originalEvent.dataTransfer; + if (dataTransfer && dataTransfer.files.length) { + e.preventDefault(); + e.stopPropagation(); + var file = dataTransfer.files[0]; + var reader = new FileReader(); + reader.onload = $.proxy(function (file, $input, event) { + $input.val(event.target.result); + toggleSubmit(); + }, this, file, $("#input")); + reader.readAsText(file); + } + } + }); + } + }; + init(); +}; diff --git a/src/SMAPI.Web/wwwroot/Content/js/log-parser.js b/src/SMAPI.Web/wwwroot/Content/js/log-parser.js index e87a1a5c..e6c7591c 100644 --- a/src/SMAPI.Web/wwwroot/Content/js/log-parser.js +++ b/src/SMAPI.Web/wwwroot/Content/js/log-parser.js @@ -23,7 +23,7 @@ smapi.logParser = function (data, sectionUrl) { } // set local time started - if(data) + if (data) data.localTimeStarted = ("0" + data.logStarted.getHours()).slice(-2) + ":" + ("0" + data.logStarted.getMinutes()).slice(-2); // init app @@ -100,7 +100,7 @@ smapi.logParser = function (data, sectionUrl) { updateModFilters(); }, - filtersAllow: function(modId, level) { + filtersAllow: function (modId, level) { return this.showMods[modId] !== false && this.showLevels[level] !== false; }, @@ -121,16 +121,15 @@ smapi.logParser = function (data, sectionUrl) { var submit = $("#submit"); // instruction OS chooser - var chooseSystem = function() { + var chooseSystem = function () { systemInstructions.hide(); systemInstructions.filter("[data-os='" + $("input[name='os']:checked").val() + "']").show(); - } + }; systemOptions.on("click", chooseSystem); chooseSystem(); // disable submit if it's empty - var toggleSubmit = function() - { + var toggleSubmit = function () { var hasText = !!input.val().trim(); submit.prop("disabled", !hasText); } @@ -139,18 +138,18 @@ smapi.logParser = function (data, sectionUrl) { // drag & drop file input.on({ - 'dragover dragenter': function(e) { + 'dragover dragenter': function (e) { e.preventDefault(); e.stopPropagation(); }, - 'drop': function(e) { + 'drop': function (e) { var dataTransfer = e.originalEvent.dataTransfer; if (dataTransfer && dataTransfer.files.length) { e.preventDefault(); e.stopPropagation(); var file = dataTransfer.files[0]; var reader = new FileReader(); - reader.onload = $.proxy(function(file, $input, event) { + reader.onload = $.proxy(function (file, $input, event) { $input.val(event.target.result); toggleSubmit(); }, this, file, $("#input")); diff --git a/src/SMAPI.Web/wwwroot/schemas/manifest.json b/src/SMAPI.Web/wwwroot/schemas/manifest.json new file mode 100644 index 00000000..06173333 --- /dev/null +++ b/src/SMAPI.Web/wwwroot/schemas/manifest.json @@ -0,0 +1,120 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://smapi.io/schemas/manifest.json", + "title": "SMAPI manifest", + "description": "Manifest file for a SMAPI mod or content pack", + "type": "object", + "properties": { + "Name": { + "title": "Mod name", + "description": "The mod's display name. SMAPI uses this in player messages, logs, and errors.", + "type": "string", + "examples": [ "Lookup Anything" ] + }, + "Author": { + "title": "Mod author", + "description": "The name of the person who created the mod. Ideally this should include the username used to publish mods.", + "type": "string", + "examples": [ "Pathoschild" ] + }, + "Version": { + "title": "Mod version", + "description": "The mod's semantic version. Make sure you update this for each release! SMAPI uses this for update checks, mod dependencies, and compatibility blacklists (if the mod breaks in a future version of the game).", + "$ref": "#/definitions/SemanticVersion" + }, + "Description": { + "title": "Mod description", + "description": "A short explanation of what your mod does (one or two sentences), shown in the SMAPI log.", + "type": "string", + "examples": [ "View metadata about anything by pressing a button." ] + }, + "UniqueID": { + "title": "Mod unique ID", + "description": "A unique identifier for your mod. The recommended format is \"Username.ModName\", with no spaces or special characters. SMAPI uses this for update checks, mod dependencies, and compatibility blacklists (if the mod breaks in a future version of the game). When another mod needs to reference this mod, it uses the unique ID.", + "$ref": "#/definitions/ModID" + }, + "EntryDll": { + "title": "Mod entry DLL", + "description": "The DLL filename SMAPI should load for this mod. Mutually exclusive with ContentPackFor.", + "type": "string", + "pattern": "^[a-zA-Z0-9_.-]+\\.dll$", + "examples": "LookupAnything.dll" + }, + "ContentPackFor": { + "title": "Content pack for", + "description": "Specifies the mod which can read this content pack.", + "type": "object", + "properties": { + "UniqueID": { + "title": "Required unique ID", + "description": "The unique ID of the mod which can read this content pack.", + "$ref": "#/definitions/ModID" + }, + "MinimumVersion": { + "title": "Required minimum version", + "description": "The minimum semantic version of the mod which can read this content pack, if applicable.", + "$ref": "#/definitions/SemanticVersion" + } + }, + + "required": [ "UniqueID" ] + }, + "Dependencies": { + "title": "Mod dependencies", + "description": "Specifies other mods to load before this mod. If a dependency is required and a player tries to use the mod without the dependency installed, the mod won't be loaded and they'll see a friendly message saying they need to install those.", + "type": "array", + "items": { + "type": "object", + "properties": { + "UniqueID": { + "title": "Dependency unique ID", + "description": "The unique ID of the mod to load first.", + "$ref": "#/definitions/ModID" + }, + "MinimumVersion": { + "title": "Dependency minimum version", + "description": "The minimum semantic version of the mod to load first, if applicable.", + "$ref": "#/definitions/SemanticVersion" + }, + "IsRequired": { + "title": "Dependency is required", + "description": "Whether the dependency is required. Default true if not specified." + } + }, + "required": [ "UniqueID" ] + } + }, + "UpdateKeys": { + "title": "Mod update keys", + "description": "Specifies where SMAPI should check for mod updates, so it can alert the user with a link to your mod page. See https://stardewvalleywiki.com/Modding:Modder_Guide/APIs/Manifest#Update_checks for more info.", + "type": "array", + "items": { + "type": "string", + "pattern": "^(Chucklefish:\\d+|Nexus:\\d+|GitHub:[A-Za-z0-9_]+/[A-Za-z0-9_]+|ModDrop:\\d+)$" + } + } + }, + + "required": [ "Name", "Author", "Version", "Description", "UniqueID" ], + "oneOf": [ + { + "required": [ "EntryDll" ] + }, + { + "required": [ "ContentPackFor" ] + } + ], + + "definitions": { + "SemanticVersion": { + "type": "string", + "pattern": "(?>(?0|[1-9]\\d*))\\.(?>(?0|[1-9]\\d*))(?>(?:\\.(?0|[1-9]\\d*))?)(?:-(?(?>[a-zA-Z0-9]+[\\-\\.]?)+))?", // derived from SMAPI.Toolkit.SemanticVersion + "examples": [ "1.0.0", "1.0.1-beta.2" ] + }, + "ModID": { + "type": "string", + "pattern": "^[a-zA-Z0-9_.-]+$", // derived from SMAPI.Toolkit.Utilities.PathUtilities.IsSlug + "examples": [ "Pathoschild.LookupAnything" ] + } + } +} -- cgit From 8b09a2776d9c0faf96fa90c923952033ce659477 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Thu, 7 Nov 2019 13:51:45 -0500 Subject: add support for CurseForge update keys (#605) --- docs/release-notes.md | 1 + .../Framework/UpdateData/ModRepositoryKey.cs | 3 + src/SMAPI.Web/Controllers/ModsApiController.cs | 5 +- .../Clients/CurseForge/CurseForgeClient.cs | 113 +++++++++++++++++++++ .../Framework/Clients/CurseForge/CurseForgeMod.cs | 23 +++++ .../Clients/CurseForge/ICurseForgeClient.cs | 17 ++++ .../CurseForge/ResponseModels/ModFileModel.cs | 12 +++ .../Clients/CurseForge/ResponseModels/ModModel.cs | 18 ++++ .../Framework/ConfigModels/ApiClientsConfig.cs | 7 ++ .../ModRepositories/CurseForgeRepository.cs | 63 ++++++++++++ src/SMAPI.Web/Startup.cs | 5 + src/SMAPI.Web/appsettings.json | 2 + 12 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 src/SMAPI.Web/Framework/Clients/CurseForge/CurseForgeClient.cs create mode 100644 src/SMAPI.Web/Framework/Clients/CurseForge/CurseForgeMod.cs create mode 100644 src/SMAPI.Web/Framework/Clients/CurseForge/ICurseForgeClient.cs create mode 100644 src/SMAPI.Web/Framework/Clients/CurseForge/ResponseModels/ModFileModel.cs create mode 100644 src/SMAPI.Web/Framework/Clients/CurseForge/ResponseModels/ModModel.cs create mode 100644 src/SMAPI.Web/Framework/ModRepositories/CurseForgeRepository.cs (limited to 'src/SMAPI.Web/Framework/ConfigModels') diff --git a/docs/release-notes.md b/docs/release-notes.md index 1d933f96..5c12c4cc 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -47,6 +47,7 @@ For modders: * Now ignores metadata files and folders (like `__MACOSX` and `__folder_managed_by_vortex`) and content files (like `.txt` or `.png`), which avoids missing-manifest errors in some common cases. * Now detects XNB mods more accurately, and consolidates multi-folder XNB mods in logged messages. * SMAPI now automatically removes invalid content when loading a save to prevent crashes. A warning is shown in-game when this happens. This applies for locations and NPCs. + * Added update checks for CurseForge mods. * Added support for configuring console colors via `smapi-internal/config.json` (intended for players with unusual consoles). * Added support for specifying SMAPI command-line arguments as environment variables for Linux/Mac compatibility. * Improved launch script compatibility on Linux (thanks to kurumushi and toastal!). diff --git a/src/SMAPI.Toolkit/Framework/UpdateData/ModRepositoryKey.cs b/src/SMAPI.Toolkit/Framework/UpdateData/ModRepositoryKey.cs index f6c402d5..765ca334 100644 --- a/src/SMAPI.Toolkit/Framework/UpdateData/ModRepositoryKey.cs +++ b/src/SMAPI.Toolkit/Framework/UpdateData/ModRepositoryKey.cs @@ -9,6 +9,9 @@ namespace StardewModdingAPI.Toolkit.Framework.UpdateData /// The Chucklefish mod repository. Chucklefish, + /// The CurseForge mod repository. + CurseForge, + /// A GitHub project containing releases. GitHub, diff --git a/src/SMAPI.Web/Controllers/ModsApiController.cs b/src/SMAPI.Web/Controllers/ModsApiController.cs index 8419b220..1412105a 100644 --- a/src/SMAPI.Web/Controllers/ModsApiController.cs +++ b/src/SMAPI.Web/Controllers/ModsApiController.cs @@ -14,6 +14,7 @@ using StardewModdingAPI.Toolkit.Framework.UpdateData; using StardewModdingAPI.Web.Framework.Caching.Mods; using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.Clients.Chucklefish; +using StardewModdingAPI.Web.Framework.Clients.CurseForge; using StardewModdingAPI.Web.Framework.Clients.GitHub; using StardewModdingAPI.Web.Framework.Clients.ModDrop; using StardewModdingAPI.Web.Framework.Clients.Nexus; @@ -61,10 +62,11 @@ namespace StardewModdingAPI.Web.Controllers /// The cache in which to store mod metadata. /// The config settings for mod update checks. /// The Chucklefish API client. + /// The CurseForge API client. /// The GitHub API client. /// The ModDrop API client. /// The Nexus API client. - public ModsApiController(IHostingEnvironment environment, IWikiCacheRepository wikiCache, IModCacheRepository modCache, IOptions configProvider, IChucklefishClient chucklefish, IGitHubClient github, IModDropClient modDrop, INexusClient nexus) + public ModsApiController(IHostingEnvironment environment, IWikiCacheRepository wikiCache, IModCacheRepository modCache, IOptions configProvider, IChucklefishClient chucklefish, ICurseForgeClient curseForge, IGitHubClient github, IModDropClient modDrop, INexusClient nexus) { this.ModDatabase = new ModToolkit().GetModDatabase(Path.Combine(environment.WebRootPath, "SMAPI.metadata.json")); ModUpdateCheckConfig config = configProvider.Value; @@ -78,6 +80,7 @@ namespace StardewModdingAPI.Web.Controllers new IModRepository[] { new ChucklefishRepository(chucklefish), + new CurseForgeRepository(curseForge), new GitHubRepository(github), new ModDropRepository(modDrop), new NexusRepository(nexus) diff --git a/src/SMAPI.Web/Framework/Clients/CurseForge/CurseForgeClient.cs b/src/SMAPI.Web/Framework/Clients/CurseForge/CurseForgeClient.cs new file mode 100644 index 00000000..140b854e --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/CurseForge/CurseForgeClient.cs @@ -0,0 +1,113 @@ +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Pathoschild.Http.Client; +using StardewModdingAPI.Toolkit; +using StardewModdingAPI.Web.Framework.Clients.CurseForge.ResponseModels; + +namespace StardewModdingAPI.Web.Framework.Clients.CurseForge +{ + /// An HTTP client for fetching mod metadata from the CurseForge API. + internal class CurseForgeClient : ICurseForgeClient + { + /********* + ** Fields + *********/ + /// The underlying HTTP client. + private readonly IClient Client; + + /// A regex pattern which matches a version number in a CurseForge mod file name. + private readonly Regex VersionInNamePattern = new Regex(@"^(?:.+? | *)v?(\d+\.\d+(?:\.\d+)?(?:-.+?)?) *(?:\.(?:zip|rar|7z))?$", RegexOptions.Compiled); + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The user agent for the API client. + /// The base URL for the CurseForge API. + public CurseForgeClient(string userAgent, string apiUrl) + { + this.Client = new FluentClient(apiUrl).SetUserAgent(userAgent); + } + + /// Get metadata about a mod. + /// The CurseForge mod ID. + /// Returns the mod info if found, else null. + public async Task GetModAsync(long id) + { + // get raw data + ModModel mod = await this.Client + .GetAsync($"addon/{id}") + .As(); + if (mod == null) + return null; + + // get latest versions + string invalidVersion = null; + ISemanticVersion latest = null; + foreach (ModFileModel file in mod.LatestFiles) + { + // extract version + ISemanticVersion version; + { + string raw = this.GetRawVersion(file); + if (raw == null) + continue; + + if (!SemanticVersion.TryParse(raw, out version)) + { + if (invalidVersion == null) + invalidVersion = raw; + continue; + } + } + + // track latest version + if (latest == null || version.IsNewerThan(latest)) + latest = version; + } + + // get error + string error = null; + if (latest == null && invalidVersion == null) + { + error = mod.LatestFiles.Any() + ? $"CurseForge mod {id} has no downloads which specify the version in a recognised format." + : $"CurseForge mod {id} has no downloads."; + } + + // generate result + return new CurseForgeMod + { + Name = mod.Name, + LatestVersion = latest?.ToString() ?? invalidVersion, + Url = mod.WebsiteUrl, + Error = error + }; + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.Client?.Dispose(); + } + + + /********* + ** Private methods + *********/ + /// Get a raw version string for a mod file, if available. + /// The file whose version to get. + private string GetRawVersion(ModFileModel file) + { + Match match = this.VersionInNamePattern.Match(file.DisplayName); + if (!match.Success) + match = this.VersionInNamePattern.Match(file.FileName); + + return match.Success + ? match.Groups[1].Value + : null; + } + } +} diff --git a/src/SMAPI.Web/Framework/Clients/CurseForge/CurseForgeMod.cs b/src/SMAPI.Web/Framework/Clients/CurseForge/CurseForgeMod.cs new file mode 100644 index 00000000..e5bb8cf1 --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/CurseForge/CurseForgeMod.cs @@ -0,0 +1,23 @@ +using Newtonsoft.Json; + +namespace StardewModdingAPI.Web.Framework.Clients.CurseForge +{ + /// Mod metadata from the CurseForge API. + internal class CurseForgeMod + { + /********* + ** Accessors + *********/ + /// The mod name. + public string Name { get; set; } + + /// The latest file version. + public string LatestVersion { get; set; } + + /// The mod's web URL. + public string Url { get; set; } + + /// A user-friendly error which indicates why fetching the mod info failed (if applicable). + public string Error { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/Clients/CurseForge/ICurseForgeClient.cs b/src/SMAPI.Web/Framework/Clients/CurseForge/ICurseForgeClient.cs new file mode 100644 index 00000000..907b4087 --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/CurseForge/ICurseForgeClient.cs @@ -0,0 +1,17 @@ +using System; +using System.Threading.Tasks; + +namespace StardewModdingAPI.Web.Framework.Clients.CurseForge +{ + /// An HTTP client for fetching mod metadata from the CurseForge API. + internal interface ICurseForgeClient : IDisposable + { + /********* + ** Methods + *********/ + /// Get metadata about a mod. + /// The CurseForge mod ID. + /// Returns the mod info if found, else null. + Task GetModAsync(long id); + } +} diff --git a/src/SMAPI.Web/Framework/Clients/CurseForge/ResponseModels/ModFileModel.cs b/src/SMAPI.Web/Framework/Clients/CurseForge/ResponseModels/ModFileModel.cs new file mode 100644 index 00000000..9de74847 --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/CurseForge/ResponseModels/ModFileModel.cs @@ -0,0 +1,12 @@ +namespace StardewModdingAPI.Web.Framework.Clients.CurseForge.ResponseModels +{ + /// Metadata from the CurseForge API about a mod file. + public class ModFileModel + { + /// The file name as downloaded. + public string FileName { get; set; } + + /// The file display name. + public string DisplayName { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/Clients/CurseForge/ResponseModels/ModModel.cs b/src/SMAPI.Web/Framework/Clients/CurseForge/ResponseModels/ModModel.cs new file mode 100644 index 00000000..48cd185b --- /dev/null +++ b/src/SMAPI.Web/Framework/Clients/CurseForge/ResponseModels/ModModel.cs @@ -0,0 +1,18 @@ +namespace StardewModdingAPI.Web.Framework.Clients.CurseForge.ResponseModels +{ + /// An mod from the CurseForge API. + public class ModModel + { + /// The mod's unique ID on CurseForge. + public int ID { get; set; } + + /// The mod name. + public string Name { get; set; } + + /// The web URL for the mod page. + public string WebsiteUrl { get; set; } + + /// The available file downloads. + public ModFileModel[] LatestFiles { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index a0a1f42a..121690c5 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -23,6 +23,13 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string ChucklefishModPageUrlFormat { get; set; } + /**** + ** CurseForge + ****/ + /// The base URL for the CurseForge API. + public string CurseForgeBaseUrl { get; set; } + + /**** ** GitHub ****/ diff --git a/src/SMAPI.Web/Framework/ModRepositories/CurseForgeRepository.cs b/src/SMAPI.Web/Framework/ModRepositories/CurseForgeRepository.cs new file mode 100644 index 00000000..93ddc1eb --- /dev/null +++ b/src/SMAPI.Web/Framework/ModRepositories/CurseForgeRepository.cs @@ -0,0 +1,63 @@ +using System; +using System.Threading.Tasks; +using StardewModdingAPI.Toolkit.Framework.UpdateData; +using StardewModdingAPI.Web.Framework.Clients.CurseForge; + +namespace StardewModdingAPI.Web.Framework.ModRepositories +{ + /// An HTTP client for fetching mod metadata from CurseForge. + internal class CurseForgeRepository : RepositoryBase + { + /********* + ** Fields + *********/ + /// The underlying CurseForge API client. + private readonly ICurseForgeClient Client; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The underlying CurseForge API client. + public CurseForgeRepository(ICurseForgeClient client) + : base(ModRepositoryKey.CurseForge) + { + this.Client = client; + } + + /// Get metadata about a mod in the repository. + /// The mod ID in this repository. + public override async Task GetModInfoAsync(string id) + { + // validate ID format + if (!uint.TryParse(id, out uint curseID)) + return new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, $"The value '{id}' isn't a valid CurseForge mod ID, must be an integer ID."); + + // fetch info + try + { + CurseForgeMod mod = await this.Client.GetModAsync(curseID); + if (mod == null) + return new ModInfoModel().SetError(RemoteModStatus.DoesNotExist, "Found no CurseForge mod with this ID."); + if (mod.Error != null) + { + RemoteModStatus remoteStatus = RemoteModStatus.InvalidData; + return new ModInfoModel().SetError(remoteStatus, mod.Error); + } + + return new ModInfoModel(name: mod.Name, version: this.NormalizeVersion(mod.LatestVersion), url: mod.Url); + } + catch (Exception ex) + { + return new ModInfoModel().SetError(RemoteModStatus.TemporaryError, ex.ToString()); + } + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public override void Dispose() + { + this.Client.Dispose(); + } + } +} diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index bf69d543..8110b696 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -16,6 +16,7 @@ using StardewModdingAPI.Web.Framework.Caching; using StardewModdingAPI.Web.Framework.Caching.Mods; using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.Clients.Chucklefish; +using StardewModdingAPI.Web.Framework.Clients.CurseForge; using StardewModdingAPI.Web.Framework.Clients.GitHub; using StardewModdingAPI.Web.Framework.Clients.ModDrop; using StardewModdingAPI.Web.Framework.Clients.Nexus; @@ -119,6 +120,10 @@ namespace StardewModdingAPI.Web baseUrl: api.ChucklefishBaseUrl, modPageUrlFormat: api.ChucklefishModPageUrlFormat )); + services.AddSingleton(new CurseForgeClient( + userAgent: userAgent, + apiUrl: api.CurseForgeBaseUrl + )); services.AddSingleton(new GitHubClient( baseUrl: api.GitHubBaseUrl, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index a440cf42..674bb672 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -30,6 +30,8 @@ "ChucklefishBaseUrl": "https://community.playstarbound.com", "ChucklefishModPageUrlFormat": "resources/{0}", + "CurseForgeBaseUrl": "https://addons-ecs.forgesvc.net/api/v2/", + "GitHubBaseUrl": "https://api.github.com", "GitHubAcceptHeader": "application/vnd.github.v3+json", "GitHubUsername": null, // see top note -- cgit