From ce6cedaf4be53d52f2e558055b91e515b92e4c83 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Fri, 19 Jul 2019 13:15:45 -0400 Subject: add background fetch for mod compatibility list (#651) --- src/SMAPI.Web/BackgroundService.cs | 92 +++++++++++ src/SMAPI.Web/Controllers/ModsController.cs | 33 ++-- .../Framework/Caching/BaseCacheRepository.cs | 6 +- .../Framework/Caching/Wiki/CachedWikiMetadata.cs | 2 +- .../Framework/Caching/Wiki/IWikiCacheRepository.cs | 4 +- .../ConfigModels/BackgroundServicesConfig.cs | 12 ++ .../ConfigModels/ModCompatibilityListConfig.cs | 6 +- src/SMAPI.Web/SMAPI.Web.csproj | 2 + src/SMAPI.Web/Startup.cs | 48 ++++-- src/SMAPI.Web/ViewModels/ModListModel.cs | 19 ++- src/SMAPI.Web/Views/Mods/Index.cshtml | 172 +++++++++++---------- src/SMAPI.Web/appsettings.Development.json | 7 - src/SMAPI.Web/appsettings.json | 9 +- src/SMAPI.Web/wwwroot/Content/css/mods.css | 57 ++++--- 14 files changed, 315 insertions(+), 154 deletions(-) create mode 100644 src/SMAPI.Web/BackgroundService.cs create mode 100644 src/SMAPI.Web/Framework/ConfigModels/BackgroundServicesConfig.cs (limited to 'src') diff --git a/src/SMAPI.Web/BackgroundService.cs b/src/SMAPI.Web/BackgroundService.cs new file mode 100644 index 00000000..2ccfd5f7 --- /dev/null +++ b/src/SMAPI.Web/BackgroundService.cs @@ -0,0 +1,92 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Hangfire; +using Microsoft.Extensions.Hosting; +using StardewModdingAPI.Toolkit; +using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; +using StardewModdingAPI.Web.Framework.Caching.Wiki; + +namespace StardewModdingAPI.Web +{ + /// A hosted service which runs background data updates. + /// Task methods need to be static, since otherwise Hangfire will try to serialise the entire instance. + internal class BackgroundService : IHostedService, IDisposable + { + /********* + ** Fields + *********/ + /// The background task server. + private static BackgroundJobServer JobServer; + + /// The cache in which to store mod metadata. + private static IWikiCacheRepository WikiCache; + + + /********* + ** Public methods + *********/ + /**** + ** Hosted service + ****/ + /// Construct an instance. + /// The cache in which to store mod metadata. + public BackgroundService(IWikiCacheRepository wikiCache) + { + BackgroundService.WikiCache = wikiCache; + } + + /// Start the service. + /// Tracks whether the start process has been aborted. + public Task StartAsync(CancellationToken cancellationToken) + { + this.TryInit(); + + // set startup tasks + BackgroundJob.Enqueue(() => BackgroundService.UpdateWikiAsync()); + + // set recurring tasks + RecurringJob.AddOrUpdate(() => BackgroundService.UpdateWikiAsync(), "*/10 * * * *"); + + return Task.CompletedTask; + } + + /// Triggered when the application host is performing a graceful shutdown. + /// Tracks whether the shutdown process should no longer be graceful. + public async Task StopAsync(CancellationToken cancellationToken) + { + if (BackgroundService.JobServer != null) + await BackgroundService.JobServer.WaitForShutdownAsync(cancellationToken); + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + BackgroundService.JobServer?.Dispose(); + } + + /**** + ** Tasks + ****/ + /// Update the cached wiki metadata. + public static async Task UpdateWikiAsync() + { + WikiModList wikiCompatList = await new ModToolkit().GetWikiCompatibilityListAsync(); + BackgroundService.WikiCache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods, out _, out _); + } + + + /********* + ** Private method + *********/ + /// Initialise the background service if it's not already initialised. + /// The background service is already initialised. + private void TryInit() + { + if (BackgroundService.JobServer != null) + throw new InvalidOperationException("The scheduler service is already started."); + + BackgroundService.JobServer = new BackgroundJobServer(); + } + } +} diff --git a/src/SMAPI.Web/Controllers/ModsController.cs b/src/SMAPI.Web/Controllers/ModsController.cs index b6040e06..b621ded0 100644 --- a/src/SMAPI.Web/Controllers/ModsController.cs +++ b/src/SMAPI.Web/Controllers/ModsController.cs @@ -1,9 +1,7 @@ using System.Linq; using System.Text.RegularExpressions; -using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; -using StardewModdingAPI.Toolkit; using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.ViewModels; @@ -19,8 +17,8 @@ namespace StardewModdingAPI.Web.Controllers /// The cache in which to store mod metadata. private readonly IWikiCacheRepository Cache; - /// The number of minutes successful update checks should be cached before refetching them. - private readonly int CacheMinutes; + /// The number of minutes before which wiki data should be considered old. + private readonly int StaleMinutes; /********* @@ -34,15 +32,15 @@ namespace StardewModdingAPI.Web.Controllers ModCompatibilityListConfig config = configProvider.Value; this.Cache = cache; - this.CacheMinutes = config.CacheMinutes; + this.StaleMinutes = config.StaleMinutes; } /// Display information for all mods. [HttpGet] [Route("mods")] - public async Task Index() + public ViewResult Index() { - return this.View("Index", await this.FetchDataAsync()); + return this.View("Index", this.FetchData()); } @@ -50,25 +48,22 @@ namespace StardewModdingAPI.Web.Controllers ** Private methods *********/ /// Asynchronously fetch mod metadata from the wiki. - public async Task FetchDataAsync() + public ModListModel FetchData() { - // refresh cache - CachedWikiMod[] mods; - if (!this.Cache.TryGetWikiMetadata(out CachedWikiMetadata metadata) || this.Cache.IsStale(metadata.LastUpdated, this.CacheMinutes)) - { - var wikiCompatList = await new ModToolkit().GetWikiCompatibilityListAsync(); - this.Cache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods, out metadata, out mods); - } - else - mods = this.Cache.GetWikiMods().ToArray(); + // fetch cached data + if (!this.Cache.TryGetWikiMetadata(out CachedWikiMetadata metadata)) + return new ModListModel(); // build model return new ModListModel( stableVersion: metadata.StableVersion, betaVersion: metadata.BetaVersion, - mods: mods + mods: this.Cache + .GetWikiMods() .Select(mod => new ModModel(mod.GetModel())) - .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")) // ignore case, spaces, and special characters when sorting + .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")), // ignore case, spaces, and special characters when sorting + lastUpdated: metadata.LastUpdated, + isStale: this.Cache.IsStale(metadata.LastUpdated, this.StaleMinutes) ); } } diff --git a/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs index 904455c5..f5354b93 100644 --- a/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs @@ -10,10 +10,10 @@ namespace StardewModdingAPI.Web.Framework.Caching *********/ /// Whether cached data is stale. /// The date when the data was updated. - /// The age in minutes before data is considered stale. - public bool IsStale(DateTimeOffset lastUpdated, int cacheMinutes) + /// The age in minutes before data is considered stale. + public bool IsStale(DateTimeOffset lastUpdated, int staleMinutes) { - return lastUpdated < DateTimeOffset.UtcNow.AddMinutes(-cacheMinutes); + return lastUpdated < DateTimeOffset.UtcNow.AddMinutes(-staleMinutes); } } } diff --git a/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs b/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs index de1ea9db..4d6b4b10 100644 --- a/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs +++ b/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs @@ -37,7 +37,7 @@ namespace StardewModdingAPI.Web.Framework.Caching.Wiki { this.StableVersion = stableVersion; this.BetaVersion = betaVersion; - this.LastUpdated = DateTimeOffset.UtcNow;; + this.LastUpdated = DateTimeOffset.UtcNow; } } } diff --git a/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs index d319db69..6031123d 100644 --- a/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs +++ b/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs @@ -17,8 +17,8 @@ namespace StardewModdingAPI.Web.Framework.Caching.Wiki /// Whether cached data is stale. /// The date when the data was updated. - /// The age in minutes before data is considered stale. - bool IsStale(DateTimeOffset lastUpdated, int cacheMinutes); + /// The age in minutes before data is considered stale. + bool IsStale(DateTimeOffset lastUpdated, int staleMinutes); /// Get the cached wiki mods. /// A filter to apply, if any. diff --git a/src/SMAPI.Web/Framework/ConfigModels/BackgroundServicesConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/BackgroundServicesConfig.cs new file mode 100644 index 00000000..de871c9a --- /dev/null +++ b/src/SMAPI.Web/Framework/ConfigModels/BackgroundServicesConfig.cs @@ -0,0 +1,12 @@ +namespace StardewModdingAPI.Web.Framework.ConfigModels +{ + /// The config settings for background services. + internal class BackgroundServicesConfig + { + /********* + ** Accessors + *********/ + /// Whether to enable background update services. + public bool Enabled { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/ConfigModels/ModCompatibilityListConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ModCompatibilityListConfig.cs index d9ac9f02..24b540cd 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ModCompatibilityListConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ModCompatibilityListConfig.cs @@ -1,12 +1,12 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels { - /// The config settings for mod compatibility list. + /// The config settings for the mod compatibility list. internal class ModCompatibilityListConfig { /********* ** Accessors *********/ - /// The number of minutes data from the wiki should be cached before refetching it. - public int CacheMinutes { get; set; } + /// The number of minutes before which wiki data should be considered old. + public int StaleMinutes { get; set; } } } diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index 2f90389b..d53914d5 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -16,6 +16,8 @@ + + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 7beb0bcc..bdfa5ed9 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -1,4 +1,6 @@ using System.Collections.Generic; +using Hangfire; +using Hangfire.Mongo; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Rewrite; @@ -49,12 +51,13 @@ namespace StardewModdingAPI.Web /// The service injection container. public void ConfigureServices(IServiceCollection services) { - // init configuration + // init basic services services + .Configure(this.Configuration.GetSection("BackgroundServices")) .Configure(this.Configuration.GetSection("ModCompatibilityList")) .Configure(this.Configuration.GetSection("ModUpdateCheck")) - .Configure(this.Configuration.GetSection("Site")) .Configure(this.Configuration.GetSection("MongoDB")) + .Configure(this.Configuration.GetSection("Site")) .Configure(options => options.ConstraintMap.Add("semanticVersion", typeof(VersionConstraint))) .AddLogging() .AddMemoryCache() @@ -69,6 +72,33 @@ namespace StardewModdingAPI.Web options.SerializerSettings.NullValueHandling = NullValueHandling.Ignore; }); + // init background service + { + BackgroundServicesConfig config = this.Configuration.GetSection("BackgroundServices").Get(); + if (config.Enabled) + services.AddHostedService(); + } + + // init MongoDB + MongoDbConfig mongoConfig = this.Configuration.GetSection("MongoDB").Get(); + string mongoConnectionStr = mongoConfig.GetConnectionString(); + services.AddSingleton(serv => new MongoClient(mongoConnectionStr).GetDatabase(mongoConfig.Database)); + services.AddSingleton(serv => new WikiCacheRepository(serv.GetService())); + + // init Hangfire (needs MongoDB) + services + .AddHangfire(config => + { + config + .SetDataCompatibilityLevel(CompatibilityLevel.Version_170) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMongoStorage(mongoConnectionStr, $"{mongoConfig.Database}-hangfire", new MongoStorageOptions + { + MigrationOptions = new MongoMigrationOptions(MongoMigrationStrategy.Drop) + }); + }); + // init API clients { ApiClientsConfig api = this.Configuration.GetSection("ApiClients").Get(); @@ -111,15 +141,6 @@ namespace StardewModdingAPI.Web devKey: api.PastebinDevKey )); } - - // init MongoDB - { - MongoDbConfig mongoConfig = this.Configuration.GetSection("MongoDB").Get(); - string connectionString = mongoConfig.GetConnectionString(); - - services.AddSingleton(serv => new MongoClient(connectionString).GetDatabase(mongoConfig.Database)); - services.AddSingleton(serv => new WikiCacheRepository(serv.GetService())); - } } /// The method called by the runtime to configure the HTTP request pipeline. @@ -127,9 +148,9 @@ namespace StardewModdingAPI.Web /// The hosting environment. public void Configure(IApplicationBuilder app, IHostingEnvironment env) { + // basic config if (env.IsDevelopment()) app.UseDeveloperExceptionPage(); - app .UseCors(policy => policy .AllowAnyHeader() @@ -140,6 +161,9 @@ namespace StardewModdingAPI.Web .UseRewriter(this.GetRedirectRules()) .UseStaticFiles() // wwwroot folder .UseMvc(); + + // config Hangfire + app.UseHangfireDashboard("/tasks"); } diff --git a/src/SMAPI.Web/ViewModels/ModListModel.cs b/src/SMAPI.Web/ViewModels/ModListModel.cs index 3b87d393..ff7513bc 100644 --- a/src/SMAPI.Web/ViewModels/ModListModel.cs +++ b/src/SMAPI.Web/ViewModels/ModListModel.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; @@ -18,19 +19,35 @@ namespace StardewModdingAPI.Web.ViewModels /// The mods to display. public ModModel[] Mods { get; set; } + /// When the data was last updated. + public DateTimeOffset LastUpdated { get; set; } + + /// Whether the data hasn't been updated in a while. + public bool IsStale { get; set; } + + /// Whether the mod metadata is available. + public bool HasData => this.Mods != null; + /********* ** Public methods *********/ + /// Construct an empty instance. + public ModListModel() { } + /// Construct an instance. /// The current stable version of the game. /// The current beta version of the game (if any). /// The mods to display. - public ModListModel(string stableVersion, string betaVersion, IEnumerable mods) + /// When the data was last updated. + /// Whether the data hasn't been updated in a while. + public ModListModel(string stableVersion, string betaVersion, IEnumerable mods, DateTimeOffset lastUpdated, bool isStale) { this.StableVersion = stableVersion; this.BetaVersion = betaVersion; this.Mods = mods.ToArray(); + this.LastUpdated = lastUpdated; + this.IsStale = isStale; } } } diff --git a/src/SMAPI.Web/Views/Mods/Index.cshtml b/src/SMAPI.Web/Views/Mods/Index.cshtml index 2d45a64d..aa7c0678 100644 --- a/src/SMAPI.Web/Views/Mods/Index.cshtml +++ b/src/SMAPI.Web/Views/Mods/Index.cshtml @@ -18,92 +18,104 @@ } -
-
-

This page shows all known SMAPI mods and (incompatible) content packs, whether they work with the latest versions of Stardew Valley and SMAPI, and how to fix them if not. If a mod doesn't work after following the instructions below, check the troubleshooting guide or ask for help.

+@if (!Model.HasData) +{ +
↻ The mod data hasn't been fetched yet; please try again in a few minutes.
+} +else +{ + @if (Model.IsStale) + { +
Showing data from @(Math.Round((DateTimeOffset.UtcNow - Model.LastUpdated).TotalMinutes)) minutes ago. (Couldn't fetch newer data; the wiki API may be offline.)
+ } -

The list is updated every few days (you can help update it!). It doesn't include XNB mods (see using XNB mods on the wiki instead) or compatible content packs.

+
+
+

This page shows all known SMAPI mods and (incompatible) content packs, whether they work with the latest versions of Stardew Valley and SMAPI, and how to fix them if not. If a mod doesn't work after following the instructions below, check the troubleshooting guide or ask for help.

- @if (Model.BetaVersion != null) - { -

Note: "SDV @Model.BetaVersion only" lines are for an unreleased version of the game, not the stable version most players have. If a mod doesn't have that line, the info applies to both versions of the game.

- } -
+

The list is updated every few days (you can help update it!). It doesn't include XNB mods (see using XNB mods on the wiki instead) or compatible content packs.

-
-
- - + @if (Model.BetaVersion != null) + { +

Note: "SDV @Model.BetaVersion only" lines are for an unreleased version of the game, not the stable version most players have. If a mod doesn't have that line, the info applies to both versions of the game.

+ }
-
- - -
-
- {{filterGroup.label}}: + +
+
+ + +
+
+ + +
+
+ {{filterGroup.label}}: +
-
-
-
- {{visibleStats.total}} mods shown ({{Math.round((visibleStats.compatible + visibleStats.workaround) / visibleStats.total * 100)}}% compatible or have a workaround, {{Math.round((visibleStats.soon + visibleStats.broken) / visibleStats.total * 100)}}% broken, {{Math.round(visibleStats.abandoned / visibleStats.total * 100)}}% obsolete). +
+
+ {{visibleStats.total}} mods shown ({{Math.round((visibleStats.compatible + visibleStats.workaround) / visibleStats.total * 100)}}% compatible or have a workaround, {{Math.round((visibleStats.soon + visibleStats.broken) / visibleStats.total * 100)}}% broken, {{Math.round(visibleStats.abandoned / visibleStats.total * 100)}}% obsolete). +
+ No matching mods found.
- No matching mods found. -
- - - - - - - - - - - - - - - - - - - - - + + + + + + + +
mod namelinksauthorcompatibilitybroke incode 
- {{mod.Name}} - (aka {{mod.AlternateNames}}) - - - {{link.Text}}{{i < mod.ModPages.length - 1 ? ', ' : ''}} - - - {{mod.Author}} - (aka {{mod.AlternateAuthors}}) - -
-
- SDV @Model.BetaVersion only: - -
-
⚠ {{warning}}
- 		- source - no source - - - # - - - - [dev note] + + + + + + + + + + + + + + + + - - -
mod namelinksauthorcompatibilitybroke incode 
+ {{mod.Name}} + (aka {{mod.AlternateNames}}) + + + {{link.Text}}{{i < mod.ModPages.length - 1 ? ', ' : ''}} - -
- + 		+ {{mod.Author}} + (aka {{mod.AlternateAuthors}}) + +
+
+ SDV @Model.BetaVersion only: + +
+
⚠ {{warning}}
+ 		+ source + no source + + + # + + + + [dev note] + + +
+
+} diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index c50557b5..5856b96c 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -8,13 +8,6 @@ */ { - "Logging": { - "IncludeScopes": false, - "LogLevel": { - "Default": "Warning" - } - }, - "Site": { "RootUrl": "http://localhost:59482/", "ModListUrl": "http://localhost:59482/mods/", diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 532ea017..ea7e9cd2 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -10,7 +10,8 @@ "Logging": { "IncludeScopes": false, "LogLevel": { - "Default": "Warning" + "Default": "Warning", + "Hangfire": "Information" } }, @@ -55,7 +56,11 @@ }, "ModCompatibilityList": { - "CacheMinutes": 10 + "StaleMinutes": 15 + }, + + "BackgroundServices": { + "Enabled": true }, "ModUpdateCheck": { diff --git a/src/SMAPI.Web/wwwroot/Content/css/mods.css b/src/SMAPI.Web/wwwroot/Content/css/mods.css index fc5fff47..1c2b8056 100644 --- a/src/SMAPI.Web/wwwroot/Content/css/mods.css +++ b/src/SMAPI.Web/wwwroot/Content/css/mods.css @@ -15,30 +15,6 @@ border: 3px solid darkgreen; } -table.wikitable { - background-color:#f8f9fa; - color:#222; - border:1px solid #a2a9b1; - border-collapse:collapse -} - -table.wikitable > tr > th, -table.wikitable > tr > td, -table.wikitable > * > tr > th, -table.wikitable > * > tr > td { - border:1px solid #a2a9b1; - padding:0.2em 0.4em -} - -table.wikitable > tr > th, -table.wikitable > * > tr > th { - background-color:#eaecf0; -} - -table.wikitable > caption { - font-weight:bold -} - #options { margin-bottom: 1em; } @@ -73,6 +49,39 @@ table.wikitable > caption { opacity: 0.5; } +div.error { + padding: 2em 0; + color: red; + font-weight: bold; +} + +/********* +** Mod list +*********/ +table.wikitable { + background-color:#f8f9fa; + color:#222; + border:1px solid #a2a9b1; + border-collapse:collapse +} + +table.wikitable > tr > th, +table.wikitable > tr > td, +table.wikitable > * > tr > th, +table.wikitable > * > tr > td { + border:1px solid #a2a9b1; + padding:0.2em 0.4em +} + +table.wikitable > tr > th, +table.wikitable > * > tr > th { + background-color:#eaecf0; +} + +table.wikitable > caption { + font-weight:bold +} + #mod-list { font-size: 0.9em; } -- cgit