From 17c6ae7ed995344111513ca91b18ec6598ec2399 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 24 Jul 2019 18:33:26 -0400 Subject: migrate update check caching to MongoDB (#651) --- src/SMAPI.Web/Controllers/ModsApiController.cs | 68 ++++++++++---------------- 1 file changed, 26 insertions(+), 42 deletions(-) (limited to 'src/SMAPI.Web/Controllers') diff --git a/src/SMAPI.Web/Controllers/ModsApiController.cs b/src/SMAPI.Web/Controllers/ModsApiController.cs index a74d0d8a..195ee5bf 100644 --- a/src/SMAPI.Web/Controllers/ModsApiController.cs +++ b/src/SMAPI.Web/Controllers/ModsApiController.cs @@ -2,17 +2,17 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit; using StardewModdingAPI.Toolkit.Framework.Clients.WebApi; using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; using StardewModdingAPI.Toolkit.Framework.ModData; using StardewModdingAPI.Toolkit.Framework.UpdateData; +using StardewModdingAPI.Web.Framework.Caching.Mods; +using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.Clients.Chucklefish; using StardewModdingAPI.Web.Framework.Clients.GitHub; using StardewModdingAPI.Web.Framework.Clients.ModDrop; @@ -33,8 +33,11 @@ namespace StardewModdingAPI.Web.Controllers /// The mod repositories which provide mod metadata. private readonly IDictionary Repositories; - /// The cache in which to store mod metadata. - private readonly IMemoryCache Cache; + /// The cache in which to store wiki data. + private readonly IWikiCacheRepository WikiCache; + + /// The cache in which to store mod data. + private readonly IModCacheRepository ModCache; /// The number of minutes successful update checks should be cached before refetching them. private readonly int SuccessCacheMinutes; @@ -42,9 +45,6 @@ namespace StardewModdingAPI.Web.Controllers /// The number of minutes failed update checks should be cached before refetching them. private readonly int ErrorCacheMinutes; - /// A regex which matches SMAPI-style semantic version. - private readonly string VersionRegex; - /// The internal mod metadata list. private readonly ModDatabase ModDatabase; @@ -57,22 +57,23 @@ namespace StardewModdingAPI.Web.Controllers *********/ /// Construct an instance. /// The web hosting environment. - /// The cache in which to store mod metadata. + /// The cache in which to store wiki data. + /// The cache in which to store mod metadata. /// The config settings for mod update checks. /// The Chucklefish API client. /// The GitHub API client. /// The ModDrop API client. /// The Nexus API client. - public ModsApiController(IHostingEnvironment environment, IMemoryCache cache, IOptions configProvider, IChucklefishClient chucklefish, IGitHubClient github, IModDropClient modDrop, INexusClient nexus) + public ModsApiController(IHostingEnvironment environment, IWikiCacheRepository wikiCache, IModCacheRepository modCache, IOptions configProvider, IChucklefishClient chucklefish, IGitHubClient github, IModDropClient modDrop, INexusClient nexus) { this.ModDatabase = new ModToolkit().GetModDatabase(Path.Combine(environment.WebRootPath, "SMAPI.metadata.json")); ModUpdateCheckConfig config = configProvider.Value; this.CompatibilityPageUrl = config.CompatibilityPageUrl; - this.Cache = cache; + this.WikiCache = wikiCache; + this.ModCache = modCache; this.SuccessCacheMinutes = config.SuccessCacheMinutes; this.ErrorCacheMinutes = config.ErrorCacheMinutes; - this.VersionRegex = config.SemanticVersionRegex; this.Repositories = new IModRepository[] { @@ -93,7 +94,7 @@ namespace StardewModdingAPI.Web.Controllers return new ModEntryModel[0]; // fetch wiki data - WikiModEntry[] wikiData = await this.GetWikiDataAsync(); + WikiModEntry[] wikiData = this.WikiCache.GetWikiMods().Select(p => p.GetModel()).ToArray(); IDictionary mods = new Dictionary(StringComparer.CurrentCultureIgnoreCase); foreach (ModSearchEntryModel mod in model.Mods) { @@ -218,26 +219,6 @@ namespace StardewModdingAPI.Web.Controllers return current != null && (other == null || other.IsOlderThan(current)); } - /// Get mod data from the wiki compatibility list. - private async Task GetWikiDataAsync() - { - ModToolkit toolkit = new ModToolkit(); - return await this.Cache.GetOrCreateAsync("_wiki", async entry => - { - try - { - WikiModEntry[] entries = (await toolkit.GetWikiCompatibilityListAsync()).Mods; - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.SuccessCacheMinutes); - return entries; - } - catch - { - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.ErrorCacheMinutes); - return new WikiModEntry[0]; - } - }); - } - /// Get the mod info for an update key. /// The namespaced update key. private async Task GetInfoForUpdateKeyAsync(string updateKey) @@ -247,24 +228,27 @@ namespace StardewModdingAPI.Web.Controllers if (!parsed.LooksValid) return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"The update key '{updateKey}' isn't in a valid format. It should contain the site key and mod ID like 'Nexus:541'."); - // get matching repository - if (!this.Repositories.TryGetValue(parsed.Repository, out IModRepository repository)) - return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{parsed.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}]."); - - // fetch mod info - return await this.Cache.GetOrCreateAsync($"{repository.VendorKey}:{parsed.ID}".ToLower(), async entry => + // get mod + if (!this.ModCache.TryGetMod(parsed.Repository, parsed.ID, out CachedMod mod) || this.ModCache.IsStale(mod.LastUpdated, mod.FetchStatus == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes)) { + // get site + if (!this.Repositories.TryGetValue(parsed.Repository, out IModRepository repository)) + return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{parsed.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}]."); + + // fetch mod ModInfoModel result = await repository.GetModInfoAsync(parsed.ID); if (result.Error == null) { if (result.Version == null) result.WithError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with no version number."); - else if (!Regex.IsMatch(result.Version, this.VersionRegex, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)) + else if (!SemanticVersion.TryParse(result.Version, out _)) result.WithError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with invalid semantic version '{result.Version}'."); } - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(result.Status == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes); - return result; - }); + + // cache mod + this.ModCache.SaveMod(repository.VendorKey, parsed.ID, result, out mod); + } + return mod.GetModel(); } /// Get update keys based on the available mod metadata, while maintaining the precedence order. -- cgit