From 2b1f607d41b3d4d071c0db0671dbc99b6982909f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 21:21:28 -0500 Subject: encapsulate file storage, also handle Pastebin rate limits in JSON validator --- .../Controllers/JsonValidatorController.cs | 58 +++---- src/SMAPI.Web/Controllers/LogParserController.cs | 178 ++------------------- 2 files changed, 38 insertions(+), 198 deletions(-) (limited to 'src/SMAPI.Web/Controllers') diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs index 40599abc..830fe839 100644 --- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -9,8 +9,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using StardewModdingAPI.Web.Framework; -using StardewModdingAPI.Web.Framework.Clients.Pastebin; -using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.Storage; using StardewModdingAPI.Web.ViewModels.JsonValidator; namespace StardewModdingAPI.Web.Controllers @@ -21,11 +20,8 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Fields *********/ - /// The underlying Pastebin client. - private readonly IPastebinClient Pastebin; - - /// The underlying text compression helper. - private readonly IGzipHelper GzipHelper; + /// Provides access to raw data storage. + private readonly IStorageProvider Storage; /// The supported JSON schemas (names indexed by ID). private readonly IDictionary SchemaFormats = new Dictionary @@ -49,12 +45,10 @@ namespace StardewModdingAPI.Web.Controllers ** Constructor ***/ /// Construct an instance. - /// The Pastebin API client. - /// The underlying text compression helper. - public JsonValidatorController(IPastebinClient pastebin, IGzipHelper gzipHelper) + /// Provides access to raw data storage. + public JsonValidatorController(IStorageProvider storage) { - this.Pastebin = pastebin; - this.GzipHelper = gzipHelper; + this.Storage = storage; } /*** @@ -62,7 +56,7 @@ namespace StardewModdingAPI.Web.Controllers ***/ /// Render the schema validator UI. /// The schema name with which to validate the JSON. - /// The paste ID. + /// The stored file ID. [HttpGet] [Route("json")] [Route("json/{schemaName}")] @@ -76,16 +70,16 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", result); // fetch raw JSON - PasteInfo paste = await this.GetAsync(id); - if (string.IsNullOrWhiteSpace(paste.Content)) + StoredFileInfo file = await this.Storage.GetAsync(id); + if (string.IsNullOrWhiteSpace(file.Content)) return this.View("Index", result.SetUploadError("The JSON file seems to be empty.")); - result.SetContent(paste.Content); + result.SetContent(file.Content, expiry: file.Expiry, uploadWarning: file.Warning); // parse JSON JToken parsed; try { - parsed = JToken.Parse(paste.Content, new JsonLoadSettings + parsed = JToken.Parse(file.Content, new JsonLoadSettings { DuplicatePropertyNameHandling = DuplicatePropertyNameHandling.Error, CommentHandling = CommentHandling.Load @@ -97,7 +91,7 @@ namespace StardewModdingAPI.Web.Controllers } // format JSON - result.SetContent(parsed.ToString(Formatting.Indented)); + result.SetContent(parsed.ToString(Formatting.Indented), expiry: file.Expiry, uploadWarning: file.Warning); // skip if no schema selected if (schemaName == "none") @@ -132,23 +126,20 @@ namespace StardewModdingAPI.Web.Controllers public async Task PostAsync(JsonValidatorRequestModel request) { if (request == null) - return this.View("Index", new JsonValidatorModel(null, null, this.SchemaFormats).SetUploadError("The request seems to be invalid.")); + return this.View("Index", this.GetModel(null, null).SetUploadError("The request seems to be invalid.")); // normalize schema name string schemaName = this.NormalizeSchemaName(request.SchemaName); - // get raw log text + // get raw text string input = request.Content; if (string.IsNullOrWhiteSpace(input)) - return this.View("Index", new JsonValidatorModel(null, schemaName, this.SchemaFormats).SetUploadError("The JSON file seems to be empty.")); - - // upload log - input = this.GzipHelper.CompressString(input); - SavePasteResult result = await this.Pastebin.PostAsync($"JSON validator {DateTime.UtcNow:s}", input); + return this.View("Index", this.GetModel(null, schemaName).SetUploadError("The JSON file seems to be empty.")); - // handle errors - if (!result.Success) - return this.View("Index", new JsonValidatorModel(result.ID, schemaName, this.SchemaFormats).SetUploadError($"Pastebin error: {result.Error ?? "unknown error"}")); + // upload file + var result = await this.Storage.SaveAsync(title: $"JSON validator {DateTime.UtcNow:s}", content: input, compress: true); + if (!result.Succeeded) + return this.View("Index", this.GetModel(result.ID, schemaName).SetUploadError(result.UploadError)); // redirect to view return this.Redirect(this.Url.PlainAction("Index", "JsonValidator", new { schemaName = schemaName, id = result.ID })); @@ -158,13 +149,12 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Private methods *********/ - /// Fetch raw text from Pastebin. - /// The Pastebin paste ID. - private async Task GetAsync(string id) + /// Build a JSON validator model. + /// The stored file ID. + /// The schema name with which the JSON was validated. + private JsonValidatorModel GetModel(string pasteID, string schemaName) { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; + return new JsonValidatorModel(pasteID, schemaName, this.SchemaFormats); } /// Get a normalized schema name, or the if blank. diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index 318b34d0..e270ae0a 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,22 +1,12 @@ using System; -using System.IO; using System.Linq; -using System.Text; using System.Threading.Tasks; -using Amazon; -using Amazon.Runtime; -using Amazon.S3; -using Amazon.S3.Model; -using Amazon.S3.Transfer; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit.Utilities; using StardewModdingAPI.Web.Framework; -using StardewModdingAPI.Web.Framework.Clients.Pastebin; -using StardewModdingAPI.Web.Framework.Compression; -using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.Framework.LogParsing; using StardewModdingAPI.Web.Framework.LogParsing.Models; +using StardewModdingAPI.Web.Framework.Storage; using StardewModdingAPI.Web.ViewModels; namespace StardewModdingAPI.Web.Controllers @@ -27,14 +17,8 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Fields *********/ - /// The API client settings. - private readonly ApiClientsConfig ClientsConfig; - - /// The underlying Pastebin client. - private readonly IPastebinClient Pastebin; - - /// The underlying text compression helper. - private readonly IGzipHelper GzipHelper; + /// Provides access to raw data storage. + private readonly IStorageProvider Storage; /********* @@ -44,21 +28,17 @@ namespace StardewModdingAPI.Web.Controllers ** Constructor ***/ /// Construct an instance. - /// The API client settings. - /// The Pastebin API client. - /// The underlying text compression helper. - public LogParserController(IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + /// Provides access to raw data storage. + public LogParserController(IStorageProvider storage) { - this.ClientsConfig = clientsConfig.Value; - this.Pastebin = pastebin; - this.GzipHelper = gzipHelper; + this.Storage = storage; } /*** ** Web UI ***/ /// Render the log parser UI. - /// The paste ID. + /// The stored file ID. /// Whether to display the raw unparsed log. [HttpGet] [Route("log")] @@ -70,12 +50,12 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(id)); // log page - PasteInfo paste = await this.GetAsync(id); - ParsedLog log = paste.Success - ? new LogParser().Parse(paste.Content) - : new ParsedLog { IsValid = false, Error = paste.Error }; + StoredFileInfo file = await this.Storage.GetAsync(id); + ParsedLog log = file.Success + ? new LogParser().Parse(file.Content) + : new ParsedLog { IsValid = false, Error = file.Error }; - return this.View("Index", this.GetModel(id, uploadWarning: paste.Warning, expiry: paste.Expiry).SetResult(log, raw)); + return this.View("Index", this.GetModel(id, uploadWarning: file.Warning, expiry: file.Expiry).SetResult(log, raw)); } /*** @@ -92,8 +72,7 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty.")); // upload log - input = this.GzipHelper.CompressString(input); - var uploadResult = await this.TrySaveLog(input); + UploadResult uploadResult = await this.Storage.SaveAsync(title: $"SMAPI log {DateTime.UtcNow:s}", content: input, compress: true); if (!uploadResult.Succeeded) return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); @@ -105,106 +84,8 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Private methods *********/ - /// Fetch raw text from Pastebin. - /// The Pastebin paste ID. - private async Task GetAsync(string id) - { - // get from Amazon S3 - if (Guid.TryParseExact(id, "N", out Guid _)) - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - - using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) - { - try - { - using (GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonLogBucket, $"logs/{id}")) - using (Stream responseStream = response.ResponseStream) - using (StreamReader reader = new StreamReader(responseStream)) - { - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - - return new PasteInfo - { - Success = true, - Content = content, - Expiry = expiry, - Warning = pastebinError - }; - } - } - catch (AmazonServiceException ex) - { - return ex.ErrorCode == "NoSuchKey" - ? new PasteInfo { Error = "There's no log with that ID." } - : new PasteInfo { Error = $"Could not fetch that log from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; - } - } - } - - // get from PasteBin - else - { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; - } - } - - /// Save a log to Pastebin or Amazon S3, if available. - /// The content to upload. - /// Returns metadata about the save attempt. - private async Task TrySaveLog(string content) - { - // save to PasteBin - string uploadError; - { - SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 - try - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - - using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) - using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) - using (TransferUtility uploader = new TransferUtility(s3)) - { - string id = Guid.NewGuid().ToString("N"); - - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonLogBucket, - Key = $"logs/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; - - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); - } - } - catch (Exception ex) - { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); - } - } - /// Build a log parser model. - /// The paste ID. + /// The stored file ID. /// When the uploaded file will no longer be available. /// A non-blocking warning while uploading the log. /// An error which occurred while uploading the log. @@ -243,36 +124,5 @@ namespace StardewModdingAPI.Web.Controllers return null; } } - - /// The result of an attempt to upload a file. - private class UploadResult - { - /********* - ** Accessors - *********/ - /// Whether the file upload succeeded. - public bool Succeeded { get; } - - /// The file ID, if applicable. - public string ID { get; } - - /// The upload error, if any. - public string UploadError { get; } - - - /********* - ** Public methods - *********/ - /// Construct an instance. - /// Whether the file upload succeeded. - /// The file ID, if applicable. - /// The upload error, if any. - public UploadResult(bool succeeded, string id, string uploadError) - { - this.Succeeded = succeeded; - this.ID = id; - this.UploadError = uploadError; - } - } } } -- cgit From 8ddb60cee636cc17291100c316df4786eb3bb448 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 23:06:42 -0500 Subject: move supporter list into environment config --- src/SMAPI.Web/Controllers/IndexController.cs | 2 +- src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs | 3 +++ src/SMAPI.Web/ViewModels/IndexModel.cs | 7 ++++++- src/SMAPI.Web/Views/Index/Index.cshtml | 24 ++++++++-------------- src/SMAPI.Web/appsettings.Development.json | 5 ----- src/SMAPI.Web/appsettings.json | 5 +++-- 6 files changed, 22 insertions(+), 24 deletions(-) (limited to 'src/SMAPI.Web/Controllers') diff --git a/src/SMAPI.Web/Controllers/IndexController.cs b/src/SMAPI.Web/Controllers/IndexController.cs index 4e3602d5..a887f14a 100644 --- a/src/SMAPI.Web/Controllers/IndexController.cs +++ b/src/SMAPI.Web/Controllers/IndexController.cs @@ -72,7 +72,7 @@ namespace StardewModdingAPI.Web.Controllers : null; // render view - var model = new IndexModel(stableVersionModel, betaVersionModel, this.SiteConfig.BetaBlurb); + var model = new IndexModel(stableVersionModel, betaVersionModel, this.SiteConfig.BetaBlurb, this.SiteConfig.SupporterList); return this.View(model); } diff --git a/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs index d379c423..43969f51 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs @@ -11,5 +11,8 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// A short sentence shown under the beta download button, if any. public string BetaBlurb { get; set; } + + /// A list of supports to credit on the main page, in Markdown format. + public string SupporterList { get; set; } } } diff --git a/src/SMAPI.Web/ViewModels/IndexModel.cs b/src/SMAPI.Web/ViewModels/IndexModel.cs index 82c4e06f..450fdc0e 100644 --- a/src/SMAPI.Web/ViewModels/IndexModel.cs +++ b/src/SMAPI.Web/ViewModels/IndexModel.cs @@ -15,6 +15,9 @@ namespace StardewModdingAPI.Web.ViewModels /// A short sentence shown under the beta download button, if any. public string BetaBlurb { get; set; } + /// A list of supports to credit on the main page, in Markdown format. + public string SupporterList { get; set; } + /********* ** Public methods @@ -26,11 +29,13 @@ namespace StardewModdingAPI.Web.ViewModels /// The latest stable SMAPI version. /// The latest prerelease SMAPI version (if newer than ). /// A short sentence shown under the beta download button, if any. - internal IndexModel(IndexVersionModel stableVersion, IndexVersionModel betaVersion, string betaBlurb) + /// A list of supports to credit on the main page, in Markdown format. + internal IndexModel(IndexVersionModel stableVersion, IndexVersionModel betaVersion, string betaBlurb, string supporterList) { this.StableVersion = stableVersion; this.BetaVersion = betaVersion; this.BetaBlurb = betaBlurb; + this.SupporterList = supporterList; } } } diff --git a/src/SMAPI.Web/Views/Index/Index.cshtml b/src/SMAPI.Web/Views/Index/Index.cshtml index ec9cfafe..5d91dc84 100644 --- a/src/SMAPI.Web/Views/Index/Index.cshtml +++ b/src/SMAPI.Web/Views/Index/Index.cshtml @@ -1,3 +1,4 @@ +@using Markdig @using Microsoft.Extensions.Options @using StardewModdingAPI.Web.Framework @using StardewModdingAPI.Web.Framework.ConfigModels @@ -94,29 +95,22 @@ else
    • -

    - Special thanks to - bwdy, - hawkfalcon, - iKeychain, - jwdred, - Karmylla, - minervamaga, - Pucklynn, - Renorien, - Robby LaFarge, - and a few anonymous users for their ongoing support on Patreon; you're awesome! -

    +@if (!string.IsNullOrWhiteSpace(Model.SupporterList)) +{ + @Html.Raw(Markdig.Markdown.ToHtml( + $"Special thanks to {Model.SupporterList}, and a few anonymous users for their ongoing support on Patreon; you're awesome!" + )) +}

    For mod creators

      diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 6b32f4ab..74ded25d 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -8,11 +8,6 @@ */ { - "Site": { - "BetaEnabled": false, - "BetaBlurb": null - }, - "ApiClients": { "AmazonAccessKey": null, "AmazonSecretKey": null, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index b3567469..2e20b299 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -16,8 +16,9 @@ }, "Site": { - "BetaEnabled": null, - "BetaBlurb": null + "BetaEnabled": false, + "BetaBlurb": null, + "SupporterList": null }, "ApiClients": { -- cgit From c4e2e94eed30f6e04312d5973322e4696ea672ea Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Mon, 16 Dec 2019 21:39:37 -0500 Subject: add option to edit & reupload in the JSON validator --- docs/release-notes.md | 11 ++--- .../Controllers/JsonValidatorController.cs | 6 ++- src/SMAPI.Web/Views/JsonValidator/Index.cshtml | 50 +++++++++++----------- 3 files changed, 37 insertions(+), 30 deletions(-) (limited to 'src/SMAPI.Web/Controllers') diff --git a/docs/release-notes.md b/docs/release-notes.md index bd377d0b..75438aaa 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -11,6 +11,12 @@ * Internal optimizations. * Updated translations. Thanks to PlussRolf (added Spanish)! +* For the web UI: + * Added option to edit & reupload in the JSON validator. + * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. + * Updated the JSON validator for Content Patcher 1.10.0. + * Fixed JSON validator no longer letting you change format when viewing results. + * For modders: * Added asset propagation for grass textures. * Added asset propagation for `Data\Bundles` changes (for added bundles only). @@ -18,11 +24,6 @@ * `helper.Read/WriteSaveData` can now be used while a save is being loaded (e.g. within a `Specialized.LoadStageChanged` event). * Fixed private textures loaded from content packs not having their `Name` field set. -* For the web UI: - * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. - * Updated the JSON validator for Content Patcher 1.10.0. - * Fixed JSON validator no longer letting you change format when viewing results. - ## 3.0.1 Released 02 December 2019 for Stardew Valley 1.4.0.1. diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs index 830fe839..e4eff0f4 100644 --- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -55,7 +55,7 @@ namespace StardewModdingAPI.Web.Controllers ** Web UI ***/ /// Render the schema validator UI. - /// The schema name with which to validate the JSON. + /// The schema name with which to validate the JSON, or 'edit' to return to the edit screen. /// The stored file ID. [HttpGet] [Route("json")] @@ -75,6 +75,10 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", result.SetUploadError("The JSON file seems to be empty.")); result.SetContent(file.Content, expiry: file.Expiry, uploadWarning: file.Warning); + // skip parsing if we're going to the edit screen + if (schemaName?.ToLower() == "edit") + return this.View("Index", result); + // parse JSON JToken parsed; try diff --git a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml index a042f024..fb43823a 100644 --- a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml +++ b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml @@ -8,7 +8,8 @@ string curPageUrl = this.Url.PlainAction("Index", "JsonValidator", new { schemaName = Model.SchemaName, id = Model.PasteID }); string newUploadUrl = this.Url.PlainAction("Index", "JsonValidator", new { schemaName = Model.SchemaName }); string schemaDisplayName = null; - bool isValidSchema = Model.SchemaName != null && Model.SchemaFormats.TryGetValue(Model.SchemaName, out schemaDisplayName) && schemaDisplayName != "None"; + bool isValidSchema = Model.SchemaName != null && Model.SchemaFormats.TryGetValue(Model.SchemaName, out schemaDisplayName) && schemaDisplayName?.ToLower() != "none"; + bool isEditView = Model.Content == null || Model.SchemaName?.ToLower() == "edit"; // build title ViewData["Title"] = "JSON validator"; @@ -60,7 +61,7 @@ else if (Model.ParseError != null) Error details: @Model.ParseError } -else if (Model.PasteID != null) +else if (!isEditView && Model.PasteID != null) {