From 0aad3f545af854619c641dc8b57ac0cf12971c8e Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 19:36:56 -0500 Subject: update Content Patcher schema --- src/SMAPI.Web/wwwroot/schemas/content-patcher.json | 23 +++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'src/SMAPI.Web/wwwroot') diff --git a/src/SMAPI.Web/wwwroot/schemas/content-patcher.json b/src/SMAPI.Web/wwwroot/schemas/content-patcher.json index 61a633cb..c0236f1e 100644 --- a/src/SMAPI.Web/wwwroot/schemas/content-patcher.json +++ b/src/SMAPI.Web/wwwroot/schemas/content-patcher.json @@ -11,9 +11,9 @@ "title": "Format version", "description": "The format version. You should always use the latest version to enable the latest features and avoid obsolete behavior.", "type": "string", - "const": "1.9", + "const": "1.10.0", "@errorMessages": { - "const": "Incorrect value '@value'. This should be set to the latest format version, currently '1.9'." + "const": "Incorrect value '@value'. This should be set to the latest format version, currently '1.10.0'." } }, "ConfigSchema": { @@ -194,6 +194,8 @@ } }, "MoveEntries": { + "title": "Move entries", + "description": "Change the entry order in a list asset like Data/MoviesReactions. (Using this with a non-list asset will cause an error, since those have no order.)", "type": "array", "items": { "type": "object", @@ -259,6 +261,14 @@ } } }, + "MapProperties": { + "title": "Map properties", + "description": "The map properties (not tile properties) to add, replace, or delete. To add an property, just specify a key that doesn't exist; to delete an entry, set the value to null (like \"some key\": null). This field supports tokens in property keys and values.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "When": { "title": "When", "description": "Only apply the patch if the given conditions match.", @@ -300,7 +310,7 @@ }, "then": { "propertyNames": { - "enum": [ "Action", "Target", "LogName", "Enabled", "When", "Fields", "Entries", "MoveEntries" ] + "enum": [ "Action", "Target", "LogName", "Enabled", "When", "FromFile", "Fields", "Entries", "MoveEntries" ] } } }, @@ -313,7 +323,7 @@ "then": { "properties": { "FromFile": { - "description": "The relative path to the map in your content pack folder from which to copy (like assets/town.tbin). This can be a .tbin or .xnb file. This field supports tokens and capitalization doesn't matter.\nContent Patcher will handle tilesheets referenced by the FromFile map for you if it's a .tbin file:\n - If a tilesheet isn't referenced by the target map, Content Patcher will add it for you (with a z_ ID prefix to avoid conflicts with hardcoded game logic). If the source map has a custom version of a tilesheet that's already referenced, it'll be added as a separate tilesheet only used by your tiles.\n - If you include the tilesheet file in your mod folder, Content Patcher will use that one automatically; otherwise it will be loaded from the game's Content/Maps folder." + "description": "The relative path to the map in your content pack folder from which to copy (like assets/town.tbin). This can be a .tbin or .xnb file. This field supports tokens and capitalization doesn't matter.\nContent Patcher will handle tilesheets referenced by the FromFile map for you:\n - If a tilesheet isn't referenced by the target map, Content Patcher will add it for you (with a z_ ID prefix to avoid conflicts with hardcoded game logic). If the source map has a custom version of a tilesheet that's already referenced, it'll be added as a separate tilesheet only used by your tiles.\n - If you include the tilesheet file in your mod folder, Content Patcher will use that one automatically; otherwise it will be loaded from the game's Content/Maps folder." }, "FromArea": { "description": "The part of the source map to copy. Defaults to the whole source map." @@ -323,9 +333,8 @@ } }, "propertyNames": { - "enum": [ "Action", "Target", "LogName", "Enabled", "When", "FromFile", "FromArea", "ToArea" ] - }, - "required": [ "FromFile", "ToArea" ] + "enum": [ "Action", "Target", "LogName", "Enabled", "When", "FromFile", "FromArea", "ToArea", "MapProperties" ] + } } } ], -- cgit From 2b1f607d41b3d4d071c0db0671dbc99b6982909f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 21:21:28 -0500 Subject: encapsulate file storage, also handle Pastebin rate limits in JSON validator --- docs/release-notes.md | 1 + .../Controllers/JsonValidatorController.cs | 58 +++---- src/SMAPI.Web/Controllers/LogParserController.cs | 178 ++------------------- .../Framework/Clients/Pastebin/PasteInfo.cs | 6 - .../Framework/ConfigModels/ApiClientsConfig.cs | 2 +- .../Framework/Storage/IStorageProvider.cs | 19 +++ src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 147 +++++++++++++++++ src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs | 23 +++ src/SMAPI.Web/Framework/Storage/UploadResult.cs | 33 ++++ src/SMAPI.Web/Startup.cs | 10 +- .../ViewModels/JsonValidator/JsonValidatorModel.cs | 19 ++- src/SMAPI.Web/Views/JsonValidator/Index.cshtml | 15 +- src/SMAPI.Web/appsettings.json | 2 +- .../wwwroot/Content/css/json-validator.css | 6 + 14 files changed, 307 insertions(+), 212 deletions(-) create mode 100644 src/SMAPI.Web/Framework/Storage/IStorageProvider.cs create mode 100644 src/SMAPI.Web/Framework/Storage/StorageProvider.cs create mode 100644 src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs create mode 100644 src/SMAPI.Web/Framework/Storage/UploadResult.cs (limited to 'src/SMAPI.Web/wwwroot') diff --git a/docs/release-notes.md b/docs/release-notes.md index 9c3b8e3e..0b0a0f9e 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -4,6 +4,7 @@ ## Upcoming release * For the web UI: + * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. * Updated the JSON validator for Content Patcher 1.10.0. ## 3.0.1 diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs index 40599abc..830fe839 100644 --- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -9,8 +9,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Schema; using StardewModdingAPI.Web.Framework; -using StardewModdingAPI.Web.Framework.Clients.Pastebin; -using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.Storage; using StardewModdingAPI.Web.ViewModels.JsonValidator; namespace StardewModdingAPI.Web.Controllers @@ -21,11 +20,8 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Fields *********/ - /// The underlying Pastebin client. - private readonly IPastebinClient Pastebin; - - /// The underlying text compression helper. - private readonly IGzipHelper GzipHelper; + /// Provides access to raw data storage. + private readonly IStorageProvider Storage; /// The supported JSON schemas (names indexed by ID). private readonly IDictionary SchemaFormats = new Dictionary @@ -49,12 +45,10 @@ namespace StardewModdingAPI.Web.Controllers ** Constructor ***/ /// Construct an instance. - /// The Pastebin API client. - /// The underlying text compression helper. - public JsonValidatorController(IPastebinClient pastebin, IGzipHelper gzipHelper) + /// Provides access to raw data storage. + public JsonValidatorController(IStorageProvider storage) { - this.Pastebin = pastebin; - this.GzipHelper = gzipHelper; + this.Storage = storage; } /*** @@ -62,7 +56,7 @@ namespace StardewModdingAPI.Web.Controllers ***/ /// Render the schema validator UI. /// The schema name with which to validate the JSON. - /// The paste ID. + /// The stored file ID. [HttpGet] [Route("json")] [Route("json/{schemaName}")] @@ -76,16 +70,16 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", result); // fetch raw JSON - PasteInfo paste = await this.GetAsync(id); - if (string.IsNullOrWhiteSpace(paste.Content)) + StoredFileInfo file = await this.Storage.GetAsync(id); + if (string.IsNullOrWhiteSpace(file.Content)) return this.View("Index", result.SetUploadError("The JSON file seems to be empty.")); - result.SetContent(paste.Content); + result.SetContent(file.Content, expiry: file.Expiry, uploadWarning: file.Warning); // parse JSON JToken parsed; try { - parsed = JToken.Parse(paste.Content, new JsonLoadSettings + parsed = JToken.Parse(file.Content, new JsonLoadSettings { DuplicatePropertyNameHandling = DuplicatePropertyNameHandling.Error, CommentHandling = CommentHandling.Load @@ -97,7 +91,7 @@ namespace StardewModdingAPI.Web.Controllers } // format JSON - result.SetContent(parsed.ToString(Formatting.Indented)); + result.SetContent(parsed.ToString(Formatting.Indented), expiry: file.Expiry, uploadWarning: file.Warning); // skip if no schema selected if (schemaName == "none") @@ -132,23 +126,20 @@ namespace StardewModdingAPI.Web.Controllers public async Task PostAsync(JsonValidatorRequestModel request) { if (request == null) - return this.View("Index", new JsonValidatorModel(null, null, this.SchemaFormats).SetUploadError("The request seems to be invalid.")); + return this.View("Index", this.GetModel(null, null).SetUploadError("The request seems to be invalid.")); // normalize schema name string schemaName = this.NormalizeSchemaName(request.SchemaName); - // get raw log text + // get raw text string input = request.Content; if (string.IsNullOrWhiteSpace(input)) - return this.View("Index", new JsonValidatorModel(null, schemaName, this.SchemaFormats).SetUploadError("The JSON file seems to be empty.")); - - // upload log - input = this.GzipHelper.CompressString(input); - SavePasteResult result = await this.Pastebin.PostAsync($"JSON validator {DateTime.UtcNow:s}", input); + return this.View("Index", this.GetModel(null, schemaName).SetUploadError("The JSON file seems to be empty.")); - // handle errors - if (!result.Success) - return this.View("Index", new JsonValidatorModel(result.ID, schemaName, this.SchemaFormats).SetUploadError($"Pastebin error: {result.Error ?? "unknown error"}")); + // upload file + var result = await this.Storage.SaveAsync(title: $"JSON validator {DateTime.UtcNow:s}", content: input, compress: true); + if (!result.Succeeded) + return this.View("Index", this.GetModel(result.ID, schemaName).SetUploadError(result.UploadError)); // redirect to view return this.Redirect(this.Url.PlainAction("Index", "JsonValidator", new { schemaName = schemaName, id = result.ID })); @@ -158,13 +149,12 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Private methods *********/ - /// Fetch raw text from Pastebin. - /// The Pastebin paste ID. - private async Task GetAsync(string id) + /// Build a JSON validator model. + /// The stored file ID. + /// The schema name with which the JSON was validated. + private JsonValidatorModel GetModel(string pasteID, string schemaName) { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; + return new JsonValidatorModel(pasteID, schemaName, this.SchemaFormats); } /// Get a normalized schema name, or the if blank. diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index 318b34d0..e270ae0a 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,22 +1,12 @@ using System; -using System.IO; using System.Linq; -using System.Text; using System.Threading.Tasks; -using Amazon; -using Amazon.Runtime; -using Amazon.S3; -using Amazon.S3.Model; -using Amazon.S3.Transfer; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit.Utilities; using StardewModdingAPI.Web.Framework; -using StardewModdingAPI.Web.Framework.Clients.Pastebin; -using StardewModdingAPI.Web.Framework.Compression; -using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.Framework.LogParsing; using StardewModdingAPI.Web.Framework.LogParsing.Models; +using StardewModdingAPI.Web.Framework.Storage; using StardewModdingAPI.Web.ViewModels; namespace StardewModdingAPI.Web.Controllers @@ -27,14 +17,8 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Fields *********/ - /// The API client settings. - private readonly ApiClientsConfig ClientsConfig; - - /// The underlying Pastebin client. - private readonly IPastebinClient Pastebin; - - /// The underlying text compression helper. - private readonly IGzipHelper GzipHelper; + /// Provides access to raw data storage. + private readonly IStorageProvider Storage; /********* @@ -44,21 +28,17 @@ namespace StardewModdingAPI.Web.Controllers ** Constructor ***/ /// Construct an instance. - /// The API client settings. - /// The Pastebin API client. - /// The underlying text compression helper. - public LogParserController(IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + /// Provides access to raw data storage. + public LogParserController(IStorageProvider storage) { - this.ClientsConfig = clientsConfig.Value; - this.Pastebin = pastebin; - this.GzipHelper = gzipHelper; + this.Storage = storage; } /*** ** Web UI ***/ /// Render the log parser UI. - /// The paste ID. + /// The stored file ID. /// Whether to display the raw unparsed log. [HttpGet] [Route("log")] @@ -70,12 +50,12 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(id)); // log page - PasteInfo paste = await this.GetAsync(id); - ParsedLog log = paste.Success - ? new LogParser().Parse(paste.Content) - : new ParsedLog { IsValid = false, Error = paste.Error }; + StoredFileInfo file = await this.Storage.GetAsync(id); + ParsedLog log = file.Success + ? new LogParser().Parse(file.Content) + : new ParsedLog { IsValid = false, Error = file.Error }; - return this.View("Index", this.GetModel(id, uploadWarning: paste.Warning, expiry: paste.Expiry).SetResult(log, raw)); + return this.View("Index", this.GetModel(id, uploadWarning: file.Warning, expiry: file.Expiry).SetResult(log, raw)); } /*** @@ -92,8 +72,7 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty.")); // upload log - input = this.GzipHelper.CompressString(input); - var uploadResult = await this.TrySaveLog(input); + UploadResult uploadResult = await this.Storage.SaveAsync(title: $"SMAPI log {DateTime.UtcNow:s}", content: input, compress: true); if (!uploadResult.Succeeded) return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); @@ -105,106 +84,8 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Private methods *********/ - /// Fetch raw text from Pastebin. - /// The Pastebin paste ID. - private async Task GetAsync(string id) - { - // get from Amazon S3 - if (Guid.TryParseExact(id, "N", out Guid _)) - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - - using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) - { - try - { - using (GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonLogBucket, $"logs/{id}")) - using (Stream responseStream = response.ResponseStream) - using (StreamReader reader = new StreamReader(responseStream)) - { - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - - return new PasteInfo - { - Success = true, - Content = content, - Expiry = expiry, - Warning = pastebinError - }; - } - } - catch (AmazonServiceException ex) - { - return ex.ErrorCode == "NoSuchKey" - ? new PasteInfo { Error = "There's no log with that ID." } - : new PasteInfo { Error = $"Could not fetch that log from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; - } - } - } - - // get from PasteBin - else - { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; - } - } - - /// Save a log to Pastebin or Amazon S3, if available. - /// The content to upload. - /// Returns metadata about the save attempt. - private async Task TrySaveLog(string content) - { - // save to PasteBin - string uploadError; - { - SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 - try - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - - using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) - using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) - using (TransferUtility uploader = new TransferUtility(s3)) - { - string id = Guid.NewGuid().ToString("N"); - - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonLogBucket, - Key = $"logs/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; - - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); - } - } - catch (Exception ex) - { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); - } - } - /// Build a log parser model. - /// The paste ID. + /// The stored file ID. /// When the uploaded file will no longer be available. /// A non-blocking warning while uploading the log. /// An error which occurred while uploading the log. @@ -243,36 +124,5 @@ namespace StardewModdingAPI.Web.Controllers return null; } } - - /// The result of an attempt to upload a file. - private class UploadResult - { - /********* - ** Accessors - *********/ - /// Whether the file upload succeeded. - public bool Succeeded { get; } - - /// The file ID, if applicable. - public string ID { get; } - - /// The upload error, if any. - public string UploadError { get; } - - - /********* - ** Public methods - *********/ - /// Construct an instance. - /// Whether the file upload succeeded. - /// The file ID, if applicable. - /// The upload error, if any. - public UploadResult(bool succeeded, string id, string uploadError) - { - this.Succeeded = succeeded; - this.ID = id; - this.UploadError = uploadError; - } - } } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs index bb2de356..1ef3ef12 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs @@ -11,12 +11,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// The fetched paste content (if is true). public string Content { get; set; } - /// When the file will no longer be available. - public DateTime? Expiry { get; set; } - - /// The error message if saving succeeded, but a non-blocking issue was encountered. - public string Warning { get; set; } - /// The error message if saving failed. public string Error { get; set; } } diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index 7119ef03..1e020840 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -26,7 +26,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string AmazonRegion { get; set; } /// The AWS bucket in which to store temporary uploaded logs. - public string AmazonLogBucket { get; set; } + public string AmazonTempBucket { get; set; } /**** diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs new file mode 100644 index 00000000..e222a235 --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -0,0 +1,19 @@ +using System.Threading.Tasks; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// Provides access to raw data storage. + internal interface IStorageProvider + { + /// Save a text file to Pastebin or Amazon S3, if available. + /// The display title, if applicable. + /// The content to upload. + /// Whether to gzip the text. + /// Returns metadata about the save attempt. + Task SaveAsync(string title, string content, bool compress = true); + + /// Fetch raw text from storage. + /// The storage ID returned by . + Task GetAsync(string id); + } +} diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs new file mode 100644 index 00000000..bbb6e06b --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -0,0 +1,147 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Microsoft.Extensions.Options; +using StardewModdingAPI.Web.Framework.Clients.Pastebin; +using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.ConfigModels; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// Provides access to raw data storage. + internal class StorageProvider : IStorageProvider + { + /********* + ** Fields + *********/ + /// The API client settings. + private readonly ApiClientsConfig ClientsConfig; + + /// The underlying Pastebin client. + private readonly IPastebinClient Pastebin; + + /// The underlying text compression helper. + private readonly IGzipHelper GzipHelper; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The API client settings. + /// The underlying Pastebin client. + /// The underlying text compression helper. + public StorageProvider(IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + { + this.ClientsConfig = clientsConfig.Value; + this.Pastebin = pastebin; + this.GzipHelper = gzipHelper; + } + + /// Save a text file to Pastebin or Amazon S3, if available. + /// The display title, if applicable. + /// The content to upload. + /// Whether to gzip the text. + /// Returns metadata about the save attempt. + public async Task SaveAsync(string title, string content, bool compress = true) + { + // save to PasteBin + string uploadError; + { + SavePasteResult result = await this.Pastebin.PostAsync(title, content); + if (result.Success) + return new UploadResult(true, result.ID, null); + + uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; + } + + // fallback to S3 + try + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + using TransferUtility uploader = new TransferUtility(s3); + + string id = Guid.NewGuid().ToString("N"); + + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = this.ClientsConfig.AmazonTempBucket, + Key = $"uploads/{id}", + InputStream = stream, + Metadata = + { + // note: AWS will lowercase keys and prefix 'x-amz-meta-' + ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), + ["pastebin-error"] = uploadError + } + }; + + await uploader.UploadAsync(uploadRequest); + + return new UploadResult(true, id, uploadError); + } + catch (Exception ex) + { + return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + } + } + + /// Fetch raw text from storage. + /// The storage ID returned by . + public async Task GetAsync(string id) + { + // get from Amazon S3 + if (Guid.TryParseExact(id, "N", out Guid _)) + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } + } + + // get from PasteBin + else + { + PasteInfo response = await this.Pastebin.GetAsync(id); + response.Content = this.GzipHelper.DecompressString(response.Content); + return new StoredFileInfo + { + Success = response.Success, + Content = response.Content, + Error = response.Error + }; + } + } + } +} diff --git a/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs b/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs new file mode 100644 index 00000000..30676c88 --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs @@ -0,0 +1,23 @@ +using System; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// The response for a get-file request. + internal class StoredFileInfo + { + /// Whether the file was successfully fetched. + public bool Success { get; set; } + + /// The fetched file content (if is true). + public string Content { get; set; } + + /// When the file will no longer be available. + public DateTime? Expiry { get; set; } + + /// The error message if saving succeeded, but a non-blocking issue was encountered. + public string Warning { get; set; } + + /// The error message if saving failed. + public string Error { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/Storage/UploadResult.cs b/src/SMAPI.Web/Framework/Storage/UploadResult.cs new file mode 100644 index 00000000..483c1769 --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/UploadResult.cs @@ -0,0 +1,33 @@ +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// The result of an attempt to upload a file. + internal class UploadResult + { + /********* + ** Accessors + *********/ + /// Whether the file upload succeeded. + public bool Succeeded { get; } + + /// The file ID, if applicable. + public string ID { get; } + + /// The upload error, if any. + public string UploadError { get; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// Whether the file upload succeeded. + /// The file ID, if applicable. + /// The upload error, if any. + public UploadResult(bool succeeded, string id, string uploadError) + { + this.Succeeded = succeeded; + this.ID = id; + this.UploadError = uploadError; + } + } +} diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 53823771..31b5e61d 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -7,6 +7,7 @@ using Microsoft.AspNetCore.Rewrite; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using MongoDB.Bson.Serialization; using MongoDB.Driver; using Newtonsoft.Json; @@ -24,6 +25,7 @@ using StardewModdingAPI.Web.Framework.Clients.Pastebin; using StardewModdingAPI.Web.Framework.Compression; using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.Framework.RewriteRules; +using StardewModdingAPI.Web.Framework.Storage; namespace StardewModdingAPI.Web { @@ -158,7 +160,13 @@ namespace StardewModdingAPI.Web } // init helpers - services.AddSingleton(new GzipHelper()); + services + .AddSingleton(new GzipHelper()) + .AddSingleton(serv => new StorageProvider( + serv.GetRequiredService>(), + serv.GetRequiredService(), + serv.GetRequiredService() + )); } /// The method called by the runtime to configure the HTTP request pipeline. diff --git a/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs index 5b18331f..c0dd7184 100644 --- a/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs +++ b/src/SMAPI.Web/ViewModels/JsonValidator/JsonValidatorModel.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; @@ -24,7 +25,13 @@ namespace StardewModdingAPI.Web.ViewModels.JsonValidator /// The schema validation errors, if any. public JsonValidatorErrorModel[] Errors { get; set; } = new JsonValidatorErrorModel[0]; - /// An error which occurred while uploading the JSON to Pastebin. + /// A non-blocking warning while uploading the file. + public string UploadWarning { get; set; } + + /// When the uploaded file will no longer be available. + public DateTime? Expiry { get; set; } + + /// An error which occurred while uploading the JSON. public string UploadError { get; set; } /// An error which occurred while parsing the JSON. @@ -41,7 +48,7 @@ namespace StardewModdingAPI.Web.ViewModels.JsonValidator public JsonValidatorModel() { } /// Construct an instance. - /// The paste ID. + /// The stored file ID. /// The schema name with which the JSON was validated. /// The supported JSON schemas (names indexed by ID). public JsonValidatorModel(string pasteID, string schemaName, IDictionary schemaFormats) @@ -53,14 +60,18 @@ namespace StardewModdingAPI.Web.ViewModels.JsonValidator /// Set the validated content. /// The validated content. - public JsonValidatorModel SetContent(string content) + /// When the uploaded file will no longer be available. + /// A non-blocking warning while uploading the log. + public JsonValidatorModel SetContent(string content, DateTime? expiry, string uploadWarning = null) { this.Content = content; + this.Expiry = expiry; + this.UploadWarning = uploadWarning; return this; } - /// Set the error which occurred while uploading the log to Pastebin. + /// Set the error which occurred while uploading the JSON. /// The error message. public JsonValidatorModel SetUploadError(string error) { diff --git a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml index de6b06a2..a5a134ac 100644 --- a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml +++ b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml @@ -1,3 +1,4 @@ +@using Humanizer @using StardewModdingAPI.Web.Framework @using StardewModdingAPI.Web.ViewModels.JsonValidator @model JsonValidatorModel @@ -26,7 +27,7 @@ { } - + @@ -67,6 +68,18 @@ else if (Model.PasteID != null) } +@* save warnings *@ +@if (Model.UploadWarning != null || Model.Expiry != null) +{ +
+ @if (Model.Expiry != null) + { + This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). + } + +
+} + @* upload new file *@ @if (Model.Content == null) { diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index f81587ef..b3567469 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -26,7 +26,7 @@ "AmazonAccessKey": null, "AmazonSecretKey": null, "AmazonRegion": "us-east-1", - "AmazonLogBucket": "smapi-log-parser", + "AmazonTempBucket": "smapi-web-temp", "ChucklefishBaseUrl": "https://community.playstarbound.com", "ChucklefishModPageUrlFormat": "resources/{0}", diff --git a/src/SMAPI.Web/wwwroot/Content/css/json-validator.css b/src/SMAPI.Web/wwwroot/Content/css/json-validator.css index cd117694..18195098 100644 --- a/src/SMAPI.Web/wwwroot/Content/css/json-validator.css +++ b/src/SMAPI.Web/wwwroot/Content/css/json-validator.css @@ -41,6 +41,12 @@ background: #FCC; } +.save-metadata { + margin-top: 1em; + font-size: 0.8em; + opacity: 0.3; +} + /********* ** Validation results *********/ -- cgit From 9465628effad6bdb1994599031a8f60c3af2452e Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 4 Dec 2019 20:52:40 -0500 Subject: fix JSON validator format selector no longer working since URL changes --- docs/release-notes.md | 1 + src/SMAPI.Web/Views/JsonValidator/Index.cshtml | 6 +++--- src/SMAPI.Web/wwwroot/Content/js/json-validator.js | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'src/SMAPI.Web/wwwroot') diff --git a/docs/release-notes.md b/docs/release-notes.md index 0b0a0f9e..dec552d6 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -6,6 +6,7 @@ * For the web UI: * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. * Updated the JSON validator for Content Patcher 1.10.0. + * Fixed JSON validator no longer letting you change format when viewing results. ## 3.0.1 Released 02 December 2019 for Stardew Valley 1.4.0.1. diff --git a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml index a5a134ac..a042f024 100644 --- a/src/SMAPI.Web/Views/JsonValidator/Index.cshtml +++ b/src/SMAPI.Web/Views/JsonValidator/Index.cshtml @@ -27,17 +27,17 @@ { } - + - + } diff --git a/src/SMAPI.Web/wwwroot/Content/js/json-validator.js b/src/SMAPI.Web/wwwroot/Content/js/json-validator.js index 76b5f6d4..401efbee 100644 --- a/src/SMAPI.Web/wwwroot/Content/js/json-validator.js +++ b/src/SMAPI.Web/wwwroot/Content/js/json-validator.js @@ -70,10 +70,10 @@ smapi.LineNumberRange = function (maxLines) { /** * UI logic for the JSON validator page. - * @param {any} sectionUrl The base JSON validator page URL. - * @param {any} pasteID The Pastebin paste ID for the content being viewed, if any. + * @param {string} urlFormat The URL format for a file, with $schemaName and $id placeholders. + * @param {string} pasteID The Pastebin paste ID for the content being viewed, if any. */ -smapi.jsonValidator = function (sectionUrl, pasteID) { +smapi.jsonValidator = function (urlFormat, pasteID) { /** * The original content element. */ @@ -138,7 +138,7 @@ smapi.jsonValidator = function (sectionUrl, pasteID) { // change format $("#output #format").on("change", function() { var schemaName = $(this).val(); - location.href = new URL(schemaName + "/" + pasteID, sectionUrl).toString(); + location.href = urlFormat.replace("$schemaName", schemaName).replace("$id", pasteID); }); // upload form -- cgit From 4da65cf4c079ee32f0a5582fb73add2362b5a70c Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Mon, 16 Dec 2019 21:51:55 -0500 Subject: update schemas for Content Patcher 1.11 --- docs/release-notes.md | 2 +- src/SMAPI.Web/wwwroot/schemas/content-patcher.json | 23 ++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) (limited to 'src/SMAPI.Web/wwwroot') diff --git a/docs/release-notes.md b/docs/release-notes.md index 75438aaa..772d95aa 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -14,7 +14,7 @@ * For the web UI: * Added option to edit & reupload in the JSON validator. * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. - * Updated the JSON validator for Content Patcher 1.10.0. + * Updated the JSON validator for Content Patcher 1.10 and 1.11. * Fixed JSON validator no longer letting you change format when viewing results. * For modders: diff --git a/src/SMAPI.Web/wwwroot/schemas/content-patcher.json b/src/SMAPI.Web/wwwroot/schemas/content-patcher.json index c0236f1e..28e38e73 100644 --- a/src/SMAPI.Web/wwwroot/schemas/content-patcher.json +++ b/src/SMAPI.Web/wwwroot/schemas/content-patcher.json @@ -11,9 +11,9 @@ "title": "Format version", "description": "The format version. You should always use the latest version to enable the latest features and avoid obsolete behavior.", "type": "string", - "const": "1.10.0", + "const": "1.11.0", "@errorMessages": { - "const": "Incorrect value '@value'. This should be set to the latest format version, currently '1.10.0'." + "const": "Incorrect value '@value'. This should be set to the latest format version, currently '1.11.0'." } }, "ConfigSchema": { @@ -276,6 +276,9 @@ } }, "allOf": [ + { + "required": [ "Action" ] + }, { "if": { "properties": { @@ -364,26 +367,26 @@ "properties": { "X": { "title": "X-Coordinate", - "description": "Location in pixels of the top-left of the rectangle", - "type": "integer", + "description": "The X position of the area's top-left corner, measured in pixels for a texture or tiles for a map. This can contain tokens.", + "type": [ "integer", "string" ], "minimum:": 0 }, "Y": { "title": "Y-Coordinate", - "description": "Location in pixels of the top-left of the rectangle", - "type": "integer", + "description": "The Y position of the area's top-left corner, measured in pixels for a texture or tiles for a map. This can contain tokens.", + "type": [ "integer", "string" ], "minimum:": 0 }, "Width": { "title": "Width", - "description": "The width of the rectangle", - "type": "integer", + "description": "The width of the area, measured in pixels for a texture or tiles for a map. This can contain tokens.", + "type": [ "integer", "string" ], "minimum:": 0 }, "Height": { "title": "Height", - "description": "The height of the rectangle", - "type": "integer", + "description": "The height of the area, measured in pixels for a texture or tiles for a map. This can contain tokens.", + "type": [ "integer", "string" ], "minimum:": 0 } }, -- cgit From 082f285bc7ce156ad0750bb48d46ed65a2e4aedb Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 22 Dec 2019 00:44:13 -0500 Subject: streamline local environments, update technical docs & privacy page --- docs/release-notes.md | 2 +- docs/technical/web.md | 69 ++++++++++++---------- .../Framework/Clients/Pastebin/PasteInfo.cs | 2 - .../Framework/Clients/Pastebin/SavePasteResult.cs | 15 ----- .../Framework/ConfigModels/MongoDbConfig.cs | 6 ++ src/SMAPI.Web/SMAPI.Web.csproj | 2 + src/SMAPI.Web/Startup.cs | 25 +++++++- src/SMAPI.Web/Views/Index/Privacy.cshtml | 8 +-- src/SMAPI.Web/appsettings.Development.json | 6 +- src/SMAPI.Web/wwwroot/Content/js/json-validator.js | 6 +- 10 files changed, 81 insertions(+), 60 deletions(-) delete mode 100644 src/SMAPI.Web/Framework/Clients/Pastebin/SavePasteResult.cs (limited to 'src/SMAPI.Web/wwwroot') diff --git a/docs/release-notes.md b/docs/release-notes.md index bbe08c13..bae1cb98 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -27,7 +27,7 @@ * Fixed asset propagation for `Characters\Farmer\farmer_girl_base_bald`. * For SMAPI developers: - * You can now run local environments without configuring Amazon, Azure, and Pastebin accounts. + * You can now run local environments without configuring Amazon, Azure, MongoDB, and Pastebin accounts. ## 3.0.1 Released 02 December 2019 for Stardew Valley 1.4.0.1. diff --git a/docs/technical/web.md b/docs/technical/web.md index 97e0704a..697ec280 100644 --- a/docs/technical/web.md +++ b/docs/technical/web.md @@ -10,17 +10,21 @@ and update check API. * [Short URLs](#short-urls) * [For SMAPI developers](#for-smapi-developers) * [Local development](#local-development) - * [Deploying to Amazon Beanstalk](#deploying-to-amazon-beanstalk) + * [Production environment](#production-environment) ## Log parser -The log parser provides a web UI for uploading, parsing, and sharing SMAPI logs. The logs are -persisted in a compressed form to Pastebin. The log parser lives at https://smapi.io/log. +The log parser at https://smapi.io/log provides a web UI for uploading, parsing, and sharing SMAPI +logs. + +The logs are saved in a compressed form to Amazon Blob storage for 30 days. ## JSON validator ### Overview -The JSON validator provides a web UI for uploading and sharing JSON files, and validating them as -plain JSON or against a predefined format like `manifest.json` or Content Patcher's `content.json`. -The JSON validator lives at https://smapi.io/json. +The JSON validator at https://smapi.io/json provides a web UI for uploading and sharing JSON files, +and validating them as plain JSON or against a predefined format like `manifest.json` or Content +Patcher's `content.json`. + +The logs are saved in a compressed form to Amazon Blob storage for 30 days. ### Schema file format Schema files are defined in `wwwroot/schemas` using the [JSON Schema](https://json-schema.org/) @@ -336,43 +340,46 @@ short url | → | target page A local environment lets you run a complete copy of the web project (including cache database) on your machine, with no external dependencies aside from the actual mod sites. -Initial setup: - -1. [Install MongoDB](https://docs.mongodb.com/manual/administration/install-community/) and add its - `bin` folder to the system PATH. -2. Create a local folder for the MongoDB data (e.g. `C:\dev\smapi-cache`). -3. Enter your credentials in the `appsettings.Development.json` file. You can leave the MongoDB - credentials as-is to use the default local instance; see the next section for the other settings. - -To launch the environment: -1. Launch MongoDB from a terminal (change the data path if applicable): - ```sh - mongod --dbpath C:\dev\smapi-cache - ``` -2. Launch `SMAPI.Web` from Visual Studio to run a local version of the site. - (Local URLs will use HTTP instead of HTTPS.) +1. Enter the Nexus credentials in `appsettings.Development.json` . You can leave the other + credentials empty to default to fetching data anonymously, and storing data in-memory and + on disk. +2. Launch `SMAPI.Web` from Visual Studio to run a local version of the site. ### Production environment A production environment includes the web servers and cache database hosted online for public -access. This section assumes you're creating a new production environment from scratch (not using -the official live environment). +access. + +This section assumes you're creating a new environment on Azure, but the app isn't tied to any +Azure services. If you want to host it on a different site, you'll need to adjust the instructions +accordingly. Initial setup: -1. Launch an empty MongoDB server (e.g. using [MongoDB Atlas](https://www.mongodb.com/cloud/atlas)). -2. Create an AWS Beanstalk .NET environment with these environment properties: +1. Launch an empty MongoDB server (e.g. using [MongoDB Atlas](https://www.mongodb.com/cloud/atlas)) + for mod data. +2. Create an Azure Blob storage account for uploaded files. +3. Create an Azure App Services environment running the latest .NET Core on Linux or Windows. +4. Add these application settings in the new App Services environment: property name | description ------------------------------- | ----------------- - `LogParser:PastebinDevKey` | The [Pastebin developer key](https://pastebin.com/api#1) used to authenticate with the Pastebin API. - `LogParser:PastebinUserKey` | The [Pastebin user key](https://pastebin.com/api#8) used to authenticate with the Pastebin API. Can be left blank to post anonymously. - `ModUpdateCheck:GitHubPassword` | The password with which to authenticate to GitHub when fetching release info. - `ModUpdateCheck:GitHubUsername` | The username with which to authenticate to GitHub when fetching release info. + `ApiClients.AzureBlobConnectionString` | The connection string for the Azure Blob storage account created in step 2. + `ApiClients.GitHubUsername`
`ApiClients.GitHubPassword` | The login credentials for the GitHub account with which to fetch release info. If these are omitted, GitHub will impose much stricter rate limits. + `ApiClients:NexusApiKey` | The [Nexus API authentication key](https://github.com/Pathoschild/FluentNexus#init-a-client). `MongoDB:Host` | The hostname for the MongoDB instance. `MongoDB:Username` | The login username for the MongoDB instance. `MongoDB:Password` | The login password for the MongoDB instance. - `MongoDB:Database` | The database name (e.g. `smapi` in production or `smapi-edge` in testing environments). + `MongoDB:Database` | The MongoDB database name (e.g. `smapi` in production or `smapi-edge` in testing environments). + + Optional settings: + + property name | description + ------------------------------- | ----------------- + `BackgroundServices:Enabled` | Set to `true` to enable background processes like fetching data from the wiki, or false to disable them. + `Site:BetaEnabled` | Set to `true` to show a separate download button if there's a beta version of SMAPI in its GitHub releases. + `Site:BetaBlurb` | If `Site:BetaEnabled` is true and there's a beta version of SMAPI in its GitHub releases, this is shown on the beta download button as explanatory subtext. + `Site:SupporterList` | A list of Patreon supports to credit on the download page. To deploy updates: -1. Deploy the web project using [AWS Toolkit for Visual Studio](https://aws.amazon.com/visualstudio/). +1. [Deploy the web project from Visual Studio](https://docs.microsoft.com/en-us/visualstudio/deployment/quickstart-deploy-to-azure). 2. If the MongoDB schema changed, delete the MongoDB database. (It'll be recreated automatically.) diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs index 1ef3ef12..813ea115 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs @@ -1,5 +1,3 @@ -using System; - namespace StardewModdingAPI.Web.Framework.Clients.Pastebin { /// The response for a get-paste request. diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/SavePasteResult.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/SavePasteResult.cs deleted file mode 100644 index 89dab697..00000000 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/SavePasteResult.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace StardewModdingAPI.Web.Framework.Clients.Pastebin -{ - /// The response for a save-log request. - internal class SavePasteResult - { - /// Whether the log was successfully saved. - public bool Success { get; set; } - - /// The saved paste ID (if is true). - public string ID { get; set; } - - /// The error message (if saving failed). - public string Error { get; set; } - } -} diff --git a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs index 3c508300..e2e18477 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs @@ -24,6 +24,12 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /********* ** Public method *********/ + /// Get whether a MongoDB instance is configured. + public bool IsConfigured() + { + return !string.IsNullOrWhiteSpace(this.Host); + } + /// Get the MongoDB connection string. public string GetConnectionString() { diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index 22f5e975..504254cd 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -14,6 +14,7 @@ + @@ -23,6 +24,7 @@ + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 07ee0c9e..338cd2d5 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -1,5 +1,7 @@ +using System; using System.Collections.Generic; using Hangfire; +using Hangfire.MemoryStorage; using Hangfire.Mongo; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -8,6 +10,7 @@ using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; +using Mongo2Go; using MongoDB.Bson.Serialization; using MongoDB.Driver; using Newtonsoft.Json; @@ -89,10 +92,20 @@ namespace StardewModdingAPI.Web } // init MongoDB + services.AddSingleton(serv => !mongoConfig.IsConfigured() + ? MongoDbRunner.Start() + : throw new InvalidOperationException("The MongoDB connection is configured, so the local development version should not be used.") + ); services.AddSingleton(serv => { + // get connection string + string connectionString = mongoConfig.IsConfigured() + ? mongoConfig.GetConnectionString() + : serv.GetRequiredService().ConnectionString; + + // get client BsonSerializer.RegisterSerializer(new UtcDateTimeOffsetSerializer()); - return new MongoClient(mongoConfig.GetConnectionString()).GetDatabase(mongoConfig.Database); + return new MongoClient(connectionString).GetDatabase(mongoConfig.Database); }); services.AddSingleton(serv => new ModCacheRepository(serv.GetRequiredService())); services.AddSingleton(serv => new WikiCacheRepository(serv.GetRequiredService())); @@ -104,12 +117,18 @@ namespace StardewModdingAPI.Web config .SetDataCompatibilityLevel(CompatibilityLevel.Version_170) .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage(mongoConfig.GetConnectionString(), $"{mongoConfig.Database}-hangfire", new MongoStorageOptions + .UseRecommendedSerializerSettings(); + + if (mongoConfig.IsConfigured()) + { + config.UseMongoStorage(mongoConfig.GetConnectionString(), $"{mongoConfig.Database}-hangfire", new MongoStorageOptions { MigrationOptions = new MongoMigrationOptions(MongoMigrationStrategy.Drop), CheckConnection = false // error on startup takes down entire process }); + } + else + config.UseMemoryStorage(); }); // init API clients diff --git a/src/SMAPI.Web/Views/Index/Privacy.cshtml b/src/SMAPI.Web/Views/Index/Privacy.cshtml index 7327de3d..fd78f908 100644 --- a/src/SMAPI.Web/Views/Index/Privacy.cshtml +++ b/src/SMAPI.Web/Views/Index/Privacy.cshtml @@ -22,10 +22,10 @@

Data collected and transmitted

Web logging

-

This website and SMAPI's web API are hosted by Amazon Web Services. Their servers may automatically collect diagnostics like your IP address, but this information is not visible to SMAPI's web application or developers. For more information, see the Amazon Privacy Notice.

+

This website and SMAPI's web API are hosted on Microsoft Azure. Their servers may automatically collect diagnostics like your IP address, but this information is not visible to SMAPI's web apps or its developers. For more information, see the Microsoft Azure legal resources.

Update checks

-

SMAPI notifies you when there's a new version of SMAPI or your mods available. To do so, it sends your game/SMAPI/mod versions and platform type to its web API. No personal information is stored by the web application, but see web logging.

+

SMAPI notifies you when there's a new version of SMAPI or your mods available. To do so, it sends basic metadata like your game/SMAPI/mod versions and platform type to its web API. No personal information is stored by the web app.

You can disable update checks, and no information will be transmitted to the web API. To do so:

    @@ -34,8 +34,8 @@
  1. change "CheckForUpdates": true to "CheckForUpdates": false.
-

Log parser

-

The log parser page lets you store a log file for analysis and sharing. The log data is stored indefinitely in an obfuscated form as unlisted pastes in Pastebin. No personal information is stored by the log parser beyond what you choose to upload, but see web logging and the Pastebin Privacy Statement.

+

Log parser and JSON validator

+

The log parser and JSON validator let you upload files to analyze and share with other users. The log data is stored for 30 days in an obfuscated form in a private Microsoft Azure Blob storage account. No personal information is stored by the log parser beyond what you choose to upload as part of those files.

Multiplayer sync

As part of its multiplayer API, SMAPI transmits basic context to players you connect to (mainly your OS, SMAPI version, game version, and installed mods). This is used to enable multiplayer features like inter-mod messages, compatibility checks, etc. Although this information is normally hidden from players, it may be visible due to mods or configuration changes.

diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 3c2001ef..a6e48c69 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -18,9 +18,13 @@ }, "MongoDB": { - "Host": "localhost", + "Host": null, "Username": null, "Password": null, "Database": "smapi-edge" + }, + + "BackgroundServices": { + "Enabled": true } } diff --git a/src/SMAPI.Web/wwwroot/Content/js/json-validator.js b/src/SMAPI.Web/wwwroot/Content/js/json-validator.js index 401efbee..72b8192b 100644 --- a/src/SMAPI.Web/wwwroot/Content/js/json-validator.js +++ b/src/SMAPI.Web/wwwroot/Content/js/json-validator.js @@ -71,9 +71,9 @@ smapi.LineNumberRange = function (maxLines) { /** * UI logic for the JSON validator page. * @param {string} urlFormat The URL format for a file, with $schemaName and $id placeholders. - * @param {string} pasteID The Pastebin paste ID for the content being viewed, if any. + * @param {string} fileId The file ID for the content being viewed, if any. */ -smapi.jsonValidator = function (urlFormat, pasteID) { +smapi.jsonValidator = function (urlFormat, fileId) { /** * The original content element. */ @@ -138,7 +138,7 @@ smapi.jsonValidator = function (urlFormat, pasteID) { // change format $("#output #format").on("change", function() { var schemaName = $(this).val(); - location.href = urlFormat.replace("$schemaName", schemaName).replace("$id", pasteID); + location.href = urlFormat.replace("$schemaName", schemaName).replace("$id", fileId); }); // upload form -- cgit From 4c471ea2154772e0eee6a50be2f0e468915a1b9d Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 22 Dec 2019 18:46:03 -0500 Subject: fix CP schema not requiring Default field if AllowBlank is omitted --- docs/release-notes.md | 1 + src/SMAPI.Web/wwwroot/schemas/content-patcher.json | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/SMAPI.Web/wwwroot') diff --git a/docs/release-notes.md b/docs/release-notes.md index 9b07160c..dc0f584d 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -21,6 +21,7 @@ * The log parser and JSON validator no longer save files to Pastebin due to ongoing performance issues. All files are now saved to Azure Blob storage instead and expire after one month. * Updated the JSON validator for Content Patcher 1.10 and 1.11. * Fixed JSON validator no longer letting you change format when viewing results. + * Fixed JSON validator for Content Patcher schema not requiring `Default` if `AllowBlank` omitted. * For modders: * Added asset propagation for... diff --git a/src/SMAPI.Web/wwwroot/schemas/content-patcher.json b/src/SMAPI.Web/wwwroot/schemas/content-patcher.json index 28e38e73..7e00c28e 100644 --- a/src/SMAPI.Web/wwwroot/schemas/content-patcher.json +++ b/src/SMAPI.Web/wwwroot/schemas/content-patcher.json @@ -51,8 +51,7 @@ "if": { "properties": { "AllowBlank": { "const": false } - }, - "required": [ "AllowBlank" ] + } }, "then": { "required": [ "Default" ] -- cgit