From 2b1f607d41b3d4d071c0db0671dbc99b6982909f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 21:21:28 -0500 Subject: encapsulate file storage, also handle Pastebin rate limits in JSON validator --- src/SMAPI.Web/Controllers/LogParserController.cs | 178 ++--------------------- 1 file changed, 14 insertions(+), 164 deletions(-) (limited to 'src/SMAPI.Web/Controllers/LogParserController.cs') diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index 318b34d0..e270ae0a 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,22 +1,12 @@ using System; -using System.IO; using System.Linq; -using System.Text; using System.Threading.Tasks; -using Amazon; -using Amazon.Runtime; -using Amazon.S3; -using Amazon.S3.Model; -using Amazon.S3.Transfer; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit.Utilities; using StardewModdingAPI.Web.Framework; -using StardewModdingAPI.Web.Framework.Clients.Pastebin; -using StardewModdingAPI.Web.Framework.Compression; -using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.Framework.LogParsing; using StardewModdingAPI.Web.Framework.LogParsing.Models; +using StardewModdingAPI.Web.Framework.Storage; using StardewModdingAPI.Web.ViewModels; namespace StardewModdingAPI.Web.Controllers @@ -27,14 +17,8 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Fields *********/ - /// The API client settings. - private readonly ApiClientsConfig ClientsConfig; - - /// The underlying Pastebin client. - private readonly IPastebinClient Pastebin; - - /// The underlying text compression helper. - private readonly IGzipHelper GzipHelper; + /// Provides access to raw data storage. + private readonly IStorageProvider Storage; /********* @@ -44,21 +28,17 @@ namespace StardewModdingAPI.Web.Controllers ** Constructor ***/ /// Construct an instance. - /// The API client settings. - /// The Pastebin API client. - /// The underlying text compression helper. - public LogParserController(IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + /// Provides access to raw data storage. + public LogParserController(IStorageProvider storage) { - this.ClientsConfig = clientsConfig.Value; - this.Pastebin = pastebin; - this.GzipHelper = gzipHelper; + this.Storage = storage; } /*** ** Web UI ***/ /// Render the log parser UI. - /// The paste ID. + /// The stored file ID. /// Whether to display the raw unparsed log. [HttpGet] [Route("log")] @@ -70,12 +50,12 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(id)); // log page - PasteInfo paste = await this.GetAsync(id); - ParsedLog log = paste.Success - ? new LogParser().Parse(paste.Content) - : new ParsedLog { IsValid = false, Error = paste.Error }; + StoredFileInfo file = await this.Storage.GetAsync(id); + ParsedLog log = file.Success + ? new LogParser().Parse(file.Content) + : new ParsedLog { IsValid = false, Error = file.Error }; - return this.View("Index", this.GetModel(id, uploadWarning: paste.Warning, expiry: paste.Expiry).SetResult(log, raw)); + return this.View("Index", this.GetModel(id, uploadWarning: file.Warning, expiry: file.Expiry).SetResult(log, raw)); } /*** @@ -92,8 +72,7 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty.")); // upload log - input = this.GzipHelper.CompressString(input); - var uploadResult = await this.TrySaveLog(input); + UploadResult uploadResult = await this.Storage.SaveAsync(title: $"SMAPI log {DateTime.UtcNow:s}", content: input, compress: true); if (!uploadResult.Succeeded) return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); @@ -105,106 +84,8 @@ namespace StardewModdingAPI.Web.Controllers /********* ** Private methods *********/ - /// Fetch raw text from Pastebin. - /// The Pastebin paste ID. - private async Task GetAsync(string id) - { - // get from Amazon S3 - if (Guid.TryParseExact(id, "N", out Guid _)) - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - - using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) - { - try - { - using (GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonLogBucket, $"logs/{id}")) - using (Stream responseStream = response.ResponseStream) - using (StreamReader reader = new StreamReader(responseStream)) - { - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - - return new PasteInfo - { - Success = true, - Content = content, - Expiry = expiry, - Warning = pastebinError - }; - } - } - catch (AmazonServiceException ex) - { - return ex.ErrorCode == "NoSuchKey" - ? new PasteInfo { Error = "There's no log with that ID." } - : new PasteInfo { Error = $"Could not fetch that log from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; - } - } - } - - // get from PasteBin - else - { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; - } - } - - /// Save a log to Pastebin or Amazon S3, if available. - /// The content to upload. - /// Returns metadata about the save attempt. - private async Task TrySaveLog(string content) - { - // save to PasteBin - string uploadError; - { - SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 - try - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - - using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) - using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) - using (TransferUtility uploader = new TransferUtility(s3)) - { - string id = Guid.NewGuid().ToString("N"); - - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonLogBucket, - Key = $"logs/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; - - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); - } - } - catch (Exception ex) - { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); - } - } - /// Build a log parser model. - /// The paste ID. + /// The stored file ID. /// When the uploaded file will no longer be available. /// A non-blocking warning while uploading the log. /// An error which occurred while uploading the log. @@ -243,36 +124,5 @@ namespace StardewModdingAPI.Web.Controllers return null; } } - - /// The result of an attempt to upload a file. - private class UploadResult - { - /********* - ** Accessors - *********/ - /// Whether the file upload succeeded. - public bool Succeeded { get; } - - /// The file ID, if applicable. - public string ID { get; } - - /// The upload error, if any. - public string UploadError { get; } - - - /********* - ** Public methods - *********/ - /// Construct an instance. - /// Whether the file upload succeeded. - /// The file ID, if applicable. - /// The upload error, if any. - public UploadResult(bool succeeded, string id, string uploadError) - { - this.Succeeded = succeeded; - this.ID = id; - this.UploadError = uploadError; - } - } } } -- cgit From c1b15fb3725661ebfd8e03cec08343ae49e5d6da Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 23:52:47 -0500 Subject: allow local dev environments without an Azure account --- docs/release-notes.md | 3 + .../Controllers/JsonValidatorController.cs | 2 +- src/SMAPI.Web/Controllers/LogParserController.cs | 2 +- .../Framework/Storage/IStorageProvider.cs | 3 +- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 116 +++++++++++++++------ src/SMAPI.Web/Views/LogParser/Index.cshtml | 21 ++-- 6 files changed, 104 insertions(+), 43 deletions(-) (limited to 'src/SMAPI.Web/Controllers/LogParserController.cs') diff --git a/docs/release-notes.md b/docs/release-notes.md index 591f4cc2..bbe08c13 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -26,6 +26,9 @@ * Fixed private textures loaded from content packs not having their `Name` field set. * Fixed asset propagation for `Characters\Farmer\farmer_girl_base_bald`. +* For SMAPI developers: + * You can now run local environments without configuring Amazon, Azure, and Pastebin accounts. + ## 3.0.1 Released 02 December 2019 for Stardew Valley 1.4.0.1. diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs index c4bfff3b..2ade3e3d 100644 --- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -141,7 +141,7 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(null, schemaName).SetUploadError("The JSON file seems to be empty.")); // upload file - UploadResult result = await this.Storage.SaveAsync(title: $"JSON validator {DateTime.UtcNow:s}", content: input, compress: true); + UploadResult result = await this.Storage.SaveAsync(input); if (!result.Succeeded) return this.View("Index", this.GetModel(result.ID, schemaName).SetUploadError(result.UploadError)); diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index e270ae0a..97c419d9 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -72,7 +72,7 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty.")); // upload log - UploadResult uploadResult = await this.Storage.SaveAsync(title: $"SMAPI log {DateTime.UtcNow:s}", content: input, compress: true); + UploadResult uploadResult = await this.Storage.SaveAsync(input); if (!uploadResult.Succeeded) return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index 12a5e421..96a34fbb 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -6,11 +6,10 @@ namespace StardewModdingAPI.Web.Framework.Storage internal interface IStorageProvider { /// Save a text file to storage. - /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. - Task SaveAsync(string title, string content, bool compress = true); + Task SaveAsync(string content, bool compress = true); /// Fetch raw text from storage. /// The storage ID returned by . diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index 12a35f18..35538443 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -27,6 +27,12 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The underlying text compression helper. private readonly IGzipHelper GzipHelper; + /// Whether Azure blob storage is configured. + private bool HasAzure => !string.IsNullOrWhiteSpace(this.ClientsConfig.AzureBlobConnectionString); + + /// The number of days since the blob's last-modified date when it will be deleted. + private int ExpiryDays => this.ClientsConfig.AzureBlobTempExpiryDays; + /********* ** Public methods @@ -43,25 +49,38 @@ namespace StardewModdingAPI.Web.Framework.Storage } /// Save a text file to storage. - /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. - public async Task SaveAsync(string title, string content, bool compress = true) + public async Task SaveAsync(string content, bool compress = true) { - try - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - string id = Guid.NewGuid().ToString("N"); + string id = Guid.NewGuid().ToString("N"); - BlobClient blob = this.GetAzureBlobClient(id); - await blob.UploadAsync(stream); + // save to Azure + if (this.HasAzure) + { + try + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + BlobClient blob = this.GetAzureBlobClient(id); + await blob.UploadAsync(stream); - return new UploadResult(true, id, null); + return new UploadResult(true, id, null); + } + catch (Exception ex) + { + return new UploadResult(false, null, ex.Message); + } } - catch (Exception ex) + + // save to local filesystem for testing + else { - return new UploadResult(false, null, ex.Message); + string path = this.GetDevFilePath(id); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + + File.WriteAllText(path, content); + return new UploadResult(true, id, null); } } @@ -69,39 +88,67 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The storage ID returned by . public async Task GetAsync(string id) { - // fetch from Azure/Amazon + // fetch from blob storage if (Guid.TryParseExact(id, "N", out Guid _)) { - // try Azure - try + // Azure Blob storage + if (this.HasAzure) { - BlobClient blob = this.GetAzureBlobClient(id); - Response response = await blob.DownloadAsync(); - using BlobDownloadInfo result = response.Value; + try + { + BlobClient blob = this.GetAzureBlobClient(id); + Response response = await blob.DownloadAsync(); + using BlobDownloadInfo result = response.Value; - using StreamReader reader = new StreamReader(result.Content); - DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays); - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + using StreamReader reader = new StreamReader(result.Content); + DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ExpiryDays); + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - return new StoredFileInfo + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry.UtcDateTime + }; + } + catch (RequestFailedException ex) { - Success = true, - Content = content, - Expiry = expiry.UtcDateTime - }; + return new StoredFileInfo + { + Error = ex.ErrorCode == "BlobNotFound" + ? "There's no file with that ID." + : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." + }; + } } - catch (RequestFailedException ex) + + // local filesystem for testing + else { + FileInfo file = new FileInfo(this.GetDevFilePath(id)); + if (file.Exists) + { + if (file.LastWriteTimeUtc.AddDays(this.ExpiryDays) < DateTime.UtcNow) + file.Delete(); + else + { + return new StoredFileInfo + { + Success = true, + Content = File.ReadAllText(file.FullName), + Expiry = DateTime.UtcNow.AddDays(this.ExpiryDays), + Warning = "This file was saved temporarily to the local computer. This should only happen in a local development environment." + }; + } + } return new StoredFileInfo { - Error = ex.ErrorCode == "BlobNotFound" - ? "There's no file with that ID." - : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." + Error = "There's no file with that ID." }; } } - // get from PasteBin + // get from Pastebin else { PasteInfo response = await this.Pastebin.GetAsync(id); @@ -116,12 +163,19 @@ namespace StardewModdingAPI.Web.Framework.Storage } /// Get a client for reading and writing to Azure Blob storage. - /// The file ID to fetch. + /// The file ID. private BlobClient GetAzureBlobClient(string id) { var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString); var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer); return container.GetBlobClient($"uploads/{id}"); } + + /// Get the absolute file path for an upload when running in a local test environment with no Azure account configured. + /// The file ID. + private string GetDevFilePath(string id) + { + return Path.Combine(Path.GetTempPath(), "smapi-web-temp", $"{id}.txt"); + } } } diff --git a/src/SMAPI.Web/Views/LogParser/Index.cshtml b/src/SMAPI.Web/Views/LogParser/Index.cshtml index 439167bc..ac951564 100644 --- a/src/SMAPI.Web/Views/LogParser/Index.cshtml +++ b/src/SMAPI.Web/Views/LogParser/Index.cshtml @@ -67,12 +67,16 @@ else if (Model.ParsedLog?.IsValid == true) @* save warnings *@ @if (Model.UploadWarning != null || Model.Expiry != null) { + @if (Model.UploadWarning != null) + { + ⚠️ @Model.UploadWarning
 + } +
@if (Model.Expiry != null) { - This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). + This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). } -
} @@ -294,10 +298,7 @@ else if (Model.ParsedLog?.IsValid == true) string sectionFilter = message.Section != null && !message.IsStartOfSection ? $"&& sectionsAllow('{message.Section}')" : null; // filter the message by section if applicable v-on:click="toggleSection('@message.Section')" - } + @if (message.IsStartOfSection) { v-on:click="toggleSection('@message.Section')" } v-show="filtersAllow('@Model.GetSlug(message.Mod)', '@levelStr') @sectionFilter"> @message.Time @message.Level.ToString().ToUpper() @@ -307,8 +308,12 @@ else if (Model.ParsedLog?.IsValid == true) @if (message.IsStartOfSection) { - - + + } -- cgit