From 2b1f607d41b3d4d071c0db0671dbc99b6982909f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 21:21:28 -0500 Subject: encapsulate file storage, also handle Pastebin rate limits in JSON validator --- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 147 +++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 src/SMAPI.Web/Framework/Storage/StorageProvider.cs (limited to 'src/SMAPI.Web/Framework/Storage/StorageProvider.cs') diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs new file mode 100644 index 00000000..bbb6e06b --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -0,0 +1,147 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Microsoft.Extensions.Options; +using StardewModdingAPI.Web.Framework.Clients.Pastebin; +using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.ConfigModels; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// Provides access to raw data storage. + internal class StorageProvider : IStorageProvider + { + /********* + ** Fields + *********/ + /// The API client settings. + private readonly ApiClientsConfig ClientsConfig; + + /// The underlying Pastebin client. + private readonly IPastebinClient Pastebin; + + /// The underlying text compression helper. + private readonly IGzipHelper GzipHelper; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The API client settings. + /// The underlying Pastebin client. + /// The underlying text compression helper. + public StorageProvider(IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + { + this.ClientsConfig = clientsConfig.Value; + this.Pastebin = pastebin; + this.GzipHelper = gzipHelper; + } + + /// Save a text file to Pastebin or Amazon S3, if available. + /// The display title, if applicable. + /// The content to upload. + /// Whether to gzip the text. + /// Returns metadata about the save attempt. + public async Task SaveAsync(string title, string content, bool compress = true) + { + // save to PasteBin + string uploadError; + { + SavePasteResult result = await this.Pastebin.PostAsync(title, content); + if (result.Success) + return new UploadResult(true, result.ID, null); + + uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; + } + + // fallback to S3 + try + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + using TransferUtility uploader = new TransferUtility(s3); + + string id = Guid.NewGuid().ToString("N"); + + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = this.ClientsConfig.AmazonTempBucket, + Key = $"uploads/{id}", + InputStream = stream, + Metadata = + { + // note: AWS will lowercase keys and prefix 'x-amz-meta-' + ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), + ["pastebin-error"] = uploadError + } + }; + + await uploader.UploadAsync(uploadRequest); + + return new UploadResult(true, id, uploadError); + } + catch (Exception ex) + { + return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + } + } + + /// Fetch raw text from storage. + /// The storage ID returned by . + public async Task GetAsync(string id) + { + // get from Amazon S3 + if (Guid.TryParseExact(id, "N", out Guid _)) + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } + } + + // get from PasteBin + else + { + PasteInfo response = await this.Pastebin.GetAsync(id); + response.Content = this.GzipHelper.DecompressString(response.Content); + return new StoredFileInfo + { + Success = response.Success, + Content = response.Content, + Error = response.Error + }; + } + } + } +} -- cgit From 02f645900eb31648376abe21df30dd956221ad90 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 11:56:19 -0500 Subject: add option to disable uploading new files to Pastebin --- src/SMAPI.Web/Controllers/JsonValidatorController.cs | 2 +- src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs | 3 +++ src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 3 ++- src/SMAPI.Web/appsettings.json | 3 ++- 4 files changed, 8 insertions(+), 3 deletions(-) (limited to 'src/SMAPI.Web/Framework/Storage/StorageProvider.cs') diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs index e4eff0f4..c4bfff3b 100644 --- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -141,7 +141,7 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(null, schemaName).SetUploadError("The JSON file seems to be empty.")); // upload file - var result = await this.Storage.SaveAsync(title: $"JSON validator {DateTime.UtcNow:s}", content: input, compress: true); + UploadResult result = await this.Storage.SaveAsync(title: $"JSON validator {DateTime.UtcNow:s}", content: input, compress: true); if (!result.Succeeded) return this.View("Index", this.GetModel(result.ID, schemaName).SetUploadError(result.UploadError)); diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index 1e020840..e88f351f 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -97,5 +97,8 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The developer key used to authenticate with the Pastebin API. public string PastebinDevKey { get; set; } + /// Whether to enable uploading new files to Pastebin. This doesn't affect fetching already-uploaded files. + public bool PastebinEnableUploads { get; set; } + } } diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index bbb6e06b..e5e71325 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -52,7 +52,8 @@ namespace StardewModdingAPI.Web.Framework.Storage public async Task SaveAsync(string title, string content, bool compress = true) { // save to PasteBin - string uploadError; + string uploadError = null; + if (this.ClientsConfig.PastebinEnableUploads) { SavePasteResult result = await this.Pastebin.PostAsync(title, content); if (result.Success) diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 2e20b299..6ba05a22 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -49,7 +49,8 @@ "PastebinBaseUrl": "https://pastebin.com/", "PastebinUserKey": null, - "PastebinDevKey": null + "PastebinDevKey": null, + "PastebinEnableUploads": true }, "MongoDB": { -- cgit From 242dc718cdedf2c7a264670008b9f760eba160d9 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 15:41:55 -0500 Subject: switch to Azure Blob storage for saving files --- .../Framework/Clients/Pastebin/IPastebinClient.cs | 5 - .../Framework/Clients/Pastebin/PastebinClient.cs | 58 +---------- .../Framework/ConfigModels/ApiClientsConfig.cs | 26 +++-- .../Framework/Storage/IStorageProvider.cs | 2 +- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 108 +++++++++++---------- src/SMAPI.Web/SMAPI.Web.csproj | 1 + src/SMAPI.Web/Startup.cs | 4 +- src/SMAPI.Web/appsettings.Development.json | 7 +- src/SMAPI.Web/appsettings.json | 9 +- 9 files changed, 87 insertions(+), 133 deletions(-) (limited to 'src/SMAPI.Web/Framework/Storage/StorageProvider.cs') diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs index a635abe3..431fed7b 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs @@ -9,10 +9,5 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// Fetch a saved paste. /// The paste ID. Task GetAsync(string id); - - /// Save a paste to Pastebin. - /// The paste name. - /// The paste content. - Task PostAsync(string name, string content); } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs index d695aab6..1be00be7 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs @@ -1,7 +1,5 @@ using System; -using System.Linq; using System.Net; -using System.Net.Http; using System.Threading.Tasks; using Pathoschild.Http.Client; @@ -16,12 +14,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// The underlying HTTP client. private readonly IClient Client; - /// The user key used to authenticate with the Pastebin API. - private readonly string UserKey; - - /// The developer key used to authenticate with the Pastebin API. - private readonly string DevKey; - /********* ** Public methods @@ -29,13 +21,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// Construct an instance. /// The base URL for the Pastebin API. /// The user agent for the API client. - /// The user key used to authenticate with the Pastebin API. - /// The developer key used to authenticate with the Pastebin API. - public PastebinClient(string baseUrl, string userAgent, string userKey, string devKey) + public PastebinClient(string baseUrl, string userAgent) { this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); - this.UserKey = userKey; - this.DevKey = devKey; } /// Fetch a saved paste. @@ -66,50 +54,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin } } - /// Save a paste to Pastebin. - /// The paste name. - /// The paste content. - public async Task PostAsync(string name, string content) - { - try - { - // validate - if (string.IsNullOrWhiteSpace(content)) - return new SavePasteResult { Error = "The log content can't be empty." }; - - // post to API - string response = await this.Client - .PostAsync("api/api_post.php") - .WithBody(p => p.FormUrlEncoded(new - { - api_option = "paste", - api_user_key = this.UserKey, - api_dev_key = this.DevKey, - api_paste_private = 1, // unlisted - api_paste_name = name, - api_paste_expire_date = "N", // never expire - api_paste_code = content - })) - .AsString(); - - // handle Pastebin errors - if (string.IsNullOrWhiteSpace(response)) - return new SavePasteResult { Error = "Received an empty response from Pastebin." }; - if (response.StartsWith("Bad API request")) - return new SavePasteResult { Error = response }; - if (!response.Contains("/")) - return new SavePasteResult { Error = $"Received an unknown response: {response}" }; - - // return paste ID - string pastebinID = response.Split("/").Last(); - return new SavePasteResult { Success = true, ID = pastebinID }; - } - catch (Exception ex) - { - return new SavePasteResult { Success = false, Error = ex.ToString() }; - } - } - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. public void Dispose() { diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index e88f351f..4a73750b 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -29,6 +29,19 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string AmazonTempBucket { get; set; } + /**** + ** Azure + ****/ + /// The connection string for the Azure Blob storage account. + public string AzureBlobConnectionString { get; set; } + + /// The Azure Blob container in which to store temporary uploaded logs. + public string AzureBlobTempContainer { get; set; } + + /// The number of days since the blob's last-modified date when it will be deleted. + public int AzureBlobTempExpiryDays { get; set; } + + /**** ** Chucklefish ****/ @@ -61,6 +74,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The password with which to authenticate to the GitHub API (if any). public string GitHubPassword { get; set; } + /**** ** ModDrop ****/ @@ -70,6 +84,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The URL for a ModDrop mod page for the user, where {0} is the mod ID. public string ModDropModPageUrl { get; set; } + /**** ** Nexus Mods ****/ @@ -85,20 +100,11 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The Nexus API authentication key. public string NexusApiKey { get; set; } + /**** ** Pastebin ****/ /// The base URL for the Pastebin API. public string PastebinBaseUrl { get; set; } - - /// The user key used to authenticate with the Pastebin API. - public string PastebinUserKey { get; set; } - - /// The developer key used to authenticate with the Pastebin API. - public string PastebinDevKey { get; set; } - - /// Whether to enable uploading new files to Pastebin. This doesn't affect fetching already-uploaded files. - public bool PastebinEnableUploads { get; set; } - } } diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index e222a235..12a5e421 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -5,7 +5,7 @@ namespace StardewModdingAPI.Web.Framework.Storage /// Provides access to raw data storage. internal interface IStorageProvider { - /// Save a text file to Pastebin or Amazon S3, if available. + /// Save a text file to storage. /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index e5e71325..b2d8ae7e 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -6,7 +6,9 @@ using Amazon; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; -using Amazon.S3.Transfer; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Microsoft.Extensions.Options; using StardewModdingAPI.Web.Framework.Clients.Pastebin; using StardewModdingAPI.Web.Framework.Compression; @@ -44,54 +46,26 @@ namespace StardewModdingAPI.Web.Framework.Storage this.GzipHelper = gzipHelper; } - /// Save a text file to Pastebin or Amazon S3, if available. + /// Save a text file to storage. /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. public async Task SaveAsync(string title, string content, bool compress = true) { - // save to PasteBin - string uploadError = null; - if (this.ClientsConfig.PastebinEnableUploads) - { - SavePasteResult result = await this.Pastebin.PostAsync(title, content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 try { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - using TransferUtility uploader = new TransferUtility(s3); - string id = Guid.NewGuid().ToString("N"); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonTempBucket, - Key = $"uploads/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; + BlobClient blob = this.GetAzureBlobClient(id); + await blob.UploadAsync(stream); - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); + return new UploadResult(true, id, null); } catch (Exception ex) { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + return new UploadResult(false, null, ex.Message); } } @@ -99,35 +73,62 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The storage ID returned by . public async Task GetAsync(string id) { - // get from Amazon S3 + // fetch from Azure/Amazon if (Guid.TryParseExact(id, "N", out Guid _)) { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - + // try Azure try { - using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); - using Stream responseStream = response.ResponseStream; - using StreamReader reader = new StreamReader(responseStream); + BlobClient blob = this.GetAzureBlobClient(id); + Response response = await blob.DownloadAsync(); + using BlobDownloadInfo result = response.Value; - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + using StreamReader reader = new StreamReader(result.Content); + DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays); string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); return new StoredFileInfo { Success = true, Content = content, - Expiry = expiry, - Warning = pastebinError + Expiry = expiry.UtcDateTime }; } - catch (AmazonServiceException ex) + catch (RequestFailedException ex) { - return ex.ErrorCode == "NoSuchKey" - ? new StoredFileInfo { Error = "There's no file with that ID." } - : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + if (ex.ErrorCode != "BlobNotFound") + return new StoredFileInfo { Error = $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }; + } + + // try legacy Amazon S3 + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } } } @@ -144,5 +145,14 @@ namespace StardewModdingAPI.Web.Framework.Storage }; } } + + /// Get a client for reading and writing to Azure Blob storage. + /// The file ID to fetch. + private BlobClient GetAzureBlobClient(string id) + { + var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString); + var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer); + return container.GetBlobClient($"uploads/{id}"); + } } } diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index 2c56fa75..773a7316 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -13,6 +13,7 @@ + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 31b5e61d..07ee0c9e 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -153,9 +153,7 @@ namespace StardewModdingAPI.Web services.AddSingleton(new PastebinClient( baseUrl: api.PastebinBaseUrl, - userAgent: userAgent, - userKey: api.PastebinUserKey, - devKey: api.PastebinDevKey + userAgent: userAgent )); } diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 74ded25d..05f0da1d 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -12,13 +12,12 @@ "AmazonAccessKey": null, "AmazonSecretKey": null, + "AzureBlobConnectionString": null, + "GitHubUsername": null, "GitHubPassword": null, - "NexusApiKey": null, - - "PastebinUserKey": null, - "PastebinDevKey": null + "NexusApiKey": null }, "MongoDB": { diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 6ba05a22..c6dc1b69 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -29,6 +29,10 @@ "AmazonRegion": "us-east-1", "AmazonTempBucket": "smapi-web-temp", + "AzureBlobConnectionString": null, + "AzureBlobTempContainer": "smapi-web-temp", + "AzureBlobTempExpiryDays": 30, + "ChucklefishBaseUrl": "https://community.playstarbound.com", "ChucklefishModPageUrlFormat": "resources/{0}", @@ -47,10 +51,7 @@ "NexusModUrlFormat": "mods/{0}", "NexusModScrapeUrlFormat": "mods/{0}?tab=files", - "PastebinBaseUrl": "https://pastebin.com/", - "PastebinUserKey": null, - "PastebinDevKey": null, - "PastebinEnableUploads": true + "PastebinBaseUrl": "https://pastebin.com/" }, "MongoDB": { -- cgit From ba46491ebc305a66a0a3a1f3cefa8bb677e57ff7 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 23:14:15 -0500 Subject: drop Amazon S3 support --- docs/release-notes.md | 2 +- .../Framework/ConfigModels/ApiClientsConfig.cs | 16 --------- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 41 +++------------------- src/SMAPI.Web/SMAPI.Web.csproj | 1 - src/SMAPI.Web/appsettings.Development.json | 3 -- src/SMAPI.Web/appsettings.json | 5 --- 6 files changed, 6 insertions(+), 62 deletions(-) (limited to 'src/SMAPI.Web/Framework/Storage/StorageProvider.cs') diff --git a/docs/release-notes.md b/docs/release-notes.md index fa64a037..591f4cc2 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -13,7 +13,7 @@ * For the web UI: * Added option to edit & reupload in the JSON validator. - * If a JSON validator upload can't be saved to Pastebin (e.g. due to rate limits), it's now uploaded to Amazon S3 instead. Files uploaded to S3 expire after one month. + * The log parser and JSON validator no longer save files to Pastebin due to ongoing performance issues. All files are now saved to Azure Blob storage instead and expire after one month. * Updated the JSON validator for Content Patcher 1.10 and 1.11. * Fixed JSON validator no longer letting you change format when viewing results. diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index 4a73750b..878130bf 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -13,22 +13,6 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string UserAgent { get; set; } - /**** - ** Amazon Web Services - ****/ - /// The access key for AWS authentication. - public string AmazonAccessKey { get; set; } - - /// The secret key for AWS authentication. - public string AmazonSecretKey { get; set; } - - /// The AWS region endpoint (like 'us-east-1'). - public string AmazonRegion { get; set; } - - /// The AWS bucket in which to store temporary uploaded logs. - public string AmazonTempBucket { get; set; } - - /**** ** Azure ****/ diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index b2d8ae7e..12a35f18 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -2,10 +2,6 @@ using System; using System.IO; using System.Text; using System.Threading.Tasks; -using Amazon; -using Amazon.Runtime; -using Amazon.S3; -using Amazon.S3.Model; using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -96,39 +92,12 @@ namespace StardewModdingAPI.Web.Framework.Storage } catch (RequestFailedException ex) { - if (ex.ErrorCode != "BlobNotFound") - return new StoredFileInfo { Error = $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }; - } - - // try legacy Amazon S3 - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - - try - { - using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); - using Stream responseStream = response.ResponseStream; - using StreamReader reader = new StreamReader(responseStream); - - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - - return new StoredFileInfo - { - Success = true, - Content = content, - Expiry = expiry, - Warning = pastebinError - }; - } - catch (AmazonServiceException ex) + return new StoredFileInfo { - return ex.ErrorCode == "NoSuchKey" - ? new StoredFileInfo { Error = "There's no file with that ID." } - : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; - } + Error = ex.ErrorCode == "BlobNotFound" + ? "There's no file with that ID." + : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." + }; } } diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index 773a7316..22f5e975 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -12,7 +12,6 @@ - diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 05f0da1d..3c2001ef 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -9,9 +9,6 @@ */ { "ApiClients": { - "AmazonAccessKey": null, - "AmazonSecretKey": null, - "AzureBlobConnectionString": null, "GitHubUsername": null, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index c6dc1b69..0f61ebb9 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -24,11 +24,6 @@ "ApiClients": { "UserAgent": "SMAPI/{0} (+https://smapi.io)", - "AmazonAccessKey": null, - "AmazonSecretKey": null, - "AmazonRegion": "us-east-1", - "AmazonTempBucket": "smapi-web-temp", - "AzureBlobConnectionString": null, "AzureBlobTempContainer": "smapi-web-temp", "AzureBlobTempExpiryDays": 30, -- cgit From c1b15fb3725661ebfd8e03cec08343ae49e5d6da Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 23:52:47 -0500 Subject: allow local dev environments without an Azure account --- docs/release-notes.md | 3 + .../Controllers/JsonValidatorController.cs | 2 +- src/SMAPI.Web/Controllers/LogParserController.cs | 2 +- .../Framework/Storage/IStorageProvider.cs | 3 +- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 116 +++++++++++++++------ src/SMAPI.Web/Views/LogParser/Index.cshtml | 21 ++-- 6 files changed, 104 insertions(+), 43 deletions(-) (limited to 'src/SMAPI.Web/Framework/Storage/StorageProvider.cs') diff --git a/docs/release-notes.md b/docs/release-notes.md index 591f4cc2..bbe08c13 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -26,6 +26,9 @@ * Fixed private textures loaded from content packs not having their `Name` field set. * Fixed asset propagation for `Characters\Farmer\farmer_girl_base_bald`. +* For SMAPI developers: + * You can now run local environments without configuring Amazon, Azure, and Pastebin accounts. + ## 3.0.1 Released 02 December 2019 for Stardew Valley 1.4.0.1. diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs index c4bfff3b..2ade3e3d 100644 --- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs +++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs @@ -141,7 +141,7 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(null, schemaName).SetUploadError("The JSON file seems to be empty.")); // upload file - UploadResult result = await this.Storage.SaveAsync(title: $"JSON validator {DateTime.UtcNow:s}", content: input, compress: true); + UploadResult result = await this.Storage.SaveAsync(input); if (!result.Succeeded) return this.View("Index", this.GetModel(result.ID, schemaName).SetUploadError(result.UploadError)); diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index e270ae0a..97c419d9 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -72,7 +72,7 @@ namespace StardewModdingAPI.Web.Controllers return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty.")); // upload log - UploadResult uploadResult = await this.Storage.SaveAsync(title: $"SMAPI log {DateTime.UtcNow:s}", content: input, compress: true); + UploadResult uploadResult = await this.Storage.SaveAsync(input); if (!uploadResult.Succeeded) return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index 12a5e421..96a34fbb 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -6,11 +6,10 @@ namespace StardewModdingAPI.Web.Framework.Storage internal interface IStorageProvider { /// Save a text file to storage. - /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. - Task SaveAsync(string title, string content, bool compress = true); + Task SaveAsync(string content, bool compress = true); /// Fetch raw text from storage. /// The storage ID returned by . diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index 12a35f18..35538443 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -27,6 +27,12 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The underlying text compression helper. private readonly IGzipHelper GzipHelper; + /// Whether Azure blob storage is configured. + private bool HasAzure => !string.IsNullOrWhiteSpace(this.ClientsConfig.AzureBlobConnectionString); + + /// The number of days since the blob's last-modified date when it will be deleted. + private int ExpiryDays => this.ClientsConfig.AzureBlobTempExpiryDays; + /********* ** Public methods @@ -43,25 +49,38 @@ namespace StardewModdingAPI.Web.Framework.Storage } /// Save a text file to storage. - /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. - public async Task SaveAsync(string title, string content, bool compress = true) + public async Task SaveAsync(string content, bool compress = true) { - try - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - string id = Guid.NewGuid().ToString("N"); + string id = Guid.NewGuid().ToString("N"); - BlobClient blob = this.GetAzureBlobClient(id); - await blob.UploadAsync(stream); + // save to Azure + if (this.HasAzure) + { + try + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + BlobClient blob = this.GetAzureBlobClient(id); + await blob.UploadAsync(stream); - return new UploadResult(true, id, null); + return new UploadResult(true, id, null); + } + catch (Exception ex) + { + return new UploadResult(false, null, ex.Message); + } } - catch (Exception ex) + + // save to local filesystem for testing + else { - return new UploadResult(false, null, ex.Message); + string path = this.GetDevFilePath(id); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + + File.WriteAllText(path, content); + return new UploadResult(true, id, null); } } @@ -69,39 +88,67 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The storage ID returned by . public async Task GetAsync(string id) { - // fetch from Azure/Amazon + // fetch from blob storage if (Guid.TryParseExact(id, "N", out Guid _)) { - // try Azure - try + // Azure Blob storage + if (this.HasAzure) { - BlobClient blob = this.GetAzureBlobClient(id); - Response response = await blob.DownloadAsync(); - using BlobDownloadInfo result = response.Value; + try + { + BlobClient blob = this.GetAzureBlobClient(id); + Response response = await blob.DownloadAsync(); + using BlobDownloadInfo result = response.Value; - using StreamReader reader = new StreamReader(result.Content); - DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays); - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + using StreamReader reader = new StreamReader(result.Content); + DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ExpiryDays); + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - return new StoredFileInfo + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry.UtcDateTime + }; + } + catch (RequestFailedException ex) { - Success = true, - Content = content, - Expiry = expiry.UtcDateTime - }; + return new StoredFileInfo + { + Error = ex.ErrorCode == "BlobNotFound" + ? "There's no file with that ID." + : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." + }; + } } - catch (RequestFailedException ex) + + // local filesystem for testing + else { + FileInfo file = new FileInfo(this.GetDevFilePath(id)); + if (file.Exists) + { + if (file.LastWriteTimeUtc.AddDays(this.ExpiryDays) < DateTime.UtcNow) + file.Delete(); + else + { + return new StoredFileInfo + { + Success = true, + Content = File.ReadAllText(file.FullName), + Expiry = DateTime.UtcNow.AddDays(this.ExpiryDays), + Warning = "This file was saved temporarily to the local computer. This should only happen in a local development environment." + }; + } + } return new StoredFileInfo { - Error = ex.ErrorCode == "BlobNotFound" - ? "There's no file with that ID." - : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." + Error = "There's no file with that ID." }; } } - // get from PasteBin + // get from Pastebin else { PasteInfo response = await this.Pastebin.GetAsync(id); @@ -116,12 +163,19 @@ namespace StardewModdingAPI.Web.Framework.Storage } /// Get a client for reading and writing to Azure Blob storage. - /// The file ID to fetch. + /// The file ID. private BlobClient GetAzureBlobClient(string id) { var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString); var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer); return container.GetBlobClient($"uploads/{id}"); } + + /// Get the absolute file path for an upload when running in a local test environment with no Azure account configured. + /// The file ID. + private string GetDevFilePath(string id) + { + return Path.Combine(Path.GetTempPath(), "smapi-web-temp", $"{id}.txt"); + } } } diff --git a/src/SMAPI.Web/Views/LogParser/Index.cshtml b/src/SMAPI.Web/Views/LogParser/Index.cshtml index 439167bc..ac951564 100644 --- a/src/SMAPI.Web/Views/LogParser/Index.cshtml +++ b/src/SMAPI.Web/Views/LogParser/Index.cshtml @@ -67,12 +67,16 @@ else if (Model.ParsedLog?.IsValid == true) @* save warnings *@ @if (Model.UploadWarning != null || Model.Expiry != null) { + @if (Model.UploadWarning != null) + { + ⚠️ @Model.UploadWarning
 + } +
@if (Model.Expiry != null) { - This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). + This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). } -
} @@ -294,10 +298,7 @@ else if (Model.ParsedLog?.IsValid == true) string sectionFilter = message.Section != null && !message.IsStartOfSection ? $"&& sectionsAllow('{message.Section}')" : null; // filter the message by section if applicable v-on:click="toggleSection('@message.Section')" - } + @if (message.IsStartOfSection) { v-on:click="toggleSection('@message.Section')" } v-show="filtersAllow('@Model.GetSlug(message.Mod)', '@levelStr') @sectionFilter"> @message.Time @message.Level.ToString().ToUpper() @@ -307,8 +308,12 @@ else if (Model.ParsedLog?.IsValid == true) @if (message.IsStartOfSection) { - - + + } -- cgit