From 2b1f607d41b3d4d071c0db0671dbc99b6982909f Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 21:21:28 -0500 Subject: encapsulate file storage, also handle Pastebin rate limits in JSON validator --- .../Framework/Clients/Pastebin/PasteInfo.cs | 6 - .../Framework/ConfigModels/ApiClientsConfig.cs | 2 +- .../Framework/Storage/IStorageProvider.cs | 19 +++ src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 147 +++++++++++++++++++++ src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs | 23 ++++ src/SMAPI.Web/Framework/Storage/UploadResult.cs | 33 +++++ 6 files changed, 223 insertions(+), 7 deletions(-) create mode 100644 src/SMAPI.Web/Framework/Storage/IStorageProvider.cs create mode 100644 src/SMAPI.Web/Framework/Storage/StorageProvider.cs create mode 100644 src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs create mode 100644 src/SMAPI.Web/Framework/Storage/UploadResult.cs (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs index bb2de356..1ef3ef12 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs @@ -11,12 +11,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// The fetched paste content (if is true). public string Content { get; set; } - /// When the file will no longer be available. - public DateTime? Expiry { get; set; } - - /// The error message if saving succeeded, but a non-blocking issue was encountered. - public string Warning { get; set; } - /// The error message if saving failed. public string Error { get; set; } } diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index 7119ef03..1e020840 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -26,7 +26,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string AmazonRegion { get; set; } /// The AWS bucket in which to store temporary uploaded logs. - public string AmazonLogBucket { get; set; } + public string AmazonTempBucket { get; set; } /**** diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs new file mode 100644 index 00000000..e222a235 --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -0,0 +1,19 @@ +using System.Threading.Tasks; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// Provides access to raw data storage. + internal interface IStorageProvider + { + /// Save a text file to Pastebin or Amazon S3, if available. + /// The display title, if applicable. + /// The content to upload. + /// Whether to gzip the text. + /// Returns metadata about the save attempt. + Task SaveAsync(string title, string content, bool compress = true); + + /// Fetch raw text from storage. + /// The storage ID returned by . + Task GetAsync(string id); + } +} diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs new file mode 100644 index 00000000..bbb6e06b --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -0,0 +1,147 @@ +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; +using Microsoft.Extensions.Options; +using StardewModdingAPI.Web.Framework.Clients.Pastebin; +using StardewModdingAPI.Web.Framework.Compression; +using StardewModdingAPI.Web.Framework.ConfigModels; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// Provides access to raw data storage. + internal class StorageProvider : IStorageProvider + { + /********* + ** Fields + *********/ + /// The API client settings. + private readonly ApiClientsConfig ClientsConfig; + + /// The underlying Pastebin client. + private readonly IPastebinClient Pastebin; + + /// The underlying text compression helper. + private readonly IGzipHelper GzipHelper; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The API client settings. + /// The underlying Pastebin client. + /// The underlying text compression helper. + public StorageProvider(IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + { + this.ClientsConfig = clientsConfig.Value; + this.Pastebin = pastebin; + this.GzipHelper = gzipHelper; + } + + /// Save a text file to Pastebin or Amazon S3, if available. + /// The display title, if applicable. + /// The content to upload. + /// Whether to gzip the text. + /// Returns metadata about the save attempt. + public async Task SaveAsync(string title, string content, bool compress = true) + { + // save to PasteBin + string uploadError; + { + SavePasteResult result = await this.Pastebin.PostAsync(title, content); + if (result.Success) + return new UploadResult(true, result.ID, null); + + uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; + } + + // fallback to S3 + try + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + using TransferUtility uploader = new TransferUtility(s3); + + string id = Guid.NewGuid().ToString("N"); + + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = this.ClientsConfig.AmazonTempBucket, + Key = $"uploads/{id}", + InputStream = stream, + Metadata = + { + // note: AWS will lowercase keys and prefix 'x-amz-meta-' + ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), + ["pastebin-error"] = uploadError + } + }; + + await uploader.UploadAsync(uploadRequest); + + return new UploadResult(true, id, uploadError); + } + catch (Exception ex) + { + return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + } + } + + /// Fetch raw text from storage. + /// The storage ID returned by . + public async Task GetAsync(string id) + { + // get from Amazon S3 + if (Guid.TryParseExact(id, "N", out Guid _)) + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } + } + + // get from PasteBin + else + { + PasteInfo response = await this.Pastebin.GetAsync(id); + response.Content = this.GzipHelper.DecompressString(response.Content); + return new StoredFileInfo + { + Success = response.Success, + Content = response.Content, + Error = response.Error + }; + } + } + } +} diff --git a/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs b/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs new file mode 100644 index 00000000..30676c88 --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/StoredFileInfo.cs @@ -0,0 +1,23 @@ +using System; + +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// The response for a get-file request. + internal class StoredFileInfo + { + /// Whether the file was successfully fetched. + public bool Success { get; set; } + + /// The fetched file content (if is true). + public string Content { get; set; } + + /// When the file will no longer be available. + public DateTime? Expiry { get; set; } + + /// The error message if saving succeeded, but a non-blocking issue was encountered. + public string Warning { get; set; } + + /// The error message if saving failed. + public string Error { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/Storage/UploadResult.cs b/src/SMAPI.Web/Framework/Storage/UploadResult.cs new file mode 100644 index 00000000..483c1769 --- /dev/null +++ b/src/SMAPI.Web/Framework/Storage/UploadResult.cs @@ -0,0 +1,33 @@ +namespace StardewModdingAPI.Web.Framework.Storage +{ + /// The result of an attempt to upload a file. + internal class UploadResult + { + /********* + ** Accessors + *********/ + /// Whether the file upload succeeded. + public bool Succeeded { get; } + + /// The file ID, if applicable. + public string ID { get; } + + /// The upload error, if any. + public string UploadError { get; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// Whether the file upload succeeded. + /// The file ID, if applicable. + /// The upload error, if any. + public UploadResult(bool succeeded, string id, string uploadError) + { + this.Succeeded = succeeded; + this.ID = id; + this.UploadError = uploadError; + } + } +} -- cgit From 8ddb60cee636cc17291100c316df4786eb3bb448 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 3 Dec 2019 23:06:42 -0500 Subject: move supporter list into environment config --- src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs index d379c423..43969f51 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/SiteConfig.cs @@ -11,5 +11,8 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// A short sentence shown under the beta download button, if any. public string BetaBlurb { get; set; } + + /// A list of supports to credit on the main page, in Markdown format. + public string SupporterList { get; set; } } } -- cgit From 02f645900eb31648376abe21df30dd956221ad90 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 11:56:19 -0500 Subject: add option to disable uploading new files to Pastebin --- src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs | 3 +++ src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index 1e020840..e88f351f 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -97,5 +97,8 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The developer key used to authenticate with the Pastebin API. public string PastebinDevKey { get; set; } + /// Whether to enable uploading new files to Pastebin. This doesn't affect fetching already-uploaded files. + public bool PastebinEnableUploads { get; set; } + } } diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index bbb6e06b..e5e71325 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -52,7 +52,8 @@ namespace StardewModdingAPI.Web.Framework.Storage public async Task SaveAsync(string title, string content, bool compress = true) { // save to PasteBin - string uploadError; + string uploadError = null; + if (this.ClientsConfig.PastebinEnableUploads) { SavePasteResult result = await this.Pastebin.PostAsync(title, content); if (result.Success) -- cgit From 242dc718cdedf2c7a264670008b9f760eba160d9 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 15:41:55 -0500 Subject: switch to Azure Blob storage for saving files --- .../Framework/Clients/Pastebin/IPastebinClient.cs | 5 - .../Framework/Clients/Pastebin/PastebinClient.cs | 58 +---------- .../Framework/ConfigModels/ApiClientsConfig.cs | 26 +++-- .../Framework/Storage/IStorageProvider.cs | 2 +- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 108 +++++++++++---------- 5 files changed, 77 insertions(+), 122 deletions(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs index a635abe3..431fed7b 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs @@ -9,10 +9,5 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// Fetch a saved paste. /// The paste ID. Task GetAsync(string id); - - /// Save a paste to Pastebin. - /// The paste name. - /// The paste content. - Task PostAsync(string name, string content); } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs index d695aab6..1be00be7 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs @@ -1,7 +1,5 @@ using System; -using System.Linq; using System.Net; -using System.Net.Http; using System.Threading.Tasks; using Pathoschild.Http.Client; @@ -16,12 +14,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// The underlying HTTP client. private readonly IClient Client; - /// The user key used to authenticate with the Pastebin API. - private readonly string UserKey; - - /// The developer key used to authenticate with the Pastebin API. - private readonly string DevKey; - /********* ** Public methods @@ -29,13 +21,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// Construct an instance. /// The base URL for the Pastebin API. /// The user agent for the API client. - /// The user key used to authenticate with the Pastebin API. - /// The developer key used to authenticate with the Pastebin API. - public PastebinClient(string baseUrl, string userAgent, string userKey, string devKey) + public PastebinClient(string baseUrl, string userAgent) { this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); - this.UserKey = userKey; - this.DevKey = devKey; } /// Fetch a saved paste. @@ -66,50 +54,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin } } - /// Save a paste to Pastebin. - /// The paste name. - /// The paste content. - public async Task PostAsync(string name, string content) - { - try - { - // validate - if (string.IsNullOrWhiteSpace(content)) - return new SavePasteResult { Error = "The log content can't be empty." }; - - // post to API - string response = await this.Client - .PostAsync("api/api_post.php") - .WithBody(p => p.FormUrlEncoded(new - { - api_option = "paste", - api_user_key = this.UserKey, - api_dev_key = this.DevKey, - api_paste_private = 1, // unlisted - api_paste_name = name, - api_paste_expire_date = "N", // never expire - api_paste_code = content - })) - .AsString(); - - // handle Pastebin errors - if (string.IsNullOrWhiteSpace(response)) - return new SavePasteResult { Error = "Received an empty response from Pastebin." }; - if (response.StartsWith("Bad API request")) - return new SavePasteResult { Error = response }; - if (!response.Contains("/")) - return new SavePasteResult { Error = $"Received an unknown response: {response}" }; - - // return paste ID - string pastebinID = response.Split("/").Last(); - return new SavePasteResult { Success = true, ID = pastebinID }; - } - catch (Exception ex) - { - return new SavePasteResult { Success = false, Error = ex.ToString() }; - } - } - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. public void Dispose() { diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index e88f351f..4a73750b 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -29,6 +29,19 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string AmazonTempBucket { get; set; } + /**** + ** Azure + ****/ + /// The connection string for the Azure Blob storage account. + public string AzureBlobConnectionString { get; set; } + + /// The Azure Blob container in which to store temporary uploaded logs. + public string AzureBlobTempContainer { get; set; } + + /// The number of days since the blob's last-modified date when it will be deleted. + public int AzureBlobTempExpiryDays { get; set; } + + /**** ** Chucklefish ****/ @@ -61,6 +74,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The password with which to authenticate to the GitHub API (if any). public string GitHubPassword { get; set; } + /**** ** ModDrop ****/ @@ -70,6 +84,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The URL for a ModDrop mod page for the user, where {0} is the mod ID. public string ModDropModPageUrl { get; set; } + /**** ** Nexus Mods ****/ @@ -85,20 +100,11 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The Nexus API authentication key. public string NexusApiKey { get; set; } + /**** ** Pastebin ****/ /// The base URL for the Pastebin API. public string PastebinBaseUrl { get; set; } - - /// The user key used to authenticate with the Pastebin API. - public string PastebinUserKey { get; set; } - - /// The developer key used to authenticate with the Pastebin API. - public string PastebinDevKey { get; set; } - - /// Whether to enable uploading new files to Pastebin. This doesn't affect fetching already-uploaded files. - public bool PastebinEnableUploads { get; set; } - } } diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index e222a235..12a5e421 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -5,7 +5,7 @@ namespace StardewModdingAPI.Web.Framework.Storage /// Provides access to raw data storage. internal interface IStorageProvider { - /// Save a text file to Pastebin or Amazon S3, if available. + /// Save a text file to storage. /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index e5e71325..b2d8ae7e 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -6,7 +6,9 @@ using Amazon; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; -using Amazon.S3.Transfer; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Microsoft.Extensions.Options; using StardewModdingAPI.Web.Framework.Clients.Pastebin; using StardewModdingAPI.Web.Framework.Compression; @@ -44,54 +46,26 @@ namespace StardewModdingAPI.Web.Framework.Storage this.GzipHelper = gzipHelper; } - /// Save a text file to Pastebin or Amazon S3, if available. + /// Save a text file to storage. /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. public async Task SaveAsync(string title, string content, bool compress = true) { - // save to PasteBin - string uploadError = null; - if (this.ClientsConfig.PastebinEnableUploads) - { - SavePasteResult result = await this.Pastebin.PostAsync(title, content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 try { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - using TransferUtility uploader = new TransferUtility(s3); - string id = Guid.NewGuid().ToString("N"); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonTempBucket, - Key = $"uploads/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; + BlobClient blob = this.GetAzureBlobClient(id); + await blob.UploadAsync(stream); - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); + return new UploadResult(true, id, null); } catch (Exception ex) { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + return new UploadResult(false, null, ex.Message); } } @@ -99,35 +73,62 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The storage ID returned by . public async Task GetAsync(string id) { - // get from Amazon S3 + // fetch from Azure/Amazon if (Guid.TryParseExact(id, "N", out Guid _)) { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - + // try Azure try { - using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); - using Stream responseStream = response.ResponseStream; - using StreamReader reader = new StreamReader(responseStream); + BlobClient blob = this.GetAzureBlobClient(id); + Response response = await blob.DownloadAsync(); + using BlobDownloadInfo result = response.Value; - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + using StreamReader reader = new StreamReader(result.Content); + DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays); string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); return new StoredFileInfo { Success = true, Content = content, - Expiry = expiry, - Warning = pastebinError + Expiry = expiry.UtcDateTime }; } - catch (AmazonServiceException ex) + catch (RequestFailedException ex) { - return ex.ErrorCode == "NoSuchKey" - ? new StoredFileInfo { Error = "There's no file with that ID." } - : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + if (ex.ErrorCode != "BlobNotFound") + return new StoredFileInfo { Error = $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }; + } + + // try legacy Amazon S3 + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } } } @@ -144,5 +145,14 @@ namespace StardewModdingAPI.Web.Framework.Storage }; } } + + /// Get a client for reading and writing to Azure Blob storage. + /// The file ID to fetch. + private BlobClient GetAzureBlobClient(string id) + { + var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString); + var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer); + return container.GetBlobClient($"uploads/{id}"); + } } } -- cgit From ba46491ebc305a66a0a3a1f3cefa8bb677e57ff7 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 23:14:15 -0500 Subject: drop Amazon S3 support --- .../Framework/ConfigModels/ApiClientsConfig.cs | 16 --------- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 41 +++------------------- 2 files changed, 5 insertions(+), 52 deletions(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index 4a73750b..878130bf 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -13,22 +13,6 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string UserAgent { get; set; } - /**** - ** Amazon Web Services - ****/ - /// The access key for AWS authentication. - public string AmazonAccessKey { get; set; } - - /// The secret key for AWS authentication. - public string AmazonSecretKey { get; set; } - - /// The AWS region endpoint (like 'us-east-1'). - public string AmazonRegion { get; set; } - - /// The AWS bucket in which to store temporary uploaded logs. - public string AmazonTempBucket { get; set; } - - /**** ** Azure ****/ diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index b2d8ae7e..12a35f18 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -2,10 +2,6 @@ using System; using System.IO; using System.Text; using System.Threading.Tasks; -using Amazon; -using Amazon.Runtime; -using Amazon.S3; -using Amazon.S3.Model; using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -96,39 +92,12 @@ namespace StardewModdingAPI.Web.Framework.Storage } catch (RequestFailedException ex) { - if (ex.ErrorCode != "BlobNotFound") - return new StoredFileInfo { Error = $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }; - } - - // try legacy Amazon S3 - { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - - try - { - using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); - using Stream responseStream = response.ResponseStream; - using StreamReader reader = new StreamReader(responseStream); - - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - - return new StoredFileInfo - { - Success = true, - Content = content, - Expiry = expiry, - Warning = pastebinError - }; - } - catch (AmazonServiceException ex) + return new StoredFileInfo { - return ex.ErrorCode == "NoSuchKey" - ? new StoredFileInfo { Error = "There's no file with that ID." } - : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; - } + Error = ex.ErrorCode == "BlobNotFound" + ? "There's no file with that ID." + : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." + }; } } -- cgit From c1b15fb3725661ebfd8e03cec08343ae49e5d6da Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 23:52:47 -0500 Subject: allow local dev environments without an Azure account --- .../Framework/Storage/IStorageProvider.cs | 3 +- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 116 +++++++++++++++------ 2 files changed, 86 insertions(+), 33 deletions(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index 12a5e421..96a34fbb 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -6,11 +6,10 @@ namespace StardewModdingAPI.Web.Framework.Storage internal interface IStorageProvider { /// Save a text file to storage. - /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. - Task SaveAsync(string title, string content, bool compress = true); + Task SaveAsync(string content, bool compress = true); /// Fetch raw text from storage. /// The storage ID returned by . diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index 12a35f18..35538443 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -27,6 +27,12 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The underlying text compression helper. private readonly IGzipHelper GzipHelper; + /// Whether Azure blob storage is configured. + private bool HasAzure => !string.IsNullOrWhiteSpace(this.ClientsConfig.AzureBlobConnectionString); + + /// The number of days since the blob's last-modified date when it will be deleted. + private int ExpiryDays => this.ClientsConfig.AzureBlobTempExpiryDays; + /********* ** Public methods @@ -43,25 +49,38 @@ namespace StardewModdingAPI.Web.Framework.Storage } /// Save a text file to storage. - /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. - public async Task SaveAsync(string title, string content, bool compress = true) + public async Task SaveAsync(string content, bool compress = true) { - try - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - string id = Guid.NewGuid().ToString("N"); + string id = Guid.NewGuid().ToString("N"); - BlobClient blob = this.GetAzureBlobClient(id); - await blob.UploadAsync(stream); + // save to Azure + if (this.HasAzure) + { + try + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + BlobClient blob = this.GetAzureBlobClient(id); + await blob.UploadAsync(stream); - return new UploadResult(true, id, null); + return new UploadResult(true, id, null); + } + catch (Exception ex) + { + return new UploadResult(false, null, ex.Message); + } } - catch (Exception ex) + + // save to local filesystem for testing + else { - return new UploadResult(false, null, ex.Message); + string path = this.GetDevFilePath(id); + Directory.CreateDirectory(Path.GetDirectoryName(path)); + + File.WriteAllText(path, content); + return new UploadResult(true, id, null); } } @@ -69,39 +88,67 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The storage ID returned by . public async Task GetAsync(string id) { - // fetch from Azure/Amazon + // fetch from blob storage if (Guid.TryParseExact(id, "N", out Guid _)) { - // try Azure - try + // Azure Blob storage + if (this.HasAzure) { - BlobClient blob = this.GetAzureBlobClient(id); - Response response = await blob.DownloadAsync(); - using BlobDownloadInfo result = response.Value; + try + { + BlobClient blob = this.GetAzureBlobClient(id); + Response response = await blob.DownloadAsync(); + using BlobDownloadInfo result = response.Value; - using StreamReader reader = new StreamReader(result.Content); - DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays); - string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + using StreamReader reader = new StreamReader(result.Content); + DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ExpiryDays); + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); - return new StoredFileInfo + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry.UtcDateTime + }; + } + catch (RequestFailedException ex) { - Success = true, - Content = content, - Expiry = expiry.UtcDateTime - }; + return new StoredFileInfo + { + Error = ex.ErrorCode == "BlobNotFound" + ? "There's no file with that ID." + : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." + }; + } } - catch (RequestFailedException ex) + + // local filesystem for testing + else { + FileInfo file = new FileInfo(this.GetDevFilePath(id)); + if (file.Exists) + { + if (file.LastWriteTimeUtc.AddDays(this.ExpiryDays) < DateTime.UtcNow) + file.Delete(); + else + { + return new StoredFileInfo + { + Success = true, + Content = File.ReadAllText(file.FullName), + Expiry = DateTime.UtcNow.AddDays(this.ExpiryDays), + Warning = "This file was saved temporarily to the local computer. This should only happen in a local development environment." + }; + } + } return new StoredFileInfo { - Error = ex.ErrorCode == "BlobNotFound" - ? "There's no file with that ID." - : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." + Error = "There's no file with that ID." }; } } - // get from PasteBin + // get from Pastebin else { PasteInfo response = await this.Pastebin.GetAsync(id); @@ -116,12 +163,19 @@ namespace StardewModdingAPI.Web.Framework.Storage } /// Get a client for reading and writing to Azure Blob storage. - /// The file ID to fetch. + /// The file ID. private BlobClient GetAzureBlobClient(string id) { var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString); var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer); return container.GetBlobClient($"uploads/{id}"); } + + /// Get the absolute file path for an upload when running in a local test environment with no Azure account configured. + /// The file ID. + private string GetDevFilePath(string id) + { + return Path.Combine(Path.GetTempPath(), "smapi-web-temp", $"{id}.txt"); + } } } -- cgit From 082f285bc7ce156ad0750bb48d46ed65a2e4aedb Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 22 Dec 2019 00:44:13 -0500 Subject: streamline local environments, update technical docs & privacy page --- src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs | 2 -- .../Framework/Clients/Pastebin/SavePasteResult.cs | 15 --------------- src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs | 6 ++++++ 3 files changed, 6 insertions(+), 17 deletions(-) delete mode 100644 src/SMAPI.Web/Framework/Clients/Pastebin/SavePasteResult.cs (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs index 1ef3ef12..813ea115 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs @@ -1,5 +1,3 @@ -using System; - namespace StardewModdingAPI.Web.Framework.Clients.Pastebin { /// The response for a get-paste request. diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/SavePasteResult.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/SavePasteResult.cs deleted file mode 100644 index 89dab697..00000000 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/SavePasteResult.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace StardewModdingAPI.Web.Framework.Clients.Pastebin -{ - /// The response for a save-log request. - internal class SavePasteResult - { - /// Whether the log was successfully saved. - public bool Success { get; set; } - - /// The saved paste ID (if is true). - public string ID { get; set; } - - /// The error message (if saving failed). - public string Error { get; set; } - } -} diff --git a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs index 3c508300..e2e18477 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs @@ -24,6 +24,12 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /********* ** Public method *********/ + /// Get whether a MongoDB instance is configured. + public bool IsConfigured() + { + return !string.IsNullOrWhiteSpace(this.Host); + } + /// Get the MongoDB connection string. public string GetConnectionString() { -- cgit From d6ef6f627ae049c29c2241d39261dee7de3da663 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 22 Dec 2019 12:08:01 -0500 Subject: configure MongoDB connection string directly --- .../Framework/ConfigModels/MongoDbConfig.cs | 25 +++------------------- 1 file changed, 3 insertions(+), 22 deletions(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs index e2e18477..c7b6cb00 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs @@ -1,5 +1,3 @@ -using System; - namespace StardewModdingAPI.Web.Framework.ConfigModels { /// The config settings for mod compatibility list. @@ -8,14 +6,8 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /********* ** Accessors *********/ - /// The MongoDB hostname. - public string Host { get; set; } - - /// The MongoDB username (if any). - public string Username { get; set; } - - /// The MongoDB password (if any). - public string Password { get; set; } + /// The MongoDB connection string. + public string ConnectionString { get; set; } /// The database name. public string Database { get; set; } @@ -27,18 +19,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// Get whether a MongoDB instance is configured. public bool IsConfigured() { - return !string.IsNullOrWhiteSpace(this.Host); - } - - /// Get the MongoDB connection string. - public string GetConnectionString() - { - bool isLocal = this.Host == "localhost"; - bool hasLogin = !string.IsNullOrWhiteSpace(this.Username) && !string.IsNullOrWhiteSpace(this.Password); - - return $"mongodb{(isLocal ? "" : "+srv")}://" - + (hasLogin ? $"{Uri.EscapeDataString(this.Username)}:{Uri.EscapeDataString(this.Password)}@" : "") - + $"{this.Host}/{this.Database}?retryWrites=true&w=majority"; + return !string.IsNullOrWhiteSpace(this.ConnectionString); } } } -- cgit From 2833ad9a621d82cfef5be3868f89c9b2db436cf6 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Tue, 24 Dec 2019 13:59:11 -0500 Subject: fix log parser & JSON validator showing relative share URL --- src/SMAPI.Web/Framework/Extensions.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/Extensions.cs b/src/SMAPI.Web/Framework/Extensions.cs index d7707924..e0da1424 100644 --- a/src/SMAPI.Web/Framework/Extensions.cs +++ b/src/SMAPI.Web/Framework/Extensions.cs @@ -1,4 +1,6 @@ +using System; using JetBrains.Annotations; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Routing; @@ -12,8 +14,9 @@ namespace StardewModdingAPI.Web.Framework /// The name of the action method. /// The name of the controller. /// An object that contains route values. + /// Get an absolute URL instead of a server-relative path/ /// The generated URL. - public static string PlainAction(this IUrlHelper helper, [AspMvcAction] string action, [AspMvcController] string controller, object values = null) + public static string PlainAction(this IUrlHelper helper, [AspMvcAction] string action, [AspMvcController] string controller, object values = null, bool absoluteUrl = false) { RouteValueDictionary valuesDict = new RouteValueDictionary(values); foreach (var value in helper.ActionContext.RouteData.Values) @@ -22,7 +25,14 @@ namespace StardewModdingAPI.Web.Framework valuesDict[value.Key] = null; // explicitly remove it from the URL } - return helper.Action(action, controller, valuesDict); + string url = helper.Action(action, controller, valuesDict); + if (absoluteUrl) + { + HttpRequest request = helper.ActionContext.HttpContext.Request; + Uri baseUri = new Uri($"{request.Scheme}://{request.Host}"); + url = new Uri(baseUri, url).ToString(); + } + return url; } } } -- cgit From 360a982336e8b250d79597e323ba267c70c404c2 Mon Sep 17 00:00:00 2001 From: Dan Volchek Date: Thu, 26 Dec 2019 20:34:39 -0800 Subject: fix log parser content pack list entry pattern, remove unneeded ternary in content pack logging --- src/SMAPI.Web/Framework/LogParsing/LogParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/LogParsing/LogParser.cs b/src/SMAPI.Web/Framework/LogParsing/LogParser.cs index 66a3687f..1210f708 100644 --- a/src/SMAPI.Web/Framework/LogParsing/LogParser.cs +++ b/src/SMAPI.Web/Framework/LogParsing/LogParser.cs @@ -37,7 +37,7 @@ namespace StardewModdingAPI.Web.Framework.LogParsing private readonly Regex ContentPackListStartPattern = new Regex(@"^Loaded \d+ content packs:$", RegexOptions.Compiled | RegexOptions.IgnoreCase); /// A regex pattern matching an entry in SMAPI's content pack list. - private readonly Regex ContentPackListEntryPattern = new Regex(@"^ (?.+) (?.+) by (?.+) \| for (?.+?)(?: \| (?.+))?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private readonly Regex ContentPackListEntryPattern = new Regex(@"^ (?.+?) (?" + SemanticVersion.UnboundedVersionPattern + @")(?: by (?[^\|]+))? \| for (?[^\|]+)(?: \| (?.+))?$", RegexOptions.Compiled | RegexOptions.IgnoreCase); /// A regex pattern matching the start of SMAPI's mod update list. private readonly Regex ModUpdateListStartPattern = new Regex(@"^You can update \d+ mods?:$", RegexOptions.Compiled | RegexOptions.IgnoreCase); -- cgit