From 242dc718cdedf2c7a264670008b9f760eba160d9 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 15:41:55 -0500 Subject: switch to Azure Blob storage for saving files --- .../Framework/Clients/Pastebin/IPastebinClient.cs | 5 - .../Framework/Clients/Pastebin/PastebinClient.cs | 58 +---------- .../Framework/ConfigModels/ApiClientsConfig.cs | 26 +++-- .../Framework/Storage/IStorageProvider.cs | 2 +- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 108 +++++++++++---------- 5 files changed, 77 insertions(+), 122 deletions(-) (limited to 'src/SMAPI.Web/Framework') diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs index a635abe3..431fed7b 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs @@ -9,10 +9,5 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// Fetch a saved paste. /// The paste ID. Task GetAsync(string id); - - /// Save a paste to Pastebin. - /// The paste name. - /// The paste content. - Task PostAsync(string name, string content); } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs index d695aab6..1be00be7 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs @@ -1,7 +1,5 @@ using System; -using System.Linq; using System.Net; -using System.Net.Http; using System.Threading.Tasks; using Pathoschild.Http.Client; @@ -16,12 +14,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// The underlying HTTP client. private readonly IClient Client; - /// The user key used to authenticate with the Pastebin API. - private readonly string UserKey; - - /// The developer key used to authenticate with the Pastebin API. - private readonly string DevKey; - /********* ** Public methods @@ -29,13 +21,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// Construct an instance. /// The base URL for the Pastebin API. /// The user agent for the API client. - /// The user key used to authenticate with the Pastebin API. - /// The developer key used to authenticate with the Pastebin API. - public PastebinClient(string baseUrl, string userAgent, string userKey, string devKey) + public PastebinClient(string baseUrl, string userAgent) { this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); - this.UserKey = userKey; - this.DevKey = devKey; } /// Fetch a saved paste. @@ -66,50 +54,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin } } - /// Save a paste to Pastebin. - /// The paste name. - /// The paste content. - public async Task PostAsync(string name, string content) - { - try - { - // validate - if (string.IsNullOrWhiteSpace(content)) - return new SavePasteResult { Error = "The log content can't be empty." }; - - // post to API - string response = await this.Client - .PostAsync("api/api_post.php") - .WithBody(p => p.FormUrlEncoded(new - { - api_option = "paste", - api_user_key = this.UserKey, - api_dev_key = this.DevKey, - api_paste_private = 1, // unlisted - api_paste_name = name, - api_paste_expire_date = "N", // never expire - api_paste_code = content - })) - .AsString(); - - // handle Pastebin errors - if (string.IsNullOrWhiteSpace(response)) - return new SavePasteResult { Error = "Received an empty response from Pastebin." }; - if (response.StartsWith("Bad API request")) - return new SavePasteResult { Error = response }; - if (!response.Contains("/")) - return new SavePasteResult { Error = $"Received an unknown response: {response}" }; - - // return paste ID - string pastebinID = response.Split("/").Last(); - return new SavePasteResult { Success = true, ID = pastebinID }; - } - catch (Exception ex) - { - return new SavePasteResult { Success = false, Error = ex.ToString() }; - } - } - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. public void Dispose() { diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index e88f351f..4a73750b 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -29,6 +29,19 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string AmazonTempBucket { get; set; } + /**** + ** Azure + ****/ + /// The connection string for the Azure Blob storage account. + public string AzureBlobConnectionString { get; set; } + + /// The Azure Blob container in which to store temporary uploaded logs. + public string AzureBlobTempContainer { get; set; } + + /// The number of days since the blob's last-modified date when it will be deleted. + public int AzureBlobTempExpiryDays { get; set; } + + /**** ** Chucklefish ****/ @@ -61,6 +74,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The password with which to authenticate to the GitHub API (if any). public string GitHubPassword { get; set; } + /**** ** ModDrop ****/ @@ -70,6 +84,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The URL for a ModDrop mod page for the user, where {0} is the mod ID. public string ModDropModPageUrl { get; set; } + /**** ** Nexus Mods ****/ @@ -85,20 +100,11 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// The Nexus API authentication key. public string NexusApiKey { get; set; } + /**** ** Pastebin ****/ /// The base URL for the Pastebin API. public string PastebinBaseUrl { get; set; } - - /// The user key used to authenticate with the Pastebin API. - public string PastebinUserKey { get; set; } - - /// The developer key used to authenticate with the Pastebin API. - public string PastebinDevKey { get; set; } - - /// Whether to enable uploading new files to Pastebin. This doesn't affect fetching already-uploaded files. - public bool PastebinEnableUploads { get; set; } - } } diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index e222a235..12a5e421 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -5,7 +5,7 @@ namespace StardewModdingAPI.Web.Framework.Storage /// Provides access to raw data storage. internal interface IStorageProvider { - /// Save a text file to Pastebin or Amazon S3, if available. + /// Save a text file to storage. /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index e5e71325..b2d8ae7e 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -6,7 +6,9 @@ using Amazon; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; -using Amazon.S3.Transfer; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Microsoft.Extensions.Options; using StardewModdingAPI.Web.Framework.Clients.Pastebin; using StardewModdingAPI.Web.Framework.Compression; @@ -44,54 +46,26 @@ namespace StardewModdingAPI.Web.Framework.Storage this.GzipHelper = gzipHelper; } - /// Save a text file to Pastebin or Amazon S3, if available. + /// Save a text file to storage. /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. public async Task SaveAsync(string title, string content, bool compress = true) { - // save to PasteBin - string uploadError = null; - if (this.ClientsConfig.PastebinEnableUploads) - { - SavePasteResult result = await this.Pastebin.PostAsync(title, content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 try { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - using TransferUtility uploader = new TransferUtility(s3); - string id = Guid.NewGuid().ToString("N"); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonTempBucket, - Key = $"uploads/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; + BlobClient blob = this.GetAzureBlobClient(id); + await blob.UploadAsync(stream); - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); + return new UploadResult(true, id, null); } catch (Exception ex) { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + return new UploadResult(false, null, ex.Message); } } @@ -99,35 +73,62 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The storage ID returned by . public async Task GetAsync(string id) { - // get from Amazon S3 + // fetch from Azure/Amazon if (Guid.TryParseExact(id, "N", out Guid _)) { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - + // try Azure try { - using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); - using Stream responseStream = response.ResponseStream; - using StreamReader reader = new StreamReader(responseStream); + BlobClient blob = this.GetAzureBlobClient(id); + Response response = await blob.DownloadAsync(); + using BlobDownloadInfo result = response.Value; - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + using StreamReader reader = new StreamReader(result.Content); + DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays); string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); return new StoredFileInfo { Success = true, Content = content, - Expiry = expiry, - Warning = pastebinError + Expiry = expiry.UtcDateTime }; } - catch (AmazonServiceException ex) + catch (RequestFailedException ex) { - return ex.ErrorCode == "NoSuchKey" - ? new StoredFileInfo { Error = "There's no file with that ID." } - : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + if (ex.ErrorCode != "BlobNotFound") + return new StoredFileInfo { Error = $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }; + } + + // try legacy Amazon S3 + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } } } @@ -144,5 +145,14 @@ namespace StardewModdingAPI.Web.Framework.Storage }; } } + + /// Get a client for reading and writing to Azure Blob storage. + /// The file ID to fetch. + private BlobClient GetAzureBlobClient(string id) + { + var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString); + var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer); + return container.GetBlobClient($"uploads/{id}"); + } } } -- cgit