From 242dc718cdedf2c7a264670008b9f760eba160d9 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sat, 21 Dec 2019 15:41:55 -0500 Subject: switch to Azure Blob storage for saving files --- .../Framework/Storage/IStorageProvider.cs | 2 +- src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 108 +++++++++++---------- 2 files changed, 60 insertions(+), 50 deletions(-) (limited to 'src/SMAPI.Web/Framework/Storage') diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index e222a235..12a5e421 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -5,7 +5,7 @@ namespace StardewModdingAPI.Web.Framework.Storage /// Provides access to raw data storage. internal interface IStorageProvider { - /// Save a text file to Pastebin or Amazon S3, if available. + /// Save a text file to storage. /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index e5e71325..b2d8ae7e 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -6,7 +6,9 @@ using Amazon; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; -using Amazon.S3.Transfer; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Microsoft.Extensions.Options; using StardewModdingAPI.Web.Framework.Clients.Pastebin; using StardewModdingAPI.Web.Framework.Compression; @@ -44,54 +46,26 @@ namespace StardewModdingAPI.Web.Framework.Storage this.GzipHelper = gzipHelper; } - /// Save a text file to Pastebin or Amazon S3, if available. + /// Save a text file to storage. /// The display title, if applicable. /// The content to upload. /// Whether to gzip the text. /// Returns metadata about the save attempt. public async Task SaveAsync(string title, string content, bool compress = true) { - // save to PasteBin - string uploadError = null; - if (this.ClientsConfig.PastebinEnableUploads) - { - SavePasteResult result = await this.Pastebin.PostAsync(title, content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 try { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - using TransferUtility uploader = new TransferUtility(s3); - string id = Guid.NewGuid().ToString("N"); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonTempBucket, - Key = $"uploads/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; + BlobClient blob = this.GetAzureBlobClient(id); + await blob.UploadAsync(stream); - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); + return new UploadResult(true, id, null); } catch (Exception ex) { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + return new UploadResult(false, null, ex.Message); } } @@ -99,35 +73,62 @@ namespace StardewModdingAPI.Web.Framework.Storage /// The storage ID returned by . public async Task GetAsync(string id) { - // get from Amazon S3 + // fetch from Azure/Amazon if (Guid.TryParseExact(id, "N", out Guid _)) { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - + // try Azure try { - using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); - using Stream responseStream = response.ResponseStream; - using StreamReader reader = new StreamReader(responseStream); + BlobClient blob = this.GetAzureBlobClient(id); + Response response = await blob.DownloadAsync(); + using BlobDownloadInfo result = response.Value; - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + using StreamReader reader = new StreamReader(result.Content); + DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays); string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); return new StoredFileInfo { Success = true, Content = content, - Expiry = expiry, - Warning = pastebinError + Expiry = expiry.UtcDateTime }; } - catch (AmazonServiceException ex) + catch (RequestFailedException ex) { - return ex.ErrorCode == "NoSuchKey" - ? new StoredFileInfo { Error = "There's no file with that ID." } - : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + if (ex.ErrorCode != "BlobNotFound") + return new StoredFileInfo { Error = $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }; + } + + // try legacy Amazon S3 + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } } } @@ -144,5 +145,14 @@ namespace StardewModdingAPI.Web.Framework.Storage }; } } + + /// Get a client for reading and writing to Azure Blob storage. + /// The file ID to fetch. + private BlobClient GetAzureBlobClient(string id) + { + var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString); + var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer); + return container.GetBlobClient($"uploads/{id}"); + } } } -- cgit