diff options
| -rw-r--r-- | src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs | 5 | ||||
| -rw-r--r-- | src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs | 58 | ||||
| -rw-r--r-- | src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs | 26 | ||||
| -rw-r--r-- | src/SMAPI.Web/Framework/Storage/IStorageProvider.cs | 2 | ||||
| -rw-r--r-- | src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 108 | ||||
| -rw-r--r-- | src/SMAPI.Web/SMAPI.Web.csproj | 1 | ||||
| -rw-r--r-- | src/SMAPI.Web/Startup.cs | 4 | ||||
| -rw-r--r-- | src/SMAPI.Web/appsettings.Development.json | 7 | ||||
| -rw-r--r-- | src/SMAPI.Web/appsettings.json | 9 |
9 files changed, 87 insertions, 133 deletions
diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs index a635abe3..431fed7b 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/IPastebinClient.cs @@ -9,10 +9,5 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// <summary>Fetch a saved paste.</summary> /// <param name="id">The paste ID.</param> Task<PasteInfo> GetAsync(string id); - - /// <summary>Save a paste to Pastebin.</summary> - /// <param name="name">The paste name.</param> - /// <param name="content">The paste content.</param> - Task<SavePasteResult> PostAsync(string name, string content); } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs index d695aab6..1be00be7 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs @@ -1,7 +1,5 @@ using System; -using System.Linq; using System.Net; -using System.Net.Http; using System.Threading.Tasks; using Pathoschild.Http.Client; @@ -16,12 +14,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// <summary>The underlying HTTP client.</summary> private readonly IClient Client; - /// <summary>The user key used to authenticate with the Pastebin API.</summary> - private readonly string UserKey; - - /// <summary>The developer key used to authenticate with the Pastebin API.</summary> - private readonly string DevKey; - /********* ** Public methods @@ -29,13 +21,9 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// <summary>Construct an instance.</summary> /// <param name="baseUrl">The base URL for the Pastebin API.</param> /// <param name="userAgent">The user agent for the API client.</param> - /// <param name="userKey">The user key used to authenticate with the Pastebin API.</param> - /// <param name="devKey">The developer key used to authenticate with the Pastebin API.</param> - public PastebinClient(string baseUrl, string userAgent, string userKey, string devKey) + public PastebinClient(string baseUrl, string userAgent) { this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); - this.UserKey = userKey; - this.DevKey = devKey; } /// <summary>Fetch a saved paste.</summary> @@ -66,50 +54,6 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin } } - /// <summary>Save a paste to Pastebin.</summary> - /// <param name="name">The paste name.</param> - /// <param name="content">The paste content.</param> - public async Task<SavePasteResult> PostAsync(string name, string content) - { - try - { - // validate - if (string.IsNullOrWhiteSpace(content)) - return new SavePasteResult { Error = "The log content can't be empty." }; - - // post to API - string response = await this.Client - .PostAsync("api/api_post.php") - .WithBody(p => p.FormUrlEncoded(new - { - api_option = "paste", - api_user_key = this.UserKey, - api_dev_key = this.DevKey, - api_paste_private = 1, // unlisted - api_paste_name = name, - api_paste_expire_date = "N", // never expire - api_paste_code = content - })) - .AsString(); - - // handle Pastebin errors - if (string.IsNullOrWhiteSpace(response)) - return new SavePasteResult { Error = "Received an empty response from Pastebin." }; - if (response.StartsWith("Bad API request")) - return new SavePasteResult { Error = response }; - if (!response.Contains("/")) - return new SavePasteResult { Error = $"Received an unknown response: {response}" }; - - // return paste ID - string pastebinID = response.Split("/").Last(); - return new SavePasteResult { Success = true, ID = pastebinID }; - } - catch (Exception ex) - { - return new SavePasteResult { Success = false, Error = ex.ToString() }; - } - } - /// <summary>Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.</summary> public void Dispose() { diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index e88f351f..4a73750b 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -30,6 +30,19 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /**** + ** Azure + ****/ + /// <summary>The connection string for the Azure Blob storage account.</summary> + public string AzureBlobConnectionString { get; set; } + + /// <summary>The Azure Blob container in which to store temporary uploaded logs.</summary> + public string AzureBlobTempContainer { get; set; } + + /// <summary>The number of days since the blob's last-modified date when it will be deleted.</summary> + public int AzureBlobTempExpiryDays { get; set; } + + + /**** ** Chucklefish ****/ /// <summary>The base URL for the Chucklefish mod site.</summary> @@ -61,6 +74,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// <summary>The password with which to authenticate to the GitHub API (if any).</summary> public string GitHubPassword { get; set; } + /**** ** ModDrop ****/ @@ -70,6 +84,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// <summary>The URL for a ModDrop mod page for the user, where {0} is the mod ID.</summary> public string ModDropModPageUrl { get; set; } + /**** ** Nexus Mods ****/ @@ -85,20 +100,11 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels /// <summary>The Nexus API authentication key.</summary> public string NexusApiKey { get; set; } + /**** ** Pastebin ****/ /// <summary>The base URL for the Pastebin API.</summary> public string PastebinBaseUrl { get; set; } - - /// <summary>The user key used to authenticate with the Pastebin API.</summary> - public string PastebinUserKey { get; set; } - - /// <summary>The developer key used to authenticate with the Pastebin API.</summary> - public string PastebinDevKey { get; set; } - - /// <summary>Whether to enable uploading new files to Pastebin. This doesn't affect fetching already-uploaded files.</summary> - public bool PastebinEnableUploads { get; set; } - } } diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs index e222a235..12a5e421 100644 --- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs @@ -5,7 +5,7 @@ namespace StardewModdingAPI.Web.Framework.Storage /// <summary>Provides access to raw data storage.</summary> internal interface IStorageProvider { - /// <summary>Save a text file to Pastebin or Amazon S3, if available.</summary> + /// <summary>Save a text file to storage.</summary> /// <param name="title">The display title, if applicable.</param> /// <param name="content">The content to upload.</param> /// <param name="compress">Whether to gzip the text.</param> diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs index e5e71325..b2d8ae7e 100644 --- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs +++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs @@ -6,7 +6,9 @@ using Amazon; using Amazon.Runtime; using Amazon.S3; using Amazon.S3.Model; -using Amazon.S3.Transfer; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; using Microsoft.Extensions.Options; using StardewModdingAPI.Web.Framework.Clients.Pastebin; using StardewModdingAPI.Web.Framework.Compression; @@ -44,54 +46,26 @@ namespace StardewModdingAPI.Web.Framework.Storage this.GzipHelper = gzipHelper; } - /// <summary>Save a text file to Pastebin or Amazon S3, if available.</summary> + /// <summary>Save a text file to storage.</summary> /// <param name="title">The display title, if applicable.</param> /// <param name="content">The content to upload.</param> /// <param name="compress">Whether to gzip the text.</param> /// <returns>Returns metadata about the save attempt.</returns> public async Task<UploadResult> SaveAsync(string title, string content, bool compress = true) { - // save to PasteBin - string uploadError = null; - if (this.ClientsConfig.PastebinEnableUploads) - { - SavePasteResult result = await this.Pastebin.PostAsync(title, content); - if (result.Success) - return new UploadResult(true, result.ID, null); - - uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; - } - - // fallback to S3 try { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - using TransferUtility uploader = new TransferUtility(s3); - string id = Guid.NewGuid().ToString("N"); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = this.ClientsConfig.AmazonTempBucket, - Key = $"uploads/{id}", - InputStream = stream, - Metadata = - { - // note: AWS will lowercase keys and prefix 'x-amz-meta-' - ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), - ["pastebin-error"] = uploadError - } - }; + BlobClient blob = this.GetAzureBlobClient(id); + await blob.UploadAsync(stream); - await uploader.UploadAsync(uploadRequest); - - return new UploadResult(true, id, uploadError); + return new UploadResult(true, id, null); } catch (Exception ex) { - return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + return new UploadResult(false, null, ex.Message); } } @@ -99,35 +73,62 @@ namespace StardewModdingAPI.Web.Framework.Storage /// <param name="id">The storage ID returned by <see cref="SaveAsync"/>.</param> public async Task<StoredFileInfo> GetAsync(string id) { - // get from Amazon S3 + // fetch from Azure/Amazon if (Guid.TryParseExact(id, "N", out Guid _)) { - var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); - using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); - + // try Azure try { - using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); - using Stream responseStream = response.ResponseStream; - using StreamReader reader = new StreamReader(responseStream); + BlobClient blob = this.GetAzureBlobClient(id); + Response<BlobDownloadInfo> response = await blob.DownloadAsync(); + using BlobDownloadInfo result = response.Value; - DateTime expiry = response.Expiration.ExpiryDateUtc; - string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + using StreamReader reader = new StreamReader(result.Content); + DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays); string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); return new StoredFileInfo { Success = true, Content = content, - Expiry = expiry, - Warning = pastebinError + Expiry = expiry.UtcDateTime }; } - catch (AmazonServiceException ex) + catch (RequestFailedException ex) { - return ex.ErrorCode == "NoSuchKey" - ? new StoredFileInfo { Error = "There's no file with that ID." } - : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + if (ex.ErrorCode != "BlobNotFound") + return new StoredFileInfo { Error = $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})." }; + } + + // try legacy Amazon S3 + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + using IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)); + + try + { + using GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonTempBucket, $"uploads/{id}"); + using Stream responseStream = response.ResponseStream; + using StreamReader reader = new StreamReader(responseStream); + + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new StoredFileInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new StoredFileInfo { Error = "There's no file with that ID." } + : new StoredFileInfo { Error = $"Could not fetch that file from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } } } @@ -144,5 +145,14 @@ namespace StardewModdingAPI.Web.Framework.Storage }; } } + + /// <summary>Get a client for reading and writing to Azure Blob storage.</summary> + /// <param name="id">The file ID to fetch.</param> + private BlobClient GetAzureBlobClient(string id) + { + var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString); + var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer); + return container.GetBlobClient($"uploads/{id}"); + } } } diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index 2c56fa75..773a7316 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -13,6 +13,7 @@ <ItemGroup> <PackageReference Include="AWSSDK.S3" Version="3.3.108.4" /> + <PackageReference Include="Azure.Storage.Blobs" Version="12.1.0" /> <PackageReference Include="Hangfire.AspNetCore" Version="1.7.7" /> <PackageReference Include="Hangfire.Mongo" Version="0.6.5" /> <PackageReference Include="HtmlAgilityPack" Version="1.11.16" /> diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 31b5e61d..07ee0c9e 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -153,9 +153,7 @@ namespace StardewModdingAPI.Web services.AddSingleton<IPastebinClient>(new PastebinClient( baseUrl: api.PastebinBaseUrl, - userAgent: userAgent, - userKey: api.PastebinUserKey, - devKey: api.PastebinDevKey + userAgent: userAgent )); } diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 74ded25d..05f0da1d 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -12,13 +12,12 @@ "AmazonAccessKey": null, "AmazonSecretKey": null, + "AzureBlobConnectionString": null, + "GitHubUsername": null, "GitHubPassword": null, - "NexusApiKey": null, - - "PastebinUserKey": null, - "PastebinDevKey": null + "NexusApiKey": null }, "MongoDB": { diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 6ba05a22..c6dc1b69 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -29,6 +29,10 @@ "AmazonRegion": "us-east-1", "AmazonTempBucket": "smapi-web-temp", + "AzureBlobConnectionString": null, + "AzureBlobTempContainer": "smapi-web-temp", + "AzureBlobTempExpiryDays": 30, + "ChucklefishBaseUrl": "https://community.playstarbound.com", "ChucklefishModPageUrlFormat": "resources/{0}", @@ -47,10 +51,7 @@ "NexusModUrlFormat": "mods/{0}", "NexusModScrapeUrlFormat": "mods/{0}?tab=files", - "PastebinBaseUrl": "https://pastebin.com/", - "PastebinUserKey": null, - "PastebinDevKey": null, - "PastebinEnableUploads": true + "PastebinBaseUrl": "https://pastebin.com/" }, "MongoDB": { |