From b1400bcb684c43790dd38628b6c131e9e7c4d400 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 27 Nov 2019 21:49:36 -0500 Subject: fallback to Amazon S3 if saving a log to Pastebin fails --- src/SMAPI.Web/Controllers/LogParserController.cs | 177 ++++++++++++++++++++--- 1 file changed, 158 insertions(+), 19 deletions(-) (limited to 'src/SMAPI.Web/Controllers') diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index f7f19cd8..32c45038 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,6 +1,13 @@ using System; +using System.IO; using System.Linq; +using System.Text; using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit.Utilities; @@ -21,7 +28,10 @@ namespace StardewModdingAPI.Web.Controllers ** Fields *********/ /// The site config settings. - private readonly SiteConfig Config; + private readonly SiteConfig SiteConfig; + + /// The API client settings. + private readonly ApiClientsConfig ClientsConfig; /// The underlying Pastebin client. private readonly IPastebinClient Pastebin; @@ -38,11 +48,13 @@ namespace StardewModdingAPI.Web.Controllers ***/ /// Construct an instance. /// The context config settings. + /// The API client settings. /// The Pastebin API client. /// The underlying text compression helper. - public LogParserController(IOptions siteConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + public LogParserController(IOptions siteConfig, IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) { - this.Config = siteConfig.Value; + this.SiteConfig = siteConfig.Value; + this.ClientsConfig = clientsConfig.Value; this.Pastebin = pastebin; this.GzipHelper = gzipHelper; } @@ -66,8 +78,9 @@ namespace StardewModdingAPI.Web.Controllers PasteInfo paste = await this.GetAsync(id); ParsedLog log = paste.Success ? new LogParser().Parse(paste.Content) - : new ParsedLog { IsValid = false, Error = "Pastebin error: " + paste.Error }; - return this.View("Index", this.GetModel(id).SetResult(log, raw)); + : new ParsedLog { IsValid = false, Error = paste.Error }; + + return this.View("Index", this.GetModel(id, uploadWarning: paste.Warning, expiry: paste.Expiry).SetResult(log, raw)); } /*** @@ -85,15 +98,13 @@ namespace StardewModdingAPI.Web.Controllers // upload log input = this.GzipHelper.CompressString(input); - SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", input); - - // handle errors - if (!result.Success) - return this.View("Index", this.GetModel(result.ID, uploadError: $"Pastebin error: {result.Error ?? "unknown error"}")); + var uploadResult = await this.TrySaveLog(input); + if (!uploadResult.Succeeded) + return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); // redirect to view - UriBuilder uri = new UriBuilder(new Uri(this.Config.LogParserUrl)); - uri.Path = uri.Path.TrimEnd('/') + '/' + result.ID; + UriBuilder uri = new UriBuilder(new Uri(this.SiteConfig.LogParserUrl)); + uri.Path = $"{uri.Path.TrimEnd('/')}/{uploadResult.ID}"; return this.Redirect(uri.Uri.ToString()); } @@ -105,19 +116,116 @@ namespace StardewModdingAPI.Web.Controllers /// The Pastebin paste ID. private async Task GetAsync(string id) { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; + // get from Amazon S3 + if (Guid.TryParseExact(id, "N", out Guid _)) + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + + using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) + { + try + { + using (GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonLogBucket, $"logs/{id}")) + using (Stream responseStream = response.ResponseStream) + using (StreamReader reader = new StreamReader(responseStream)) + { + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new PasteInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new PasteInfo { Error = "There's no log with that ID." } + : new PasteInfo { Error = $"Could not fetch that log from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } + } + } + + // get from PasteBin + else + { + PasteInfo response = await this.Pastebin.GetAsync(id); + response.Content = this.GzipHelper.DecompressString(response.Content); + return response; + } + } + + /// Save a log to Pastebin or Amazon S3, if available. + /// The content to upload. + /// Returns metadata about the save attempt. + private async Task TrySaveLog(string content) + { + // save to PasteBin + string uploadError; + { + SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", content); + if (result.Success) + return new UploadResult(true, result.ID, null); + + uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; + } + + // fallback to S3 + try + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + + using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) + using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) + using (TransferUtility uploader = new TransferUtility(s3)) + { + string id = Guid.NewGuid().ToString("N"); + + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = this.ClientsConfig.AmazonLogBucket, + Key = $"logs/{id}", + InputStream = stream, + Metadata = + { + // note: AWS will lowercase keys and prefix 'x-amz-meta-' + ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), + ["pastebin-error"] = uploadError + } + }; + + await uploader.UploadAsync(uploadRequest); + + return new UploadResult(true, id, uploadError); + } + } + catch (Exception ex) + { + return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + } } /// Build a log parser model. /// The paste ID. - /// An error which occurred while uploading the log to Pastebin. - private LogParserModel GetModel(string pasteID, string uploadError = null) + /// When the uploaded file will no longer be available. + /// A non-blocking warning while uploading the log. + /// An error which occurred while uploading the log. + private LogParserModel GetModel(string pasteID, DateTime? expiry = null, string uploadWarning = null, string uploadError = null) { - string sectionUrl = this.Config.LogParserUrl; + string sectionUrl = this.SiteConfig.LogParserUrl; Platform? platform = this.DetectClientPlatform(); - return new LogParserModel(sectionUrl, pasteID, platform) { UploadError = uploadError }; + + return new LogParserModel(sectionUrl, pasteID, platform) + { + UploadWarning = uploadWarning, + UploadError = uploadError, + Expiry = expiry + }; } /// Detect the viewer's OS. @@ -143,5 +251,36 @@ namespace StardewModdingAPI.Web.Controllers return null; } } + + /// The result of an attempt to upload a file. + private class UploadResult + { + /********* + ** Accessors + *********/ + /// Whether the file upload succeeded. + public bool Succeeded { get; } + + /// The file ID, if applicable. + public string ID { get; } + + /// The upload error, if any. + public string UploadError { get; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// Whether the file upload succeeded. + /// The file ID, if applicable. + /// The upload error, if any. + public UploadResult(bool succeeded, string id, string uploadError) + { + this.Succeeded = succeeded; + this.ID = id; + this.UploadError = uploadError; + } + } } } -- cgit