From b1400bcb684c43790dd38628b6c131e9e7c4d400 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Wed, 27 Nov 2019 21:49:36 -0500 Subject: fallback to Amazon S3 if saving a log to Pastebin fails --- src/SMAPI.Web/Controllers/LogParserController.cs | 177 ++++++++++++++++++--- .../Framework/Clients/Pastebin/PasteInfo.cs | 10 +- .../Framework/Clients/Pastebin/PastebinClient.cs | 2 +- .../Framework/ConfigModels/ApiClientsConfig.cs | 16 ++ src/SMAPI.Web/SMAPI.Web.csproj | 1 + src/SMAPI.Web/Startup.cs | 1 + src/SMAPI.Web/ViewModels/LogParserModel.cs | 9 +- src/SMAPI.Web/Views/LogParser/Index.cshtml | 21 ++- src/SMAPI.Web/appsettings.Development.json | 3 + src/SMAPI.Web/appsettings.json | 35 ++-- src/SMAPI.Web/wwwroot/Content/css/log-parser.css | 6 + 11 files changed, 240 insertions(+), 41 deletions(-) (limited to 'src/SMAPI.Web') diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index f7f19cd8..32c45038 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,6 +1,13 @@ using System; +using System.IO; using System.Linq; +using System.Text; using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit.Utilities; @@ -21,7 +28,10 @@ namespace StardewModdingAPI.Web.Controllers ** Fields *********/ /// The site config settings. - private readonly SiteConfig Config; + private readonly SiteConfig SiteConfig; + + /// The API client settings. + private readonly ApiClientsConfig ClientsConfig; /// The underlying Pastebin client. private readonly IPastebinClient Pastebin; @@ -38,11 +48,13 @@ namespace StardewModdingAPI.Web.Controllers ***/ /// Construct an instance. /// The context config settings. + /// The API client settings. /// The Pastebin API client. /// The underlying text compression helper. - public LogParserController(IOptions siteConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) + public LogParserController(IOptions siteConfig, IOptions clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper) { - this.Config = siteConfig.Value; + this.SiteConfig = siteConfig.Value; + this.ClientsConfig = clientsConfig.Value; this.Pastebin = pastebin; this.GzipHelper = gzipHelper; } @@ -66,8 +78,9 @@ namespace StardewModdingAPI.Web.Controllers PasteInfo paste = await this.GetAsync(id); ParsedLog log = paste.Success ? new LogParser().Parse(paste.Content) - : new ParsedLog { IsValid = false, Error = "Pastebin error: " + paste.Error }; - return this.View("Index", this.GetModel(id).SetResult(log, raw)); + : new ParsedLog { IsValid = false, Error = paste.Error }; + + return this.View("Index", this.GetModel(id, uploadWarning: paste.Warning, expiry: paste.Expiry).SetResult(log, raw)); } /*** @@ -85,15 +98,13 @@ namespace StardewModdingAPI.Web.Controllers // upload log input = this.GzipHelper.CompressString(input); - SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", input); - - // handle errors - if (!result.Success) - return this.View("Index", this.GetModel(result.ID, uploadError: $"Pastebin error: {result.Error ?? "unknown error"}")); + var uploadResult = await this.TrySaveLog(input); + if (!uploadResult.Succeeded) + return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError)); // redirect to view - UriBuilder uri = new UriBuilder(new Uri(this.Config.LogParserUrl)); - uri.Path = uri.Path.TrimEnd('/') + '/' + result.ID; + UriBuilder uri = new UriBuilder(new Uri(this.SiteConfig.LogParserUrl)); + uri.Path = $"{uri.Path.TrimEnd('/')}/{uploadResult.ID}"; return this.Redirect(uri.Uri.ToString()); } @@ -105,19 +116,116 @@ namespace StardewModdingAPI.Web.Controllers /// The Pastebin paste ID. private async Task GetAsync(string id) { - PasteInfo response = await this.Pastebin.GetAsync(id); - response.Content = this.GzipHelper.DecompressString(response.Content); - return response; + // get from Amazon S3 + if (Guid.TryParseExact(id, "N", out Guid _)) + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + + using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) + { + try + { + using (GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonLogBucket, $"logs/{id}")) + using (Stream responseStream = response.ResponseStream) + using (StreamReader reader = new StreamReader(responseStream)) + { + DateTime expiry = response.Expiration.ExpiryDateUtc; + string pastebinError = response.Metadata["x-amz-meta-pastebin-error"]; + string content = this.GzipHelper.DecompressString(reader.ReadToEnd()); + + return new PasteInfo + { + Success = true, + Content = content, + Expiry = expiry, + Warning = pastebinError + }; + } + } + catch (AmazonServiceException ex) + { + return ex.ErrorCode == "NoSuchKey" + ? new PasteInfo { Error = "There's no log with that ID." } + : new PasteInfo { Error = $"Could not fetch that log from AWS S3 ({ex.ErrorCode}: {ex.Message})." }; + } + } + } + + // get from PasteBin + else + { + PasteInfo response = await this.Pastebin.GetAsync(id); + response.Content = this.GzipHelper.DecompressString(response.Content); + return response; + } + } + + /// Save a log to Pastebin or Amazon S3, if available. + /// The content to upload. + /// Returns metadata about the save attempt. + private async Task TrySaveLog(string content) + { + // save to PasteBin + string uploadError; + { + SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", content); + if (result.Success) + return new UploadResult(true, result.ID, null); + + uploadError = $"Pastebin error: {result.Error ?? "unknown error"}"; + } + + // fallback to S3 + try + { + var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey); + + using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content))) + using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion))) + using (TransferUtility uploader = new TransferUtility(s3)) + { + string id = Guid.NewGuid().ToString("N"); + + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = this.ClientsConfig.AmazonLogBucket, + Key = $"logs/{id}", + InputStream = stream, + Metadata = + { + // note: AWS will lowercase keys and prefix 'x-amz-meta-' + ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"), + ["pastebin-error"] = uploadError + } + }; + + await uploader.UploadAsync(uploadRequest); + + return new UploadResult(true, id, uploadError); + } + } + catch (Exception ex) + { + return new UploadResult(false, null, $"{uploadError}\n{ex.Message}"); + } } /// Build a log parser model. /// The paste ID. - /// An error which occurred while uploading the log to Pastebin. - private LogParserModel GetModel(string pasteID, string uploadError = null) + /// When the uploaded file will no longer be available. + /// A non-blocking warning while uploading the log. + /// An error which occurred while uploading the log. + private LogParserModel GetModel(string pasteID, DateTime? expiry = null, string uploadWarning = null, string uploadError = null) { - string sectionUrl = this.Config.LogParserUrl; + string sectionUrl = this.SiteConfig.LogParserUrl; Platform? platform = this.DetectClientPlatform(); - return new LogParserModel(sectionUrl, pasteID, platform) { UploadError = uploadError }; + + return new LogParserModel(sectionUrl, pasteID, platform) + { + UploadWarning = uploadWarning, + UploadError = uploadError, + Expiry = expiry + }; } /// Detect the viewer's OS. @@ -143,5 +251,36 @@ namespace StardewModdingAPI.Web.Controllers return null; } } + + /// The result of an attempt to upload a file. + private class UploadResult + { + /********* + ** Accessors + *********/ + /// Whether the file upload succeeded. + public bool Succeeded { get; } + + /// The file ID, if applicable. + public string ID { get; } + + /// The upload error, if any. + public string UploadError { get; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// Whether the file upload succeeded. + /// The file ID, if applicable. + /// The upload error, if any. + public UploadResult(bool succeeded, string id, string uploadError) + { + this.Succeeded = succeeded; + this.ID = id; + this.UploadError = uploadError; + } + } } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs index 955156eb..bb2de356 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs @@ -1,3 +1,5 @@ +using System; + namespace StardewModdingAPI.Web.Framework.Clients.Pastebin { /// The response for a get-paste request. @@ -9,7 +11,13 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin /// The fetched paste content (if is true). public string Content { get; set; } - /// The error message (if saving failed). + /// When the file will no longer be available. + public DateTime? Expiry { get; set; } + + /// The error message if saving succeeded, but a non-blocking issue was encountered. + public string Warning { get; set; } + + /// The error message if saving failed. public string Error { get; set; } } } diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs index 2e8a8c68..d695aab6 100644 --- a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs +++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs @@ -62,7 +62,7 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin } catch (Exception ex) { - return new PasteInfo { Error = ex.ToString() }; + return new PasteInfo { Error = $"Pastebin error: {ex}" }; } } diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs index 121690c5..7119ef03 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs @@ -13,6 +13,22 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels public string UserAgent { get; set; } + /**** + ** Amazon Web Services + ****/ + /// The access key for AWS authentication. + public string AmazonAccessKey { get; set; } + + /// The secret key for AWS authentication. + public string AmazonSecretKey { get; set; } + + /// The AWS region endpoint (like 'us-east-1'). + public string AmazonRegion { get; set; } + + /// The AWS bucket in which to store temporary uploaded logs. + public string AmazonLogBucket { get; set; } + + /**** ** Chucklefish ****/ diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index 30209146..863c586e 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -12,6 +12,7 @@ + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index 8110b696..fc6161b5 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -58,6 +58,7 @@ namespace StardewModdingAPI.Web { // init basic services services + .Configure(this.Configuration.GetSection("ApiClients")) .Configure(this.Configuration.GetSection("BackgroundServices")) .Configure(this.Configuration.GetSection("ModCompatibilityList")) .Configure(this.Configuration.GetSection("ModUpdateCheck")) diff --git a/src/SMAPI.Web/ViewModels/LogParserModel.cs b/src/SMAPI.Web/ViewModels/LogParserModel.cs index f4c5214b..b06b5b2d 100644 --- a/src/SMAPI.Web/ViewModels/LogParserModel.cs +++ b/src/SMAPI.Web/ViewModels/LogParserModel.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; @@ -34,12 +35,18 @@ namespace StardewModdingAPI.Web.ViewModels /// Whether to show the raw unparsed log. public bool ShowRaw { get; set; } - /// An error which occurred while uploading the log to Pastebin. + /// A non-blocking warning while uploading the log. + public string UploadWarning { get; set; } + + /// An error which occurred while uploading the log. public string UploadError { get; set; } /// An error which occurred while parsing the log file. public string ParseError => this.ParsedLog?.Error; + /// When the uploaded file will no longer be available. + public DateTime? Expiry { get; set; } + /********* ** Public methods diff --git a/src/SMAPI.Web/Views/LogParser/Index.cshtml b/src/SMAPI.Web/Views/LogParser/Index.cshtml index f98ffdf9..df2ac115 100644 --- a/src/SMAPI.Web/Views/LogParser/Index.cshtml +++ b/src/SMAPI.Web/Views/LogParser/Index.cshtml @@ -1,3 +1,4 @@ +@using Humanizer @using Newtonsoft.Json @using StardewModdingAPI.Toolkit.Utilities @using StardewModdingAPI.Web.Framework.LogParsing.Models @@ -18,7 +19,7 @@ { } - + @@ -62,6 +63,18 @@ else if (Model.ParsedLog?.IsValid == true) } +@* save warnings *@ +@if (Model.UploadWarning != null || Model.Expiry != null) +{ +
+ @if (Model.Expiry != null) + { + This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). + } + +
+} + @* upload new log *@ @if (Model.ParsedLog == null) { @@ -71,7 +84,7 @@ else if (Model.ParsedLog?.IsValid == true) @foreach (Platform platform in new[] { Platform.Android, Platform.Linux, Platform.Mac, Platform.Windows }) {
  • - +
    • } @@ -151,7 +164,7 @@ else if (Model.ParsedLog?.IsValid == true)
    @foreach (LogModInfo contentPack in contentPackList.Where(pack => pack.HasUpdate)) { - + @contentPack.Name
    + + @contentPack.Name
    }
    } @@ -173,7 +186,7 @@ else if (Model.ParsedLog?.IsValid == true)
    @foreach (LogModInfo contentPack in contentPackList.Where(pack => pack.HasUpdate)) { - @contentPack.Version → @contentPack.UpdateVersion
    + @contentPack.Version → @contentPack.UpdateVersion
    }
    } diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index baf7efb7..8e863591 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -18,6 +18,9 @@ }, "ApiClients": { + "AmazonAccessKey": null, + "AmazonSecretKey": null, + "GitHubUsername": null, "GitHubPassword": null, diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 674bb672..3b6f8fbd 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -16,17 +16,22 @@ }, "Site": { - "RootUrl": null, // see top note - "ModListUrl": null, // see top note - "LogParserUrl": null, // see top note - "JsonValidatorUrl": null, // see top note - "BetaEnabled": null, // see top note - "BetaBlurb": null // see top note + "RootUrl": null, + "ModListUrl": null, + "LogParserUrl": null, + "JsonValidatorUrl": null, + "BetaEnabled": null, + "BetaBlurb": null }, "ApiClients": { "UserAgent": "SMAPI/{0} (+https://smapi.io)", + "AmazonAccessKey": null, + "AmazonSecretKey": null, + "AmazonRegion": "us-east-1", + "AmazonLogBucket": "smapi-log-parser", + "ChucklefishBaseUrl": "https://community.playstarbound.com", "ChucklefishModPageUrlFormat": "resources/{0}", @@ -34,27 +39,27 @@ "GitHubBaseUrl": "https://api.github.com", "GitHubAcceptHeader": "application/vnd.github.v3+json", - "GitHubUsername": null, // see top note - "GitHubPassword": null, // see top note + "GitHubUsername": null, + "GitHubPassword": null, "ModDropApiUrl": "https://www.moddrop.com/api/mods/data", "ModDropModPageUrl": "https://www.moddrop.com/sdv/mod/{0}", - "NexusApiKey": null, // see top note + "NexusApiKey": null, "NexusBaseUrl": "https://www.nexusmods.com/stardewvalley/", "NexusModUrlFormat": "mods/{0}", "NexusModScrapeUrlFormat": "mods/{0}?tab=files", "PastebinBaseUrl": "https://pastebin.com/", - "PastebinUserKey": null, // see top note - "PastebinDevKey": null // see top note + "PastebinUserKey": null, + "PastebinDevKey": null }, "MongoDB": { - "Host": null, // see top note - "Username": null, // see top note - "Password": null, // see top note - "Database": null // see top note + "Host": null, + "Username": null, + "Password": null, + "Database": null }, "ModCompatibilityList": { diff --git a/src/SMAPI.Web/wwwroot/Content/css/log-parser.css b/src/SMAPI.Web/wwwroot/Content/css/log-parser.css index d5013207..4d4ab326 100644 --- a/src/SMAPI.Web/wwwroot/Content/css/log-parser.css +++ b/src/SMAPI.Web/wwwroot/Content/css/log-parser.css @@ -47,6 +47,12 @@ table caption { background: #FCC; } +.save-metadata { + margin-top: 1em; + font-size: 0.8em; + opacity: 0.3; +} + /********* ** Log metadata & filters *********/ -- cgit