fallback to Amazon S3 if saving a log to Pastebin fails

11 files changed, 240 insertions, 41 deletions

diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs
index f7f19cd8..32c45038 100644
--- a/src/SMAPI.Web/Controllers/LogParserController.cs
+++ b/src/SMAPI.Web/Controllers/LogParserController.cs
@@ -1,6 +1,13 @@
 using System;
+using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading.Tasks;
+using Amazon;
+using Amazon.Runtime;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Amazon.S3.Transfer;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Options;
 using StardewModdingAPI.Toolkit.Utilities;
@@ -21,7 +28,10 @@ namespace StardewModdingAPI.Web.Controllers
         ** Fields
         *********/
         /// <summary>The site config settings.</summary>
-        private readonly SiteConfig Config;
+        private readonly SiteConfig SiteConfig;
+
+        /// <summary>The API client settings.</summary>
+        private readonly ApiClientsConfig ClientsConfig;
 
         /// <summary>The underlying Pastebin client.</summary>
         private readonly IPastebinClient Pastebin;
@@ -38,11 +48,13 @@ namespace StardewModdingAPI.Web.Controllers
         ***/
         /// <summary>Construct an instance.</summary>
         /// <param name="siteConfig">The context config settings.</param>
+        /// <param name="clientsConfig">The API client settings.</param>
         /// <param name="pastebin">The Pastebin API client.</param>
         /// <param name="gzipHelper">The underlying text compression helper.</param>
-        public LogParserController(IOptions<SiteConfig> siteConfig, IPastebinClient pastebin, IGzipHelper gzipHelper)
+        public LogParserController(IOptions<SiteConfig> siteConfig, IOptions<ApiClientsConfig> clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper)
         {
-            this.Config = siteConfig.Value;
+            this.SiteConfig = siteConfig.Value;
+            this.ClientsConfig = clientsConfig.Value;
             this.Pastebin = pastebin;
             this.GzipHelper = gzipHelper;
         }
@@ -66,8 +78,9 @@ namespace StardewModdingAPI.Web.Controllers
             PasteInfo paste = await this.GetAsync(id);
             ParsedLog log = paste.Success
                 ? new LogParser().Parse(paste.Content)
-                : new ParsedLog { IsValid = false, Error = "Pastebin error: " + paste.Error };
-            return this.View("Index", this.GetModel(id).SetResult(log, raw));
+                : new ParsedLog { IsValid = false, Error = paste.Error };
+
+            return this.View("Index", this.GetModel(id, uploadWarning: paste.Warning, expiry: paste.Expiry).SetResult(log, raw));
         }
 
         /***
@@ -85,15 +98,13 @@ namespace StardewModdingAPI.Web.Controllers
 
             // upload log
             input = this.GzipHelper.CompressString(input);
-            SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", input);
-
-            // handle errors
-            if (!result.Success)
-                return this.View("Index", this.GetModel(result.ID, uploadError: $"Pastebin error: {result.Error ?? "unknown error"}"));
+            var uploadResult = await this.TrySaveLog(input);
+            if (!uploadResult.Succeeded)
+                return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError));
 
             // redirect to view
-            UriBuilder uri = new UriBuilder(new Uri(this.Config.LogParserUrl));
-            uri.Path = uri.Path.TrimEnd('/') + '/' + result.ID;
+            UriBuilder uri = new UriBuilder(new Uri(this.SiteConfig.LogParserUrl));
+            uri.Path = $"{uri.Path.TrimEnd('/')}/{uploadResult.ID}";
             return this.Redirect(uri.Uri.ToString());
         }
 
@@ -105,19 +116,116 @@ namespace StardewModdingAPI.Web.Controllers
         /// <param name="id">The Pastebin paste ID.</param>
         private async Task<PasteInfo> GetAsync(string id)
         {
-            PasteInfo response = await this.Pastebin.GetAsync(id);
-            response.Content = this.GzipHelper.DecompressString(response.Content);
-            return response;
+            // get from Amazon S3
+            if (Guid.TryParseExact(id, "N", out Guid _))
+            {
+                var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey);
+
+                using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)))
+                {
+                    try
+                    {
+                        using (GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonLogBucket, $"logs/{id}"))
+                        using (Stream responseStream = response.ResponseStream)
+                        using (StreamReader reader = new StreamReader(responseStream))
+                        {
+                            DateTime expiry = response.Expiration.ExpiryDateUtc;
+                            string pastebinError = response.Metadata["x-amz-meta-pastebin-error"];
+                            string content = this.GzipHelper.DecompressString(reader.ReadToEnd());
+
+                            return new PasteInfo
+                            {
+                                Success = true,
+                                Content = content,
+                                Expiry = expiry,
+                                Warning = pastebinError
+                            };
+                        }
+                    }
+                    catch (AmazonServiceException ex)
+                    {
+                        return ex.ErrorCode == "NoSuchKey"
+                            ? new PasteInfo { Error = "There's no log with that ID." }
+                            : new PasteInfo { Error = $"Could not fetch that log from AWS S3 ({ex.ErrorCode}: {ex.Message})." };
+                    }
+                }
+            }
+
+            // get from PasteBin
+            else
+            {
+                PasteInfo response = await this.Pastebin.GetAsync(id);
+                response.Content = this.GzipHelper.DecompressString(response.Content);
+                return response;
+            }
+        }
+
+        /// <summary>Save a log to Pastebin or Amazon S3, if available.</summary>
+        /// <param name="content">The content to upload.</param>
+        /// <returns>Returns metadata about the save attempt.</returns>
+        private async Task<UploadResult> TrySaveLog(string content)
+        {
+            // save to PasteBin
+            string uploadError;
+            {
+                SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", content);
+                if (result.Success)
+                    return new UploadResult(true, result.ID, null);
+
+                uploadError = $"Pastebin error: {result.Error ?? "unknown error"}";
+            }
+
+            // fallback to S3
+            try
+            {
+                var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey);
+
+                using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)))
+                using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)))
+                using (TransferUtility uploader = new TransferUtility(s3))
+                {
+                    string id = Guid.NewGuid().ToString("N");
+
+                    var uploadRequest = new TransferUtilityUploadRequest
+                    {
+                        BucketName = this.ClientsConfig.AmazonLogBucket,
+                        Key = $"logs/{id}",
+                        InputStream = stream,
+                        Metadata =
+                        {
+                            // note: AWS will lowercase keys and prefix 'x-amz-meta-'
+                            ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"),
+                            ["pastebin-error"] = uploadError
+                        }
+                    };
+
+                    await uploader.UploadAsync(uploadRequest);
+
+                    return new UploadResult(true, id, uploadError);
+                }
+            }
+            catch (Exception ex)
+            {
+                return new UploadResult(false, null, $"{uploadError}\n{ex.Message}");
+            }
         }
 
         /// <summary>Build a log parser model.</summary>
         /// <param name="pasteID">The paste ID.</param>
-        /// <param name="uploadError">An error which occurred while uploading the log to Pastebin.</param>
-        private LogParserModel GetModel(string pasteID, string uploadError = null)
+        /// <param name="expiry">When the uploaded file will no longer be available.</param>
+        /// <param name="uploadWarning">A non-blocking warning while uploading the log.</param>
+        /// <param name="uploadError">An error which occurred while uploading the log.</param>
+        private LogParserModel GetModel(string pasteID, DateTime? expiry = null, string uploadWarning = null, string uploadError = null)
         {
-            string sectionUrl = this.Config.LogParserUrl;
+            string sectionUrl = this.SiteConfig.LogParserUrl;
             Platform? platform = this.DetectClientPlatform();
-            return new LogParserModel(sectionUrl, pasteID, platform) { UploadError = uploadError };
+
+            return new LogParserModel(sectionUrl, pasteID, platform)
+            {
+                UploadWarning = uploadWarning,
+                UploadError = uploadError,
+                Expiry = expiry
+            };
         }
 
         /// <summary>Detect the viewer's OS.</summary>
@@ -143,5 +251,36 @@ namespace StardewModdingAPI.Web.Controllers
                     return null;
             }
         }
+
+        /// <summary>The result of an attempt to upload a file.</summary>
+        private class UploadResult
+        {
+            /*********
+            ** Accessors
+            *********/
+            /// <summary>Whether the file upload succeeded.</summary>
+            public bool Succeeded { get; }
+
+            /// <summary>The file ID, if applicable.</summary>
+            public string ID { get; }
+
+            /// <summary>The upload error, if any.</summary>
+            public string UploadError { get; }
+
+
+            /*********
+            ** Public methods
+            *********/
+            /// <summary>Construct an instance.</summary>
+            /// <param name="succeeded">Whether the file upload succeeded.</param>
+            /// <param name="id">The file ID, if applicable.</param>
+            /// <param name="uploadError">The upload error, if any.</param>
+            public UploadResult(bool succeeded, string id, string uploadError)
+            {
+                this.Succeeded = succeeded;
+                this.ID = id;
+                this.UploadError = uploadError;
+            }
+        }
     }
 }
diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs
index 955156eb..bb2de356 100644
--- a/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs
+++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PasteInfo.cs
@@ -1,3 +1,5 @@
+using System;
+
 namespace StardewModdingAPI.Web.Framework.Clients.Pastebin
 {
     /// <summary>The response for a get-paste request.</summary>
@@ -9,7 +11,13 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin
         /// <summary>The fetched paste content (if <see cref="Success"/> is <c>true</c>).</summary>
         public string Content { get; set; }
 
-        /// <summary>The error message (if saving failed).</summary>
+        /// <summary>When the file will no longer be available.</summary>
+        public DateTime? Expiry { get; set; }
+
+        /// <summary>The error message if saving succeeded, but a non-blocking issue was encountered.</summary>
+        public string Warning { get; set; }
+
+        /// <summary>The error message if saving failed.</summary>
         public string Error { get; set; }
     }
 }
diff --git a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs
index 2e8a8c68..d695aab6 100644
--- a/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs
+++ b/src/SMAPI.Web/Framework/Clients/Pastebin/PastebinClient.cs
@@ -62,7 +62,7 @@ namespace StardewModdingAPI.Web.Framework.Clients.Pastebin
             }
             catch (Exception ex)
             {
-                return new PasteInfo { Error = ex.ToString() };
+                return new PasteInfo { Error = $"Pastebin error: {ex}" };
             }
         }
 
diff --git a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs
index 121690c5..7119ef03 100644
--- a/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs
+++ b/src/SMAPI.Web/Framework/ConfigModels/ApiClientsConfig.cs
@@ -14,6 +14,22 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels
 
 
         /****
+        ** Amazon Web Services
+        ****/
+        /// <summary>The access key for AWS authentication.</summary>
+        public string AmazonAccessKey { get; set; }
+
+        /// <summary>The secret key for AWS authentication.</summary>
+        public string AmazonSecretKey { get; set; }
+
+        /// <summary>The AWS region endpoint (like 'us-east-1').</summary>
+        public string AmazonRegion { get; set; }
+
+        /// <summary>The AWS bucket in which to store temporary uploaded logs.</summary>
+        public string AmazonLogBucket { get; set; }
+
+
+        /****
         ** Chucklefish
         ****/
         /// <summary>The base URL for the Chucklefish mod site.</summary>
diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj
index 30209146..863c586e 100644
--- a/src/SMAPI.Web/SMAPI.Web.csproj
+++ b/src/SMAPI.Web/SMAPI.Web.csproj
@@ -12,6 +12,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.3.108.4" />
     <PackageReference Include="Hangfire.AspNetCore" Version="1.7.7" />
     <PackageReference Include="Hangfire.Mongo" Version="0.6.5" />
     <PackageReference Include="HtmlAgilityPack" Version="1.11.16" />
diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs
index 8110b696..fc6161b5 100644
--- a/src/SMAPI.Web/Startup.cs
+++ b/src/SMAPI.Web/Startup.cs
@@ -58,6 +58,7 @@ namespace StardewModdingAPI.Web
         {
             // init basic services
             services
+                .Configure<ApiClientsConfig>(this.Configuration.GetSection("ApiClients"))
                 .Configure<BackgroundServicesConfig>(this.Configuration.GetSection("BackgroundServices"))
                 .Configure<ModCompatibilityListConfig>(this.Configuration.GetSection("ModCompatibilityList"))
                 .Configure<ModUpdateCheckConfig>(this.Configuration.GetSection("ModUpdateCheck"))
diff --git a/src/SMAPI.Web/ViewModels/LogParserModel.cs b/src/SMAPI.Web/ViewModels/LogParserModel.cs
index f4c5214b..b06b5b2d 100644
--- a/src/SMAPI.Web/ViewModels/LogParserModel.cs
+++ b/src/SMAPI.Web/ViewModels/LogParserModel.cs
@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
@@ -34,12 +35,18 @@ namespace StardewModdingAPI.Web.ViewModels
         /// <summary>Whether to show the raw unparsed log.</summary>
         public bool ShowRaw { get; set; }
 
-        /// <summary>An error which occurred while uploading the log to Pastebin.</summary>
+        /// <summary>A non-blocking warning while uploading the log.</summary>
+        public string UploadWarning { get; set; }
+
+        /// <summary>An error which occurred while uploading the log.</summary>
         public string UploadError { get; set; }
 
         /// <summary>An error which occurred while parsing the log file.</summary>
         public string ParseError => this.ParsedLog?.Error;
 
+        /// <summary>When the uploaded file will no longer be available.</summary>
+        public DateTime? Expiry { get; set; }
+
 
         /*********
         ** Public methods
diff --git a/src/SMAPI.Web/Views/LogParser/Index.cshtml b/src/SMAPI.Web/Views/LogParser/Index.cshtml
index f98ffdf9..df2ac115 100644
--- a/src/SMAPI.Web/Views/LogParser/Index.cshtml
+++ b/src/SMAPI.Web/Views/LogParser/Index.cshtml
@@ -1,3 +1,4 @@
+@using Humanizer
 @using Newtonsoft.Json
 @using StardewModdingAPI.Toolkit.Utilities
 @using StardewModdingAPI.Web.Framework.LogParsing.Models
@@ -18,7 +19,7 @@
     {
         <meta name="robots" content="noindex" />
     }
-    <link rel="stylesheet" href="~/Content/css/log-parser.css?r=20190515" />
+    <link rel="stylesheet" href="~/Content/css/log-parser.css?r=20191127" />
     <script src="https://cdn.jsdelivr.net/npm/vue@2.5.17/dist/vue.min.js" crossorigin="anonymous"></script>
     <script src="https://cdn.jsdelivr.net/npm/jquery@3.3.1/dist/jquery.min.js" crossorigin="anonymous"></script>
     <script src="~/Content/js/log-parser.js?r=20190515"></script>
@@ -62,6 +63,18 @@ else if (Model.ParsedLog?.IsValid == true)
     </div>
 }
 
+@* save warnings *@
+@if (Model.UploadWarning != null || Model.Expiry != null)
+{
+    <div class="save-metadata" v-pre>
+        @if (Model.Expiry != null)
+        {
+            <text>This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). </text>
+        }
+        <!--@Model.UploadWarning-->
+    </div>
+}
+
 @* upload new log *@
 @if (Model.ParsedLog == null)
 {
@@ -71,7 +84,7 @@ else if (Model.ParsedLog?.IsValid == true)
         @foreach (Platform platform in new[] { Platform.Android, Platform.Linux, Platform.Mac, Platform.Windows })
         {
             <li>
-                <input type="radio" name="os" value="@platform" id="os-@platform" checked="@(Model.DetectedPlatform == platform)"/>
+                <input type="radio" name="os" value="@platform" id="os-@platform" checked="@(Model.DetectedPlatform == platform)" />
                 <label for="os-@platform">@platform</label>
             </li>
         }
@@ -151,7 +164,7 @@ else if (Model.ParsedLog?.IsValid == true)
                                         <div class="content-packs">
                                             @foreach (LogModInfo contentPack in contentPackList.Where(pack => pack.HasUpdate))
                                             {
-                                                <text>+ @contentPack.Name</text><br/>
+                                                <text>+ @contentPack.Name</text><br />
                                             }
                                         </div>
                                     }
@@ -173,7 +186,7 @@ else if (Model.ParsedLog?.IsValid == true)
                                         <div>
                                             @foreach (LogModInfo contentPack in contentPackList.Where(pack => pack.HasUpdate))
                                             {
-                                                <a href="@contentPack.UpdateLink" target="_blank">@contentPack.Version → @contentPack.UpdateVersion</a><br/>
+                                                <a href="@contentPack.UpdateLink" target="_blank">@contentPack.Version → @contentPack.UpdateVersion</a><br />
                                             }
                                         </div>
                                     }
diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json
index baf7efb7..8e863591 100644
--- a/src/SMAPI.Web/appsettings.Development.json
+++ b/src/SMAPI.Web/appsettings.Development.json
@@ -18,6 +18,9 @@
     },
 
     "ApiClients": {
+        "AmazonAccessKey": null,
+        "AmazonSecretKey": null,
+
         "GitHubUsername": null,
         "GitHubPassword": null,
 
diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json
index 674bb672..3b6f8fbd 100644
--- a/src/SMAPI.Web/appsettings.json
+++ b/src/SMAPI.Web/appsettings.json
@@ -16,17 +16,22 @@
     },
 
     "Site": {
-        "RootUrl": null, // see top note
-        "ModListUrl": null, // see top note
-        "LogParserUrl": null, // see top note
-        "JsonValidatorUrl": null, // see top note
-        "BetaEnabled": null, // see top note
-        "BetaBlurb": null // see top note
+        "RootUrl": null,
+        "ModListUrl": null,
+        "LogParserUrl": null,
+        "JsonValidatorUrl": null,
+        "BetaEnabled": null,
+        "BetaBlurb": null
     },
 
     "ApiClients": {
         "UserAgent": "SMAPI/{0} (+https://smapi.io)",
 
+        "AmazonAccessKey": null,
+        "AmazonSecretKey": null,
+        "AmazonRegion": "us-east-1",
+        "AmazonLogBucket": "smapi-log-parser",
+
         "ChucklefishBaseUrl": "https://community.playstarbound.com",
         "ChucklefishModPageUrlFormat": "resources/{0}",
 
@@ -34,27 +39,27 @@
 
         "GitHubBaseUrl": "https://api.github.com",
         "GitHubAcceptHeader": "application/vnd.github.v3+json",
-        "GitHubUsername": null, // see top note
-        "GitHubPassword": null, // see top note
+        "GitHubUsername": null,
+        "GitHubPassword": null,
 
         "ModDropApiUrl": "https://www.moddrop.com/api/mods/data",
         "ModDropModPageUrl": "https://www.moddrop.com/sdv/mod/{0}",
 
-        "NexusApiKey": null, // see top note
+        "NexusApiKey": null,
         "NexusBaseUrl": "https://www.nexusmods.com/stardewvalley/",
         "NexusModUrlFormat": "mods/{0}",
         "NexusModScrapeUrlFormat": "mods/{0}?tab=files",
 
         "PastebinBaseUrl": "https://pastebin.com/",
-        "PastebinUserKey": null, // see top note
-        "PastebinDevKey": null // see top note
+        "PastebinUserKey": null,
+        "PastebinDevKey": null
     },
 
     "MongoDB": {
-        "Host": null, // see top note
-        "Username": null, // see top note
-        "Password": null, // see top note
-        "Database": null // see top note
+        "Host": null,
+        "Username": null,
+        "Password": null,
+        "Database": null
     },
 
     "ModCompatibilityList": {
diff --git a/src/SMAPI.Web/wwwroot/Content/css/log-parser.css b/src/SMAPI.Web/wwwroot/Content/css/log-parser.css
index d5013207..4d4ab326 100644
--- a/src/SMAPI.Web/wwwroot/Content/css/log-parser.css
+++ b/src/SMAPI.Web/wwwroot/Content/css/log-parser.css
@@ -47,6 +47,12 @@ table caption {
     background: #FCC;
 }
 
+.save-metadata {
+    margin-top: 1em;
+    font-size: 0.8em;
+    opacity: 0.3;
+}
+
 /*********
 ** Log metadata & filters
 *********/