From ad5bb5b49af49c4668fd30fb2a0e606dcefe4ec0 Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Fri, 27 Oct 2017 19:39:13 -0400 Subject: proxy Pastebin requests through our API instead of third parties, improve error-handling (#358) --- src/SMAPI.Web/Controllers/LogParserController.cs | 54 +++++++++- .../Framework/AllowLargePostsAttribute.cs | 52 ++++++++++ .../Framework/ConfigModels/LogParserConfig.cs | 18 ++++ .../Framework/ConfigModels/ModUpdateCheckConfig.cs | 2 +- .../Framework/LogParser/GetPasteResponse.cs | 15 +++ .../Framework/LogParser/PastebinClient.cs | 110 +++++++++++++++++++++ .../Framework/LogParser/SavePasteResponse.cs | 15 +++ src/SMAPI.Web/Startup.cs | 1 + src/SMAPI.Web/appsettings.json | 5 + src/SMAPI.Web/wwwroot/Content/js/log-parser.js | 65 +++++------- 10 files changed, 296 insertions(+), 41 deletions(-) create mode 100644 src/SMAPI.Web/Framework/AllowLargePostsAttribute.cs create mode 100644 src/SMAPI.Web/Framework/ConfigModels/LogParserConfig.cs create mode 100644 src/SMAPI.Web/Framework/LogParser/GetPasteResponse.cs create mode 100644 src/SMAPI.Web/Framework/LogParser/PastebinClient.cs create mode 100644 src/SMAPI.Web/Framework/LogParser/SavePasteResponse.cs (limited to 'src') diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index 4ed8898a..893d9a52 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,19 +1,69 @@ +using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Options; +using StardewModdingAPI.Web.Framework; +using StardewModdingAPI.Web.Framework.ConfigModels; +using StardewModdingAPI.Web.Framework.LogParser; namespace StardewModdingAPI.Web.Controllers { /// Provides a web UI and API for parsing SMAPI log files. - [Route("log")] internal class LogParserController : Controller { + /********* + ** Properties + *********/ + /// The underlying Pastebin client. + private readonly PastebinClient PastebinClient; + + /********* ** Public methods *********/ - /// Render the web UI to upload a log file. + /*** + ** Constructor + ***/ + /// Construct an instance. + /// The log parser config settings. + public LogParserController(IOptions configProvider) + { + // init Pastebin client + LogParserConfig config = configProvider.Value; + string version = this.GetType().Assembly.GetName().Version.ToString(3); + string userAgent = string.Format(config.PastebinUserAgent, version); + this.PastebinClient = new PastebinClient(config.PastebinBaseUrl, userAgent, config.PastebinDevKey); + } + + /*** + ** Web UI + ***/ + /// Render the log parser UI. [HttpGet] + [Route("log")] public ViewResult Index() { return this.View("Index"); } + + /*** + ** JSON + ***/ + /// Fetch raw text from Pastebin. + /// The Pastebin paste ID. + [HttpGet, Produces("application/json")] + [Route("log/fetch/{id}")] + public async Task GetAsync(string id) + { + return await this.PastebinClient.GetAsync(id); + } + + /// Save raw log data. + /// The log content to save. + [HttpPost, Produces("application/json"), AllowLargePosts] + [Route("log/save")] + public async Task PostAsync([FromBody] string content) + { + return await this.PastebinClient.PostAsync(content); + } } } diff --git a/src/SMAPI.Web/Framework/AllowLargePostsAttribute.cs b/src/SMAPI.Web/Framework/AllowLargePostsAttribute.cs new file mode 100644 index 00000000..68ead3c2 --- /dev/null +++ b/src/SMAPI.Web/Framework/AllowLargePostsAttribute.cs @@ -0,0 +1,52 @@ +using System; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Mvc.Filters; + +namespace StardewModdingAPI.Web.Framework +{ + /// A filter which increases the maximum request size for an endpoint. + /// Derived from . + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)] + public class AllowLargePostsAttribute : Attribute, IAuthorizationFilter, IOrderedFilter + { + /********* + ** Properties + *********/ + /// The underlying form options. + private readonly FormOptions FormOptions; + + + /********* + ** Accessors + *********/ + /// The attribute order. + public int Order { get; set; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + public AllowLargePostsAttribute() + { + this.FormOptions = new FormOptions + { + ValueLengthLimit = 200 * 1024 * 1024 // 200MB + }; + } + + /// Called early in the filter pipeline to confirm request is authorized. + /// The authorisation filter context. + public void OnAuthorization(AuthorizationFilterContext context) + { + IFeatureCollection features = context.HttpContext.Features; + IFormFeature formFeature = features.Get(); + + if (formFeature?.Form == null) + { + // Request form has not been read yet, so set the limits + features.Set(new FormFeature(context.HttpContext.Request, this.FormOptions)); + } + } + } +} diff --git a/src/SMAPI.Web/Framework/ConfigModels/LogParserConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/LogParserConfig.cs new file mode 100644 index 00000000..5cb0cf95 --- /dev/null +++ b/src/SMAPI.Web/Framework/ConfigModels/LogParserConfig.cs @@ -0,0 +1,18 @@ +namespace StardewModdingAPI.Web.Framework.ConfigModels +{ + /// The config settings for the log parser. + internal class LogParserConfig + { + /********* + ** Accessors + *********/ + /// The base URL for the Pastebin API. + public string PastebinBaseUrl { get; set; } + + /// The user agent for the Pastebin API client, where {0} is the SMAPI version. + public string PastebinUserAgent { get; set; } + + /// The developer key used to authenticate with the Pastebin API. + public string PastebinDevKey { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs index 03de639e..2fb5b97e 100644 --- a/src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs +++ b/src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs @@ -1,7 +1,7 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels { /// The config settings for mod update checks. - public class ModUpdateCheckConfig + internal class ModUpdateCheckConfig { /********* ** Accessors diff --git a/src/SMAPI.Web/Framework/LogParser/GetPasteResponse.cs b/src/SMAPI.Web/Framework/LogParser/GetPasteResponse.cs new file mode 100644 index 00000000..4f8794db --- /dev/null +++ b/src/SMAPI.Web/Framework/LogParser/GetPasteResponse.cs @@ -0,0 +1,15 @@ +namespace StardewModdingAPI.Web.Framework.LogParser +{ + /// The response for a get-paste request. + internal class GetPasteResponse + { + /// Whether the log was successfully fetched. + public bool Success { get; set; } + + /// The fetched paste content (if is true). + public string Content { get; set; } + + /// The error message (if saving failed). + public string Error { get; set; } + } +} diff --git a/src/SMAPI.Web/Framework/LogParser/PastebinClient.cs b/src/SMAPI.Web/Framework/LogParser/PastebinClient.cs new file mode 100644 index 00000000..8536f249 --- /dev/null +++ b/src/SMAPI.Web/Framework/LogParser/PastebinClient.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Pathoschild.Http.Client; + +namespace StardewModdingAPI.Web.Framework.LogParser +{ + /// An API client for Pastebin. + internal class PastebinClient : IDisposable + { + /********* + ** Properties + *********/ + /// The underlying HTTP client. + private readonly IClient Client; + + /// The developer key used to authenticate with the Pastebin API. + private readonly string DevKey; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The base URL for the Pastebin API. + /// The user agent for the API client. + /// The developer key used to authenticate with the Pastebin API. + public PastebinClient(string baseUrl, string userAgent, string devKey) + { + this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent); + this.DevKey = devKey; + } + + /// Fetch a saved paste. + /// The paste ID. + public async Task GetAsync(string id) + { + try + { + // get from API + string content = await this.Client + .GetAsync($"raw/{id}") + .AsString(); + + // handle Pastebin errors + if (string.IsNullOrWhiteSpace(content)) + return new GetPasteResponse { Error = "Received an empty response from Pastebin." }; + if (content.StartsWith(" PostAsync(string content) + { + try + { + // validate + if (string.IsNullOrWhiteSpace(content)) + return new SavePasteResponse { Error = "The log content can't be empty." }; + + // post to API + string response = await this.Client + .PostAsync("api/api_post.php") + .WithBodyContent(new FormUrlEncodedContent(new Dictionary + { + ["api_dev_key"] = "b8219d942109d1e60ebb14fbb45f06f9", + ["api_option"] = "paste", + ["api_paste_private"] = "1", + ["api_paste_code"] = content, + ["api_paste_expire_date"] = "1W" + })) + .AsString(); + + // handle Pastebin errors + if (string.IsNullOrWhiteSpace(response)) + return new SavePasteResponse { Error = "Received an empty response from Pastebin." }; + if (response.StartsWith("Bad API request")) + return new SavePasteResponse { Error = response }; + if (!response.Contains("/")) + return new SavePasteResponse { Error = $"Received an unknown response: {response}" }; + + // return paste ID + string pastebinID = response.Split("/").Last(); + return new SavePasteResponse { Success = true, ID = pastebinID }; + } + catch (Exception ex) + { + return new SavePasteResponse { Success = false, Error = ex.ToString() }; + } + } + + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + public void Dispose() + { + this.Client.Dispose(); + } + } +} diff --git a/src/SMAPI.Web/Framework/LogParser/SavePasteResponse.cs b/src/SMAPI.Web/Framework/LogParser/SavePasteResponse.cs new file mode 100644 index 00000000..1c0960a4 --- /dev/null +++ b/src/SMAPI.Web/Framework/LogParser/SavePasteResponse.cs @@ -0,0 +1,15 @@ +namespace StardewModdingAPI.Web.Framework.LogParser +{ + /// The response for a save-log request. + internal class SavePasteResponse + { + /// Whether the log was successfully saved. + public bool Success { get; set; } + + /// The saved paste ID (if is true). + public string ID { get; set; } + + /// The error message (if saving failed). + public string Error { get; set; } + } +} diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index abce8f28..c0ea90da 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -43,6 +43,7 @@ namespace StardewModdingAPI.Web { services .Configure(this.Configuration.GetSection("ModUpdateCheck")) + .Configure(this.Configuration.GetSection("LogParser")) .Configure(options => options.ConstraintMap.Add("semanticVersion", typeof(VersionConstraint))) .AddMemoryCache() .AddMvc() diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 852f6f71..ca1299ce 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -26,5 +26,10 @@ "NexusUserAgent": "Nexus Client v0.63.15", "NexusBaseUrl": "http://www.nexusmods.com/stardewvalley", "NexusModUrlFormat": "mods/{0}" + }, + "LogParser": { + "PastebinBaseUrl": "https://pastebin.com/", + "PastebinUserAgent": "SMAPI/{0} (+https://github.com/Pathoschild/SMAPI)", + "PastebinDevKey": "b8219d942109d1e60ebb14fbb45f06f9" } } diff --git a/src/SMAPI.Web/wwwroot/Content/js/log-parser.js b/src/SMAPI.Web/wwwroot/Content/js/log-parser.js index 4597392c..b1f8f5c6 100644 --- a/src/SMAPI.Web/wwwroot/Content/js/log-parser.js +++ b/src/SMAPI.Web/wwwroot/Content/js/log-parser.js @@ -36,10 +36,6 @@ $(function() { $("#input").val(""); $("#popup-upload").fadeIn(); }); - var proxies = [ - "https://cors-anywhere.herokuapp.com/", - "https://galvanize-cors-proxy.herokuapp.com/" - ]; $("#popup-upload").on({ 'dragover dragenter': function(e) { e.preventDefault(); @@ -66,38 +62,35 @@ $(function() { $("#submit").on("click", function() { $("#popup-upload").fadeOut(); - if ($("#input").val()) { + var raw = $("#input").val(); + if (raw) { memory = ""; - var raw = $("#input").val(); var paste = LZString.compressToUTF16(raw); - logSize("Raw", raw); - logSize("Compressed", paste); if (paste.length * 2 > 524288) { $("#output").html('

Unable to save!

This log cannot be saved due to its size.
' + $("#input").val() + "
"); return; } - console.log("paste:", paste); - var packet = { - api_dev_key: "b8219d942109d1e60ebb14fbb45f06f9", - api_option: "paste", - api_paste_private: 1, - api_paste_code: paste, - api_paste_expire_date: "1W" - }; $("#uploader").attr("data-text", "Saving..."); $("#uploader").fadeIn(); - var uri = proxies[Math.floor(Math.random() * proxies.length)] + "pastebin.com/api/api_post.php"; - console.log(packet, uri); - $.post(uri, packet, function(data) { - $("#uploader").fadeOut(); - console.log("Result: ", data); - if (data.substring(0, 15) === "Bad API request") - $("#output").html('

Parsing failed!

Parsing of the log failed, details follow.
 

Stage: Upload

Error: ' + data + "
" + $("#input").val() + "
"); - else if (data) - location.href = "?" + data.split("/").pop(); - else - $("#output").html('

Parsing failed!

Parsing of the log failed, details follow.
 

Stage: Upload

Error: Received null response
' + $("#input").val() + "
"); - }); + $ + .ajax({ + type: "POST", + url: "/log/save", + data: JSON.stringify(paste), + contentType: "application/json" // sent to API + }) + .fail(function(xhr, textStatus) { + $("#uploader").fadeOut(); + $("#output").html('

Parsing failed!

Parsing of the log failed, details follow.
 

Stage: Upload

Error: ' + textStatus + ': ' + xhr.responseText + "
" + $("#input").val() + "
"); + }) + .then(function(data) { + $("#uploader").fadeOut(); + console.log("Result: ", data); + if (!data.success) + $("#output").html('

Parsing failed!

Parsing of the log failed, details follow.
 

Stage: Upload

Error: ' + data.error + "
" + $("#input").val() + "
"); + else + location.href = "?" + data.id; + }); } else { alert("Unable to parse log, the input is empty!"); $("#uploader").fadeOut(); @@ -122,10 +115,6 @@ $(function() { /********* ** Helpers *********/ - function logSize(id, str) { - console.log(id + ":", str.length * 2, "bytes", Math.round(str.length / 5.12) / 100, "kb"); - } - function modClicked(evt) { var id = $(evt.currentTarget).attr("id").split("-")[1], cls = "mod-" + id; @@ -284,13 +273,13 @@ $(function() { function getData() { $("#uploader").attr("data-text", "Loading..."); $("#uploader").fadeIn(); - $.get("https://cors-anywhere.herokuapp.com/pastebin.com/raw/" + location.search.substring(1) + "/?nocache=" + Math.random(), function(data) { - if (data.substring(0, 9) === "

Captcha required!

The pastebin server is asking for a captcha, but their API doesnt let us show it to you directly.
Instead, to finish saving the log, you need to solve the captcha in a new tab, once you have done so, reload this page.'); - } - else { - $("#input").val(LZString.decompressFromUTF16(data) || data); + $.get("/log/fetch/" + location.search.substring(1), function(data) { + if (data.success) { + $("#input").val(LZString.decompressFromUTF16(data.content) || data.content); loadData(); + } else { + $("#output").html('

Fetching the log failed!

' + data.error + '

'); + $("#rawlog").text($("#input").val()); } $("#uploader").fadeOut(); }); -- cgit