From 2b1f607d41b3d4d071c0db0671dbc99b6982909f Mon Sep 17 00:00:00 2001
From: Jesse Plamondon-Willard <Pathoschild@users.noreply.github.com>
Date: Tue, 3 Dec 2019 21:21:28 -0500
Subject: encapsulate file storage, also handle Pastebin rate limits in JSON
validator
---
src/SMAPI.Web/Controllers/LogParserController.cs | 178 ++---------------------
1 file changed, 14 insertions(+), 164 deletions(-)
(limited to 'src/SMAPI.Web/Controllers/LogParserController.cs')
diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs
index 318b34d0..e270ae0a 100644
--- a/src/SMAPI.Web/Controllers/LogParserController.cs
+++ b/src/SMAPI.Web/Controllers/LogParserController.cs
@@ -1,22 +1,12 @@
using System;
-using System.IO;
using System.Linq;
-using System.Text;
using System.Threading.Tasks;
-using Amazon;
-using Amazon.Runtime;
-using Amazon.S3;
-using Amazon.S3.Model;
-using Amazon.S3.Transfer;
using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Options;
using StardewModdingAPI.Toolkit.Utilities;
using StardewModdingAPI.Web.Framework;
-using StardewModdingAPI.Web.Framework.Clients.Pastebin;
-using StardewModdingAPI.Web.Framework.Compression;
-using StardewModdingAPI.Web.Framework.ConfigModels;
using StardewModdingAPI.Web.Framework.LogParsing;
using StardewModdingAPI.Web.Framework.LogParsing.Models;
+using StardewModdingAPI.Web.Framework.Storage;
using StardewModdingAPI.Web.ViewModels;
namespace StardewModdingAPI.Web.Controllers
@@ -27,14 +17,8 @@ namespace StardewModdingAPI.Web.Controllers
/*********
** Fields
*********/
- /// <summary>The API client settings.</summary>
- private readonly ApiClientsConfig ClientsConfig;
-
- /// <summary>The underlying Pastebin client.</summary>
- private readonly IPastebinClient Pastebin;
-
- /// <summary>The underlying text compression helper.</summary>
- private readonly IGzipHelper GzipHelper;
+ /// <summary>Provides access to raw data storage.</summary>
+ private readonly IStorageProvider Storage;
/*********
@@ -44,21 +28,17 @@ namespace StardewModdingAPI.Web.Controllers
** Constructor
***/
/// <summary>Construct an instance.</summary>
- /// <param name="clientsConfig">The API client settings.</param>
- /// <param name="pastebin">The Pastebin API client.</param>
- /// <param name="gzipHelper">The underlying text compression helper.</param>
- public LogParserController(IOptions<ApiClientsConfig> clientsConfig, IPastebinClient pastebin, IGzipHelper gzipHelper)
+ /// <param name="storage">Provides access to raw data storage.</param>
+ public LogParserController(IStorageProvider storage)
{
- this.ClientsConfig = clientsConfig.Value;
- this.Pastebin = pastebin;
- this.GzipHelper = gzipHelper;
+ this.Storage = storage;
}
/***
** Web UI
***/
/// <summary>Render the log parser UI.</summary>
- /// <param name="id">The paste ID.</param>
+ /// <param name="id">The stored file ID.</param>
/// <param name="raw">Whether to display the raw unparsed log.</param>
[HttpGet]
[Route("log")]
@@ -70,12 +50,12 @@ namespace StardewModdingAPI.Web.Controllers
return this.View("Index", this.GetModel(id));
// log page
- PasteInfo paste = await this.GetAsync(id);
- ParsedLog log = paste.Success
- ? new LogParser().Parse(paste.Content)
- : new ParsedLog { IsValid = false, Error = paste.Error };
+ StoredFileInfo file = await this.Storage.GetAsync(id);
+ ParsedLog log = file.Success
+ ? new LogParser().Parse(file.Content)
+ : new ParsedLog { IsValid = false, Error = file.Error };
- return this.View("Index", this.GetModel(id, uploadWarning: paste.Warning, expiry: paste.Expiry).SetResult(log, raw));
+ return this.View("Index", this.GetModel(id, uploadWarning: file.Warning, expiry: file.Expiry).SetResult(log, raw));
}
/***
@@ -92,8 +72,7 @@ namespace StardewModdingAPI.Web.Controllers
return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty."));
// upload log
- input = this.GzipHelper.CompressString(input);
- var uploadResult = await this.TrySaveLog(input);
+ UploadResult uploadResult = await this.Storage.SaveAsync(title: $"SMAPI log {DateTime.UtcNow:s}", content: input, compress: true);
if (!uploadResult.Succeeded)
return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError));
@@ -105,106 +84,8 @@ namespace StardewModdingAPI.Web.Controllers
/*********
** Private methods
*********/
- /// <summary>Fetch raw text from Pastebin.</summary>
- /// <param name="id">The Pastebin paste ID.</param>
- private async Task<PasteInfo> GetAsync(string id)
- {
- // get from Amazon S3
- if (Guid.TryParseExact(id, "N", out Guid _))
- {
- var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey);
-
- using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)))
- {
- try
- {
- using (GetObjectResponse response = await s3.GetObjectAsync(this.ClientsConfig.AmazonLogBucket, $"logs/{id}"))
- using (Stream responseStream = response.ResponseStream)
- using (StreamReader reader = new StreamReader(responseStream))
- {
- DateTime expiry = response.Expiration.ExpiryDateUtc;
- string pastebinError = response.Metadata["x-amz-meta-pastebin-error"];
- string content = this.GzipHelper.DecompressString(reader.ReadToEnd());
-
- return new PasteInfo
- {
- Success = true,
- Content = content,
- Expiry = expiry,
- Warning = pastebinError
- };
- }
- }
- catch (AmazonServiceException ex)
- {
- return ex.ErrorCode == "NoSuchKey"
- ? new PasteInfo { Error = "There's no log with that ID." }
- : new PasteInfo { Error = $"Could not fetch that log from AWS S3 ({ex.ErrorCode}: {ex.Message})." };
- }
- }
- }
-
- // get from PasteBin
- else
- {
- PasteInfo response = await this.Pastebin.GetAsync(id);
- response.Content = this.GzipHelper.DecompressString(response.Content);
- return response;
- }
- }
-
- /// <summary>Save a log to Pastebin or Amazon S3, if available.</summary>
- /// <param name="content">The content to upload.</param>
- /// <returns>Returns metadata about the save attempt.</returns>
- private async Task<UploadResult> TrySaveLog(string content)
- {
- // save to PasteBin
- string uploadError;
- {
- SavePasteResult result = await this.Pastebin.PostAsync($"SMAPI log {DateTime.UtcNow:s}", content);
- if (result.Success)
- return new UploadResult(true, result.ID, null);
-
- uploadError = $"Pastebin error: {result.Error ?? "unknown error"}";
- }
-
- // fallback to S3
- try
- {
- var credentials = new BasicAWSCredentials(accessKey: this.ClientsConfig.AmazonAccessKey, secretKey: this.ClientsConfig.AmazonSecretKey);
-
- using (Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)))
- using (IAmazonS3 s3 = new AmazonS3Client(credentials, RegionEndpoint.GetBySystemName(this.ClientsConfig.AmazonRegion)))
- using (TransferUtility uploader = new TransferUtility(s3))
- {
- string id = Guid.NewGuid().ToString("N");
-
- var uploadRequest = new TransferUtilityUploadRequest
- {
- BucketName = this.ClientsConfig.AmazonLogBucket,
- Key = $"logs/{id}",
- InputStream = stream,
- Metadata =
- {
- // note: AWS will lowercase keys and prefix 'x-amz-meta-'
- ["smapi-uploaded"] = DateTime.UtcNow.ToString("O"),
- ["pastebin-error"] = uploadError
- }
- };
-
- await uploader.UploadAsync(uploadRequest);
-
- return new UploadResult(true, id, uploadError);
- }
- }
- catch (Exception ex)
- {
- return new UploadResult(false, null, $"{uploadError}\n{ex.Message}");
- }
- }
-
/// <summary>Build a log parser model.</summary>
- /// <param name="pasteID">The paste ID.</param>
+ /// <param name="pasteID">The stored file ID.</param>
/// <param name="expiry">When the uploaded file will no longer be available.</param>
/// <param name="uploadWarning">A non-blocking warning while uploading the log.</param>
/// <param name="uploadError">An error which occurred while uploading the log.</param>
@@ -243,36 +124,5 @@ namespace StardewModdingAPI.Web.Controllers
return null;
}
}
-
- /// <summary>The result of an attempt to upload a file.</summary>
- private class UploadResult
- {
- /*********
- ** Accessors
- *********/
- /// <summary>Whether the file upload succeeded.</summary>
- public bool Succeeded { get; }
-
- /// <summary>The file ID, if applicable.</summary>
- public string ID { get; }
-
- /// <summary>The upload error, if any.</summary>
- public string UploadError { get; }
-
-
- /*********
- ** Public methods
- *********/
- /// <summary>Construct an instance.</summary>
- /// <param name="succeeded">Whether the file upload succeeded.</param>
- /// <param name="id">The file ID, if applicable.</param>
- /// <param name="uploadError">The upload error, if any.</param>
- public UploadResult(bool succeeded, string id, string uploadError)
- {
- this.Succeeded = succeeded;
- this.ID = id;
- this.UploadError = uploadError;
- }
- }
}
}
--
cgit
From c1b15fb3725661ebfd8e03cec08343ae49e5d6da Mon Sep 17 00:00:00 2001
From: Jesse Plamondon-Willard <Pathoschild@users.noreply.github.com>
Date: Sat, 21 Dec 2019 23:52:47 -0500
Subject: allow local dev environments without an Azure account
---
docs/release-notes.md | 3 +
.../Controllers/JsonValidatorController.cs | 2 +-
src/SMAPI.Web/Controllers/LogParserController.cs | 2 +-
.../Framework/Storage/IStorageProvider.cs | 3 +-
src/SMAPI.Web/Framework/Storage/StorageProvider.cs | 116 +++++++++++++++------
src/SMAPI.Web/Views/LogParser/Index.cshtml | 21 ++--
6 files changed, 104 insertions(+), 43 deletions(-)
(limited to 'src/SMAPI.Web/Controllers/LogParserController.cs')
diff --git a/docs/release-notes.md b/docs/release-notes.md
index 591f4cc2..bbe08c13 100644
--- a/docs/release-notes.md
+++ b/docs/release-notes.md
@@ -26,6 +26,9 @@
* Fixed private textures loaded from content packs not having their `Name` field set.
* Fixed asset propagation for `Characters\Farmer\farmer_girl_base_bald`.
+* For SMAPI developers:
+ * You can now run local environments without configuring Amazon, Azure, and Pastebin accounts.
+
## 3.0.1
Released 02 December 2019 for Stardew Valley 1.4.0.1.
diff --git a/src/SMAPI.Web/Controllers/JsonValidatorController.cs b/src/SMAPI.Web/Controllers/JsonValidatorController.cs
index c4bfff3b..2ade3e3d 100644
--- a/src/SMAPI.Web/Controllers/JsonValidatorController.cs
+++ b/src/SMAPI.Web/Controllers/JsonValidatorController.cs
@@ -141,7 +141,7 @@ namespace StardewModdingAPI.Web.Controllers
return this.View("Index", this.GetModel(null, schemaName).SetUploadError("The JSON file seems to be empty."));
// upload file
- UploadResult result = await this.Storage.SaveAsync(title: $"JSON validator {DateTime.UtcNow:s}", content: input, compress: true);
+ UploadResult result = await this.Storage.SaveAsync(input);
if (!result.Succeeded)
return this.View("Index", this.GetModel(result.ID, schemaName).SetUploadError(result.UploadError));
diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs
index e270ae0a..97c419d9 100644
--- a/src/SMAPI.Web/Controllers/LogParserController.cs
+++ b/src/SMAPI.Web/Controllers/LogParserController.cs
@@ -72,7 +72,7 @@ namespace StardewModdingAPI.Web.Controllers
return this.View("Index", this.GetModel(null, uploadError: "The log file seems to be empty."));
// upload log
- UploadResult uploadResult = await this.Storage.SaveAsync(title: $"SMAPI log {DateTime.UtcNow:s}", content: input, compress: true);
+ UploadResult uploadResult = await this.Storage.SaveAsync(input);
if (!uploadResult.Succeeded)
return this.View("Index", this.GetModel(null, uploadError: uploadResult.UploadError));
diff --git a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs
index 12a5e421..96a34fbb 100644
--- a/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs
+++ b/src/SMAPI.Web/Framework/Storage/IStorageProvider.cs
@@ -6,11 +6,10 @@ namespace StardewModdingAPI.Web.Framework.Storage
internal interface IStorageProvider
{
/// <summary>Save a text file to storage.</summary>
- /// <param name="title">The display title, if applicable.</param>
/// <param name="content">The content to upload.</param>
/// <param name="compress">Whether to gzip the text.</param>
/// <returns>Returns metadata about the save attempt.</returns>
- Task<UploadResult> SaveAsync(string title, string content, bool compress = true);
+ Task<UploadResult> SaveAsync(string content, bool compress = true);
/// <summary>Fetch raw text from storage.</summary>
/// <param name="id">The storage ID returned by <see cref="SaveAsync"/>.</param>
diff --git a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs
index 12a35f18..35538443 100644
--- a/src/SMAPI.Web/Framework/Storage/StorageProvider.cs
+++ b/src/SMAPI.Web/Framework/Storage/StorageProvider.cs
@@ -27,6 +27,12 @@ namespace StardewModdingAPI.Web.Framework.Storage
/// <summary>The underlying text compression helper.</summary>
private readonly IGzipHelper GzipHelper;
+ /// <summary>Whether Azure blob storage is configured.</summary>
+ private bool HasAzure => !string.IsNullOrWhiteSpace(this.ClientsConfig.AzureBlobConnectionString);
+
+ /// <summary>The number of days since the blob's last-modified date when it will be deleted.</summary>
+ private int ExpiryDays => this.ClientsConfig.AzureBlobTempExpiryDays;
+
/*********
** Public methods
@@ -43,25 +49,38 @@ namespace StardewModdingAPI.Web.Framework.Storage
}
/// <summary>Save a text file to storage.</summary>
- /// <param name="title">The display title, if applicable.</param>
/// <param name="content">The content to upload.</param>
/// <param name="compress">Whether to gzip the text.</param>
/// <returns>Returns metadata about the save attempt.</returns>
- public async Task<UploadResult> SaveAsync(string title, string content, bool compress = true)
+ public async Task<UploadResult> SaveAsync(string content, bool compress = true)
{
- try
- {
- using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
- string id = Guid.NewGuid().ToString("N");
+ string id = Guid.NewGuid().ToString("N");
- BlobClient blob = this.GetAzureBlobClient(id);
- await blob.UploadAsync(stream);
+ // save to Azure
+ if (this.HasAzure)
+ {
+ try
+ {
+ using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
+ BlobClient blob = this.GetAzureBlobClient(id);
+ await blob.UploadAsync(stream);
- return new UploadResult(true, id, null);
+ return new UploadResult(true, id, null);
+ }
+ catch (Exception ex)
+ {
+ return new UploadResult(false, null, ex.Message);
+ }
}
- catch (Exception ex)
+
+ // save to local filesystem for testing
+ else
{
- return new UploadResult(false, null, ex.Message);
+ string path = this.GetDevFilePath(id);
+ Directory.CreateDirectory(Path.GetDirectoryName(path));
+
+ File.WriteAllText(path, content);
+ return new UploadResult(true, id, null);
}
}
@@ -69,39 +88,67 @@ namespace StardewModdingAPI.Web.Framework.Storage
/// <param name="id">The storage ID returned by <see cref="SaveAsync"/>.</param>
public async Task<StoredFileInfo> GetAsync(string id)
{
- // fetch from Azure/Amazon
+ // fetch from blob storage
if (Guid.TryParseExact(id, "N", out Guid _))
{
- // try Azure
- try
+ // Azure Blob storage
+ if (this.HasAzure)
{
- BlobClient blob = this.GetAzureBlobClient(id);
- Response<BlobDownloadInfo> response = await blob.DownloadAsync();
- using BlobDownloadInfo result = response.Value;
+ try
+ {
+ BlobClient blob = this.GetAzureBlobClient(id);
+ Response<BlobDownloadInfo> response = await blob.DownloadAsync();
+ using BlobDownloadInfo result = response.Value;
- using StreamReader reader = new StreamReader(result.Content);
- DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ClientsConfig.AzureBlobTempExpiryDays);
- string content = this.GzipHelper.DecompressString(reader.ReadToEnd());
+ using StreamReader reader = new StreamReader(result.Content);
+ DateTimeOffset expiry = result.Details.LastModified + TimeSpan.FromDays(this.ExpiryDays);
+ string content = this.GzipHelper.DecompressString(reader.ReadToEnd());
- return new StoredFileInfo
+ return new StoredFileInfo
+ {
+ Success = true,
+ Content = content,
+ Expiry = expiry.UtcDateTime
+ };
+ }
+ catch (RequestFailedException ex)
{
- Success = true,
- Content = content,
- Expiry = expiry.UtcDateTime
- };
+ return new StoredFileInfo
+ {
+ Error = ex.ErrorCode == "BlobNotFound"
+ ? "There's no file with that ID."
+ : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})."
+ };
+ }
}
- catch (RequestFailedException ex)
+
+ // local filesystem for testing
+ else
{
+ FileInfo file = new FileInfo(this.GetDevFilePath(id));
+ if (file.Exists)
+ {
+ if (file.LastWriteTimeUtc.AddDays(this.ExpiryDays) < DateTime.UtcNow)
+ file.Delete();
+ else
+ {
+ return new StoredFileInfo
+ {
+ Success = true,
+ Content = File.ReadAllText(file.FullName),
+ Expiry = DateTime.UtcNow.AddDays(this.ExpiryDays),
+ Warning = "This file was saved temporarily to the local computer. This should only happen in a local development environment."
+ };
+ }
+ }
return new StoredFileInfo
{
- Error = ex.ErrorCode == "BlobNotFound"
- ? "There's no file with that ID."
- : $"Could not fetch that file from storage ({ex.ErrorCode}: {ex.Message})."
+ Error = "There's no file with that ID."
};
}
}
- // get from PasteBin
+ // get from Pastebin
else
{
PasteInfo response = await this.Pastebin.GetAsync(id);
@@ -116,12 +163,19 @@ namespace StardewModdingAPI.Web.Framework.Storage
}
/// <summary>Get a client for reading and writing to Azure Blob storage.</summary>
- /// <param name="id">The file ID to fetch.</param>
+ /// <param name="id">The file ID.</param>
private BlobClient GetAzureBlobClient(string id)
{
var azure = new BlobServiceClient(this.ClientsConfig.AzureBlobConnectionString);
var container = azure.GetBlobContainerClient(this.ClientsConfig.AzureBlobTempContainer);
return container.GetBlobClient($"uploads/{id}");
}
+
+ /// <summary>Get the absolute file path for an upload when running in a local test environment with no Azure account configured.</summary>
+ /// <param name="id">The file ID.</param>
+ private string GetDevFilePath(string id)
+ {
+ return Path.Combine(Path.GetTempPath(), "smapi-web-temp", $"{id}.txt");
+ }
}
}
diff --git a/src/SMAPI.Web/Views/LogParser/Index.cshtml b/src/SMAPI.Web/Views/LogParser/Index.cshtml
index 439167bc..ac951564 100644
--- a/src/SMAPI.Web/Views/LogParser/Index.cshtml
+++ b/src/SMAPI.Web/Views/LogParser/Index.cshtml
@@ -67,12 +67,16 @@ else if (Model.ParsedLog?.IsValid == true)
@* save warnings *@
@if (Model.UploadWarning != null || Model.Expiry != null)
{
+ @if (Model.UploadWarning != null)
+ {
+ <text>⚠️ @Model.UploadWarning<br /></text>
+ }
+
<div class="save-metadata" v-pre>
@if (Model.Expiry != null)
{
- <text>This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()). </text>
+ <text>This log will expire in @((DateTime.UtcNow - Model.Expiry.Value).Humanize()).</text>
}
- <!--@Model.UploadWarning-->
</div>
}
@@ -294,10 +298,7 @@ else if (Model.ParsedLog?.IsValid == true)
string sectionFilter = message.Section != null && !message.IsStartOfSection ? $"&& sectionsAllow('{message.Section}')" : null; // filter the message by section if applicable
<tr class="mod @levelStr @sectionStartClass"
- @if (message.IsStartOfSection)
- {
- <text>v-on:click="toggleSection('@message.Section')"</text>
- }
+ @if (message.IsStartOfSection) { <text> v-on:click="toggleSection('@message.Section')" </text> }
v-show="filtersAllow('@Model.GetSlug(message.Mod)', '@levelStr') @sectionFilter">
<td v-pre>@message.Time</td>
<td v-pre>@message.Level.ToString().ToUpper()</td>
@@ -307,8 +308,12 @@ else if (Model.ParsedLog?.IsValid == true)
@if (message.IsStartOfSection)
{
<span class="section-toggle-message">
- <template v-if="sectionsAllow('@message.Section')">This section is shown. Click here to hide it.</template>
- <template v-else>This section is hidden. Click here to show it.</template>
+ <template v-if="sectionsAllow('@message.Section')">
+ This section is shown. Click here to hide it.
+ </template>
+ <template v-else>
+ This section is hidden. Click here to show it.
+ </template>
</span>
}
</td>
--
cgit