add mod code analyzers to detect implicit net field conversion issues (#471)

18 files changed, 1045 insertions, 28 deletions

diff --git a/build/prepare-nuget-package.targets b/build/prepare-nuget-package.targets
new file mode 100644
index 00000000..5dbc5508
--- /dev/null
+++ b/build/prepare-nuget-package.targets
@@ -0,0 +1,20 @@
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <!--
+  
+  This build task is run from the ModBuildConfig project after it's been compiled, and copies the
+  package files to the bin\Pathoschild.Stardew.ModBuildConfig folder.
+  
+  -->
+  <Target Name="AfterBuild">
+    <PropertyGroup>
+      <PackagePath>$(SolutionDir)\..\bin\Pathoschild.Stardew.ModBuildConfig</PackagePath>
+    </PropertyGroup>
+    <RemoveDir Directories="$(PackagePath)" />
+    <Copy SourceFiles="$(ProjectDir)/package.nuspec" DestinationFolder="$(PackagePath)" />
+    <Copy SourceFiles="$(ProjectDir)/build/smapi.targets" DestinationFiles="$(PackagePath)/build/Pathoschild.Stardew.ModBuildConfig.targets" />
+    <Copy SourceFiles="$(TargetDir)/StardewModdingAPI.ModBuildConfig.dll" DestinationFiles="$(PackagePath)/build/StardewModdingAPI.ModBuildConfig.dll" />
+    <Copy SourceFiles="$(SolutionDir)/SMAPI.ModBuildConfig.Analyzer/bin/netstandard1.3/StardewModdingAPI.ModBuildConfig.Analyzer.dll" DestinationFiles="$(PackagePath)/analyzers/dotnet/cs/StardewModdingAPI.ModBuildConfig.Analyzer.dll" />
+    <Copy SourceFiles="$(SolutionDir)/SMAPI.ModBuildConfig.Analyzer/tools/install.ps1" DestinationFiles="$(PackagePath)/tools/install.ps1" />
+    <Copy SourceFiles="$(SolutionDir)/SMAPI.ModBuildConfig.Analyzer/tools/uninstall.ps1" DestinationFiles="$(PackagePath)/tools/uninstall.ps1" />
+  </Target>
+</Project>
diff --git a/docs/mod-build-config.md b/docs/mod-build-config.md
index 0d72e4d9..71a2518a 100644
--- a/docs/mod-build-config.md
+++ b/docs/mod-build-config.md
@@ -3,15 +3,16 @@ for SMAPI mods.
 
 The package...
 
-* lets your code compile on any computer (Linux/Mac/Windows) without needing to change the assembly
-  references or game path.
-* packages the mod into the game's `Mods` folder when you rebuild the code (configurable).
-* configures Visual Studio so you can debug into the mod code when the game is running (_Windows
-  only_).
+* detects your game install path;
+* adds the assembly references you need (with automatic support for Linux/Mac/Windows);
+* packages the mod into your `Mods` folder when you rebuild the code (configurable);
+* configures Visual Studio to enable debugging into the code when the game is running (_Windows only_);
+* adds C# analyzers to warn for Stardew Valley-specific issues.
 
 ## Contents
 * [Install](#install)
 * [Configure](#configure)
+* [Code analysis warnings](#code-analysis-warnings)
 * [Troubleshoot](#troubleshoot)
 * [Release notes](#release-notes)
 
@@ -121,7 +122,7 @@ The configuration will check your custom path first, then fall back to the defau
 still compile on a different computer).
 
 ### Unit test projects
-**(upcoming in 2.0.3)**
+**(upcoming in 2.1)**
 
 You can use the package in unit test projects too. Its optional unit test mode...
 
@@ -134,15 +135,63 @@ To enable it, add this above the first `</PropertyGroup>` in your `.csproj`:
 <ModUnitTests>True</ModUnitTests>
 ```
 
+## Code warnings
+### Overview
+The NuGet package adds code warnings in Visual Studio specific to Stardew Valley. For example:  
+![](screenshots/code-analyzer-example.png)
+
+You can hide the warnings...
+* [for specific code](https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/preprocessor-directives/preprocessor-pragma-warning);
+* for a method using this attribute:
+  ```cs
+  [System.Diagnostics.CodeAnalysis.SuppressMessage("SMAPI.CommonErrors", "SMAPI001")] // implicit net field conversion
+  ```
+* for an entire project:
+  1. Expand the _References_ node for the project in Visual Studio.
+  2. Right-click on _Analyzers_ and choose _Open Active Rule Set_.
+  4. Expand _StardewModdingAPI.ModBuildConfig.Analyzer_ and uncheck the warnings you want to hide.
+
+See below for help with each specific warning.
+
+### SMAPI001
+**Implicit net field conversion:**
+> This implicitly converts '{{expression}}' from {{net type}} to {{other type}}, but
+> {{net type}} has unintuitive implicit conversion rules. Consider comparing against the actual
+> value instead to avoid bugs.
+
+Stardew Valley uses a set of net fields (like `NetBool` or `NetInt`) to handle multiplayer sync.
+These types can implicitly convert to their equivalent normal values (like `bool x = new NetBool()`),
+but their conversion rules can be unintuitive and error-prone. For example,
+`item?.category == null && item?.category != null` can both be true at once, and
+`building.indoors != null` will be true for a null value in some cases.
+
+Suggested fix:
+* For a reference type (i.e. one that can contain `null`), you can use the `.Value` property:
+  ```c#
+  if (building.indoors.Value == null)
+  ```
+  Or convert the value before comparison:
+  ```c#
+  GameLocation indoors = building.indoors;
+  if(indoors == null)
+     // ...
+  ```
+* For a value type (i.e. one that can't contain `null`), make sure to check for null first if
+  applicable:
+  ```cs
+  if (item != null && item.category == 0)
+  ```
+
 ## Troubleshoot
 ### "Failed to find the game install path"
 That error means the package couldn't find your game. You can specify the game path yourself; see
 _[Game path](#game-path)_ above.
 
 ## Release notes
-### 2.0.3 alpha
+### 2.1 alpha
 * Added support for Stardew Valley 1.3.
 * Added support for unit test projects.
+* Added C# analyzers to warn about implicit conversions of Netcode fields in Stardew Valley 1.3.
 
 ### 2.0.2
 * Fixed compatibility issue on Linux.
diff --git a/docs/screenshots/code-analyzer-example.png b/docs/screenshots/code-analyzer-example.png
new file mode 100644
index 00000000..2723b164
--- /dev/null
+++ b/docs/screenshots/code-analyzer-example.png
diff --git a/docs/technical-docs.md b/docs/technical-docs.md
index 37ec7f69..9e1a49e7 100644
--- a/docs/technical-docs.md
+++ b/docs/technical-docs.md
@@ -20,6 +20,7 @@ mods, this section isn't relevant to you; see the main README to use or create m
   * [Development](#development-2)
     * [Local development](#local-development)
     * [Deploying to Amazon Beanstalk](#deploying-to-amazon-beanstalk)
+* [Mod build config package](#mod-build-config-package)
 
 # SMAPI
 ## Development
@@ -222,3 +223,31 @@ property name                   | description
 `LogParser:SectionUrl`          | The root URL of the log page, like `https://log.smapi.io/`.
 `ModUpdateCheck:GitHubPassword` | The password with which to authenticate to GitHub when fetching release info.
 `ModUpdateCheck:GitHubUsername` | The username with which to authenticate to GitHub when fetching release info.
+
+## Mod build config package
+### Overview
+The mod build config package is a NuGet package that mods reference to automatically set up
+references, configure the build, and add analyzers specific to Stardew Valley mods.
+
+This involves three projects:
+
+project                                           | purpose
+------------------------------------------------- | ----------------
+`StardewModdingAPI.ModBuildConfig`                | Configures the build (references, deploying the mod files, setting up debugging, etc).
+`StardewModdingAPI.ModBuildConfig.Analyzer`       | Adds C# analyzers which show code warnings in Visual Studio.
+`StardewModdingAPI.ModBuildConfig.Analyzer.Tests` | Unit tests for the C# analyzers.
+
+When the projects are built, the relevant files are copied into `bin/Pathoschild.Stardew.ModBuildConfig`.
+
+### Preparing a build
+To prepare a build of the NuGet package:
+1. Install the [NuGet CLI](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools#nugetexe-cli).
+1. Change the version and release notes in `package.nuspec`.
+2. Rebuild the solution in _Release_ mode.
+3. Open a terminal in the `bin/Pathoschild.Stardew.ModBuildConfig` package and run this command:
+   ```bash
+   nuget.exe pack
+   ```
+
+That will create a `Pathoschild.Stardew.ModBuildConfig-<version>.nupkg` file in the same directory
+which can be uploaded to NuGet or referenced directly.
diff --git a/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticResult.cs b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticResult.cs
new file mode 100644
index 00000000..896c2cb8
--- /dev/null
+++ b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticResult.cs
@@ -0,0 +1,88 @@
+// <generated />
+using Microsoft.CodeAnalysis;
+using System;
+
+namespace SMAPI.ModBuildConfig.Analyzer.Tests.Framework
+{
+    /// <summary>
+    /// Location where the diagnostic appears, as determined by path, line number, and column number.
+    /// </summary>
+    public struct DiagnosticResultLocation
+    {
+        public DiagnosticResultLocation(string path, int line, int column)
+        {
+            if (line < -1)
+            {
+                throw new ArgumentOutOfRangeException(nameof(line), "line must be >= -1");
+            }
+
+            if (column < -1)
+            {
+                throw new ArgumentOutOfRangeException(nameof(column), "column must be >= -1");
+            }
+
+            this.Path = path;
+            this.Line = line;
+            this.Column = column;
+        }
+
+        public string Path { get; }
+        public int Line { get; }
+        public int Column { get; }
+    }
+
+    /// <summary>
+    /// Struct that stores information about a Diagnostic appearing in a source
+    /// </summary>
+    public struct DiagnosticResult
+    {
+        private DiagnosticResultLocation[] locations;
+
+        public DiagnosticResultLocation[] Locations
+        {
+            get
+            {
+                if (this.locations == null)
+                {
+                    this.locations = new DiagnosticResultLocation[] { };
+                }
+                return this.locations;
+            }
+
+            set
+            {
+                this.locations = value;
+            }
+        }
+
+        public DiagnosticSeverity Severity { get; set; }
+
+        public string Id { get; set; }
+
+        public string Message { get; set; }
+
+        public string Path
+        {
+            get
+            {
+                return this.Locations.Length > 0 ? this.Locations[0].Path : "";
+            }
+        }
+
+        public int Line
+        {
+            get
+            {
+                return this.Locations.Length > 0 ? this.Locations[0].Line : -1;
+            }
+        }
+
+        public int Column
+        {
+            get
+            {
+                return this.Locations.Length > 0 ? this.Locations[0].Column : -1;
+            }
+        }
+    }
+}
diff --git a/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.Helper.cs b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.Helper.cs
new file mode 100644
index 00000000..d33455fc
--- /dev/null
+++ b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.Helper.cs
@@ -0,0 +1,171 @@
+// <generated />
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.Text;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+
+namespace SMAPI.ModBuildConfig.Analyzer.Tests.Framework
+{
+    /// <summary>
+    /// Class for turning strings into documents and getting the diagnostics on them
+    /// All methods are static
+    /// </summary>
+    public abstract partial class DiagnosticVerifier
+    {
+        private static readonly MetadataReference CorlibReference = MetadataReference.CreateFromFile(typeof(object).Assembly.Location);
+        private static readonly MetadataReference SystemCoreReference = MetadataReference.CreateFromFile(typeof(Enumerable).Assembly.Location);
+        private static readonly MetadataReference CSharpSymbolsReference = MetadataReference.CreateFromFile(typeof(CSharpCompilation).Assembly.Location);
+        private static readonly MetadataReference CodeAnalysisReference = MetadataReference.CreateFromFile(typeof(Compilation).Assembly.Location);
+
+        internal static string DefaultFilePathPrefix = "Test";
+        internal static string CSharpDefaultFileExt = "cs";
+        internal static string VisualBasicDefaultExt = "vb";
+        internal static string TestProjectName = "TestProject";
+
+        #region  Get Diagnostics
+
+        /// <summary>
+        /// Given classes in the form of strings, their language, and an IDiagnosticAnalyzer to apply to it, return the diagnostics found in the string after converting it to a document.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source classes are in</param>
+        /// <param name="analyzer">The analyzer to be run on the sources</param>
+        /// <returns>An IEnumerable of Diagnostics that surfaced in the source code, sorted by Location</returns>
+        private static Diagnostic[] GetSortedDiagnostics(string[] sources, string language, DiagnosticAnalyzer analyzer)
+        {
+            return GetSortedDiagnosticsFromDocuments(analyzer, GetDocuments(sources, language));
+        }
+
+        /// <summary>
+        /// Given an analyzer and a document to apply it to, run the analyzer and gather an array of diagnostics found in it.
+        /// The returned diagnostics are then ordered by location in the source document.
+        /// </summary>
+        /// <param name="analyzer">The analyzer to run on the documents</param>
+        /// <param name="documents">The Documents that the analyzer will be run on</param>
+        /// <returns>An IEnumerable of Diagnostics that surfaced in the source code, sorted by Location</returns>
+        protected static Diagnostic[] GetSortedDiagnosticsFromDocuments(DiagnosticAnalyzer analyzer, Document[] documents)
+        {
+            var projects = new HashSet<Project>();
+            foreach (var document in documents)
+            {
+                projects.Add(document.Project);
+            }
+
+            var diagnostics = new List<Diagnostic>();
+            foreach (var project in projects)
+            {
+                var compilationWithAnalyzers = project.GetCompilationAsync().Result.WithAnalyzers(ImmutableArray.Create(analyzer));
+                var diags = compilationWithAnalyzers.GetAnalyzerDiagnosticsAsync().Result;
+                foreach (var diag in diags)
+                {
+                    if (diag.Location == Location.None || diag.Location.IsInMetadata)
+                    {
+                        diagnostics.Add(diag);
+                    }
+                    else
+                    {
+                        for (int i = 0; i < documents.Length; i++)
+                        {
+                            var document = documents[i];
+                            var tree = document.GetSyntaxTreeAsync().Result;
+                            if (tree == diag.Location.SourceTree)
+                            {
+                                diagnostics.Add(diag);
+                            }
+                        }
+                    }
+                }
+            }
+
+            var results = SortDiagnostics(diagnostics);
+            diagnostics.Clear();
+            return results;
+        }
+
+        /// <summary>
+        /// Sort diagnostics by location in source document
+        /// </summary>
+        /// <param name="diagnostics">The list of Diagnostics to be sorted</param>
+        /// <returns>An IEnumerable containing the Diagnostics in order of Location</returns>
+        private static Diagnostic[] SortDiagnostics(IEnumerable<Diagnostic> diagnostics)
+        {
+            return diagnostics.OrderBy(d => d.Location.SourceSpan.Start).ToArray();
+        }
+
+        #endregion
+
+        #region Set up compilation and documents
+        /// <summary>
+        /// Given an array of strings as sources and a language, turn them into a project and return the documents and spans of it.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Tuple containing the Documents produced from the sources and their TextSpans if relevant</returns>
+        private static Document[] GetDocuments(string[] sources, string language)
+        {
+            if (language != LanguageNames.CSharp && language != LanguageNames.VisualBasic)
+            {
+                throw new ArgumentException("Unsupported Language");
+            }
+
+            var project = CreateProject(sources, language);
+            var documents = project.Documents.ToArray();
+
+            if (sources.Length != documents.Length)
+            {
+                throw new InvalidOperationException("Amount of sources did not match amount of Documents created");
+            }
+
+            return documents;
+        }
+
+        /// <summary>
+        /// Create a Document from a string through creating a project that contains it.
+        /// </summary>
+        /// <param name="source">Classes in the form of a string</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Document created from the source string</returns>
+        protected static Document CreateDocument(string source, string language = LanguageNames.CSharp)
+        {
+            return CreateProject(new[] { source }, language).Documents.First();
+        }
+
+        /// <summary>
+        /// Create a project using the inputted strings as sources.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Project created out of the Documents created from the source strings</returns>
+        private static Project CreateProject(string[] sources, string language = LanguageNames.CSharp)
+        {
+            string fileNamePrefix = DefaultFilePathPrefix;
+            string fileExt = language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt;
+
+            var projectId = ProjectId.CreateNewId(debugName: TestProjectName);
+
+            var solution = new AdhocWorkspace()
+                .CurrentSolution
+                .AddProject(projectId, TestProjectName, TestProjectName, language)
+                .AddMetadataReference(projectId, CorlibReference)
+                .AddMetadataReference(projectId, SystemCoreReference)
+                .AddMetadataReference(projectId, CSharpSymbolsReference)
+                .AddMetadataReference(projectId, CodeAnalysisReference);
+
+            int count = 0;
+            foreach (var source in sources)
+            {
+                var newFileName = fileNamePrefix + count + "." + fileExt;
+                var documentId = DocumentId.CreateNewId(projectId, debugName: newFileName);
+                solution = solution.AddDocument(documentId, newFileName, SourceText.From(source));
+                count++;
+            }
+            return solution.GetProject(projectId);
+        }
+        #endregion
+    }
+}
+
diff --git a/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.cs b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.cs
new file mode 100644
index 00000000..edaaabd4
--- /dev/null
+++ b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.cs
@@ -0,0 +1,248 @@
+// <generated />
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Diagnostics;
+using NUnit.Framework;
+
+namespace SMAPI.ModBuildConfig.Analyzer.Tests.Framework
+{
+    /// <summary>
+    /// Superclass of all Unit Tests for DiagnosticAnalyzers
+    /// </summary>
+    public abstract partial class DiagnosticVerifier
+    {
+        #region To be implemented by Test classes
+        /// <summary>
+        /// Get the CSharp analyzer being tested - to be implemented in non-abstract class
+        /// </summary>
+        protected virtual DiagnosticAnalyzer GetCSharpDiagnosticAnalyzer()
+        {
+            return null;
+        }
+
+        /// <summary>
+        /// Get the Visual Basic analyzer being tested (C#) - to be implemented in non-abstract class
+        /// </summary>
+        protected virtual DiagnosticAnalyzer GetBasicDiagnosticAnalyzer()
+        {
+            return null;
+        }
+        #endregion
+
+        #region Verifier wrappers
+
+        /// <summary>
+        /// Called to test a C# DiagnosticAnalyzer when applied on the single inputted string as a source
+        /// Note: input a DiagnosticResult for each Diagnostic expected
+        /// </summary>
+        /// <param name="source">A class in the form of a string to run the analyzer on</param>
+        /// <param name="expected"> DiagnosticResults that should appear after the analyzer is run on the source</param>
+        protected void VerifyCSharpDiagnostic(string source, params DiagnosticResult[] expected)
+        {
+            VerifyDiagnostics(new[] { source }, LanguageNames.CSharp, GetCSharpDiagnosticAnalyzer(), expected);
+        }
+
+        /// <summary>
+        /// Called to test a C# DiagnosticAnalyzer when applied on the inputted strings as a source
+        /// Note: input a DiagnosticResult for each Diagnostic expected
+        /// </summary>
+        /// <param name="sources">An array of strings to create source documents from to run the analyzers on</param>
+        /// <param name="expected">DiagnosticResults that should appear after the analyzer is run on the sources</param>
+        protected void VerifyCSharpDiagnostic(string[] sources, params DiagnosticResult[] expected)
+        {
+            VerifyDiagnostics(sources, LanguageNames.CSharp, GetCSharpDiagnosticAnalyzer(), expected);
+        }
+
+        /// <summary>
+        /// General method that gets a collection of actual diagnostics found in the source after the analyzer is run, 
+        /// then verifies each of them.
+        /// </summary>
+        /// <param name="sources">An array of strings to create source documents from to run the analyzers on</param>
+        /// <param name="language">The language of the classes represented by the source strings</param>
+        /// <param name="analyzer">The analyzer to be run on the source code</param>
+        /// <param name="expected">DiagnosticResults that should appear after the analyzer is run on the sources</param>
+        private void VerifyDiagnostics(string[] sources, string language, DiagnosticAnalyzer analyzer, params DiagnosticResult[] expected)
+        {
+            var diagnostics = GetSortedDiagnostics(sources, language, analyzer);
+            VerifyDiagnosticResults(diagnostics, analyzer, expected);
+        }
+
+        #endregion
+
+        #region Actual comparisons and verifications
+        /// <summary>
+        /// Checks each of the actual Diagnostics found and compares them with the corresponding DiagnosticResult in the array of expected results.
+        /// Diagnostics are considered equal only if the DiagnosticResultLocation, Id, Severity, and Message of the DiagnosticResult match the actual diagnostic.
+        /// </summary>
+        /// <param name="actualResults">The Diagnostics found by the compiler after running the analyzer on the source code</param>
+        /// <param name="analyzer">The analyzer that was being run on the sources</param>
+        /// <param name="expectedResults">Diagnostic Results that should have appeared in the code</param>
+        private static void VerifyDiagnosticResults(IEnumerable<Diagnostic> actualResults, DiagnosticAnalyzer analyzer, params DiagnosticResult[] expectedResults)
+        {
+            int expectedCount = expectedResults.Count();
+            int actualCount = actualResults.Count();
+
+            if (expectedCount != actualCount)
+            {
+                string diagnosticsOutput = actualResults.Any() ? FormatDiagnostics(analyzer, actualResults.ToArray()) : "    NONE.";
+
+                Assert.IsTrue(false,
+                    string.Format("Mismatch between number of diagnostics returned, expected \"{0}\" actual \"{1}\"\r\n\r\nDiagnostics:\r\n{2}\r\n", expectedCount, actualCount, diagnosticsOutput));
+            }
+
+            for (int i = 0; i < expectedResults.Length; i++)
+            {
+                var actual = actualResults.ElementAt(i);
+                var expected = expectedResults[i];
+
+                if (expected.Line == -1 && expected.Column == -1)
+                {
+                    if (actual.Location != Location.None)
+                    {
+                        Assert.IsTrue(false,
+                            string.Format("Expected:\nA project diagnostic with No location\nActual:\n{0}",
+                            FormatDiagnostics(analyzer, actual)));
+                    }
+                }
+                else
+                {
+                    VerifyDiagnosticLocation(analyzer, actual, actual.Location, expected.Locations.First());
+                    var additionalLocations = actual.AdditionalLocations.ToArray();
+
+                    if (additionalLocations.Length != expected.Locations.Length - 1)
+                    {
+                        Assert.IsTrue(false,
+                            string.Format("Expected {0} additional locations but got {1} for Diagnostic:\r\n    {2}\r\n",
+                                expected.Locations.Length - 1, additionalLocations.Length,
+                                FormatDiagnostics(analyzer, actual)));
+                    }
+
+                    for (int j = 0; j < additionalLocations.Length; ++j)
+                    {
+                        VerifyDiagnosticLocation(analyzer, actual, additionalLocations[j], expected.Locations[j + 1]);
+                    }
+                }
+
+                if (actual.Id != expected.Id)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic id to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Id, actual.Id, FormatDiagnostics(analyzer, actual)));
+                }
+
+                if (actual.Severity != expected.Severity)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic severity to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Severity, actual.Severity, FormatDiagnostics(analyzer, actual)));
+                }
+
+                if (actual.GetMessage() != expected.Message)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic message to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Message, actual.GetMessage(), FormatDiagnostics(analyzer, actual)));
+                }
+            }
+        }
+
+        /// <summary>
+        /// Helper method to VerifyDiagnosticResult that checks the location of a diagnostic and compares it with the location in the expected DiagnosticResult.
+        /// </summary>
+        /// <param name="analyzer">The analyzer that was being run on the sources</param>
+        /// <param name="diagnostic">The diagnostic that was found in the code</param>
+        /// <param name="actual">The Location of the Diagnostic found in the code</param>
+        /// <param name="expected">The DiagnosticResultLocation that should have been found</param>
+        private static void VerifyDiagnosticLocation(DiagnosticAnalyzer analyzer, Diagnostic diagnostic, Location actual, DiagnosticResultLocation expected)
+        {
+            var actualSpan = actual.GetLineSpan();
+
+            Assert.IsTrue(actualSpan.Path == expected.Path || (actualSpan.Path != null && actualSpan.Path.Contains("Test0.") && expected.Path.Contains("Test.")),
+                string.Format("Expected diagnostic to be in file \"{0}\" was actually in file \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                    expected.Path, actualSpan.Path, FormatDiagnostics(analyzer, diagnostic)));
+
+            var actualLinePosition = actualSpan.StartLinePosition;
+
+            // Only check line position if there is an actual line in the real diagnostic
+            if (actualLinePosition.Line > 0)
+            {
+                if (actualLinePosition.Line + 1 != expected.Line)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic to be on line \"{0}\" was actually on line \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Line, actualLinePosition.Line + 1, FormatDiagnostics(analyzer, diagnostic)));
+                }
+            }
+
+            // Only check column position if there is an actual column position in the real diagnostic
+            if (actualLinePosition.Character > 0)
+            {
+                if (actualLinePosition.Character + 1 != expected.Column)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic to start at column \"{0}\" was actually at column \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Column, actualLinePosition.Character + 1, FormatDiagnostics(analyzer, diagnostic)));
+                }
+            }
+        }
+        #endregion
+
+        #region Formatting Diagnostics
+        /// <summary>
+        /// Helper method to format a Diagnostic into an easily readable string
+        /// </summary>
+        /// <param name="analyzer">The analyzer that this verifier tests</param>
+        /// <param name="diagnostics">The Diagnostics to be formatted</param>
+        /// <returns>The Diagnostics formatted as a string</returns>
+        private static string FormatDiagnostics(DiagnosticAnalyzer analyzer, params Diagnostic[] diagnostics)
+        {
+            var builder = new StringBuilder();
+            for (int i = 0; i < diagnostics.Length; ++i)
+            {
+                builder.AppendLine("// " + diagnostics[i].ToString());
+
+                var analyzerType = analyzer.GetType();
+                var rules = analyzer.SupportedDiagnostics;
+
+                foreach (var rule in rules)
+                {
+                    if (rule != null && rule.Id == diagnostics[i].Id)
+                    {
+                        var location = diagnostics[i].Location;
+                        if (location == Location.None)
+                        {
+                            builder.AppendFormat("GetGlobalResult({0}.{1})", analyzerType.Name, rule.Id);
+                        }
+                        else
+                        {
+                            Assert.IsTrue(location.IsInSource,
+                                $"Test base does not currently handle diagnostics in metadata locations. Diagnostic in metadata: {diagnostics[i]}\r\n");
+
+                            string resultMethodName = diagnostics[i].Location.SourceTree.FilePath.EndsWith(".cs") ? "GetCSharpResultAt" : "GetBasicResultAt";
+                            var linePosition = diagnostics[i].Location.GetLineSpan().StartLinePosition;
+
+                            builder.AppendFormat("{0}({1}, {2}, {3}.{4})",
+                                resultMethodName,
+                                linePosition.Line + 1,
+                                linePosition.Character + 1,
+                                analyzerType.Name,
+                                rule.Id);
+                        }
+
+                        if (i != diagnostics.Length - 1)
+                        {
+                            builder.Append(',');
+                        }
+
+                        builder.AppendLine();
+                        break;
+                    }
+                }
+            }
+            return builder.ToString();
+        }
+        #endregion
+    }
+}
diff --git a/src/SMAPI.ModBuildConfig.Analyzer.Tests/SMAPI.ModBuildConfig.Analyzer.Tests.csproj b/src/SMAPI.ModBuildConfig.Analyzer.Tests/SMAPI.ModBuildConfig.Analyzer.Tests.csproj
new file mode 100644
index 00000000..8d3815c3
--- /dev/null
+++ b/src/SMAPI.ModBuildConfig.Analyzer.Tests/SMAPI.ModBuildConfig.Analyzer.Tests.csproj
@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="2.4.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.3.0" />
+    <PackageReference Include="NUnit" Version="3.10.1" />
+    <PackageReference Include="NUnit3TestAdapter" Version="3.10.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\SMAPI.ModBuildConfig.Analyzer\StardewModdingAPI.ModBuildConfig.Analyzer.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/src/SMAPI.ModBuildConfig.Analyzer.Tests/UnitTests.cs b/src/SMAPI.ModBuildConfig.Analyzer.Tests/UnitTests.cs
new file mode 100644
index 00000000..d5e4ef52
--- /dev/null
+++ b/