add mod code analyzers to detect implicit net field conversion issues (#471)

3 files changed, 507 insertions, 0 deletions

diff --git a/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticResult.cs b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticResult.cs
new file mode 100644
index 00000000..896c2cb8
--- /dev/null
+++ b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticResult.cs
@@ -0,0 +1,88 @@
+// <generated />
+using Microsoft.CodeAnalysis;
+using System;
+
+namespace SMAPI.ModBuildConfig.Analyzer.Tests.Framework
+{
+    /// <summary>
+    /// Location where the diagnostic appears, as determined by path, line number, and column number.
+    /// </summary>
+    public struct DiagnosticResultLocation
+    {
+        public DiagnosticResultLocation(string path, int line, int column)
+        {
+            if (line < -1)
+            {
+                throw new ArgumentOutOfRangeException(nameof(line), "line must be >= -1");
+            }
+
+            if (column < -1)
+            {
+                throw new ArgumentOutOfRangeException(nameof(column), "column must be >= -1");
+            }
+
+            this.Path = path;
+            this.Line = line;
+            this.Column = column;
+        }
+
+        public string Path { get; }
+        public int Line { get; }
+        public int Column { get; }
+    }
+
+    /// <summary>
+    /// Struct that stores information about a Diagnostic appearing in a source
+    /// </summary>
+    public struct DiagnosticResult
+    {
+        private DiagnosticResultLocation[] locations;
+
+        public DiagnosticResultLocation[] Locations
+        {
+            get
+            {
+                if (this.locations == null)
+                {
+                    this.locations = new DiagnosticResultLocation[] { };
+                }
+                return this.locations;
+            }
+
+            set
+            {
+                this.locations = value;
+            }
+        }
+
+        public DiagnosticSeverity Severity { get; set; }
+
+        public string Id { get; set; }
+
+        public string Message { get; set; }
+
+        public string Path
+        {
+            get
+            {
+                return this.Locations.Length > 0 ? this.Locations[0].Path : "";
+            }
+        }
+
+        public int Line
+        {
+            get
+            {
+                return this.Locations.Length > 0 ? this.Locations[0].Line : -1;
+            }
+        }
+
+        public int Column
+        {
+            get
+            {
+                return this.Locations.Length > 0 ? this.Locations[0].Column : -1;
+            }
+        }
+    }
+}
diff --git a/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.Helper.cs b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.Helper.cs
new file mode 100644
index 00000000..d33455fc
--- /dev/null
+++ b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.Helper.cs
@@ -0,0 +1,171 @@
+// <generated />
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Diagnostics;
+using Microsoft.CodeAnalysis.Text;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+
+namespace SMAPI.ModBuildConfig.Analyzer.Tests.Framework
+{
+    /// <summary>
+    /// Class for turning strings into documents and getting the diagnostics on them
+    /// All methods are static
+    /// </summary>
+    public abstract partial class DiagnosticVerifier
+    {
+        private static readonly MetadataReference CorlibReference = MetadataReference.CreateFromFile(typeof(object).Assembly.Location);
+        private static readonly MetadataReference SystemCoreReference = MetadataReference.CreateFromFile(typeof(Enumerable).Assembly.Location);
+        private static readonly MetadataReference CSharpSymbolsReference = MetadataReference.CreateFromFile(typeof(CSharpCompilation).Assembly.Location);
+        private static readonly MetadataReference CodeAnalysisReference = MetadataReference.CreateFromFile(typeof(Compilation).Assembly.Location);
+
+        internal static string DefaultFilePathPrefix = "Test";
+        internal static string CSharpDefaultFileExt = "cs";
+        internal static string VisualBasicDefaultExt = "vb";
+        internal static string TestProjectName = "TestProject";
+
+        #region  Get Diagnostics
+
+        /// <summary>
+        /// Given classes in the form of strings, their language, and an IDiagnosticAnalyzer to apply to it, return the diagnostics found in the string after converting it to a document.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source classes are in</param>
+        /// <param name="analyzer">The analyzer to be run on the sources</param>
+        /// <returns>An IEnumerable of Diagnostics that surfaced in the source code, sorted by Location</returns>
+        private static Diagnostic[] GetSortedDiagnostics(string[] sources, string language, DiagnosticAnalyzer analyzer)
+        {
+            return GetSortedDiagnosticsFromDocuments(analyzer, GetDocuments(sources, language));
+        }
+
+        /// <summary>
+        /// Given an analyzer and a document to apply it to, run the analyzer and gather an array of diagnostics found in it.
+        /// The returned diagnostics are then ordered by location in the source document.
+        /// </summary>
+        /// <param name="analyzer">The analyzer to run on the documents</param>
+        /// <param name="documents">The Documents that the analyzer will be run on</param>
+        /// <returns>An IEnumerable of Diagnostics that surfaced in the source code, sorted by Location</returns>
+        protected static Diagnostic[] GetSortedDiagnosticsFromDocuments(DiagnosticAnalyzer analyzer, Document[] documents)
+        {
+            var projects = new HashSet<Project>();
+            foreach (var document in documents)
+            {
+                projects.Add(document.Project);
+            }
+
+            var diagnostics = new List<Diagnostic>();
+            foreach (var project in projects)
+            {
+                var compilationWithAnalyzers = project.GetCompilationAsync().Result.WithAnalyzers(ImmutableArray.Create(analyzer));
+                var diags = compilationWithAnalyzers.GetAnalyzerDiagnosticsAsync().Result;
+                foreach (var diag in diags)
+                {
+                    if (diag.Location == Location.None || diag.Location.IsInMetadata)
+                    {
+                        diagnostics.Add(diag);
+                    }
+                    else
+                    {
+                        for (int i = 0; i < documents.Length; i++)
+                        {
+                            var document = documents[i];
+                            var tree = document.GetSyntaxTreeAsync().Result;
+                            if (tree == diag.Location.SourceTree)
+                            {
+                                diagnostics.Add(diag);
+                            }
+                        }
+                    }
+                }
+            }
+
+            var results = SortDiagnostics(diagnostics);
+            diagnostics.Clear();
+            return results;
+        }
+
+        /// <summary>
+        /// Sort diagnostics by location in source document
+        /// </summary>
+        /// <param name="diagnostics">The list of Diagnostics to be sorted</param>
+        /// <returns>An IEnumerable containing the Diagnostics in order of Location</returns>
+        private static Diagnostic[] SortDiagnostics(IEnumerable<Diagnostic> diagnostics)
+        {
+            return diagnostics.OrderBy(d => d.Location.SourceSpan.Start).ToArray();
+        }
+
+        #endregion
+
+        #region Set up compilation and documents
+        /// <summary>
+        /// Given an array of strings as sources and a language, turn them into a project and return the documents and spans of it.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Tuple containing the Documents produced from the sources and their TextSpans if relevant</returns>
+        private static Document[] GetDocuments(string[] sources, string language)
+        {
+            if (language != LanguageNames.CSharp && language != LanguageNames.VisualBasic)
+            {
+                throw new ArgumentException("Unsupported Language");
+            }
+
+            var project = CreateProject(sources, language);
+            var documents = project.Documents.ToArray();
+
+            if (sources.Length != documents.Length)
+            {
+                throw new InvalidOperationException("Amount of sources did not match amount of Documents created");
+            }
+
+            return documents;
+        }
+
+        /// <summary>
+        /// Create a Document from a string through creating a project that contains it.
+        /// </summary>
+        /// <param name="source">Classes in the form of a string</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Document created from the source string</returns>
+        protected static Document CreateDocument(string source, string language = LanguageNames.CSharp)
+        {
+            return CreateProject(new[] { source }, language).Documents.First();
+        }
+
+        /// <summary>
+        /// Create a project using the inputted strings as sources.
+        /// </summary>
+        /// <param name="sources">Classes in the form of strings</param>
+        /// <param name="language">The language the source code is in</param>
+        /// <returns>A Project created out of the Documents created from the source strings</returns>
+        private static Project CreateProject(string[] sources, string language = LanguageNames.CSharp)
+        {
+            string fileNamePrefix = DefaultFilePathPrefix;
+            string fileExt = language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt;
+
+            var projectId = ProjectId.CreateNewId(debugName: TestProjectName);
+
+            var solution = new AdhocWorkspace()
+                .CurrentSolution
+                .AddProject(projectId, TestProjectName, TestProjectName, language)
+                .AddMetadataReference(projectId, CorlibReference)
+                .AddMetadataReference(projectId, SystemCoreReference)
+                .AddMetadataReference(projectId, CSharpSymbolsReference)
+                .AddMetadataReference(projectId, CodeAnalysisReference);
+
+            int count = 0;
+            foreach (var source in sources)
+            {
+                var newFileName = fileNamePrefix + count + "." + fileExt;
+                var documentId = DocumentId.CreateNewId(projectId, debugName: newFileName);
+                solution = solution.AddDocument(documentId, newFileName, SourceText.From(source));
+                count++;
+            }
+            return solution.GetProject(projectId);
+        }
+        #endregion
+    }
+}
+
diff --git a/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.cs b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.cs
new file mode 100644
index 00000000..edaaabd4
--- /dev/null
+++ b/src/SMAPI.ModBuildConfig.Analyzer.Tests/Framework/DiagnosticVerifier.cs
@@ -0,0 +1,248 @@
+// <generated />
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Diagnostics;
+using NUnit.Framework;
+
+namespace SMAPI.ModBuildConfig.Analyzer.Tests.Framework
+{
+    /// <summary>
+    /// Superclass of all Unit Tests for DiagnosticAnalyzers
+    /// </summary>
+    public abstract partial class DiagnosticVerifier
+    {
+        #region To be implemented by Test classes
+        /// <summary>
+        /// Get the CSharp analyzer being tested - to be implemented in non-abstract class
+        /// </summary>
+        protected virtual DiagnosticAnalyzer GetCSharpDiagnosticAnalyzer()
+        {
+            return null;
+        }
+
+        /// <summary>
+        /// Get the Visual Basic analyzer being tested (C#) - to be implemented in non-abstract class
+        /// </summary>
+        protected virtual DiagnosticAnalyzer GetBasicDiagnosticAnalyzer()
+        {
+            return null;
+        }
+        #endregion
+
+        #region Verifier wrappers
+
+        /// <summary>
+        /// Called to test a C# DiagnosticAnalyzer when applied on the single inputted string as a source
+        /// Note: input a DiagnosticResult for each Diagnostic expected
+        /// </summary>
+        /// <param name="source">A class in the form of a string to run the analyzer on</param>
+        /// <param name="expected"> DiagnosticResults that should appear after the analyzer is run on the source</param>
+        protected void VerifyCSharpDiagnostic(string source, params DiagnosticResult[] expected)
+        {
+            VerifyDiagnostics(new[] { source }, LanguageNames.CSharp, GetCSharpDiagnosticAnalyzer(), expected);
+        }
+
+        /// <summary>
+        /// Called to test a C# DiagnosticAnalyzer when applied on the inputted strings as a source
+        /// Note: input a DiagnosticResult for each Diagnostic expected
+        /// </summary>
+        /// <param name="sources">An array of strings to create source documents from to run the analyzers on</param>
+        /// <param name="expected">DiagnosticResults that should appear after the analyzer is run on the sources</param>
+        protected void VerifyCSharpDiagnostic(string[] sources, params DiagnosticResult[] expected)
+        {
+            VerifyDiagnostics(sources, LanguageNames.CSharp, GetCSharpDiagnosticAnalyzer(), expected);
+        }
+
+        /// <summary>
+        /// General method that gets a collection of actual diagnostics found in the source after the analyzer is run, 
+        /// then verifies each of them.
+        /// </summary>
+        /// <param name="sources">An array of strings to create source documents from to run the analyzers on</param>
+        /// <param name="language">The language of the classes represented by the source strings</param>
+        /// <param name="analyzer">The analyzer to be run on the source code</param>
+        /// <param name="expected">DiagnosticResults that should appear after the analyzer is run on the sources</param>
+        private void VerifyDiagnostics(string[] sources, string language, DiagnosticAnalyzer analyzer, params DiagnosticResult[] expected)
+        {
+            var diagnostics = GetSortedDiagnostics(sources, language, analyzer);
+            VerifyDiagnosticResults(diagnostics, analyzer, expected);
+        }
+
+        #endregion
+
+        #region Actual comparisons and verifications
+        /// <summary>
+        /// Checks each of the actual Diagnostics found and compares them with the corresponding DiagnosticResult in the array of expected results.
+        /// Diagnostics are considered equal only if the DiagnosticResultLocation, Id, Severity, and Message of the DiagnosticResult match the actual diagnostic.
+        /// </summary>
+        /// <param name="actualResults">The Diagnostics found by the compiler after running the analyzer on the source code</param>
+        /// <param name="analyzer">The analyzer that was being run on the sources</param>
+        /// <param name="expectedResults">Diagnostic Results that should have appeared in the code</param>
+        private static void VerifyDiagnosticResults(IEnumerable<Diagnostic> actualResults, DiagnosticAnalyzer analyzer, params DiagnosticResult[] expectedResults)
+        {
+            int expectedCount = expectedResults.Count();
+            int actualCount = actualResults.Count();
+
+            if (expectedCount != actualCount)
+            {
+                string diagnosticsOutput = actualResults.Any() ? FormatDiagnostics(analyzer, actualResults.ToArray()) : "    NONE.";
+
+                Assert.IsTrue(false,
+                    string.Format("Mismatch between number of diagnostics returned, expected \"{0}\" actual \"{1}\"\r\n\r\nDiagnostics:\r\n{2}\r\n", expectedCount, actualCount, diagnosticsOutput));
+            }
+
+            for (int i = 0; i < expectedResults.Length; i++)
+            {
+                var actual = actualResults.ElementAt(i);
+                var expected = expectedResults[i];
+
+                if (expected.Line == -1 && expected.Column == -1)
+                {
+                    if (actual.Location != Location.None)
+                    {
+                        Assert.IsTrue(false,
+                            string.Format("Expected:\nA project diagnostic with No location\nActual:\n{0}",
+                            FormatDiagnostics(analyzer, actual)));
+                    }
+                }
+                else
+                {
+                    VerifyDiagnosticLocation(analyzer, actual, actual.Location, expected.Locations.First());
+                    var additionalLocations = actual.AdditionalLocations.ToArray();
+
+                    if (additionalLocations.Length != expected.Locations.Length - 1)
+                    {
+                        Assert.IsTrue(false,
+                            string.Format("Expected {0} additional locations but got {1} for Diagnostic:\r\n    {2}\r\n",
+                                expected.Locations.Length - 1, additionalLocations.Length,
+                                FormatDiagnostics(analyzer, actual)));
+                    }
+
+                    for (int j = 0; j < additionalLocations.Length; ++j)
+                    {
+                        VerifyDiagnosticLocation(analyzer, actual, additionalLocations[j], expected.Locations[j + 1]);
+                    }
+                }
+
+                if (actual.Id != expected.Id)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic id to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Id, actual.Id, FormatDiagnostics(analyzer, actual)));
+                }
+
+                if (actual.Severity != expected.Severity)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic severity to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Severity, actual.Severity, FormatDiagnostics(analyzer, actual)));
+                }
+
+                if (actual.GetMessage() != expected.Message)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic message to be \"{0}\" was \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Message, actual.GetMessage(), FormatDiagnostics(analyzer, actual)));
+                }
+            }
+        }
+
+        /// <summary>
+        /// Helper method to VerifyDiagnosticResult that checks the location of a diagnostic and compares it with the location in the expected DiagnosticResult.
+        /// </summary>
+        /// <param name="analyzer">The analyzer that was being run on the sources</param>
+        /// <param name="diagnostic">The diagnostic that was found in the code</param>
+        /// <param name="actual">The Location of the Diagnostic found in the code</param>
+        /// <param name="expected">The DiagnosticResultLocation that should have been found</param>
+        private static void VerifyDiagnosticLocation(DiagnosticAnalyzer analyzer, Diagnostic diagnostic, Location actual, DiagnosticResultLocation expected)
+        {
+            var actualSpan = actual.GetLineSpan();
+
+            Assert.IsTrue(actualSpan.Path == expected.Path || (actualSpan.Path != null && actualSpan.Path.Contains("Test0.") && expected.Path.Contains("Test.")),
+                string.Format("Expected diagnostic to be in file \"{0}\" was actually in file \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                    expected.Path, actualSpan.Path, FormatDiagnostics(analyzer, diagnostic)));
+
+            var actualLinePosition = actualSpan.StartLinePosition;
+
+            // Only check line position if there is an actual line in the real diagnostic
+            if (actualLinePosition.Line > 0)
+            {
+                if (actualLinePosition.Line + 1 != expected.Line)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic to be on line \"{0}\" was actually on line \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Line, actualLinePosition.Line + 1, FormatDiagnostics(analyzer, diagnostic)));
+                }
+            }
+
+            // Only check column position if there is an actual column position in the real diagnostic
+            if (actualLinePosition.Character > 0)
+            {
+                if (actualLinePosition.Character + 1 != expected.Column)
+                {
+                    Assert.IsTrue(false,
+                        string.Format("Expected diagnostic to start at column \"{0}\" was actually at column \"{1}\"\r\n\r\nDiagnostic:\r\n    {2}\r\n",
+                            expected.Column, actualLinePosition.Character + 1, FormatDiagnostics(analyzer, diagnostic)));
+                }
+            }
+        }
+        #endregion
+
+        #region Formatting Diagnostics
+        /// <summary>
+        /// Helper method to format a Diagnostic into an easily readable string
+        /// </summary>
+        /// <param name="analyzer">The analyzer that this verifier tests</param>
+        /// <param name="diagnostics">The Diagnostics to be formatted</param>
+        /// <returns>The Diagnostics formatted as a string</returns>
+        private static string FormatDiagnostics(DiagnosticAnalyzer analyzer, params Diagnostic[] diagnostics)
+        {
+            var builder = new StringBuilder();
+            for (int i = 0; i < diagnostics.Length; ++i)
+            {
+                builder.AppendLine("// " + diagnostics[i].ToString());
+
+                var analyzerType = analyzer.GetType();
+                var rules = analyzer.SupportedDiagnostics;
+
+                foreach (var rule in rules)
+                {
+                    if (rule != null && rule.Id == diagnostics[i].Id)
+                    {
+                        var location = diagnostics[i].Location;
+                        if (location == Location.None)
+                        {
+                            builder.AppendFormat("GetGlobalResult({0}.{1})", analyzerType.Name, rule.Id);
+                        }
+                        else
+                        {
+                            Assert.IsTrue(location.IsInSource,
+                                $"Test base does not currently handle diagnostics in metadata locations. Diagnostic in metadata: {diagnostics[i]}\r\n");
+
+                            string resultMethodName = diagnostics[i].Location.SourceTree.FilePath.EndsWith(".cs") ? "GetCSharpResultAt" : "GetBasicResultAt";
+                            var linePosition = diagnostics[i].Location.GetLineSpan().StartLinePosition;
+
+                            builder.AppendFormat("{0}({1}, {2}, {3}.{4})",
+                                resultMethodName,
+                                linePosition.Line + 1,
+                                linePosition.Character + 1,
+                                analyzerType.Name,
+                                rule.Id);
+                        }
+
+                        if (i != diagnostics.Length - 1)
+                        {
+                            builder.Append(',');
+                        }
+
+                        builder.AppendLine();
+                        break;
+                    }
+                }
+            }
+            return builder.ToString();
+        }
+        #endregion
+    }
+}