Add output capture and more test reports

10 files changed, 137 insertions, 44 deletions

diff --git a/build.gradle.kts b/build.gradle.kts
index 48834ea..c0e28f1 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -13,3 +13,32 @@ publishing {
         }
     }
 }
+
+val testReportFile = layout.buildDirectory.file("test-results/nealisp/results.xml")
+
+tasks.create("testLisps", JavaExec::class) {
+    javaLauncher.set(javaToolchains.launcherFor(java.toolchain))
+    classpath(sourceSets.test.get().runtimeClasspath)
+    mainClass.set("TestMain")
+    dependsOn(tasks.testClasses)
+    dependsOn(tasks.processTestResources)
+    outputs.file(testReportFile)
+    systemProperty("test.report", testReportFile.map { it.asFile.absolutePath }.get())
+    systemProperty("test.suites", "test")
+    systemProperty("test.imports", "secondary")
+    group = "verification"
+}
+
+
+tasks.create("testLispsHtml", Exec::class) {
+    dependsOn("testLisps")
+    executable("xunit-viewer")
+    inputs.file(testReportFile)
+    val testReportHtmlFile = layout.buildDirectory.file("reports/nealisp/tests/index.html")
+    outputs.file(testReportHtmlFile)
+    args(
+        "-r", testReportFile.map { it.asFile.absolutePath }.get(),
+        "-o", testReportHtmlFile.map { it.asFile.absolutePath }.get()
+    )
+    group = "verification"
+}
diff --git a/src/CoreBindings.kt b/src/CoreBindings.kt
index d88c950..7db5175 100644
--- a/src/CoreBindings.kt
+++ b/src/CoreBindings.kt
@@ -3,14 +3,14 @@ package moe.nea.lisp
 object CoreBindings {
     val def = LispData.externalRawCall("def") { context, callsite, stackFrame, args ->
         if (args.size != 2) {
-            return@externalRawCall context.reportError("Function define expects exactly two arguments", callsite)
+            return@externalRawCall stackFrame.reportError("Function define expects exactly two arguments", callsite)
         }
         val (name, value) = args
         if (name !is LispAst.Reference) {
-            return@externalRawCall context.reportError("Define expects a name as first argument", name)
+            return@externalRawCall stackFrame.reportError("Define expects a name as first argument", name)
         }
         if (name.label in stackFrame.variables) {
-            return@externalRawCall context.reportError("Cannot redefine value in local context", name)
+            return@externalRawCall stackFrame.reportError("Cannot redefine value in local context", name)
         }
         return@externalRawCall stackFrame.setValueLocal(name.label, context.resolveValue(stackFrame, value))
     }
@@ -26,13 +26,13 @@ object CoreBindings {
 
     val ifFun = LispData.externalRawCall("if") { context, callsite, stackFrame, args ->
         if (args.size != 3) {
-            return@externalRawCall context.reportError("if requires 3 arguments", callsite)
+            return@externalRawCall stackFrame.reportError("if requires 3 arguments", callsite)
         }
         val (cond, ifTrue, ifFalse) = args
 
         val c = isTruthy(context.resolveValue(stackFrame, cond))
         if (c == null) {
-            return@externalRawCall context.reportError("Non boolean value $c used as condition for if", cond)
+            return@externalRawCall stackFrame.reportError("Non boolean value $c used as condition for if", cond)
         }
         if (c) {
             return@externalRawCall context.resolveValue(stackFrame, ifTrue)
@@ -54,46 +54,46 @@ object CoreBindings {
 
     val lambda = LispData.externalRawCall("lambda") { context, callsite, stackFrame, args ->
         if (args.size != 2) {
-            return@externalRawCall context.reportError("Lambda needs exactly 2 arguments", callsite)
+            return@externalRawCall stackFrame.reportError("Lambda needs exactly 2 arguments", callsite)
         }
         val (argumentNames, body) = args
         if (argumentNames !is LispAst.Parenthesis) {
-            return@externalRawCall context.reportError("Lambda has invalid argument declaration", argumentNames)
+            return@externalRawCall stackFrame.reportError("Lambda has invalid argument declaration", argumentNames)
         }
         val argumentNamesString = argumentNames.items.map {
             val ref = it as? LispAst.Reference
             if (ref == null) {
-                return@externalRawCall context.reportError("Lambda has invalid argument declaration", it)
+                return@externalRawCall stackFrame.reportError("Lambda has invalid argument declaration", it)
             }
             ref.label
         }
         if (body !is LispAst.Parenthesis) {
-            return@externalRawCall context.reportError("Lambda has invalid body declaration", body)
+            return@externalRawCall stackFrame.reportError("Lambda has invalid body declaration", body)
         }
         LispData.createLambda(stackFrame, argumentNamesString, body)
     }
 
     val defun = LispData.externalRawCall("defun") { context, callSite, stackFrame, lispAsts ->
         if (lispAsts.size != 3) {
-            return@externalRawCall context.reportError("Invalid function definition", callSite)
+            return@externalRawCall stackFrame.reportError("Invalid function definition", callSite)
         }
         val (name, args, body) = lispAsts
         if (name !is LispAst.Reference) {
-            return@externalRawCall context.reportError("Invalid function definition name", name)
+            return@externalRawCall stackFrame.reportError("Invalid function definition name", name)
         }
         if (name.label in stackFrame.variables) {
-            return@externalRawCall context.reportError("Cannot redefine function in local context", name)
+            return@externalRawCall stackFrame.reportError("Cannot redefine function in local context", name)
         }
         if (args !is LispAst.Parenthesis) {
-            return@externalRawCall context.reportError("Invalid function definition arguments", args)
+            return@externalRawCall stackFrame.reportError("Invalid function definition arguments", args)
         }
         val argumentNames = args.items.map {
             val ref = it as? LispAst.Reference
-                ?: return@externalRawCall context.reportError("Invalid function definition argument name", it)
+                ?: return@externalRawCall stackFrame.reportError("Invalid function definition argument name", it)
             ref.label
         }
         if (body !is LispAst.Parenthesis) {
-            return@externalRawCall context.reportError("Invalid function definition body", body)
+            return@externalRawCall stackFrame.reportError("Invalid function definition body", body)
         }
         return@externalRawCall stackFrame.setValueLocal(
             name.label,
@@ -105,7 +105,7 @@ object CoreBindings {
         for (arg in args) {
             lastResult = context.executeLisp(stackFrame, arg)
         }
-        lastResult ?: context.reportError("Seq cannot be invoked with 0 argumens", callsite)
+        lastResult ?: stackFrame.reportError("Seq cannot be invoked with 0 argumens", callsite)
     }
 
     internal fun stringify(thing: LispData): String {
@@ -125,8 +125,10 @@ object CoreBindings {
         LispData.LispString(args.joinToString(" ") { stringify(it) })
     }
 
-    val debuglog = LispData.externalCall("debuglog") { args, reportError ->
-        println(args.joinToString(" ") { stringify(it) })
+    val debuglog = LispData.externalRawCall("debuglog") { context, callsite, stackFrame, args ->
+        OutputCapture.print(
+            stackFrame,
+            args.joinToString(" ", postfix = "\n") { stringify(context.resolveValue(stackFrame, it)) })
         LispData.LispNil
     }
     val add = LispData.externalCall("add") { args, reportError ->
@@ -192,13 +194,13 @@ object CoreBindings {
     }
     val import = LispData.externalRawCall("import") { context, callsite, stackFrame, args ->
         if (args.size != 1) {
-            return@externalRawCall context.reportError("import needs at least one argument", callsite)
+            return@externalRawCall stackFrame.reportError("import needs at least one argument", callsite)
         }
         // TODO: aliased / namespaced imports
         val moduleName = when (val moduleObject = context.resolveValue(stackFrame, args[0])) {
             is LispData.Atom -> moduleObject.label
             is LispData.LispString -> moduleObject.string
-            else -> return@externalRawCall context.reportError("import needs a string or atom as argument", callsite)
+            else -> return@externalRawCall stackFrame.reportError("import needs a string or atom as argument", callsite)
         }
         context.importModule(moduleName, stackFrame, callsite)
         return@externalRawCall LispData.LispNil
diff --git a/src/LispData.kt b/src/LispData.kt
index 8eac3f7..97fcc36 100644
--- a/src/LispData.kt
+++ b/src/LispData.kt
@@ -43,7 +43,7 @@ sealed class LispData {
                         args.drop(argNames.size - 1).map { executionContext.resolveValue(stackFrame, it) })
                 )
             } else if (argNames.size != args.size) {
-                return executionContext.reportError(
+                return stackFrame.reportError(
                     "Expected ${argNames.size} arguments, got ${args.size} instead",
                     callsite
                 )
@@ -85,7 +85,7 @@ sealed class LispData {
                     args: List<LispAst.LispNode>
                 ): LispData {
                     val mappedArgs = args.map { executionContext.resolveValue(stackFrame, it) }
-                    return callable.invoke(mappedArgs) { executionContext.reportError(it, callsite) }
+                    return callable.invoke(mappedArgs) { stackFrame.reportError(it, callsite) }
                 }
             }
         }
diff --git a/src/LispExecutionContext.kt b/src/LispExecutionContext.kt
index 9c2d9f8..6398cc4 100644
--- a/src/LispExecutionContext.kt
+++ b/src/LispExecutionContext.kt
@@ -7,12 +7,6 @@ class LispExecutionContext() {
     val unloadedModules = mutableMapOf<String, LispAst.Program>()
     val modules = mutableMapOf<String, Map<String, LispData>>()
 
-    fun reportError(name: String, position: HasLispPosition): LispData.LispNil {
-        println("Error: $name ${position.position}")
-        return LispData.LispNil
-    }
-
-
     fun genBindings(): StackFrame {
         return StackFrame(rootStackFrame)
     }
@@ -36,8 +30,10 @@ class LispExecutionContext() {
         isWhitelist: Boolean = false
     ): TestFramework.TestSuite {
         val testSuite = TestFramework.setup(stackFrame, name, testList, isWhitelist)
+        val output = OutputCapture.captureOutput(stackFrame)
         executeProgram(stackFrame, program)
         testSuite.isTesting = false
+        testSuite.otherOutput = output.asString
         return testSuite
     }
 
@@ -54,7 +50,7 @@ class LispExecutionContext() {
         if (exports == null) {
             val module = unloadedModules[moduleName]
             if (module == null) {
-                reportError("Could not find module $moduleName", position)
+                into.reportError("Could not find module $moduleName", position)
                 return
             }
             exports = realizeModule(moduleName)
@@ -70,7 +66,7 @@ class LispExecutionContext() {
         stackFrame.setValueLocal("export", LispData.externalRawCall("export") { context, callsite, stackFrame, args ->
             args.forEach { name ->
                 if (name !is LispAst.Reference) {
-                    context.reportError("Invalid export", name)
+                    stackFrame.reportError("Invalid export", name)
                     return@forEach
                 }
                 map[name.label] = context.resolveValue(stackFrame, name)
@@ -94,7 +90,7 @@ class LispExecutionContext() {
         when (node) {
             is LispAst.Parenthesis -> {
                 val first = node.items.firstOrNull()
-                    ?: return reportError("Cannot execute empty parenthesis ()", node)
+                    ?: return stackFrame.reportError("Cannot execute empty parenthesis ()", node)
 
                 val rest = node.items.drop(1)
                 return when (val resolvedValue = resolveValue(stackFrame, first)) {
@@ -102,12 +98,12 @@ class LispExecutionContext() {
                         resolvedValue.execute(this, node, stackFrame, rest)
                     }
 
-                    else -> reportError("Cannot evaluate expression of type $resolvedValue", node)
+                    else -> stackFrame.reportError("Cannot evaluate expression of type $resolvedValue", node)
                 }
 
             }
 
-            else -> return reportError("Expected invocation", node)
+            else -> return stackFrame.reportError("Expected invocation", node)
         }
     }
 
@@ -116,7 +112,7 @@ class LispExecutionContext() {
             is LispAst.Atom -> LispData.Atom(node.label)
             is LispAst.Parenthesis -> executeLisp(stackFrame, node)
             is LispAst.Reference -> stackFrame.resolveReference(node.label)
-                ?: reportError("Could not resolve variable ${node.label}", node)
+                ?: stackFrame.reportError("Could not resolve variable ${node.label}", node)
 
             is LispAst.NumberLiteral -> LispData.LispNumber(node.numberValue)
             is LispAst.StringLiteral -> LispData.LispString(node.parsedString)
diff --git a/src/OutputCapture.kt b/src/OutputCapture.kt
new file mode 100644
index 0000000..7fc8849
--- /dev/null
+++ b/src/OutputCapture.kt
@@ -0,0 +1,23 @@
+package moe.nea.lisp
+
+object OutputCapture {
+    data class CapturedOutput(
+        internal var string: StringBuilder,
+    ) {
+        val asString get() = string.toString()
+    }
+
+    object Meta : StackFrame.MetaKey<CapturedOutput>
+
+    fun captureOutput(stackFrame: StackFrame): CapturedOutput {
+        val output = CapturedOutput(StringBuilder())
+        stackFrame.setMeta(Meta, output)
+        return output
+    }
+
+    fun print(stackFrame: StackFrame, text: String) {
+        val output = stackFrame.getMeta(Meta)
+        output?.string?.append(text)
+        print(text)
+    }
+}
\ No newline at end of file
diff --git a/src/StackFrame.kt b/src/StackFrame.kt
index 9a0832f..c91299f 100644
--- a/src/StackFrame.kt
+++ b/src/StackFrame.kt
@@ -12,6 +12,11 @@ class StackFrame(val parent: StackFrame?) {
         return meta[key]?.let { it as T } ?: parent?.meta?.get(key)?.let { it as T }
     }
 
+    fun reportError(name: String, position: HasLispPosition): LispData.LispNil {
+        OutputCapture.print(this, "Error: $name ${position.position}\n")
+        return LispData.LispNil
+    }
+
     fun <T : Any> setMeta(key: MetaKey<T>, value: T) {
         meta[key] = value
     }
diff --git a/src/TestFramework.kt b/src/TestFramework.kt
index b35020c..7c2b0cc 100644
--- a/src/TestFramework.kt
+++ b/src/TestFramework.kt
@@ -12,6 +12,7 @@ object TestFramework {
         val name: String,
         val failures: List<TestFailure>,
         val wasSkipped: Boolean,
+        val stdout: String,
     )
 
     data class TestSuite(
@@ -21,6 +22,7 @@ object TestFramework {
         val allTests: MutableList<TestResult>,
         val testList: List<String>,
         val isWhitelist: Boolean,
+        var otherOutput: String = "",
     )
 
     data class ActiveTest(
@@ -42,7 +44,7 @@ object TestFramework {
         val message = CoreBindings.stringify(args.singleOrNull() ?: return@externalCall reportError("Needs a message"))
         LispData.externalRawCall("ntest.fail.r") { context, callsite, stackFrame, args ->
             val activeTest = stackFrame.getMeta(ActiveTestMeta)
-                ?: return@externalRawCall context.reportError("No active test", callsite)
+                ?: return@externalRawCall stackFrame.reportError("No active test", callsite)
             activeTest.currentFailures.add(TestFailure(callsite, message))
             return@externalRawCall LispData.LispNil
         }
@@ -62,7 +64,7 @@ object TestFramework {
         val meta = stackFrame.getMeta(TestSuiteMeta) ?: return
         if (!meta.isTesting) return
         if (args.size != 2) {
-            context.reportError("Test case needs to be defined by a name and an executable", callsite)
+            stackFrame.reportError("Test case needs to be defined by a name and an executable", callsite)
             return
         }
         val (name, prog) = args
@@ -70,19 +72,20 @@ object TestFramework {
             is LispData.Atom -> n.label
             is LispData.LispString -> n.string
             else -> {
-                context.reportError("Test case needs an atom or string as name", name)
+                stackFrame.reportError("Test case needs an atom or string as name", name)
                 return
             }
         }
         if (testName in meta.testList != meta.isWhitelist) {
-            meta.allTests.add(TestResult(testName, listOf(), true))
+            meta.allTests.add(TestResult(testName, listOf(), true, ""))
             return
         }
         val child = stackFrame.fork()
         val test = ActiveTest(testName, mutableListOf(), false, meta)
+        val output = OutputCapture.captureOutput(child)
         child.setMeta(ActiveTestMeta, test)
         context.resolveValue(child, prog)
-        meta.allTests.add(TestResult(test.testName, test.currentFailures, false))
+        meta.allTests.add(TestResult(test.testName, test.currentFailures, false, output.asString))
     }
 
     fun setup(stackFrame: StackFrame, name: String, testList: List<String>, isWhitelist: Boolean): TestSuite {
diff --git a/src/TestResultFormatter.kt b/src/TestResultFormatter.kt
index 8a5af9c..3a19444 100644
--- a/src/TestResultFormatter.kt
+++ b/src/TestResultFormatter.kt
@@ -30,6 +30,9 @@ class TestResultFormatter(private val writer: XMLStreamWriter) {
         testSuite.allTests.forEach {
             writeTestCase(it)
         }
+        writer.writeStartElement("system-out")
+        writer.writeCData(testSuite.otherOutput)
+        writer.writeEndElement()
 
         writer.writeEndElement()
     }
@@ -45,6 +48,9 @@ class TestResultFormatter(private val writer: XMLStreamWriter) {
         for (fail in test.failures) {
             writeFailure(fail)
         }
+        writer.writeStartElement("system-out")
+        writer.writeCData(test.stdout)
+        writer.writeEndElement()
 
         writer.writeEndElement()
     }
diff --git a/test/res/test.lisp b/test/res/test.lisp
index 79d259d..9def7f3 100644
--- a/test/res/test.lisp
+++ b/test/res/test.lisp
@@ -25,11 +25,10 @@
 (debuglog "============")
 (debuglog "Running tests")
 (import :test)
-(test.test "Funny test" (seq
-    (debuglog "Funny test running")
-    ((test.assert false "False failed"))
-    ((test.assert-eq "funny" "unfunny"))
-    ((test.fail "Test failed"))))
+(test.test "unfunny test" (seq
+    (debuglog "Funny test not running")
+    ((test.assert-eq "unfunny" "funny"))
+))
 
 (test.test "Test equality" (seq
     ((test.assert-eq false false))
diff --git a/test/src/TestMain.kt b/test/src/TestMain.kt
new file mode 100644
index 0000000..fc95780
--- /dev/null
+++ b/test/src/TestMain.kt
@@ -0,0 +1,30 @@
+import moe.nea.lisp.LispExecutionContext
+import moe.nea.lisp.LispParser
+import moe.nea.lisp.TestResultFormatter
+import java.io.File
+import javax.xml.stream.XMLOutputFactory
+
+object TestMain {
+    @JvmStatic
+    fun main(args: Array<String>) {
+        val reportFile = System.getProperty("test.report")
+        val modulesToTest = System.getProperty("test.suites").split(":")
+        val modulePath = System.getProperty("test.imports").split(":")
+        val executionContext = LispExecutionContext()
+        executionContext.setupStandardBindings()
+        modulePath.forEach {
+            executionContext.registerModule(it, LispParser.parse(File(T::class.java.getResource("/$it.lisp")!!.file)))
+        }
+        val allResults = modulesToTest.map {
+            executionContext.runTests(
+                LispParser.parse(File(T::class.java.getResource("/$it.lisp")!!.file)),
+                it,
+            )
+        }
+        val w = XMLOutputFactory.newFactory()
+            .createXMLStreamWriter(File(reportFile).bufferedWriter())
+        TestResultFormatter.write(w, allResults)
+        w.close()
+
+    }
+}
\ No newline at end of file