Add output capture and more test reports

7 files changed, 74 insertions, 39 deletions

diff --git a/src/CoreBindings.kt b/src/CoreBindings.kt
index d88c950..7db5175 100644
--- a/src/CoreBindings.kt
+++ b/src/CoreBindings.kt
@@ -3,14 +3,14 @@ package moe.nea.lisp
 object CoreBindings {
     val def = LispData.externalRawCall("def") { context, callsite, stackFrame, args ->
         if (args.size != 2) {
-            return@externalRawCall context.reportError("Function define expects exactly two arguments", callsite)
+            return@externalRawCall stackFrame.reportError("Function define expects exactly two arguments", callsite)
         }
         val (name, value) = args
         if (name !is LispAst.Reference) {
-            return@externalRawCall context.reportError("Define expects a name as first argument", name)
+            return@externalRawCall stackFrame.reportError("Define expects a name as first argument", name)
         }
         if (name.label in stackFrame.variables) {
-            return@externalRawCall context.reportError("Cannot redefine value in local context", name)
+            return@externalRawCall stackFrame.reportError("Cannot redefine value in local context", name)
         }
         return@externalRawCall stackFrame.setValueLocal(name.label, context.resolveValue(stackFrame, value))
     }
@@ -26,13 +26,13 @@ object CoreBindings {
 
     val ifFun = LispData.externalRawCall("if") { context, callsite, stackFrame, args ->
         if (args.size != 3) {
-            return@externalRawCall context.reportError("if requires 3 arguments", callsite)
+            return@externalRawCall stackFrame.reportError("if requires 3 arguments", callsite)
         }
         val (cond, ifTrue, ifFalse) = args
 
         val c = isTruthy(context.resolveValue(stackFrame, cond))
         if (c == null) {
-            return@externalRawCall context.reportError("Non boolean value $c used as condition for if", cond)
+            return@externalRawCall stackFrame.reportError("Non boolean value $c used as condition for if", cond)
         }
         if (c) {
             return@externalRawCall context.resolveValue(stackFrame, ifTrue)
@@ -54,46 +54,46 @@ object CoreBindings {
 
     val lambda = LispData.externalRawCall("lambda") { context, callsite, stackFrame, args ->
         if (args.size != 2) {
-            return@externalRawCall context.reportError("Lambda needs exactly 2 arguments", callsite)
+            return@externalRawCall stackFrame.reportError("Lambda needs exactly 2 arguments", callsite)
         }
         val (argumentNames, body) = args
         if (argumentNames !is LispAst.Parenthesis) {
-            return@externalRawCall context.reportError("Lambda has invalid argument declaration", argumentNames)
+            return@externalRawCall stackFrame.reportError("Lambda has invalid argument declaration", argumentNames)
         }
         val argumentNamesString = argumentNames.items.map {
             val ref = it as? LispAst.Reference
             if (ref == null) {
-                return@externalRawCall context.reportError("Lambda has invalid argument declaration", it)
+                return@externalRawCall stackFrame.reportError("Lambda has invalid argument declaration", it)
             }
             ref.label
         }
         if (body !is LispAst.Parenthesis) {
-            return@externalRawCall context.reportError("Lambda has invalid body declaration", body)
+            return@externalRawCall stackFrame.reportError("Lambda has invalid body declaration", body)
         }
         LispData.createLambda(stackFrame, argumentNamesString, body)
     }
 
     val defun = LispData.externalRawCall("defun") { context, callSite, stackFrame, lispAsts ->
         if (lispAsts.size != 3) {
-            return@externalRawCall context.reportError("Invalid function definition", callSite)
+            return@externalRawCall stackFrame.reportError("Invalid function definition", callSite)
         }
         val (name, args, body) = lispAsts
         if (name !is LispAst.Reference) {
-            return@externalRawCall context.reportError("Invalid function definition name", name)
+            return@externalRawCall stackFrame.reportError("Invalid function definition name", name)
         }
         if (name.label in stackFrame.variables) {
-            return@externalRawCall context.reportError("Cannot redefine function in local context", name)
+            return@externalRawCall stackFrame.reportError("Cannot redefine function in local context", name)
         }
         if (args !is LispAst.Parenthesis) {
-            return@externalRawCall context.reportError("Invalid function definition arguments", args)
+            return@externalRawCall stackFrame.reportError("Invalid function definition arguments", args)
         }
         val argumentNames = args.items.map {
             val ref = it as? LispAst.Reference
-                ?: return@externalRawCall context.reportError("Invalid function definition argument name", it)
+                ?: return@externalRawCall stackFrame.reportError("Invalid function definition argument name", it)
             ref.label
         }
         if (body !is LispAst.Parenthesis) {
-            return@externalRawCall context.reportError("Invalid function definition body", body)
+            return@externalRawCall stackFrame.reportError("Invalid function definition body", body)
         }
         return@externalRawCall stackFrame.setValueLocal(
             name.label,
@@ -105,7 +105,7 @@ object CoreBindings {
         for (arg in args) {
             lastResult = context.executeLisp(stackFrame, arg)
         }
-        lastResult ?: context.reportError("Seq cannot be invoked with 0 argumens", callsite)
+        lastResult ?: stackFrame.reportError("Seq cannot be invoked with 0 argumens", callsite)
     }
 
     internal fun stringify(thing: LispData): String {
@@ -125,8 +125,10 @@ object CoreBindings {
         LispData.LispString(args.joinToString(" ") { stringify(it) })
     }
 
-    val debuglog = LispData.externalCall("debuglog") { args, reportError ->
-        println(args.joinToString(" ") { stringify(it) })
+    val debuglog = LispData.externalRawCall("debuglog") { context, callsite, stackFrame, args ->
+        OutputCapture.print(
+            stackFrame,
+            args.joinToString(" ", postfix = "\n") { stringify(context.resolveValue(stackFrame, it)) })
         LispData.LispNil
     }
     val add = LispData.externalCall("add") { args, reportError ->
@@ -192,13 +194,13 @@ object CoreBindings {
     }
     val import = LispData.externalRawCall("import") { context, callsite, stackFrame, args ->
         if (args.size != 1) {
-            return@externalRawCall context.reportError("import needs at least one argument", callsite)
+            return@externalRawCall stackFrame.reportError("import needs at least one argument", callsite)
         }
         // TODO: aliased / namespaced imports
         val moduleName = when (val moduleObject = context.resolveValue(stackFrame, args[0])) {
             is LispData.Atom -> moduleObject.label
             is LispData.LispString -> moduleObject.string
-            else -> return@externalRawCall context.reportError("import needs a string or atom as argument", callsite)
+            else -> return@externalRawCall stackFrame.reportError("import needs a string or atom as argument", callsite)
         }
         context.importModule(moduleName, stackFrame, callsite)
         return@externalRawCall LispData.LispNil
diff --git a/src/LispData.kt b/src/LispData.kt
index 8eac3f7..97fcc36 100644
--- a/src/LispData.kt
+++ b/src/LispData.kt
@@ -43,7 +43,7 @@ sealed class LispData {
                         args.drop(argNames.size - 1).map { executionContext.resolveValue(stackFrame, it) })
                 )
             } else if (argNames.size != args.size) {
-                return executionContext.reportError(
+                return stackFrame.reportError(
                     "Expected ${argNames.size} arguments, got ${args.size} instead",
                     callsite
                 )
@@ -85,7 +85,7 @@ sealed class LispData {
                     args: List<LispAst.LispNode>
                 ): LispData {
                     val mappedArgs = args.map { executionContext.resolveValue(stackFrame, it) }
-                    return callable.invoke(mappedArgs) { executionContext.reportError(it, callsite) }
+                    return callable.invoke(mappedArgs) { stackFrame.reportError(it, callsite) }
                 }
             }
         }
diff --git a/src/LispExecutionContext.kt b/src/LispExecutionContext.kt
index 9c2d9f8..6398cc4 100644
--- a/src/LispExecutionContext.kt
+++ b/src/LispExecutionContext.kt
@@ -7,12 +7,6 @@ class LispExecutionContext() {
     val unloadedModules = mutableMapOf<String, LispAst.Program>()
     val modules = mutableMapOf<String, Map<String, LispData>>()
 
-    fun reportError(name: String, position: HasLispPosition): LispData.LispNil {
-        println("Error: $name ${position.position}")
-        return LispData.LispNil
-    }
-
-
     fun genBindings(): StackFrame {
         return StackFrame(rootStackFrame)
     }
@@ -36,8 +30,10 @@ class LispExecutionContext() {
         isWhitelist: Boolean = false
     ): TestFramework.TestSuite {
         val testSuite = TestFramework.setup(stackFrame, name, testList, isWhitelist)
+        val output = OutputCapture.captureOutput(stackFrame)
         executeProgram(stackFrame, program)
         testSuite.isTesting = false
+        testSuite.otherOutput = output.asString
         return testSuite
     }
 
@@ -54,7 +50,7 @@ class LispExecutionContext() {
         if (exports == null) {
             val module = unloadedModules[moduleName]
             if (module == null) {
-                reportError("Could not find module $moduleName", position)
+                into.reportError("Could not find module $moduleName", position)
                 return
             }
             exports = realizeModule(moduleName)
@@ -70,7 +66,7 @@ class LispExecutionContext() {
         stackFrame.setValueLocal("export", LispData.externalRawCall("export") { context, callsite, stackFrame, args ->
             args.forEach { name ->
                 if (name !is LispAst.Reference) {
-                    context.reportError("Invalid export", name)
+                    stackFrame.reportError("Invalid export", name)
                     return@forEach
                 }
                 map[name.label] = context.resolveValue(stackFrame, name)
@@ -94,7 +90,7 @@ class LispExecutionContext() {
         when (node) {
             is LispAst.Parenthesis -> {
                 val first = node.items.firstOrNull()
-                    ?: return reportError("Cannot execute empty parenthesis ()", node)
+                    ?: return stackFrame.reportError("Cannot execute empty parenthesis ()", node)
 
                 val rest = node.items.drop(1)
                 return when (val resolvedValue = resolveValue(stackFrame, first)) {
@@ -102,12 +98,12 @@ class LispExecutionContext() {
                         resolvedValue.execute(this, node, stackFrame, rest)
                     }
 
-                    else -> reportError("Cannot evaluate expression of type $resolvedValue", node)
+                    else -> stackFrame.reportError("Cannot evaluate expression of type $resolvedValue", node)
                 }
 
             }
 
-            else -> return reportError("Expected invocation", node)
+            else -> return stackFrame.reportError("Expected invocation", node)
         }
     }
 
@@ -116,7 +112,7 @@ class LispExecutionContext() {
             is LispAst.Atom -> LispData.Atom(node.label)
             is LispAst.Parenthesis -> executeLisp(stackFrame, node)
             is LispAst.Reference -> stackFrame.resolveReference(node.label)
-                ?: reportError("Could not resolve variable ${node.label}", node)
+                ?: stackFrame.reportError("Could not resolve variable ${node.label}", node)
 
             is LispAst.NumberLiteral -> LispData.LispNumber(node.numberValue)
             is LispAst.StringLiteral -> LispData.LispString(node.parsedString)
diff --git a/src/OutputCapture.kt b/src/OutputCapture.kt
new file mode 100644
index 0000000..7fc8849
--- /dev/null
+++ b/src/OutputCapture.kt
@@ -0,0 +1,23 @@
+package moe.nea.lisp
+
+object OutputCapture {
+    data class CapturedOutput(
+        internal var string: StringBuilder,
+    ) {
+        val asString get() = string.toString()
+    }
+
+    object Meta : StackFrame.MetaKey<CapturedOutput>
+
+    fun captureOutput(stackFrame: StackFrame): CapturedOutput {
+        val output = CapturedOutput(StringBuilder())
+        stackFrame.setMeta(Meta, output)
+        return output
+    }
+
+    fun print(stackFrame: StackFrame, text: String) {
+        val output = stackFrame.getMeta(Meta)
+        output?.string?.append(text)
+        print(text)
+    }
+}
\ No newline at end of file
diff --git a/src/StackFrame.kt b/src/StackFrame.kt
index 9a0832f..c91299f 100644
--- a/src/StackFrame.kt
+++ b/src/StackFrame.kt
@@ -12,6 +12,11 @@ class StackFrame(val parent: StackFrame?) {
         return meta[key]?.let { it as T } ?: parent?.meta?.get(key)?.let { it as T }
     }
 
+    fun reportError(name: String, position: HasLispPosition): LispData.LispNil {
+        OutputCapture.print(this, "Error: $name ${position.position}\n")
+        return LispData.LispNil
+    }
+
     fun <T : Any> setMeta(key: MetaKey<T>, value: T) {
         meta[key] = value
     }
diff --git a/src/TestFramework.kt b/src/TestFramework.kt
index b35020c..7c2b0cc 100644
--- a/src/TestFramework.kt
+++ b/src/TestFramework.kt
@@ -12,6 +12,7 @@ object TestFramework {
         val name: String,
         val failures: List<TestFailure>,
         val wasSkipped: Boolean,
+        val stdout: String,
     )
 
     data class TestSuite(
@@ -21,6 +22,7 @@ object TestFramework {
         val allTests: MutableList<TestResult>,
         val testList: List<String>,
         val isWhitelist: Boolean,
+        var otherOutput: String = "",
     )
 
     data class ActiveTest(
@@ -42,7 +44,7 @@ object TestFramework {
         val message = CoreBindings.stringify(args.singleOrNull() ?: return@externalCall reportError("Needs a message"))
         LispData.externalRawCall("ntest.fail.r") { context, callsite, stackFrame, args ->
             val activeTest = stackFrame.getMeta(ActiveTestMeta)
-                ?: return@externalRawCall context.reportError("No active test", callsite)
+                ?: return@externalRawCall stackFrame.reportError("No active test", callsite)
             activeTest.currentFailures.add(TestFailure(callsite, message))
             return@externalRawCall LispData.LispNil
         }
@@ -62,7 +64,7 @@ object TestFramework {
         val meta = stackFrame.getMeta(TestSuiteMeta) ?: return
         if (!meta.isTesting) return
         if (args.size != 2) {
-            context.reportError("Test case needs to be defined by a name and an executable", callsite)
+            stackFrame.reportError("Test case needs to be defined by a name and an executable", callsite)
             return
         }
         val (name, prog) = args
@@ -70,19 +72,20 @@ object TestFramework {
             is LispData.Atom -> n.label
             is LispData.LispString -> n.string
             else -> {
-                context.reportError("Test case needs an atom or string as name", name)
+                stackFrame.reportError("Test case needs an atom or string as name", name)
                 return
             }
         }
         if (testName in meta.testList != meta.isWhitelist) {
-            meta.allTests.add(TestResult(testName, listOf(), true))
+            meta.allTests.add(TestResult(testName, listOf(), true, ""))
             return
         }
         val child = stackFrame.fork()
         val test = ActiveTest(testName, mutableListOf(), false, meta)
+        val output = OutputCapture.captureOutput(child)
         child.setMeta(ActiveTestMeta, test)
         context.resolveValue(child, prog)
-        meta.allTests.add(TestResult(test.testName, test.currentFailures, false))
+        meta.allTests.add(TestResult(test.testName, test.currentFailures, false, output.asString))
     }
 
     fun setup(stackFrame: StackFrame, name: String, testList: List<String>, isWhitelist: Boolean): TestSuite {
diff --git a/src/TestResultFormatter.kt b/src/TestResultFormatter.kt
index 8a5af9c..3a19444 100644
--- a/src/TestResultFormatter.kt
+++ b/src/TestResultFormatter.kt
@@ -30,6 +30,9 @@ class TestResultFormatter(private val writer: XMLStreamWriter) {
         testSuite.allTests.forEach {
             writeTestCase(it)
         }
+        writer.writeStartElement("system-out")
+        writer.writeCData(testSuite.otherOutput)
+        writer.writeEndElement()
 
         writer.writeEndElement()
     }
@@ -45,6 +48,9 @@ class TestResultFormatter(private val writer: XMLStreamWriter) {
         for (fail in test.failures) {
             writeFailure(fail)
         }
+        writer.writeStartElement("system-out")
+        writer.writeCData(test.stdout)
+        writer.writeEndElement()
 
         writer.writeEndElement()
     }