Add test junit formatter

10 files changed, 156 insertions, 33 deletions

diff --git a/res/builtins.lisp b/res/builtins.lisp
index 4348e52..e355c2f 100644
--- a/res/builtins.lisp
+++ b/res/builtins.lisp
@@ -1,5 +1,6 @@
 (defun comment (...) ((pure nil)))
 (comment "comment is a noop function for documentation")
+(export comment)
 
 
 (comment "if! a strict version of a regular if, meaning it evaluates both the falsy and the truthy case, instead of only one.")
diff --git a/res/stdtest.lisp b/res/stdtest.lisp
new file mode 100644
index 0000000..8de5164
--- /dev/null
+++ b/res/stdtest.lisp
@@ -0,0 +1,16 @@
+
+(import :ntest)
+
+
+(comment "Re-export ntest.test as test.test")
+(def test.test ntest.test)
+(export test.test)
+
+(comment "Fail a test with a certain message. Returns a closure; needs to be invoked as ((test.fail \"Fail Message\"))")
+(defun test.fail (message) (ntest.fail message))
+(export test.fail)
+
+(comment "Assert true or fail with message. Returns a closure")
+(defun test.assert (cond message) (if cond noop (ntest.fail message)))
+(export test.assert)
+
diff --git a/src/Builtins.kt b/src/Builtins.kt
index f9bf329..21da218 100644
--- a/src/Builtins.kt
+++ b/src/Builtins.kt
@@ -1,6 +1,13 @@
 package moe.nea.lisp
 
 object Builtins {
-    val builtinSource = Builtins::class.java.getResourceAsStream("/builtins.lisp")!!.bufferedReader().readText()
-    val builtinProgram = LispParser.parse("builtins.lisp", builtinSource)
+
+    private fun builtin(name: String) =
+        LispParser.parse(
+            "$name.lisp",
+            Builtins::class.java.getResourceAsStream("/$name.lisp")!!.bufferedReader().readText()
+        )
+
+    val builtinProgram = builtin("builtins")
+    val testProgram = builtin("stdtest")
 }
\ No newline at end of file
diff --git a/src/CoreBindings.kt b/src/CoreBindings.kt
index 48ecdad..571b499 100644
--- a/src/CoreBindings.kt
+++ b/src/CoreBindings.kt
@@ -1,7 +1,7 @@
 package moe.nea.lisp
 
 object CoreBindings {
-    val def = LispData.externalRawCall { context, callsite, stackFrame, args ->
+    val def = LispData.externalRawCall("def") { context, callsite, stackFrame, args ->
         if (args.size != 2) {
             return@externalRawCall context.reportError("Function define expects exactly two arguments", callsite)
         }
@@ -24,7 +24,7 @@ object CoreBindings {
     val trueValue = LispData.Atom("true")
     val falseValue = LispData.Atom("false")
 
-    val ifFun = LispData.externalRawCall { context, callsite, stackFrame, args ->
+    val ifFun = LispData.externalRawCall("if") { context, callsite, stackFrame, args ->
         if (args.size != 3) {
             return@externalRawCall context.reportError("if requires 3 arguments", callsite)
         }
@@ -41,9 +41,9 @@ object CoreBindings {
         }
     }
 
-    val pure = LispData.externalCall { args, reportError ->
+    val pure = LispData.externalCall("pure") { args, reportError ->
         return@externalCall args.singleOrNull()?.let { value ->
-            LispData.externalCall { args, reportError ->
+            LispData.externalCall("pure.r") { args, reportError ->
                 if (args.isNotEmpty())
                     reportError("Pure function does not expect arguments")
                 else
@@ -52,7 +52,7 @@ object CoreBindings {
         } ?: reportError("Function pure expects exactly one argument")
     }
 
-    val lambda = LispData.externalRawCall { context, callsite, stackFrame, args ->
+    val lambda = LispData.externalRawCall("lambda") { context, callsite, stackFrame, args ->
         if (args.size != 2) {
             return@externalRawCall context.reportError("Lambda needs exactly 2 arguments", callsite)
         }
@@ -73,7 +73,7 @@ object CoreBindings {
         LispData.createLambda(stackFrame, argumentNamesString, body)
     }
 
-    val defun = LispData.externalRawCall { context, callSite, stackFrame, lispAsts ->
+    val defun = LispData.externalRawCall("defun") { context, callSite, stackFrame, lispAsts ->
         if (lispAsts.size != 3) {
             return@externalRawCall context.reportError("Invalid function definition", callSite)
         }
@@ -100,7 +100,7 @@ object CoreBindings {
             LispData.createLambda(stackFrame, argumentNames, body, name.label)
         )
     }
-    val seq = LispData.externalRawCall { context, callsite, stackFrame, args ->
+    val seq = LispData.externalRawCall("seq") { context, callsite, stackFrame, args ->
         var lastResult: LispData? = null
         for (arg in args) {
             lastResult = context.executeLisp(stackFrame, arg)
@@ -111,7 +111,7 @@ object CoreBindings {
     internal fun stringify(thing: LispData): String {
         return when (thing) {
             is LispData.Atom -> ":${thing.label}"
-            is LispData.JavaExecutable -> "<native code>"
+            is LispData.JavaExecutable -> "<native function ${thing.name}>"
             LispData.LispNil -> "nil"
             is LispData.LispNode -> thing.node.toSource()
             is LispData.LispList -> thing.elements.joinToString(", ", "[", "]") { stringify(it) }
@@ -121,11 +121,15 @@ object CoreBindings {
         }
     }
 
-    val debuglog = LispData.externalRawCall { context, callsite, stackFrame, args ->
-        println(args.joinToString(" ") { stringify(context.resolveValue(stackFrame, it)) })
+    val tostring = LispData.externalCall("tostring") { args, reportError ->
+        LispData.LispString(args.joinToString(" ") { stringify(it) })
+    }
+
+    val debuglog = LispData.externalCall("debuglog") { args, reportError ->
+        println(args.joinToString(" ") { stringify(it) })
         LispData.LispNil
     }
-    val add = LispData.externalCall { args, reportError ->
+    val add = LispData.externalCall("add") { args, reportError ->
         if (args.size == 0) {
             return@externalCall reportError("Cannot call add without at least 1 argument")
         }
@@ -134,7 +138,7 @@ object CoreBindings {
                 ?: return@externalCall reportError("Unexpected argument $b, expected number")).value
         })
     }
-    val sub = LispData.externalCall { args, reportError ->
+    val sub = LispData.externalCall("sub") { args, reportError ->
         if (args.size == 0) {
             return@externalCall reportError("Cannot call sub without at least 1 argument")
         }
@@ -148,7 +152,7 @@ object CoreBindings {
                 ?: return@externalCall reportError("Unexpected argument $b, expected number")).value
         })
     }
-    val mul = LispData.externalCall { args, reportError ->
+    val mul = LispData.externalCall("mul") { args, reportError ->
         if (args.size == 0) {
             return@externalCall reportError("Cannot call mul without at least 1 argument")
         }
@@ -157,7 +161,7 @@ object CoreBindings {
                 ?: return@externalCall reportError("Unexpected argument $b, expected number")).value
         })
     }
-    val div = LispData.externalCall { args, reportError ->
+    val div = LispData.externalCall("div") { args, reportError ->
         if (args.size == 0) {
             return@externalCall reportError("Cannot call div without at least 1 argument")
         }
@@ -171,7 +175,7 @@ object CoreBindings {
                 ?: return@externalCall reportError("Unexpected argument $b, expected number")).value
         })
     }
-    val import = LispData.externalRawCall { context, callsite, stackFrame, args ->
+    val import = LispData.externalRawCall("import") { context, callsite, stackFrame, args ->
         if (args.size != 1) {
             return@externalRawCall context.reportError("import needs at least one argument", callsite)
         }
@@ -196,6 +200,7 @@ object CoreBindings {
         bindings.setValueLocal("if", ifFun)
         bindings.setValueLocal("nil", LispData.LispNil)
         bindings.setValueLocal("def", def)
+        bindings.setValueLocal("tostring", tostring)
         bindings.setValueLocal("pure", pure)
         bindings.setValueLocal("lambda", lambda)
         bindings.setValueLocal("defun", defun)
diff --git a/src/LispData.kt b/src/LispData.kt
index 1745762..5af3dd4 100644
--- a/src/LispData.kt
+++ b/src/LispData.kt
@@ -17,7 +17,7 @@ sealed class LispData {
         ): LispData
     }
 
-    abstract class JavaExecutable : LispExecutable()
+    abstract class JavaExecutable(val name: String) : LispExecutable()
 
     data class LispInterpretedCallable(
         val declarationStackFrame: StackFrame,
@@ -57,8 +57,11 @@ sealed class LispData {
 
 
     companion object {
-        fun externalRawCall(callable: (context: LispExecutionContext, callsite: LispAst.LispNode, stackFrame: StackFrame, args: List<LispAst.LispNode>) -> LispData): LispExecutable {
-            return object : JavaExecutable() {
+        fun externalRawCall(
+            name: String,
+            callable: (context: LispExecutionContext, callsite: LispAst.LispNode, stackFrame: StackFrame, args: List<LispAst.LispNode>) -> LispData
+        ): LispExecutable {
+            return object : JavaExecutable(name) {
                 override fun execute(
                     executionContext: LispExecutionContext,
                     callsite: LispAst.LispNode,
@@ -70,8 +73,11 @@ sealed class LispData {
             }
         }
 
-        fun externalCall(callable: (args: List<LispData>, reportError: (String) -> LispData) -> LispData): LispExecutable {
-            return object : JavaExecutable() {
+        fun externalCall(
+            name: String,
+            callable: (args: List<LispData>, reportError: (String) -> LispData) -> LispData
+        ): LispExecutable {
+            return object : JavaExecutable(name) {
                 override fun execute(
                     executionContext: LispExecutionContext,
                     callsite: LispAst.LispNode,
diff --git a/src/LispExecutionContext.kt b/src/LispExecutionContext.kt
index ed5c091..9c2d9f8 100644
--- a/src/LispExecutionContext.kt
+++ b/src/LispExecutionContext.kt
@@ -20,6 +20,7 @@ class LispExecutionContext() {
     fun setupStandardBindings() {
         CoreBindings.offerAllTo(rootStackFrame)
         registerModule("builtins", Builtins.builtinProgram)
+        registerModule("test", Builtins.testProgram)
         modules["ntest"] = TestFramework.realizedTestModule
         importModule("builtins", rootStackFrame, object : HasLispPosition {
             override val position: LispPosition
@@ -29,12 +30,14 @@ class LispExecutionContext() {
 
     fun runTests(
         program: LispAst.Program,
+        name: String,
         stackFrame: StackFrame = genBindings(),
         testList: List<String> = emptyList(),
         isWhitelist: Boolean = false
     ): TestFramework.TestSuite {
-        val testSuite = TestFramework.setup(stackFrame, testList, isWhitelist)
+        val testSuite = TestFramework.setup(stackFrame, name, testList, isWhitelist)
         executeProgram(stackFrame, program)
+        testSuite.isTesting = false
         return testSuite
     }
 
@@ -64,7 +67,7 @@ class LispExecutionContext() {
         modules[moduleName] = map
         val module = unloadedModules.remove(moduleName) ?: error("Could not find module $moduleName")
         val stackFrame = genBindings()
-        stackFrame.setValueLocal("export", LispData.externalRawCall { context, callsite, stackFrame, args ->
+        stackFrame.setValueLocal("export", LispData.externalRawCall("export") { context, callsite, stackFrame, args ->
             args.forEach { name ->
                 if (name !is LispAst.Reference) {
                     context.reportError("Invalid export", name)
diff --git a/src/TestFramework.kt b/src/TestFramework.kt
index 8406a79..b35020c 100644
--- a/src/TestFramework.kt
+++ b/src/TestFramework.kt
@@ -1,5 +1,7 @@
 package moe.nea.lisp
 
+import java.time.Instant
+
 object TestFramework {
     data class TestFailure(
         val callsite: LispAst,
@@ -14,6 +16,7 @@ object TestFramework {
 
     data class TestSuite(
         val name: String,
+        val startTime: Instant,
         var isTesting: Boolean,
         val allTests: MutableList<TestResult>,
         val testList: List<String>,
@@ -30,14 +33,14 @@ object TestFramework {
     object TestSuiteMeta : StackFrame.MetaKey<TestSuite>
     object ActiveTestMeta : StackFrame.MetaKey<ActiveTest>
 
-    val testBinding = LispData.externalRawCall { context, callsite, stackFrame, args ->
+    val testBinding = LispData.externalRawCall("ntest.test") { context, callsite, stackFrame, args ->
         runTest(context, callsite, stackFrame, args)
         return@externalRawCall LispData.LispNil
     }
 
-    val failTestBinding = LispData.externalCall { args, reportError ->
+    val failTestBinding = LispData.externalCall("ntest.fail") { args, reportError ->
         val message = CoreBindings.stringify(args.singleOrNull() ?: return@externalCall reportError("Needs a message"))
-        LispData.externalRawCall { context, callsite, stackFrame, args ->
+        LispData.externalRawCall("ntest.fail.r") { context, callsite, stackFrame, args ->
             val activeTest = stackFrame.getMeta(ActiveTestMeta)
                 ?: return@externalRawCall context.reportError("No active test", callsite)
             activeTest.currentFailures.add(TestFailure(callsite, message))
@@ -83,9 +86,8 @@ object TestFramework {
     }
 
     fun setup(stackFrame: StackFrame, name: String, testList: List<String>, isWhitelist: Boolean): TestSuite {
-        val ts = TestSuite(name, true, mutableListOf(), testList, isWhitelist)
+        val ts = TestSuite(name, Instant.now(), true, mutableListOf(), testList, isWhitelist)
         stackFrame.setMeta(TestSuiteMeta, ts)
-        ts.isTesting = false
         return ts
     }
 }
diff --git a/src/TestResultFormatter.kt b/src/TestResultFormatter.kt
new file mode 100644
index 0000000..8a5af9c
--- /dev/null
+++ b/src/TestResultFormatter.kt
@@ -0,0 +1,75 @@
+package moe.nea.lisp
+
+import java.text.SimpleDateFormat
+import java.util.*
+import javax.xml.stream.XMLStreamWriter
+
+class TestResultFormatter(private val writer: XMLStreamWriter) {
+    companion object {
+        private val timestampFormatter = SimpleDateFormat(
+            "yyyy-MM-dd'T'hh:mm:ss"
+        )
+
+        fun write(writer: XMLStreamWriter, testResults: List<TestFramework.TestSuite>) {
+            TestResultFormatter(writer).writeAll(testResults)
+        }
+    }
+
+    fun writeTestSuite(testSuite: TestFramework.TestSuite) {
+        writer.writeStartElement("testsuite")
+        writer.writeAttribute("name", testSuite.name)
+        writer.writeAttribute("tests", testSuite.allTests.size.toString())
+        writer.writeAttribute("skipped", testSuite.allTests.count { it.wasSkipped }.toString())
+        writer.writeAttribute("failures", testSuite.allTests.count { it.failures.isNotEmpty() }.toString())
+        writer.writeAttribute("errors", "0") // TODO: figure out how to differentiate errors and failures
+        writer.writeAttribute("timestamp", timestampFormatter.format(Date.from(testSuite.startTime)))
+
+        writer.writeStartElement("properties")
+        writer.writeEndElement()
+
+        testSuite.allTests.forEach {
+            writeTestCase(it)
+        }
+
+        writer.writeEndElement()
+    }
+
+    fun writeTestCase(test: TestFramework.TestResult) {
+        writer.writeStartElement("testcase")
+        writer.writeAttribute("name", test.name)
+        writer.writeAttribute("time", "0.0") // TODO: proper timestamping
+
+        if (test.wasSkipped) {
+            writeSkipped()
+        }
+        for (fail in test.failures) {
+            writeFailure(fail)
+        }
+
+        writer.writeEndElement()
+    }
+
+    fun writeSkipped() {
+        writer.writeStartElement("skipped")
+        writer.writeEndElement()
+    }
+
+    fun writeFailure(fail: TestFramework.TestFailure) {
+        writer.writeStartElement("failure")
+        writer.writeAttribute("message", fail.message)
+        writer.writeCData(fail.callsite.toSource())
+        writer.writeEndElement()
+    }
+
+    fun writeAll(testResults: List<TestFramework.TestSuite>) {
+        writer.writeStartDocument()
+        writer.writeStartElement("testsuites")
+
+        testResults.forEach {
+            writeTestSuite(it)
+        }
+
+        writer.writeEndElement()
+        writer.writeEndDocument()
+    }
+}
\ No newline at end of file
diff --git a/test/res/test.lisp b/test/res/test.lisp
index f3b870c..eea180c 100644
--- a/test/res/test.lisp
+++ b/test/res/test.lisp
@@ -24,9 +24,10 @@
 
 (debuglog "============")
 (debuglog "Running tests")
-(import :ntest)
-(ntest.test "Funny test" (seq
+(import :test)
+(test.test "Funny test" (seq
     (debuglog "Funny test running")
-    (debuglog ((ntest.fail "Test failed")))))
+    ((test.assert false "False failed"))
+    ((test.fail "Test failed"))))
 
 
diff --git a/test/src/TestLisp.kt b/test/src/TestLisp.kt
index fa90468..1f0c424 100644
--- a/test/src/TestLisp.kt
+++ b/test/src/TestLisp.kt
@@ -1,9 +1,12 @@
 import moe.nea.lisp.LispExecutionContext
 import moe.nea.lisp.LispParser
+import moe.nea.lisp.TestResultFormatter
 import java.io.File
+import javax.xml.stream.XMLOutputFactory
 
 object T
 
+
 fun main() {
     val otherP = LispParser.parse(File(T::class.java.getResource("/test.lisp")!!.file))
     val executionContext = LispExecutionContext()
@@ -13,5 +16,9 @@ fun main() {
         LispParser.parse(File(T::class.java.getResource("/secondary.lisp")!!.file))
     )
     val bindings = executionContext.genBindings()
-    println("The results are in: ${executionContext.runTests(otherP, bindings)}")
+    val testResults = executionContext.runTests(otherP, "Test", bindings)
+    val w = XMLOutputFactory.newFactory()
+        .createXMLStreamWriter(File("TestOutput.xml").bufferedWriter())
+    TestResultFormatter.write(w, listOf(testResults))
+    w.close()
 }