From 873864974127ec4f123c7db0560235e806c3faab Mon Sep 17 00:00:00 2001 From: nea Date: Thu, 10 Aug 2023 18:01:37 +0200 Subject: Add test junit formatter --- res/builtins.lisp | 1 + res/stdtest.lisp | 16 ++++++++++ src/Builtins.kt | 11 +++++-- src/CoreBindings.kt | 35 ++++++++++++--------- src/LispData.kt | 16 +++++++--- src/LispExecutionContext.kt | 7 +++-- src/TestFramework.kt | 12 +++++--- src/TestResultFormatter.kt | 75 +++++++++++++++++++++++++++++++++++++++++++++ test/res/test.lisp | 7 +++-- test/src/TestLisp.kt | 9 +++++- 10 files changed, 156 insertions(+), 33 deletions(-) create mode 100644 res/stdtest.lisp create mode 100644 src/TestResultFormatter.kt diff --git a/res/builtins.lisp b/res/builtins.lisp index 4348e52..e355c2f 100644 --- a/res/builtins.lisp +++ b/res/builtins.lisp @@ -1,5 +1,6 @@ (defun comment (...) ((pure nil))) (comment "comment is a noop function for documentation") +(export comment) (comment "if! a strict version of a regular if, meaning it evaluates both the falsy and the truthy case, instead of only one.") diff --git a/res/stdtest.lisp b/res/stdtest.lisp new file mode 100644 index 0000000..8de5164 --- /dev/null +++ b/res/stdtest.lisp @@ -0,0 +1,16 @@ + +(import :ntest) + + +(comment "Re-export ntest.test as test.test") +(def test.test ntest.test) +(export test.test) + +(comment "Fail a test with a certain message. Returns a closure; needs to be invoked as ((test.fail \"Fail Message\"))") +(defun test.fail (message) (ntest.fail message)) +(export test.fail) + +(comment "Assert true or fail with message. Returns a closure") +(defun test.assert (cond message) (if cond noop (ntest.fail message))) +(export test.assert) + diff --git a/src/Builtins.kt b/src/Builtins.kt index f9bf329..21da218 100644 --- a/src/Builtins.kt +++ b/src/Builtins.kt @@ -1,6 +1,13 @@ package moe.nea.lisp object Builtins { - val builtinSource = Builtins::class.java.getResourceAsStream("/builtins.lisp")!!.bufferedReader().readText() - val builtinProgram = LispParser.parse("builtins.lisp", builtinSource) + + private fun builtin(name: String) = + LispParser.parse( + "$name.lisp", + Builtins::class.java.getResourceAsStream("/$name.lisp")!!.bufferedReader().readText() + ) + + val builtinProgram = builtin("builtins") + val testProgram = builtin("stdtest") } \ No newline at end of file diff --git a/src/CoreBindings.kt b/src/CoreBindings.kt index 48ecdad..571b499 100644 --- a/src/CoreBindings.kt +++ b/src/CoreBindings.kt @@ -1,7 +1,7 @@ package moe.nea.lisp object CoreBindings { - val def = LispData.externalRawCall { context, callsite, stackFrame, args -> + val def = LispData.externalRawCall("def") { context, callsite, stackFrame, args -> if (args.size != 2) { return@externalRawCall context.reportError("Function define expects exactly two arguments", callsite) } @@ -24,7 +24,7 @@ object CoreBindings { val trueValue = LispData.Atom("true") val falseValue = LispData.Atom("false") - val ifFun = LispData.externalRawCall { context, callsite, stackFrame, args -> + val ifFun = LispData.externalRawCall("if") { context, callsite, stackFrame, args -> if (args.size != 3) { return@externalRawCall context.reportError("if requires 3 arguments", callsite) } @@ -41,9 +41,9 @@ object CoreBindings { } } - val pure = LispData.externalCall { args, reportError -> + val pure = LispData.externalCall("pure") { args, reportError -> return@externalCall args.singleOrNull()?.let { value -> - LispData.externalCall { args, reportError -> + LispData.externalCall("pure.r") { args, reportError -> if (args.isNotEmpty()) reportError("Pure function does not expect arguments") else @@ -52,7 +52,7 @@ object CoreBindings { } ?: reportError("Function pure expects exactly one argument") } - val lambda = LispData.externalRawCall { context, callsite, stackFrame, args -> + val lambda = LispData.externalRawCall("lambda") { context, callsite, stackFrame, args -> if (args.size != 2) { return@externalRawCall context.reportError("Lambda needs exactly 2 arguments", callsite) } @@ -73,7 +73,7 @@ object CoreBindings { LispData.createLambda(stackFrame, argumentNamesString, body) } - val defun = LispData.externalRawCall { context, callSite, stackFrame, lispAsts -> + val defun = LispData.externalRawCall("defun") { context, callSite, stackFrame, lispAsts -> if (lispAsts.size != 3) { return@externalRawCall context.reportError("Invalid function definition", callSite) } @@ -100,7 +100,7 @@ object CoreBindings { LispData.createLambda(stackFrame, argumentNames, body, name.label) ) } - val seq = LispData.externalRawCall { context, callsite, stackFrame, args -> + val seq = LispData.externalRawCall("seq") { context, callsite, stackFrame, args -> var lastResult: LispData? = null for (arg in args) { lastResult = context.executeLisp(stackFrame, arg) @@ -111,7 +111,7 @@ object CoreBindings { internal fun stringify(thing: LispData): String { return when (thing) { is LispData.Atom -> ":${thing.label}" - is LispData.JavaExecutable -> "" + is LispData.JavaExecutable -> "" LispData.LispNil -> "nil" is LispData.LispNode -> thing.node.toSource() is LispData.LispList -> thing.elements.joinToString(", ", "[", "]") { stringify(it) } @@ -121,11 +121,15 @@ object CoreBindings { } } - val debuglog = LispData.externalRawCall { context, callsite, stackFrame, args -> - println(args.joinToString(" ") { stringify(context.resolveValue(stackFrame, it)) }) + val tostring = LispData.externalCall("tostring") { args, reportError -> + LispData.LispString(args.joinToString(" ") { stringify(it) }) + } + + val debuglog = LispData.externalCall("debuglog") { args, reportError -> + println(args.joinToString(" ") { stringify(it) }) LispData.LispNil } - val add = LispData.externalCall { args, reportError -> + val add = LispData.externalCall("add") { args, reportError -> if (args.size == 0) { return@externalCall reportError("Cannot call add without at least 1 argument") } @@ -134,7 +138,7 @@ object CoreBindings { ?: return@externalCall reportError("Unexpected argument $b, expected number")).value }) } - val sub = LispData.externalCall { args, reportError -> + val sub = LispData.externalCall("sub") { args, reportError -> if (args.size == 0) { return@externalCall reportError("Cannot call sub without at least 1 argument") } @@ -148,7 +152,7 @@ object CoreBindings { ?: return@externalCall reportError("Unexpected argument $b, expected number")).value }) } - val mul = LispData.externalCall { args, reportError -> + val mul = LispData.externalCall("mul") { args, reportError -> if (args.size == 0) { return@externalCall reportError("Cannot call mul without at least 1 argument") } @@ -157,7 +161,7 @@ object CoreBindings { ?: return@externalCall reportError("Unexpected argument $b, expected number")).value }) } - val div = LispData.externalCall { args, reportError -> + val div = LispData.externalCall("div") { args, reportError -> if (args.size == 0) { return@externalCall reportError("Cannot call div without at least 1 argument") } @@ -171,7 +175,7 @@ object CoreBindings { ?: return@externalCall reportError("Unexpected argument $b, expected number")).value }) } - val import = LispData.externalRawCall { context, callsite, stackFrame, args -> + val import = LispData.externalRawCall("import") { context, callsite, stackFrame, args -> if (args.size != 1) { return@externalRawCall context.reportError("import needs at least one argument", callsite) } @@ -196,6 +200,7 @@ object CoreBindings { bindings.setValueLocal("if", ifFun) bindings.setValueLocal("nil", LispData.LispNil) bindings.setValueLocal("def", def) + bindings.setValueLocal("tostring", tostring) bindings.setValueLocal("pure", pure) bindings.setValueLocal("lambda", lambda) bindings.setValueLocal("defun", defun) diff --git a/src/LispData.kt b/src/LispData.kt index 1745762..5af3dd4 100644 --- a/src/LispData.kt +++ b/src/LispData.kt @@ -17,7 +17,7 @@ sealed class LispData { ): LispData } - abstract class JavaExecutable : LispExecutable() + abstract class JavaExecutable(val name: String) : LispExecutable() data class LispInterpretedCallable( val declarationStackFrame: StackFrame, @@ -57,8 +57,11 @@ sealed class LispData { companion object { - fun externalRawCall(callable: (context: LispExecutionContext, callsite: LispAst.LispNode, stackFrame: StackFrame, args: List) -> LispData): LispExecutable { - return object : JavaExecutable() { + fun externalRawCall( + name: String, + callable: (context: LispExecutionContext, callsite: LispAst.LispNode, stackFrame: StackFrame, args: List) -> LispData + ): LispExecutable { + return object : JavaExecutable(name) { override fun execute( executionContext: LispExecutionContext, callsite: LispAst.LispNode, @@ -70,8 +73,11 @@ sealed class LispData { } } - fun externalCall(callable: (args: List, reportError: (String) -> LispData) -> LispData): LispExecutable { - return object : JavaExecutable() { + fun externalCall( + name: String, + callable: (args: List, reportError: (String) -> LispData) -> LispData + ): LispExecutable { + return object : JavaExecutable(name) { override fun execute( executionContext: LispExecutionContext, callsite: LispAst.LispNode, diff --git a/src/LispExecutionContext.kt b/src/LispExecutionContext.kt index ed5c091..9c2d9f8 100644 --- a/src/LispExecutionContext.kt +++ b/src/LispExecutionContext.kt @@ -20,6 +20,7 @@ class LispExecutionContext() { fun setupStandardBindings() { CoreBindings.offerAllTo(rootStackFrame) registerModule("builtins", Builtins.builtinProgram) + registerModule("test", Builtins.testProgram) modules["ntest"] = TestFramework.realizedTestModule importModule("builtins", rootStackFrame, object : HasLispPosition { override val position: LispPosition @@ -29,12 +30,14 @@ class LispExecutionContext() { fun runTests( program: LispAst.Program, + name: String, stackFrame: StackFrame = genBindings(), testList: List = emptyList(), isWhitelist: Boolean = false ): TestFramework.TestSuite { - val testSuite = TestFramework.setup(stackFrame, testList, isWhitelist) + val testSuite = TestFramework.setup(stackFrame, name, testList, isWhitelist) executeProgram(stackFrame, program) + testSuite.isTesting = false return testSuite } @@ -64,7 +67,7 @@ class LispExecutionContext() { modules[moduleName] = map val module = unloadedModules.remove(moduleName) ?: error("Could not find module $moduleName") val stackFrame = genBindings() - stackFrame.setValueLocal("export", LispData.externalRawCall { context, callsite, stackFrame, args -> + stackFrame.setValueLocal("export", LispData.externalRawCall("export") { context, callsite, stackFrame, args -> args.forEach { name -> if (name !is LispAst.Reference) { context.reportError("Invalid export", name) diff --git a/src/TestFramework.kt b/src/TestFramework.kt index 8406a79..b35020c 100644 --- a/src/TestFramework.kt +++ b/src/TestFramework.kt @@ -1,5 +1,7 @@ package moe.nea.lisp +import java.time.Instant + object TestFramework { data class TestFailure( val callsite: LispAst, @@ -14,6 +16,7 @@ object TestFramework { data class TestSuite( val name: String, + val startTime: Instant, var isTesting: Boolean, val allTests: MutableList, val testList: List, @@ -30,14 +33,14 @@ object TestFramework { object TestSuiteMeta : StackFrame.MetaKey object ActiveTestMeta : StackFrame.MetaKey - val testBinding = LispData.externalRawCall { context, callsite, stackFrame, args -> + val testBinding = LispData.externalRawCall("ntest.test") { context, callsite, stackFrame, args -> runTest(context, callsite, stackFrame, args) return@externalRawCall LispData.LispNil } - val failTestBinding = LispData.externalCall { args, reportError -> + val failTestBinding = LispData.externalCall("ntest.fail") { args, reportError -> val message = CoreBindings.stringify(args.singleOrNull() ?: return@externalCall reportError("Needs a message")) - LispData.externalRawCall { context, callsite, stackFrame, args -> + LispData.externalRawCall("ntest.fail.r") { context, callsite, stackFrame, args -> val activeTest = stackFrame.getMeta(ActiveTestMeta) ?: return@externalRawCall context.reportError("No active test", callsite) activeTest.currentFailures.add(TestFailure(callsite, message)) @@ -83,9 +86,8 @@ object TestFramework { } fun setup(stackFrame: StackFrame, name: String, testList: List, isWhitelist: Boolean): TestSuite { - val ts = TestSuite(name, true, mutableListOf(), testList, isWhitelist) + val ts = TestSuite(name, Instant.now(), true, mutableListOf(), testList, isWhitelist) stackFrame.setMeta(TestSuiteMeta, ts) - ts.isTesting = false return ts } } diff --git a/src/TestResultFormatter.kt b/src/TestResultFormatter.kt new file mode 100644 index 0000000..8a5af9c --- /dev/null +++ b/src/TestResultFormatter.kt @@ -0,0 +1,75 @@ +package moe.nea.lisp + +import java.text.SimpleDateFormat +import java.util.* +import javax.xml.stream.XMLStreamWriter + +class TestResultFormatter(private val writer: XMLStreamWriter) { + companion object { + private val timestampFormatter = SimpleDateFormat( + "yyyy-MM-dd'T'hh:mm:ss" + ) + + fun write(writer: XMLStreamWriter, testResults: List) { + TestResultFormatter(writer).writeAll(testResults) + } + } + + fun writeTestSuite(testSuite: TestFramework.TestSuite) { + writer.writeStartElement("testsuite") + writer.writeAttribute("name", testSuite.name) + writer.writeAttribute("tests", testSuite.allTests.size.toString()) + writer.writeAttribute("skipped", testSuite.allTests.count { it.wasSkipped }.toString()) + writer.writeAttribute("failures", testSuite.allTests.count { it.failures.isNotEmpty() }.toString()) + writer.writeAttribute("errors", "0") // TODO: figure out how to differentiate errors and failures + writer.writeAttribute("timestamp", timestampFormatter.format(Date.from(testSuite.startTime))) + + writer.writeStartElement("properties") + writer.writeEndElement() + + testSuite.allTests.forEach { + writeTestCase(it) + } + + writer.writeEndElement() + } + + fun writeTestCase(test: TestFramework.TestResult) { + writer.writeStartElement("testcase") + writer.writeAttribute("name", test.name) + writer.writeAttribute("time", "0.0") // TODO: proper timestamping + + if (test.wasSkipped) { + writeSkipped() + } + for (fail in test.failures) { + writeFailure(fail) + } + + writer.writeEndElement() + } + + fun writeSkipped() { + writer.writeStartElement("skipped") + writer.writeEndElement() + } + + fun writeFailure(fail: TestFramework.TestFailure) { + writer.writeStartElement("failure") + writer.writeAttribute("message", fail.message) + writer.writeCData(fail.callsite.toSource()) + writer.writeEndElement() + } + + fun writeAll(testResults: List) { + writer.writeStartDocument() + writer.writeStartElement("testsuites") + + testResults.forEach { + writeTestSuite(it) + } + + writer.writeEndElement() + writer.writeEndDocument() + } +} \ No newline at end of file diff --git a/test/res/test.lisp b/test/res/test.lisp index f3b870c..eea180c 100644 --- a/test/res/test.lisp +++ b/test/res/test.lisp @@ -24,9 +24,10 @@ (debuglog "============") (debuglog "Running tests") -(import :ntest) -(ntest.test "Funny test" (seq +(import :test) +(test.test "Funny test" (seq (debuglog "Funny test running") - (debuglog ((ntest.fail "Test failed"))))) + ((test.assert false "False failed")) + ((test.fail "Test failed")))) diff --git a/test/src/TestLisp.kt b/test/src/TestLisp.kt index fa90468..1f0c424 100644 --- a/test/src/TestLisp.kt +++ b/test/src/TestLisp.kt @@ -1,9 +1,12 @@ import moe.nea.lisp.LispExecutionContext import moe.nea.lisp.LispParser +import moe.nea.lisp.TestResultFormatter import java.io.File +import javax.xml.stream.XMLOutputFactory object T + fun main() { val otherP = LispParser.parse(File(T::class.java.getResource("/test.lisp")!!.file)) val executionContext = LispExecutionContext() @@ -13,5 +16,9 @@ fun main() { LispParser.parse(File(T::class.java.getResource("/secondary.lisp")!!.file)) ) val bindings = executionContext.genBindings() - println("The results are in: ${executionContext.runTests(otherP, bindings)}") + val testResults = executionContext.runTests(otherP, "Test", bindings) + val w = XMLOutputFactory.newFactory() + .createXMLStreamWriter(File("TestOutput.xml").bufferedWriter()) + TestResultFormatter.write(w, listOf(testResults)) + w.close() } -- cgit