Fix for GH-136 GH-137 Blank lines inside code blocks are not preserved

14 files changed, 101 insertions, 12 deletions

diff --git a/core/src/main/kotlin/Formats/StructuredFormatService.kt b/core/src/main/kotlin/Formats/StructuredFormatService.kt
index 7896bcd8..0e2085c9 100644
--- a/core/src/main/kotlin/Formats/StructuredFormatService.kt
+++ b/core/src/main/kotlin/Formats/StructuredFormatService.kt
@@ -144,12 +144,9 @@ abstract class StructuredOutputBuilder(val to: StringBuilder,
             is ContentBlockSampleCode, is ContentBlockCode -> {
                 content as ContentBlockCode
                 fun ContentBlockCode.appendBlockCodeContent() {
-                    for ((index, contentNode) in this.children.withIndex()) {
-                        appendContent(contentNode)
-                        if (index < this.children.size - 1) {
-                            to.append("\n")
-                        }
-                    }
+                    children
+                            .dropWhile { it is ContentText && it.text.isBlank() }
+                            .forEach { appendContent(it) }
                 }
                 when (content) {
                     is ContentBlockSampleCode ->
diff --git a/core/src/main/kotlin/Kotlin/ContentBuilder.kt b/core/src/main/kotlin/Kotlin/ContentBuilder.kt
index 53afebaf..9c081d17 100644
--- a/core/src/main/kotlin/Kotlin/ContentBuilder.kt
+++ b/core/src/main/kotlin/Kotlin/ContentBuilder.kt
@@ -148,7 +148,7 @@ fun buildContentTo(tree: MarkdownNode, target: ContentBlock, linkResolver: (Stri
 
 private fun MarkdownNode.getLabelText() = children.filter { it.type == MarkdownTokenTypes.TEXT || it.type == MarkdownTokenTypes.EMPH }.joinToString("") { it.text }
 
-private fun keepWhitespace(node: ContentNode) = node is ContentParagraph || node is ContentSection
+private fun keepWhitespace(node: ContentNode) = node is ContentParagraph || node is ContentSection || node is ContentBlockCode
 
 fun buildInlineContentTo(tree: MarkdownNode, target: ContentBlock, linkResolver: (String) -> ContentBlock) {
     val inlineContent = tree.children.singleOrNull { it.type == MarkdownElementTypes.PARAGRAPH }?.children ?: listOf(tree)
diff --git a/core/src/main/kotlin/Markdown/MarkdownProcessor.kt b/core/src/main/kotlin/Markdown/MarkdownProcessor.kt
index 46b72c03..d1d40dd4 100644
--- a/core/src/main/kotlin/Markdown/MarkdownProcessor.kt
+++ b/core/src/main/kotlin/Markdown/MarkdownProcessor.kt
@@ -4,13 +4,14 @@ import org.intellij.markdown.IElementType
 import org.intellij.markdown.MarkdownElementTypes
 import org.intellij.markdown.ast.ASTNode
 import org.intellij.markdown.ast.LeafASTNode
+import org.intellij.markdown.ast.getTextInNode
 import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
 import org.intellij.markdown.parser.MarkdownParser
 
 class MarkdownNode(val node: ASTNode, val parent: MarkdownNode?, val markdown: String) {
     val children: List<MarkdownNode> = node.children.map { MarkdownNode(it, this, markdown) }
     val type: IElementType get() = node.type
-    val text: String get() = markdown.substring(node.startOffset, node.endOffset)
+    val text: String get() = node.getTextInNode(markdown).toString()
     fun child(type: IElementType): MarkdownNode? = children.firstOrNull { it.type == type }
 
     override fun toString(): String = StringBuilder().apply { presentTo(this) }.toString()
@@ -30,6 +31,7 @@ fun MarkdownNode.toTestString(): String {
     visit { node, visitChildren ->
         sb.append(" ".repeat(level * 2))
         node.presentTo(sb)
+        sb.appendln()
         level++
         visitChildren()
         level--
@@ -40,7 +42,6 @@ fun MarkdownNode.toTestString(): String {
 private fun MarkdownNode.presentTo(sb: StringBuilder) {
     sb.append(type.toString())
     sb.append(":" + text.replace("\n", "\u23CE"))
-    sb.appendln()
 }
 
 fun parseMarkdown(markdown: String): MarkdownNode {
diff --git a/core/src/main/kotlin/Samples/KotlinWebsiteSampleProcessingService.kt b/core/src/main/kotlin/Samples/KotlinWebsiteSampleProcessingService.kt
index baf44904..ffc5c9c7 100644
--- a/core/src/main/kotlin/Samples/KotlinWebsiteSampleProcessingService.kt
+++ b/core/src/main/kotlin/Samples/KotlinWebsiteSampleProcessingService.kt
@@ -70,6 +70,7 @@ open class KotlinWebsiteSampleProcessingService
         val psiFile = psiElement.containingFile
         if (psiFile is KtFile) {
             return ContentBlockCode("kotlin").apply {
+                append(ContentText("\n"))
                 psiFile.importList?.let {
                     it.children.filter {
                         it !is KtImportDirective || it.importPath !in importsToIgnore
diff --git a/core/src/test/kotlin/format/HtmlFormatTest.kt b/core/src/test/kotlin/format/HtmlFormatTest.kt
index 4b4eff59..f2b8624d 100644
--- a/core/src/test/kotlin/format/HtmlFormatTest.kt
+++ b/core/src/test/kotlin/format/HtmlFormatTest.kt
@@ -138,6 +138,10 @@ class HtmlFormatTest {
         verifyHtmlNode("functionalTypeWithNamedParameters")
     }
 
+    @Test fun blankLineInsideCodeBlock() {
+        verifyHtmlNode("blankLineInsideCodeBlock")
+    }
+
     private fun verifyHtmlNode(fileName: String, withKotlinRuntime: Boolean = false) {
         verifyHtmlNodes(fileName, withKotlinRuntime) { model -> model.members.single().members }
     }
diff --git a/core/src/test/kotlin/format/MarkdownFormatTest.kt b/core/src/test/kotlin/format/MarkdownFormatTest.kt
index f870d898..d788c84d 100644
--- a/core/src/test/kotlin/format/MarkdownFormatTest.kt
+++ b/core/src/test/kotlin/format/MarkdownFormatTest.kt
@@ -241,6 +241,10 @@ class MarkdownFormatTest {
         verifyMarkdownPackage("suspendParam")
     }
 
+    @Test fun blankLineInsideCodeBlock() {
+        verifyMarkdownNode("blankLineInsideCodeBlock")
+    }
+
     private fun verifyMarkdownPackage(fileName: String, withKotlinRuntime: Boolean = false) {
         verifyOutput("testdata/format/$fileName.kt", ".package.md", withKotlinRuntime = withKotlinRuntime) { model, output ->
             markdownService.createOutputBuilder(output, tempLocation).appendNodes(model.members)
diff --git a/core/src/test/kotlin/javadoc/JavadocTest.kt b/core/src/test/kotlin/javadoc/JavadocTest.kt
index f7f86881..359c5fef 100644
--- a/core/src/test/kotlin/javadoc/JavadocTest.kt
+++ b/core/src/test/kotlin/javadoc/JavadocTest.kt
@@ -1,7 +1,9 @@
 package org.jetbrains.dokka.javadoc
 
+import com.sun.javadoc.Tag
 import com.sun.javadoc.Type
 import org.jetbrains.dokka.DokkaConsoleLogger
+import org.jetbrains.dokka.tests.assertEqualsIgnoringSeparators
 import org.jetbrains.dokka.tests.verifyModel
 import org.junit.Assert.*
 import org.junit.Test
@@ -132,6 +134,23 @@ class JavadocTest {
         }
     }
 
+    @Test
+    fun testBlankLineInsideCodeBlock() {
+        verifyJavadoc("testdata/javadoc/blankLineInsideCodeBlock.kt", withKotlinRuntime = true) { doc ->
+            val method = doc.classNamed("BlankLineInsideCodeBlockKt")!!.methods()[0]
+            val text = method.inlineTags().joinToString(separator = "", transform = Tag::text)
+            assertEqualsIgnoringSeparators("""
+                <p><code><pre>
+                This is a test
+                    of Dokka's code blocks.
+                Here is a blank line.
+
+                The previous line was blank.
+                </pre></code></p>
+            """.trimIndent(), text)
+        }
+    }
+
     private fun verifyJavadoc(name: String,
                               withJdk: Boolean = false,
                               withKotlinRuntime: Boolean = false,
diff --git a/core/src/test/kotlin/model/CommentTest.kt b/core/src/test/kotlin/model/CommentTest.kt
index 25fea48d..2b2bc87e 100644
--- a/core/src/test/kotlin/model/CommentTest.kt
+++ b/core/src/test/kotlin/model/CommentTest.kt
@@ -10,14 +10,18 @@ public class CommentTest {
         verifyModel("testdata/comments/codeBlockComment.kt") { model ->
             with(model.members.single().members.first()) {
                 assertEqualsIgnoringSeparators("""[code lang=brainfuck]
+                                |
                                 |++++++++++[>+++++++>++++++++++>+++>+<<<<-]>++.>+.+++++++..+++.>++.<<+++++++++++++++.>.+++.------.--------.>+.>.
+                                |
                                 |[/code]
                                 |""".trimMargin(),
                         content.toTestString())
             }
             with(model.members.single().members.last()) {
                 assertEqualsIgnoringSeparators("""[code]
+                                |
                                 |a + b - c
+                                |
                                 |[/code]
                                 |""".trimMargin(),
                         content.toTestString())
diff --git a/core/testdata/format/blankLineInsideCodeBlock.html b/core/testdata/format/blankLineInsideCodeBlock.html
new file mode 100644
index 00000000..168dd0dd
--- /dev/null
+++ b/core/testdata/format/blankLineInsideCodeBlock.html
@@ -0,0 +1,18 @@
+<HTML>
+<HEAD>
+<meta charset="UTF-8">
+<title>u - test</title>
+</HEAD>
+<BODY>
+<a href="test/index">test</a>&nbsp;/&nbsp;<a href="test/u">u</a><br/>
+<br/>
+<h1>u</h1>
+<a name="$u()"></a>
+<code><span class="keyword">fun </span><span class="identifier">u</span><span class="symbol">(</span><span class="symbol">)</span><span class="symbol">: </span><span class="identifier">Unit</span></code><pre><code>This is a test
+    of Dokka's code blocks.
+Here is a blank line.
+
+The previous line was blank.
+</code></pre>
+</BODY>
+</HTML>
diff --git a/core/testdata/format/blankLineInsideCodeBlock.kt b/core/testdata/format/blankLineInsideCodeBlock.kt
new file mode 100644
index 00000000..9430f4d5
--- /dev/null
+++ b/core/testdata/format/blankLineInsideCodeBlock.kt
@@ -0,0 +1,12 @@
+/**
+ * ```
+ * This is a test
+ *     of Dokka's code blocks.
+ * Here is a blank line.
+ *
+ * The previous line was blank.
+ * ```
+ */
+fun u() {
+
+}
\ No newline at end of file
diff --git a/core/testdata/format/blankLineInsideCodeBlock.md b/core/testdata/format/blankLineInsideCodeBlock.md
new file mode 100644
index 00000000..66f4d65f
--- /dev/null
+++ b/core/testdata/format/blankLineInsideCodeBlock.md
@@ -0,0 +1,14 @@
+[test](test/index) / [u](test/u)
+
+# u
+
+`fun u(): Unit`
+
+```
+This is a test
+    of Dokka's code blocks.
+Here is a blank line.
+
+The previous line was blank.
+```
+
diff --git a/core/testdata/format/codeBlock.html b/core/testdata/format/codeBlock.html
index b3b65dba..48c2ffd2 100644
--- a/core/testdata/format/codeBlock.html
+++ b/core/testdata/format/codeBlock.html
@@ -10,11 +10,13 @@
 <p>This annotation indicates what exceptions should be declared by a function when compiled to a JVM method.</p>
 <p>Example:</p>
 <pre><code>Throws(IOException::class)
-fun readFile(name: String): String {...}</code></pre><a href="test/index">test</a>&nbsp;/&nbsp;<a href="test/-it-does-some-obfuscated-thing/index">ItDoesSomeObfuscatedThing</a><br/>
+fun readFile(name: String): String {...}
+</code></pre><a href="test/index">test</a>&nbsp;/&nbsp;<a href="test/-it-does-some-obfuscated-thing/index">ItDoesSomeObfuscatedThing</a><br/>
 <br/>
 <h1>ItDoesSomeObfuscatedThing</h1>
 <code><span class="keyword">class </span><span class="identifier">ItDoesSomeObfuscatedThing</span></code>
 <p>Check output of</p>
-<pre><code class="lang-brainfuck">++++++++++[&gt;+++++++&gt;++++++++++&gt;+++&gt;+&lt;&lt;&lt;&lt;-]&gt;++.&gt;+.+++++++..+++.&gt;++.&lt;&lt;+++++++++++++++.&gt;.+++.------.--------.&gt;+.&gt;.</code></pre>
+<pre><code class="lang-brainfuck">++++++++++[&gt;+++++++&gt;++++++++++&gt;+++&gt;+&lt;&lt;&lt;&lt;-]&gt;++.&gt;+.+++++++..+++.&gt;++.&lt;&lt;+++++++++++++++.&gt;.+++.------.--------.&gt;+.&gt;.
+</code></pre>
 </BODY>
 </HTML>
diff --git a/core/testdata/format/tripleBackticks.html b/core/testdata/format/tripleBackticks.html
index 5b3aef82..bac3385c 100644
--- a/core/testdata/format/tripleBackticks.html
+++ b/core/testdata/format/tripleBackticks.html
@@ -10,6 +10,7 @@
 <a name="$f()"></a>
 <code><span class="keyword">fun </span><span class="identifier">f</span><span class="symbol">(</span><span class="symbol">)</span><span class="symbol">: </span><span class="identifier">Unit</span></code>
 <p>Description</p>
-<pre><code>code sample</code></pre>
+<pre><code>code sample
+</code></pre>
 </BODY>
 </HTML>
diff --git a/core/testdata/javadoc/blankLineInsideCodeBlock.kt b/core/testdata/javadoc/blankLineInsideCodeBlock.kt
new file mode 100644
index 00000000..9430f4d5
--- /dev/null
+++ b/core/testdata/javadoc/blankLineInsideCodeBlock.kt
@@ -0,0 +1,12 @@
+/**
+ * ```
+ * This is a test
+ *     of Dokka's code blocks.
+ * Here is a blank line.
+ *
+ * The previous line was blank.
+ * ```
+ */
+fun u() {
+
+}
\ No newline at end of file