Fix missing space between Markdown elements (#2640)

4 files changed, 98 insertions, 23 deletions

diff --git a/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt b/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt
index ebcb1b43..2e301f1e 100644
--- a/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt
+++ b/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt
@@ -195,7 +195,9 @@ open class MarkdownParser(
     private fun markdownFileHandler(node: ASTNode) =
         DocTagsFromIElementFactory.getInstance(
             node.type,
-            children = node.children.evaluateChildren()
+            children = node.children
+                .filterSpacesAndEOL()
+                .evaluateChildren()
         )
 
     private fun autoLinksHandler(node: ASTNode): List<DocTag> {
@@ -250,17 +252,23 @@ open class MarkdownParser(
 
     private fun tableHandler(node: ASTNode) = DocTagsFromIElementFactory.getInstance(
         GFMElementTypes.TABLE,
-        children = node.children.filterTabSeparators().evaluateChildren()
+        children = node.children
+            .filter { it.type == GFMElementTypes.ROW || it.type == GFMElementTypes.HEADER }
+            .evaluateChildren()
     )
 
     private fun headerHandler(node: ASTNode) = DocTagsFromIElementFactory.getInstance(
         GFMElementTypes.HEADER,
-        children = node.children.filterTabSeparators().evaluateChildren()
+        children = node.children
+            .filter { it.type == GFMTokenTypes.CELL }
+            .evaluateChildren()
     )
 
     private fun rowHandler(node: ASTNode) = DocTagsFromIElementFactory.getInstance(
         GFMElementTypes.ROW,
-        children = node.children.filterTabSeparators().evaluateChildren()
+        children = node.children
+            .filter { it.type == GFMTokenTypes.CELL }
+            .evaluateChildren()
     )
 
     private fun cellHandler(node: ASTNode) = DocTagsFromIElementFactory.getInstance(
@@ -391,6 +399,9 @@ open class MarkdownParser(
     private fun List<ASTNode>.filterTabSeparators() =
         this.filterNot { it.type == GFMTokenTypes.TABLE_SEPARATOR }
 
+    private fun List<ASTNode>.filterSpacesAndEOL() =
+        this.filterNot { it.type == MarkdownTokenTypes.WHITE_SPACE || it.type == MarkdownTokenTypes.EOL }
+
     private fun List<ASTNode>.evaluateChildren(keepAllFormatting: Boolean = false): List<DocTag> =
         this.removeUselessTokens().swapImagesThatShouldBeLinks(keepAllFormatting).mergeLeafASTNodes().flatMap { visitNode(it, keepAllFormatting) }
 
@@ -430,9 +441,11 @@ open class MarkdownParser(
         MarkdownTokenTypes.HTML_BLOCK_CONTENT
     )
 
+    private fun ASTNode.isNotLeaf() = this is CompositeASTNode || this.type in notLeafNodes
+
     private fun List<ASTNode>.isNotLeaf(index: Int): Boolean =
         if (index in 0..this.lastIndex)
-            (this[index] is CompositeASTNode) || this[index].type in notLeafNodes
+            this[index].isNotLeaf()
         else
             false
 
@@ -447,17 +460,13 @@ open class MarkdownParser(
                 val sIndex = index
                 while (index < this.lastIndex) {
                     if (this.isNotLeaf(index + 1) || this[index + 1].startOffset != this[index].endOffset) {
-                        mergedLeafNode(this, index, startOffset, sIndex)?.run {
-                            children += this
-                        }
+                        children += mergedLeafNode(this, index, startOffset, sIndex)
                         break
                     }
                     index++
                 }
                 if (index == this.lastIndex) {
-                    mergedLeafNode(this, index, startOffset, sIndex)?.run {
-                        children += this
-                    }
+                    children += mergedLeafNode(this, index, startOffset, sIndex)
                 }
             }
             index++
@@ -465,15 +474,12 @@ open class MarkdownParser(
         return children
     }
 
-    private fun mergedLeafNode(nodes: List<ASTNode>, index: Int, startOffset: Int, sIndex: Int): LeafASTNode? {
+    private fun mergedLeafNode(nodes: List<ASTNode>, index: Int, startOffset: Int, sIndex: Int): LeafASTNode {
         val endOffset = nodes[index].endOffset
-        if (text.substring(startOffset, endOffset).transform().trim().isNotEmpty()) {
-            val type = if (nodes.subList(sIndex, index)
-                    .any { it.type == MarkdownTokenTypes.CODE_LINE }
-            ) MarkdownTokenTypes.CODE_LINE else MarkdownTokenTypes.TEXT
-            return LeafASTNode(type, startOffset, endOffset)
-        }
-        return null
+        val type = if (nodes.subList(sIndex, index)
+                .any { it.type == MarkdownTokenTypes.CODE_LINE }
+        ) MarkdownTokenTypes.CODE_LINE else MarkdownTokenTypes.TEXT
+        return LeafASTNode(type, startOffset, endOffset)
     }
 
     private fun String.transform() = this
diff --git a/plugins/base/src/main/kotlin/parsers/factories/DocTagsFromIElementFactory.kt b/plugins/base/src/main/kotlin/parsers/factories/DocTagsFromIElementFactory.kt
index ea87dce8..fed3f7eb 100644
--- a/plugins/base/src/main/kotlin/parsers/factories/DocTagsFromIElementFactory.kt
+++ b/plugins/base/src/main/kotlin/parsers/factories/DocTagsFromIElementFactory.kt
@@ -9,7 +9,6 @@ import org.intellij.markdown.flavours.gfm.GFMTokenTypes
 import org.jetbrains.dokka.base.translators.parseWithNormalisedSpaces
 import org.jetbrains.dokka.links.DRI
 import org.jetbrains.dokka.model.doc.DocTag.Companion.contentTypeParam
-import java.lang.NullPointerException
 
 object DocTagsFromIElementFactory {
 
@@ -40,7 +39,12 @@ object DocTagsFromIElementFactory {
                 body.orEmpty(),
                 children,
                 params
-            ) else body?.parseWithNormalisedSpaces(renderWhiteCharactersAsSpaces = false).orEmpty()
+            ) else {
+                // corner case: there are only spaces between two Markdown nodes
+                val containsOnlySpaces = body?.isNotEmpty() == true && body.all { it.isWhitespace() }
+                if (containsOnlySpaces) Text(" ", children, params)
+                else body?.parseWithNormalisedSpaces(renderWhiteCharactersAsSpaces = false).orEmpty()
+            }
             MarkdownTokenTypes.HORIZONTAL_RULE          -> HorizontalRule
             MarkdownTokenTypes.HARD_LINE_BREAK          -> Br
             GFMElementTypes.STRIKETHROUGH               -> Strikethrough(children, params)
diff --git a/plugins/base/src/test/kotlin/markdown/ParserTest.kt b/plugins/base/src/test/kotlin/markdown/ParserTest.kt
index 3498f73f..8e813eba 100644
--- a/plugins/base/src/test/kotlin/markdown/ParserTest.kt
+++ b/plugins/base/src/test/kotlin/markdown/ParserTest.kt
@@ -2,16 +2,18 @@ package org.jetbrains.dokka.tests
 
 import markdown.KDocTest
 import org.intellij.markdown.MarkdownElementTypes
+import org.jetbrains.dokka.base.parsers.MarkdownParser
 import org.jetbrains.dokka.model.doc.*
 import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.assertThrows
 import kotlin.test.assertEquals
-import kotlin.test.assertTrue
 
 
 class ParserTest : KDocTest() {
 
+    private fun parseMarkdownToDocNode(text: String) =
+        MarkdownParser( { null }, "").parseStringToDocNode(text)
+
     @Test
     fun `Simple text`() {
         val kdoc = """
@@ -1519,5 +1521,52 @@ class ParserTest : KDocTest() {
         )
         executeTest(kdoc, expectedDocumentationNode)
     }
+
+    @Test
+    fun `code with backticks`() {
+        val kdoc = "` `` `  ` ``` `"
+        val expectedDocumentationNode = DocumentationNode(
+            listOf(
+                Description(
+                    CustomDocTag(
+                        listOf(
+                            P(
+                                listOf(
+                                    CodeInline(listOf(Text("`` "))),
+                                    Text(" "),
+                                    CodeInline(listOf(Text("``` "))),
+                                )
+                            )
+                        ), name = MarkdownElementTypes.MARKDOWN_FILE.name
+                    )
+                )
+            )
+        )
+        executeTest(kdoc, expectedDocumentationNode)
+    }
+
+    @Test
+    fun `should filter spaces in markdown`() {
+        val markdown = """
+        | sdsdds f,*()hhh
+        |    dssd hf
+        | 
+        | sdsdsds sdd
+        | 
+        | 
+        | eweww  
+        | 
+        | 
+        | 
+        """.trimMargin()
+        val actualDocumentationNode = parseMarkdownToDocNode(markdown).children
+        val expectedDocumentationNode = listOf(
+            P(listOf(Text(" sdsdds f,*()hhh     dssd hf"))),
+            P(listOf(Text(" sdsdsds sdd"))),
+            P(listOf(Text(" eweww  ")))
+        )
+        print(expectedDocumentationNode)
+        assertEquals(actualDocumentationNode, expectedDocumentationNode)
+    }
 }
 
diff --git a/plugins/base/src/test/kotlin/model/CommentTest.kt b/plugins/base/src/test/kotlin/model/CommentTest.kt
index 6a7b2d89..0742587a 100644
--- a/plugins/base/src/test/kotlin/model/CommentTest.kt
+++ b/plugins/base/src/test/kotlin/model/CommentTest.kt
@@ -251,4 +251,20 @@ class CommentTest : AbstractModelTest("/src/main/kotlin/comment/Test.kt", "comme
             }
         }
     }
+
+    @Test
+    fun `should be space between Markdown nodes`() {
+        inlineModelTest(
+            """
+            |/**
+            | * Rotates paths by `amount` **radians** around (`x`, `y`).
+            | */
+            |val property = "test"
+        """
+        ) {
+            with((this / "comment" / "property").cast<DProperty>()) {
+                comments() equals "Rotates paths by amount radians around (x, y).\n"
+            }
+        }
+    }
 }