Add abstract structure for MD/HTML comments and MD parser

6 files changed, 122 insertions, 264 deletions

diff --git a/core/src/main/kotlin/pages/DefaultMarkdownToContentConverter.kt b/core/src/main/kotlin/pages/DefaultMarkdownToContentConverter.kt
deleted file mode 100644
index bb8a826d..00000000
--- a/core/src/main/kotlin/pages/DefaultMarkdownToContentConverter.kt
+++ /dev/null
@@ -1,229 +0,0 @@
-package org.jetbrains.dokka.pages
-
-import org.intellij.markdown.MarkdownElementTypes
-import org.intellij.markdown.MarkdownTokenTypes
-import org.jetbrains.dokka.markdown.MarkdownNode
-import org.jetbrains.dokka.links.DRI
-import org.jetbrains.dokka.plugability.DokkaContext
-
-class DefaultMarkdownToContentConverter(
-    private val context: DokkaContext
-) : MarkdownToContentConverter {
-    override fun buildContent(
-        node: MarkdownNode,
-        dci: DCI,
-        platforms: Set<PlatformData>,
-        links: Map<String, DRI>,
-        styles: Set<Style>,
-        extras: Set<Extra>
-
-    ): List<ContentNode> {
-//    println(tree.toTestString())
-
-        fun buildChildren(node: MarkdownNode, newStyles: Set<Style> = emptySet(), newExtras: Set<Extra> = emptySet()) =
-            node.children.flatMap {
-                buildContent(it, dci, platforms, links, styles + newStyles, extras + newExtras)
-            }.coalesceText(platforms, styles + newStyles, extras + newExtras)
-
-        fun buildHeader(level: Int) =
-            ContentHeader(buildChildren(node), level, dci, platforms, styles)
-
-        return when (node.type) {
-            MarkdownElementTypes.ATX_1 -> listOf(buildHeader(1))
-            MarkdownElementTypes.ATX_2 -> listOf(buildHeader(2))
-            MarkdownElementTypes.ATX_3 -> listOf(buildHeader(3))
-            MarkdownElementTypes.ATX_4 -> listOf(buildHeader(4))
-            MarkdownElementTypes.ATX_5 -> listOf(buildHeader(5))
-            MarkdownElementTypes.ATX_6 -> listOf(buildHeader(6))
-            MarkdownElementTypes.UNORDERED_LIST -> listOf(
-                ContentList(
-                    buildChildren(node),
-                    false,
-                    dci,
-                    platforms,
-                    styles,
-                    extras
-                )
-            )
-            MarkdownElementTypes.ORDERED_LIST -> listOf(
-                ContentList(
-                    buildChildren(node),
-                    true,
-                    dci,
-                    platforms,
-                    styles,
-                    extras
-                )
-            )
-            MarkdownElementTypes.LIST_ITEM -> TODO()
-            MarkdownElementTypes.STRONG,
-            MarkdownTokenTypes.EMPH,
-            MarkdownElementTypes.EMPH ->
-                buildChildren(node, setOf(TextStyle.Strong))
-            // TODO
-            MarkdownElementTypes.CODE_SPAN -> TODO()
-//            val startDelimiter = node.child(MarkdownTokenTypes.BACKTICK)?.text
-//            if (startDelimiter != null) {
-//                val text = node.text.substring(startDelimiter.length).removeSuffix(startDelimiter)
-//                val codeSpan = ContentCode().apply { append(ContentText(text)) }
-//                parent.append(codeSpan)
-//            }
-
-            MarkdownElementTypes.CODE_BLOCK,
-            MarkdownElementTypes.CODE_FENCE -> {
-                val language = node.child(MarkdownTokenTypes.FENCE_LANG)?.text?.trim() ?: ""
-                listOf(ContentCode(buildChildren(node), language, dci, platforms, styles, extras)) // TODO
-            }
-            MarkdownElementTypes.PARAGRAPH -> buildChildren(node, newStyles = setOf(TextStyle.Paragraph))
-
-            MarkdownElementTypes.INLINE_LINK -> {
-//            val linkTextNode = node.child(MarkdownElementTypes.LINK_TEXT)
-//            val destination = node.child(MarkdownElementTypes.LINK_DESTINATION)
-//            if (linkTextNode != null) {
-//                if (destination != null) {
-//                    val link = ContentExternalLink(destination.text)
-//                    renderLinkTextTo(linkTextNode, link, linkResolver)
-//                    parent.append(link)
-//                } else {
-//                    val link = ContentExternalLink(linkTextNode.getLabelText())
-//                    renderLinkTextTo(linkTextNode, link, linkResolver)
-//                    parent.append(link)
-//                }
-//            }
-                //TODO: Linking!!!
-//            ContentLink()
-                TODO()
-            }
-            MarkdownElementTypes.SHORT_REFERENCE_LINK,
-            MarkdownElementTypes.FULL_REFERENCE_LINK -> {
-                val destinationNode = node.children.find { it.type == MarkdownElementTypes.LINK_DESTINATION }
-                        ?: node.children.first { it.type == MarkdownElementTypes.LINK_LABEL }
-                val destination = destinationNode.children.find { it.type == MarkdownTokenTypes.TEXT }?.text
-                        ?: destinationNode.text
-                links[destination]?.let { dri ->
-                    listOf(
-                        ContentDRILink( // TODO: differentiate between KDoc link and some external link (http://...)
-                            buildChildren(node),
-                            dri,
-                            DCI(dri, ContentKind.Symbol),
-                            platforms,
-                            styles,
-                            extras
-                        )
-                    )
-                } ?: let {
-                    context.logger.error("Apparently there is no link resolved for $destination")
-                    emptyList<ContentNode>()
-                }
-            }
-            MarkdownTokenTypes.WHITE_SPACE -> {
-                // Don't append first space if start of header (it is added during formatting later)
-                //                   v
-                //               #### Some Heading
-//            if (nodeStack.peek() !is ContentHeading || node.parent?.children?.first() != node) {
-//                parent.append(ContentText(node.text))
-//            }
-                listOf(ContentText(" ", dci, platforms, styles, extras))
-            }
-            MarkdownTokenTypes.EOL -> {
-//            if ((keepEol(nodeStack.peek()) && node.parent?.children?.last() != node) ||
-//                // Keep extra blank lines when processing lists (affects Markdown formatting)
-//                (processingList(nodeStack.peek()) && node.previous?.type == MarkdownTokenTypes.EOL)) {
-//                parent.append(ContentText(node.text))
-//            }
-                listOf(ContentText(" ", dci, platforms, styles, extras))
-            }
-
-            MarkdownTokenTypes.CODE_LINE -> {
-                listOf(ContentText(node.text, dci, platforms, styles, extras)) // TODO check
-//            if (parent is ContentBlockCode) {
-//                parent.append(content)
-//            } else {
-//                parent.append(ContentBlockCode().apply { append(content) })
-//            }
-            }
-
-            MarkdownTokenTypes.TEXT ->
-//            fun createEntityOrText(text: String): ContentNode {
-//                if (text == "&amp;" || text == "&quot;" || text == "&lt;" || text == "&gt;") {
-//                    return ContentEntity(text)
-//                }
-//                if (text == "&") {
-//                    return ContentEntity("&amp;")
-//                }
-//                val decodedText = EntityConverter.replaceEntities(text, true, true)
-//                if (decodedText != text) {
-//                    return ContentEntity(text)
-//                }
-//                return ContentText(text)
-//            }
-//
-//            parent.append(createEntityOrText(node.text))
-                listOf(ContentText(node.text, dci, platforms, styles, extras)) // TODO
-
-            MarkdownTokenTypes.COLON,
-            MarkdownTokenTypes.SINGLE_QUOTE,
-            MarkdownTokenTypes.DOUBLE_QUOTE,
-            MarkdownTokenTypes.LT,
-            MarkdownTokenTypes.GT,
-            MarkdownTokenTypes.LPAREN,
-            MarkdownTokenTypes.RPAREN,
-            MarkdownTokenTypes.LBRACKET,
-            MarkdownTokenTypes.RBRACKET,
-            MarkdownTokenTypes.EXCLAMATION_MARK,
-            MarkdownTokenTypes.BACKTICK,
-            MarkdownTokenTypes.CODE_FENCE_CONTENT -> {
-                listOf(ContentText(node.text, dci, platforms, styles, extras))
-            }
-
-            MarkdownElementTypes.LINK_DEFINITION -> TODO()
-
-            MarkdownTokenTypes.EMAIL_AUTOLINK ->
-                listOf(
-                    ContentResolvedLink(
-                        listOf(ContentText(node.text, dci, platforms, styles, extras)),
-                        "mailto:${node.text}",
-                        dci, platforms, styles, extras
-                    )
-                )
-
-            else -> buildChildren(node)
-        }
-    }
-
-    private fun Collection<ContentNode>.coalesceText(
-        platforms: Set<PlatformData>,
-        styles: Set<Style>,
-        extras: Set<Extra>
-    ) =
-        this
-            .sliceWhen { prev, next -> prev::class != next::class }
-            .flatMap { nodes ->
-                when (nodes.first()) {
-                    is ContentText -> listOf(
-                        ContentText(
-                            nodes.joinToString("") { (it as ContentText).text },
-                            nodes.first().dci, platforms, styles, extras
-                        )
-                    )
-                    else -> nodes
-                }
-            }
-}
-
-fun <T> Collection<T>.sliceWhen(predicate: (before: T, after: T) -> Boolean): Collection<Collection<T>> {
-    val newCollection = mutableListOf<Collection<T>>()
-    var currentSlice = mutableListOf<T>()
-    for ((prev, next) in this.windowed(2, 1, false)) {
-        currentSlice.add(prev)
-        if (predicate(prev, next)) {
-            newCollection.add(currentSlice)
-            currentSlice = mutableListOf<T>()
-        }
-    }
-    if (this.isNotEmpty()) {
-        currentSlice.add(this.last())
-        newCollection.add(currentSlice)
-    }
-    return newCollection
-}
\ No newline at end of file
diff --git a/core/src/main/kotlin/pages/DocNodeToContentConverter.kt b/core/src/main/kotlin/pages/DocNodeToContentConverter.kt
new file mode 100644
index 00000000..dce69114
--- /dev/null
+++ b/core/src/main/kotlin/pages/DocNodeToContentConverter.kt
@@ -0,0 +1,82 @@
+package org.jetbrains.dokka.pages
+
+import model.doc.*
+import org.intellij.markdown.MarkdownElementTypes
+import org.intellij.markdown.MarkdownTokenTypes
+import org.jetbrains.dokka.markdown.MarkdownNode
+import org.jetbrains.dokka.plugability.DokkaContext
+
+class DocNodeToContentConverter(
+    private val context: DokkaContext
+) : MarkdownToContentConverter {
+    override fun buildContent(
+        docNode: DocNode,
+        dci: DCI,
+        platforms: Set<PlatformData>,
+        styles: Set<Style>,
+        extras: Set<Extra>
+
+    ): List<ContentNode> {
+
+        fun buildChildren(docNode: DocNode, newStyles: Set<Style> = emptySet(), newExtras: Set<Extra> = emptySet()) =
+            docNode.children.flatMap {
+                buildContent(it, dci, platforms, styles + newStyles, extras + newExtras)
+            }
+
+        fun buildHeader(level: Int) =
+            listOf(ContentHeader(buildChildren(docNode), level, dci, platforms, styles, extras))
+
+        fun buildList(ordered: Boolean) =
+            listOf(ContentList(buildChildren(docNode), ordered, dci, platforms, styles, extras))
+
+        return when (docNode) {
+            is H1 -> buildHeader(1)
+            is H2 -> buildHeader(2)
+            is H3 -> buildHeader(3)
+            is H4 -> buildHeader(4)
+            is H5 -> buildHeader(5)
+            is H6 -> buildHeader(6)
+            is Ul -> buildList(false)
+            is Ol -> buildList(true)
+            is Li -> buildChildren(docNode)
+            is B -> buildChildren(docNode, setOf(TextStyle.Strong))
+            is I -> buildChildren(docNode, setOf(TextStyle.Italic))
+            is P -> buildChildren(docNode, newStyles = setOf(TextStyle.Paragraph))
+            is A -> listOf(
+                ContentResolvedLink(
+                    buildChildren(docNode),
+                    docNode.params.get("href")!!,
+                    dci,
+                    platforms,
+                    styles,
+                    extras
+                )
+            )
+            is DocumentationLink -> listOf(
+                ContentDRILink(
+                    buildChildren(docNode),
+                    docNode.dri,
+                    DCI(docNode.dri, ContentKind.Symbol),
+                    platforms,
+                    styles,
+                    extras
+                )
+            )
+            is BlockQuote -> throw NotImplementedError("Implement DocNotToContent BlockQuote!")
+            is Code -> listOf(
+                ContentCode(
+                    buildChildren(docNode),
+                    "",
+                    dci,
+                    platforms,
+                    styles,
+                    extras
+                )
+            )
+            is Img -> throw NotImplementedError("Implement DocNotToContent Img!")
+            is HorizontalRule -> listOf(ContentText("", dci, platforms, setOf()))
+            is Text -> listOf(ContentText(docNode.body, dci, platforms, styles, extras))
+            else -> buildChildren(docNode)
+        }
+    }
+}
diff --git a/core/src/main/kotlin/pages/MarkdownToContentConverter.kt b/core/src/main/kotlin/pages/MarkdownToContentConverter.kt
index cd96ff79..321a3f02 100644
--- a/core/src/main/kotlin/pages/MarkdownToContentConverter.kt
+++ b/core/src/main/kotlin/pages/MarkdownToContentConverter.kt
@@ -1,14 +1,14 @@
 package org.jetbrains.dokka.pages
 
+import model.doc.DocNode
 import org.jetbrains.dokka.markdown.MarkdownNode
 import org.jetbrains.dokka.links.DRI
 
 interface MarkdownToContentConverter {
     fun buildContent(
-        node: MarkdownNode,
+        docNode: DocNode,
         dci: DCI,
         platforms: Set<PlatformData>,
-        links: Map<String, DRI> = emptyMap(),
         styles: Set<Style> = emptySet(),
         extras: Set<Extra> = emptySet()
     ): List<ContentNode>
diff --git a/core/src/main/kotlin/pages/PageBuilder.kt b/core/src/main/kotlin/pages/PageBuilder.kt
index 92e2c5fe..8951219a 100644
--- a/core/src/main/kotlin/pages/PageBuilder.kt
+++ b/core/src/main/kotlin/pages/PageBuilder.kt
@@ -1,5 +1,6 @@
 package org.jetbrains.dokka.pages
 
+import model.doc.DocType
 import org.jetbrains.dokka.model.*
 import org.jetbrains.dokka.model.Function
 
@@ -29,7 +30,7 @@ class DefaultPageBuilder(
             else -> throw IllegalStateException("$m should not be present here")
         }
 
-    private fun group(node: DocumentationNode, content: PageContentBuilderFunction) =
+    private fun group(node: Documentable, content: PageContentBuilderFunction) =
         rootContentGroup(node, ContentKind.Main, content)
 
     private fun contentForModule(m: Module) = group(m) {
@@ -60,7 +61,13 @@ class DefaultPageBuilder(
             header(2) { text("SuperInterfaces") }
             linkTable(it)
         }
-        c.commentsData.forEach { (doc, links) -> comment(doc, links) }
+        c.commentsData.forEach {
+            it.children.forEach {
+                header(3) { text(it.toHeaderString()) }
+                comment(it.root)
+                text("\n")
+            }
+        }
         block("Constructors", 2, ContentKind.Functions, c.constructors, c.platformData) {
             link(it.name, it.dri)
             signature(it)
@@ -71,20 +78,26 @@ class DefaultPageBuilder(
             signature(it)
             text(it.briefDocstring)
         }
+        block("Properties", 2, ContentKind.Properties, c.properties, c.platformData) {
+            link(it.name, it.dri)
+            text(it.briefDocstring)
+        }
     }
 
     private fun contentForFunction(f: Function) = group(f) {
         header(1) { text(f.name) }
         signature(f)
-        f.commentsData.forEach { (doc, links) -> markdown(doc, links) }
+        f.commentsData.forEach { it.children.forEach { comment(it.root) } }
         block("Parameters", 2, ContentKind.Parameters, f.children, f.platformData) {
             text(it.name ?: "<receiver>")
-            it.commentsData.forEach { (doc, links) -> markdown(doc, links) }
+            it.commentsData.forEach { it.children.forEach { comment(it.root) } }
         }
     }
+
+    private fun DocType.toHeaderString() = this.javaClass.toGenericString().split('.').last()
 }
 
-typealias RootContentBuilder = (DocumentationNode, Kind, PageContentBuilderFunction) -> ContentGroup
+typealias RootContentBuilder = (Documentable, Kind, PageContentBuilderFunction) -> ContentGroup
 
 interface PageBuilder {
     val rootContentGroup: RootContentBuilder
diff --git a/core/src/main/kotlin/pages/PageContentBuilder.kt b/core/src/main/kotlin/pages/PageContentBuilder.kt
index 3e852306..e3c924d1 100644
--- a/core/src/main/kotlin/pages/PageContentBuilder.kt
+++ b/core/src/main/kotlin/pages/PageContentBuilder.kt
@@ -1,12 +1,12 @@
 package org.jetbrains.dokka.pages
 
+import model.doc.DocNode
 import org.jetbrains.dokka.utilities.DokkaLogger
-import org.jetbrains.dokka.model.DocumentationNode
+import org.jetbrains.dokka.model.Documentable
 import org.jetbrains.dokka.model.Function
 import org.jetbrains.dokka.model.Parameter
 import org.jetbrains.dokka.model.TypeWrapper
 import org.jetbrains.dokka.links.DRI
-import org.jetbrains.dokka.markdown.parseMarkdown
 
 class DefaultPageContentBuilder(
     private val dri: DRI,
@@ -73,7 +73,7 @@ class DefaultPageContentBuilder(
         )
     }
 
-    override fun <T : DocumentationNode> block(
+    override fun <T : Documentable> block(
         name: String,
         level: Int,
         kind: Kind,
@@ -127,27 +127,18 @@ class DefaultPageContentBuilder(
         )
     }
 
-    override fun comment(raw: String, links: Map<String, DRI>) {
+    override fun comment(docNode: DocNode) {
         contents += group(ContentKind.Comment) {
             with(this as DefaultPageContentBuilder) {
                 contents += markdownConverter.buildContent(
-                    parseMarkdown(raw),
+                    docNode,
                     DCI(dri, ContentKind.Comment),
-                    platformData,
-                    links
+                    platformData
                 )
             }
         }
     }
 
-    override fun markdown(raw: String, links: Map<String, DRI>) {
-        contents += markdownConverter.buildContent(
-            parseMarkdown(raw), DCI(dri, ContentKind.Sample),
-            platformData,
-            links
-        )
-    }
-
     fun group(kind: Kind, block: PageContentBuilderFunction): ContentGroup =
         group(dri, platformData, kind, block)
 
@@ -200,8 +191,7 @@ interface PageContentBuilder {
     fun link(text: String, address: DRI, kind: Kind = ContentKind.Symbol)
     fun link(address: DRI, kind: Kind = ContentKind.Symbol, block: PageContentBuilderFunction)
     fun linkTable(elements: List<DRI>)
-    fun comment(raw: String, links: Map<String, DRI>)
-    fun markdown(raw: String, links: Map<String, DRI>)
+    fun comment(docNode: DocNode)
     fun header(level: Int, block: PageContentBuilderFunction)
     fun <T> list(
         elements: List<T>,
@@ -211,7 +201,7 @@ interface PageContentBuilder {
         operation: PageContentBuilder.(T) -> Unit
     )
 
-    fun <T : DocumentationNode> block(
+    fun <T : Documentable> block(
         name: String,
         level: Int,
         kind: Kind,
diff --git a/core/src/main/kotlin/pages/PageNodes.kt b/core/src/main/kotlin/pages/PageNodes.kt
index f1ad430f..0aa439de 100644
--- a/core/src/main/kotlin/pages/PageNodes.kt
+++ b/core/src/main/kotlin/pages/PageNodes.kt
@@ -1,6 +1,6 @@
 package org.jetbrains.dokka.pages
 
-import org.jetbrains.dokka.model.DocumentationNode
+import org.jetbrains.dokka.model.Documentable
 import org.jetbrains.dokka.Platform
 import org.jetbrains.dokka.links.DRI
 import java.util.*
@@ -9,7 +9,7 @@ interface PageNode {
     val name: String
     val content: ContentNode
     val dri: DRI
-    val documentationNode: DocumentationNode?
+    val documentable: Documentable?
     val embeddedResources: List<String>
     val children: List<PageNode>
 
@@ -24,7 +24,8 @@ interface PageNode {
 class ModulePageNode(
     override val name: String,
     override val content: ContentNode,
-    override val documentationNode: DocumentationNode?,
+
+    override val documentable: Documentable?,
     override val children: List<PageNode>,
     override val embeddedResources: List<String> = listOf()
 ) : PageNode {
@@ -37,7 +38,7 @@ class ModulePageNode(
         children: List<PageNode>
     ): ModulePageNode =
         if (name == this.name && content === this.content && embeddedResources === this.embeddedResources && children shallowEq this.children) this
-        else ModulePageNode(name, content, documentationNode, children, embeddedResources)
+        else ModulePageNode(name, content, documentable, children, embeddedResources)
 
     private fun PageNode.transformNode(operation: (PageNode) -> PageNode): PageNode =
         operation(this).let { newNode ->
@@ -64,7 +65,8 @@ class PackagePageNode(
     override val name: String,
     override val content: ContentNode,
     override val dri: DRI,
-    override val documentationNode: DocumentationNode?,
+
+    override val documentable: Documentable?,
     override val children: List<PageNode>,
     override val embeddedResources: List<String> = listOf()
 ) : PageNode {
@@ -76,14 +78,14 @@ class PackagePageNode(
         children: List<PageNode>
     ): PackagePageNode =
         if (name == this.name && content === this.content && embeddedResources === this.embeddedResources && children shallowEq this.children) this
-        else PackagePageNode(name, content, dri, documentationNode, children, embeddedResources)
+        else PackagePageNode(name, content, dri, documentable, children, embeddedResources)
 }
 
 class ClassPageNode(
     override val name: String,
     override val content: ContentNode,
     override val dri: DRI,
-    override val documentationNode: DocumentationNode?,
+    override val documentable: Documentable?,
     override val children: List<PageNode>,
     override val embeddedResources: List<String> = listOf()
 ) : PageNode {
@@ -95,14 +97,14 @@ class ClassPageNode(
         children: List<PageNode>
     ): ClassPageNode =
         if (name == this.name && content === this.content && embeddedResources === this.embeddedResources && children shallowEq this.children) this
-        else ClassPageNode(name, content, dri, documentationNode, children, embeddedResources)
+        else ClassPageNode(name, content, dri, documentable, children, embeddedResources)
 }
 
 class MemberPageNode(
     override val name: String,
     override val content: ContentNode,
     override val dri: DRI,
-    override val documentationNode: DocumentationNode?,
+    override val documentable: Documentable?,
     override val children: List<PageNode> = emptyList(),
     override val embeddedResources: List<String> = listOf()
 ) : PageNode {
@@ -114,7 +116,7 @@ class MemberPageNode(
         children: List<PageNode>
     ): MemberPageNode =
         if (name == this.name && content === this.content && embeddedResources === this.embeddedResources && children shallowEq this.children) this
-        else MemberPageNode(name, content, dri, documentationNode, children, embeddedResources)
+        else MemberPageNode(name, content, dri, documentable, children, embeddedResources)
 }
 
 data class PlatformData(val platformType: Platform, val targets: List<String>) {