load sections from KDoc PSI, not through Markdown extensions

8 files changed, 104 insertions, 139 deletions

diff --git a/src/Analysis/CommentsAPI.kt b/src/Analysis/CommentsAPI.kt
deleted file mode 100644
index ce453113..00000000
--- a/src/Analysis/CommentsAPI.kt
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.jetbrains.dokka
-
-import org.jetbrains.kotlin.descriptors.*
-import org.jetbrains.kotlin.resolve.*
-import org.jetbrains.kotlin.kdoc.psi.api.*
-
-/**
- * Retrieves PSI elements representing documentation from the [DeclarationDescriptor]
- *
- * $$receiver: [DeclarationDescriptor] to get documentation nodes from
- */
-fun DeclarationDescriptor.getDocumentationElements(): List<KDoc> {
-    val psiElement = DescriptorToSourceUtils.descriptorToDeclaration(this)
-    if (psiElement == null)
-        return listOf()
-
-    return psiElement.children() // visit children
-            .takeWhile { it is KDoc } // all KDoc
-            .map { it as KDoc } // cast
-            .toList()
-            .reverse() // make reversed list
-}
-
-/**
- * Extracts text from KDoc, removes comment symbols and trims whitespace
- */
-fun KDoc?.extractText(): String {
-    if (this == null)
-        return ""
-    val text = getText()
-    if (text == null)
-        return ""
-    val lines = text.replace("\r", "").split("\n")
-    return lines.map {
-        val comment = it.trim().dropWhile { it == '/' || it == '*' }
-        (if (comment.endsWith("*/"))
-            comment.substring(0, comment.length() - 2)
-        else
-            comment).trim()
-    }.join("\n").trim("\n")
-}
\ No newline at end of file
diff --git a/src/Formats/StructuredFormatService.kt b/src/Formats/StructuredFormatService.kt
index d2fbdc10..fe127732 100644
--- a/src/Formats/StructuredFormatService.kt
+++ b/src/Formats/StructuredFormatService.kt
@@ -81,36 +81,48 @@ public abstract class StructuredFormatService(val locationService: LocationServi
     }
 
     fun appendDescription(location: Location, to: StringBuilder, nodes: Iterable<DocumentationNode>) {
-        val described = nodes.filter { it.hasDescription() }
+        val described = nodes.filter { it.hasDescriptionOrTags() }
         if (described.any()) {
             val single = described.size() == 1
-            appendHeader(to, "Description", 3)
+            if (described.any { it.content.description != ContentEmpty }) {
+                appendHeader(to, "Description", 3)
+            }
             for (node in described) {
                 if (!single) {
                     appendBlockCode(to, formatText(location, languageService.render(node)))
                 }
                 appendLine(to, formatText(location, node.content.description))
                 appendLine(to)
-                for ((label, section) in node.content.sections) {
-                    if (!isDescriptionSection(label, node)) continue
-                    appendLine(to, formatStrong(formatText(label)))
+
+                node.content.getSectionsWithSubjects().forEach {
+                    appendSectionWithSubject(it.getKey(), location, it.getValue(), to)
+                }
+
+                for (section in node.content.sections) {
+                    if (section.subjectName != null) continue
+                    appendLine(to, formatStrong(formatText(section.label)))
                     appendLine(to, formatText(location, section))
                 }
             }
         }
     }
 
-    private fun DocumentationNode.hasDescription() =
-       content.description != ContentEmpty || content.sections.any { isDescriptionSection(it.key, this) }
-
-    private fun isDescriptionSection(label: String, node: DocumentationNode): Boolean {
-        if (label.startsWith("$"))
-            return false
-        if (node.members.any { it.name == label })
-            return false
-        return true
+    fun appendSectionWithSubject(title: String, location: Location, parameterSections: List<ContentSection>, to: StringBuilder) {
+        appendHeader(to, title, 3)
+        parameterSections.forEach {
+            val parameterName = it.subjectName
+            if (parameterName != null) {
+                to.append(formatCode(parameterName)).append(" - ")
+                val formatted = formatText(location, it)
+                to.append(formatted)
+                appendLine(to)
+            }
+        }
     }
 
+    private fun DocumentationNode.hasDescriptionOrTags() =
+            content.description != ContentEmpty || !content.sections.isEmpty()
+
     fun appendSummary(location: Location, to: StringBuilder, nodes: Iterable<DocumentationNode>) {
         val breakdownBySummary = nodes.groupByTo(LinkedHashMap()) { node ->
             formatText(location, node.summary)
diff --git a/src/Kotlin/ContentBuilder.kt b/src/Kotlin/ContentBuilder.kt
index c943a505..95e08c92 100644
--- a/src/Kotlin/ContentBuilder.kt
+++ b/src/Kotlin/ContentBuilder.kt
@@ -9,10 +9,16 @@ import org.intellij.markdown.*
 import org.jetbrains.kotlin.psi.JetDeclarationWithBody
 import org.jetbrains.kotlin.psi.JetBlockExpression
 
-public fun DocumentationBuilder.buildContent(tree: MarkdownNode, descriptor: DeclarationDescriptor): Content {
+public fun DocumentationBuilder.buildContent(tree: MarkdownNode): Content {
+    val result = Content()
+    buildContentTo(tree, result)
+    return result
+}
+
+public fun DocumentationBuilder.buildContentTo(tree: MarkdownNode, target: ContentNode) {
 //    println(tree.toTestString())
     val nodeStack = ArrayDeque<ContentNode>()
-    nodeStack.push(Content())
+    nodeStack.push(target)
 
     tree.visit {(node, processChildren) ->
         val parent = nodeStack.peek()!!
@@ -47,22 +53,6 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownNode, descriptor: Dec
                 processChildren()
                 parent.append(nodeStack.pop())
             }
-            MarkdownTokenTypes.SECTION_ID -> {
-                if (parent !is ContentSection) {
-                    val text = node.text.trimLeading("$").trim("{", "}")
-                    val name = text.substringBefore(" ")
-                    val params = text.substringAfter(" ")
-                    when (name) {
-                        "code" -> parent.append(functionBody(descriptor, params))
-                    }
-                }
-            }
-            MarkdownElementTypes.SECTION -> {
-                val label = node.child(MarkdownTokenTypes.SECTION_ID)?.let { it.text.trimLeading("$").trim("{", "}") } ?: ""
-                nodeStack.push(ContentSection(label))
-                processChildren()
-                parent.append(nodeStack.pop())
-            }
             MarkdownElementTypes.INLINE_LINK -> {
                 val label = node.child(MarkdownElementTypes.LINK_TEXT)?.child(MarkdownTokenTypes.TEXT)
                 val destination = node.child(MarkdownElementTypes.LINK_DESTINATION)
@@ -88,7 +78,7 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownNode, descriptor: Dec
             }
             MarkdownTokenTypes.WHITE_SPACE,
             MarkdownTokenTypes.EOL -> {
-                if (nodeStack.peek() is ContentParagraph && node.parent?.children?.last() != node) {
+                if (keepWhitespace(nodeStack.peek()) && node.parent?.children?.last() != node) {
                     nodeStack.push(ContentText(node.text))
                     processChildren()
                     parent.append(nodeStack.pop())
@@ -105,10 +95,7 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownNode, descriptor: Dec
                 parent.append(nodeStack.pop())
             }
             MarkdownTokenTypes.COLON -> {
-                // TODO fix markdown parser
-                if (!isColonAfterSectionLabel(node)) {
-                    parent.append(ContentText(node.text))
-                }
+                parent.append(ContentText(node.text))
             }
             MarkdownTokenTypes.DOUBLE_QUOTE,
             MarkdownTokenTypes.LT,
@@ -120,9 +107,16 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownNode, descriptor: Dec
             }
         }
     }
-    return nodeStack.pop() as Content
 }
 
+private fun keepWhitespace(node: ContentNode) = node is ContentParagraph || node is ContentSection
+
+public fun DocumentationBuilder.buildInlineContentTo(tree: MarkdownNode, target: ContentNode) {
+    val inlineContent = tree.children.firstOrNull { it.type == MarkdownElementTypes.PARAGRAPH }?.children ?: listOf(tree)
+    inlineContent.forEach {
+        buildContentTo(it, target)
+    }
+}
 
 fun DocumentationBuilder.functionBody(descriptor: DeclarationDescriptor, functionName: String): ContentNode {
     val scope = getResolutionScope(descriptor)
@@ -176,9 +170,3 @@ private fun DocumentationBuilder.resolveInScope(functionName: String, scope: Jet
 
     return symbol
 }
-
-private fun isColonAfterSectionLabel(node: MarkdownNode): Boolean {
-    val parent = node.parent
-    return parent != null && parent.type == MarkdownElementTypes.SECTION && parent.children.size() >= 2 &&
-            node == parent.children[1];
-}
diff --git a/src/Kotlin/DocumentationBuilder.kt b/src/Kotlin/DocumentationBuilder.kt
index be6d5d7c..053615bc 100644
--- a/src/Kotlin/DocumentationBuilder.kt
+++ b/src/Kotlin/DocumentationBuilder.kt
@@ -18,6 +18,10 @@ import com.intellij.psi.PsiNameIdentifierOwner
 import com.intellij.psi.PsiElement
 import org.jetbrains.kotlin.resolve.source.getPsi
 import org.jetbrains.kotlin.psi.JetParameter
+import org.jetbrains.kotlin.kdoc.findKDoc
+import org.jetbrains.kotlin.kdoc.psi.impl.KDocSection
+import com.intellij.psi.util.PsiTreeUtil
+import org.jetbrains.kotlin.kdoc.psi.impl.KDocTag
 
 public data class DocumentationOptions(val includeNonPublic: Boolean = false,
                                        val sourceLinks: List<SourceLinkDefinition>)
@@ -36,13 +40,34 @@ class DocumentationBuilder(val session: ResolveSession, val options: Documentati
     val packages = hashMapOf<FqName, DocumentationNode>()
 
     fun parseDocumentation(descriptor: DeclarationDescriptor): Content {
-        val docText = descriptor.getDocumentationElements().map { it.extractText() }.join("\n")
+        val kdoc = findKDoc(descriptor)
+        val docText = kdoc?.getContent() ?: ""
         val tree = parseMarkdown(docText)
         //println(tree.toTestString())
-        val content = buildContent(tree, descriptor)
+        val content = buildContent(tree)
+        if (kdoc is KDocSection) {
+            val tags = PsiTreeUtil.getChildrenOfType(kdoc, javaClass<KDocTag>())
+            tags?.forEach {
+                if (it.getName() == "code") {
+                    content.append(functionBody(descriptor, it.getContent()))
+                } else {
+                    val section = content.addSection(displayName(it.getName()), it.getSubjectName())
+                    val sectionContent = it.getContent()
+                    val markdownNode = parseMarkdown(sectionContent)
+                    buildInlineContentTo(markdownNode, section)
+                }
+            }
+        }
         return content
     }
 
+    fun displayName(sectionName: String?): String? =
+            when(sectionName) {
+                "param" -> "Parameters"
+                "throws", "exception" -> "Exceptions"
+                else -> sectionName
+            }
+
     fun link(node: DocumentationNode, descriptor: DeclarationDescriptor) {
         links.put(node, descriptor)
     }
diff --git a/src/Markdown/MarkdownProcessor.kt b/src/Markdown/MarkdownProcessor.kt
index 05c4a7ec..7e601905 100644
--- a/src/Markdown/MarkdownProcessor.kt
+++ b/src/Markdown/MarkdownProcessor.kt
@@ -3,7 +3,7 @@ package org.jetbrains.dokka
 import org.intellij.markdown.*
 import org.intellij.markdown.ast.*
 import org.intellij.markdown.parser.*
-import org.intellij.markdown.parser.dialects.KDocMarkerProcessor
+import org.intellij.markdown.parser.dialects.commonmark.CommonMarkMarkerProcessor
 
 class MarkdownNode(val node: ASTNode, val parent: MarkdownNode?, val markdown: String) {
     val children: List<MarkdownNode> = node.children.map { MarkdownNode(it, this, markdown) }
@@ -134,12 +134,12 @@ public fun MarkdownNode.toHtml(): String {
 fun parseMarkdown(markdown: String): MarkdownNode {
     if (markdown.isEmpty())
         return MarkdownNode(LeafASTNode(MarkdownElementTypes.MARKDOWN_FILE, 0, 0), null, markdown)
-    return MarkdownNode(MarkdownParser(KDocMarkerProcessor.Factory()).buildMarkdownTreeFromString(markdown), null, markdown)
+    return MarkdownNode(MarkdownParser(CommonMarkMarkerProcessor.Factory()).buildMarkdownTreeFromString(markdown), null, markdown)
 }
 
 fun markdownToHtml(markdown: String): String {
 
-    val tree = MarkdownParser(KDocMarkerProcessor.Factory()).buildMarkdownTreeFromString(markdown)
+    val tree = MarkdownParser(CommonMarkMarkerProcessor.Factory()).buildMarkdownTreeFromString(markdown)
     val markdownTree = MarkdownNode(tree, null, markdown)
     val ast = markdownTree.toTestString()
     return markdownTree.toHtml()
diff --git a/src/Model/Content.kt b/src/Model/Content.kt
index 8491fd88..2fae317d 100644
--- a/src/Model/Content.kt
+++ b/src/Model/Content.kt
@@ -1,7 +1,5 @@
 package org.jetbrains.dokka
 
-import kotlin.properties.Delegates
-
 public abstract class ContentNode {
     val children = arrayListOf<ContentNode>()
 
@@ -34,7 +32,7 @@ public class ContentNodeLink(val node : DocumentationNode) : ContentBlock()
 public class ContentExternalLink(val href : String) : ContentBlock()
 public class ContentList() : ContentBlock()
 public class ContentListItem() : ContentBlock()
-public class ContentSection(public val label: String) : ContentBlock()
+public class ContentSection(public val label: String, public val subjectName: String?) : ContentBlock()
 
 fun content(body: ContentNode.() -> Unit): ContentNode {
     val block = ContentBlock()
@@ -54,53 +52,38 @@ fun ContentNode.link(to: DocumentationNode, body: ContentNode.() -> Unit) {
 }
 
 public class Content() : ContentNode() {
-    public val sections: Map<String, ContentSection> by Delegates.lazy {
-        val map = linkedMapOf<String, ContentSection>()
-        for (child in children) {
-            if (child is ContentSection)
-                map.put(child.label, child)
-        }
-
-        if ("\$summary" !in map && "\$description" !in map) {
-            // no explicit summary and description, convert anonymous section
-            val anonymous = map[""]
-            if (anonymous != null) {
-                map.remove("")
-                val summary = ContentSection("\$summary")
-                val description = ContentSection("\$description")
-
-                val summaryNodes = anonymous.children.take(1)
-                val descriptionNodes = anonymous.children.drop(1)
-
-                if (summaryNodes.any()) {
-                    summary.children.addAll(summaryNodes)
-                    map.put("\$summary", summary)
-                }
-
-                if (descriptionNodes.any()) {
-                    description.children.addAll(descriptionNodes)
-                    map.put("\$description", description)
-                }
-            }
-        }
-        map
+    private val sectionList = arrayListOf<ContentSection>()
+    public val sections: List<ContentSection>
+        get() = sectionList
+
+    fun addSection(name: String?, subjectName: String?): ContentSection {
+        val section = ContentSection(name ?: "", subjectName)
+        sectionList.add(section)
+        return section
     }
 
-    public val summary: ContentNode get()  {
-        return sections["\$summary"] ?: ContentNode.empty
+    fun findSectionByName(name: String): ContentSection? =
+        sections.firstOrNull { it.label == name }
+
+    fun getSectionsWithSubjects(): Map<String, List<ContentSection>> =
+        sections.filter { it.subjectName != null }.groupBy { it.label }
+
+    public val summary: ContentNode get() = children.firstOrNull() ?: ContentEmpty
+
+    public val description: ContentNode get() {
+        val descriptionNodes = children.drop(1)
+        if (descriptionNodes.isEmpty()) {
+            return ContentEmpty
+        }
+        val result = ContentSection("\$description", null)
+        result.children.addAll(descriptionNodes)
+        return result
     }
-    public val description: ContentNode get() = sections["\$description"] ?: ContentNode.empty
 
     override fun equals(other: Any?): Boolean {
         if (other !is Content)
             return false
-        if (sections.size != other.sections.size)
-            return false
-        for (keys in sections.keySet())
-            if (sections[keys] != other.sections[keys])
-                return false
-
-        return true
+        return sections == other.sections && children == other.children
     }
 
     override fun hashCode(): Int {
@@ -110,7 +93,7 @@ public class Content() : ContentNode() {
     override fun toString(): String {
         if (sections.isEmpty())
             return "<empty>"
-        return sections.values().joinToString()
+        return sections.joinToString()
     }
 
     val isEmpty: Boolean
diff --git a/src/Model/DocumentationNode.kt b/src/Model/DocumentationNode.kt
index a666b486..3a61b8fb 100644
--- a/src/Model/DocumentationNode.kt
+++ b/src/Model/DocumentationNode.kt
@@ -8,13 +8,11 @@ public open class DocumentationNode(val name: String,
 
     private val references = LinkedHashSet<DocumentationReference>()
 
+
     public val summary: ContentNode get()  {
-        val contentSection = content.sections["\$summary"]
+        val contentSection = content.summary
         if (contentSection != null)
             return contentSection
-        val ownerSection = owner?.content?.sections?.get(name)
-        if (ownerSection != null)
-            return ownerSection
         return ContentNode.empty
     }
 
diff --git a/src/main.kt b/src/main.kt
index b075d4e9..9ebe0b1e 100644
--- a/src/main.kt
+++ b/src/main.kt
@@ -111,7 +111,7 @@ public fun main(args: Array<String>) {
                 if (file.exists()) {
                     val text = file.readText()
                     val tree = parseMarkdown(text)
-                    val content = buildContent(tree, session.getPackageFragment(FqName.ROOT))
+                    val content = buildContent(tree)
                     moduleContent.children.addAll(content.children)
                 } else {
                     println("WARN: Include file $file was not found.")