Extract content model, make doc model independent from descriptors, parse doccomments with custom parser, some tests failing due to hanging new lines.

5 files changed, 465 insertions, 0 deletions

diff --git a/src/Kotlin/ContentBuilder.kt b/src/Kotlin/ContentBuilder.kt
new file mode 100644
index 00000000..78bd7eaf
--- /dev/null
+++ b/src/Kotlin/ContentBuilder.kt
@@ -0,0 +1,79 @@
+package org.jetbrains.dokka
+
+import org.jetbrains.markdown.MarkdownElementTypes
+import java.util.ArrayDeque
+
+public fun MarkdownTree.toContent(): Content {
+    val nodeStack = ArrayDeque<ContentNode>()
+    nodeStack.push(Content())
+
+    visit {(node, text, processChildren) ->
+        val parent = nodeStack.peek()!!
+        val nodeType = node.getTokenType()
+        val nodeText = getNodeText(node)
+        when (nodeType) {
+            MarkdownElementTypes.BULLET_LIST -> {
+                nodeStack.push(ContentList())
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.ORDERED_LIST -> {
+                nodeStack.push(ContentList()) // TODO: add list kind
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.HORIZONTAL_RULE -> {
+            }
+            MarkdownElementTypes.LIST_BLOCK -> {
+                nodeStack.push(ContentBlock())
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.EMPH -> {
+                nodeStack.push(ContentEmphasis())
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.STRONG -> {
+                nodeStack.push(ContentStrong())
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.ANONYMOUS_SECTION -> {
+                nodeStack.push(ContentSection(""))
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.NAMED_SECTION -> {
+                val label = findChildByType(node, MarkdownElementTypes.SECTION_NAME)?.let { getNodeText(it) } ?: ""
+                nodeStack.push(ContentSection(label))
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.PLAIN_TEXT -> {
+                nodeStack.push(ContentText(nodeText))
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.END_LINE -> {
+                nodeStack.push(ContentText(nodeText))
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            MarkdownElementTypes.BLANK_LINE -> {
+                processChildren()
+            }
+            MarkdownElementTypes.PARA -> {
+                nodeStack.push(ContentBlock())
+                processChildren()
+                parent.children.add(nodeStack.pop())
+            }
+            else -> {
+                processChildren()
+            }
+        }
+    }
+    return nodeStack.pop() as Content
+}
+
+
diff --git a/src/Kotlin/Diagnostics.kt b/src/Kotlin/Diagnostics.kt
new file mode 100644
index 00000000..5548bc01
--- /dev/null
+++ b/src/Kotlin/Diagnostics.kt
@@ -0,0 +1,40 @@
+package org.jetbrains.dokka
+
+import org.jetbrains.jet.lang.descriptors.*
+import org.jetbrains.jet.lang.resolve.name.*
+import org.jetbrains.jet.lang.resolve.BindingContext
+
+fun DocumentationContext.checkResolveChildren(node: DocumentationNode) {
+    if (node.kind != DocumentationNode.Kind.Module && node.kind != DocumentationNode.Kind.Package) {
+        // TODO: we don't resolve packages and modules for now
+
+        val parentScope = getResolutionScope(node)
+        for (item in node.details + node.members) {
+            val symbolName = item.name
+            val symbol: DeclarationDescriptor? = when (item.kind) {
+                DocumentationNode.Kind.Modifier -> continue // do not resolve modifiers, they are not names
+                DocumentationNode.Kind.Receiver -> continue // what is receiver's name in platform?
+                DocumentationNode.Kind.Parameter -> parentScope.getLocalVariable(Name.guess(symbolName))
+                DocumentationNode.Kind.Function -> parentScope.getFunctions(Name.guess(symbolName)).firstOrNull()
+                DocumentationNode.Kind.Property -> parentScope.getProperties(Name.guess(symbolName)).firstOrNull()
+                DocumentationNode.Kind.Constructor -> parentScope.getFunctions(Name.guess(symbolName)).firstOrNull()
+                else -> parentScope.getClassifier(Name.guess(symbolName))
+            }
+
+            if (symbol == null)
+                println("WARNING: Cannot resolve $item in ${path(node)}")
+        }
+    }
+
+    for (reference in node.allReferences().filterNot { it.kind == DocumentationReference.Kind.Owner }) {
+        checkResolveChildren(reference.to)
+    }
+}
+
+fun path(node: DocumentationNode): String {
+    val owner = node.owner
+    if (owner != null)
+        return "$node in ${path(owner)}"
+    else
+        return "$node"
+}
\ No newline at end of file
diff --git a/src/Kotlin/DocumentationBuildingVisitor.kt b/src/Kotlin/DocumentationBuildingVisitor.kt
new file mode 100644
index 00000000..5654903b
--- /dev/null
+++ b/src/Kotlin/DocumentationBuildingVisitor.kt
@@ -0,0 +1,122 @@
+package org.jetbrains.dokka
+
+import org.jetbrains.jet.lang.descriptors.*
+import org.jetbrains.jet.lang.resolve.name.*
+import org.jetbrains.jet.lang.resolve.*
+
+public data class DocumentationOptions(val includeNonPublic : Boolean = false)
+
+class DocumentationBuildingVisitor(val context: BindingContext,
+                                   val options: DocumentationOptions,
+                                   private val worker: DeclarationDescriptorVisitor<DocumentationNode, DocumentationNode>)
+: DeclarationDescriptorVisitor<DocumentationNode, DocumentationNode> {
+
+    private fun visitChildren(descriptors: Collection<DeclarationDescriptor>, data: DocumentationNode) {
+        for (descriptor in descriptors) {
+            visitChild(descriptor, data)
+        }
+    }
+
+    private fun visitChild(descriptor: DeclarationDescriptor?, data: DocumentationNode) {
+        if (descriptor != null && descriptor.isUserCode()) {
+            if (options.includeNonPublic || descriptor !is MemberDescriptor || descriptor.getVisibility().isPublicAPI()) {
+                descriptor.accept(this, data)
+            }
+        }
+    }
+
+    private fun createDocumentation(descriptor: DeclarationDescriptor, data: DocumentationNode): DocumentationNode {
+        return descriptor.accept(worker, data)
+    }
+
+    private fun processCallable(descriptor: CallableDescriptor, data: DocumentationNode): DocumentationNode {
+        val node = createDocumentation(descriptor, data)
+        visitChildren(descriptor.getTypeParameters(), node)
+        visitChild(descriptor.getReceiverParameter(), node)
+        visitChildren(descriptor.getValueParameters(), node)
+        return node
+    }
+
+    public override fun visitPackageFragmentDescriptor(descriptor: PackageFragmentDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = createDocumentation(descriptor!!, data!!)
+        visitChildren(descriptor.getMemberScope().getAllDescriptors(), node)
+        return node
+    }
+
+    public override fun visitPackageViewDescriptor(descriptor: PackageViewDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = createDocumentation(descriptor!!, data!!)
+        visitChildren(descriptor.getMemberScope().getAllDescriptors(), node)
+        return node
+    }
+
+    public override fun visitVariableDescriptor(descriptor: VariableDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = processCallable(descriptor!!, data!!)
+        return node
+    }
+
+    public override fun visitPropertyDescriptor(descriptor: PropertyDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = processCallable(descriptor!!, data!!)
+        visitChild(descriptor.getGetter(), node)
+        visitChild(descriptor.getSetter(), node)
+        return node
+    }
+
+    public override fun visitFunctionDescriptor(descriptor: FunctionDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = processCallable(descriptor!!, data!!)
+        return node
+    }
+
+    public override fun visitTypeParameterDescriptor(descriptor: TypeParameterDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = createDocumentation(descriptor!!, data!!)
+        return node
+    }
+
+    public override fun visitClassDescriptor(descriptor: ClassDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = createDocumentation(descriptor!!, data!!)
+        if (descriptor.getKind() != ClassKind.OBJECT) {
+            // do not go inside object for class object and constructors, they are generated
+            visitChildren(descriptor.getTypeConstructor().getParameters(), node)
+            visitChildren(descriptor.getConstructors(), node)
+            visitChild(descriptor.getClassObjectDescriptor(), node)
+        }
+        visitChildren(descriptor.getDefaultType().getMemberScope().getAllDescriptors(), node)
+        return node
+    }
+
+    public override fun visitModuleDeclaration(descriptor: ModuleDescriptor, data: DocumentationNode): DocumentationNode {
+        val node = createDocumentation(descriptor, data)
+        visitChild(descriptor.getPackage(FqName.ROOT), node)
+        return node
+    }
+
+    public override fun visitConstructorDescriptor(descriptor: ConstructorDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = visitFunctionDescriptor(descriptor!!, data)
+        return node
+    }
+
+    public override fun visitScriptDescriptor(scriptDescriptor: ScriptDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val classDescriptor = scriptDescriptor!!.getClassDescriptor()
+        val node = visitClassDescriptor(classDescriptor, data)
+        return node
+    }
+
+    public override fun visitValueParameterDescriptor(descriptor: ValueParameterDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = visitVariableDescriptor(descriptor!!, data)
+        return node
+    }
+
+    public override fun visitPropertyGetterDescriptor(descriptor: PropertyGetterDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = visitFunctionDescriptor(descriptor!!, data)
+        return node
+    }
+
+    public override fun visitPropertySetterDescriptor(descriptor: PropertySetterDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = visitFunctionDescriptor(descriptor!!, data)
+        return node
+    }
+
+    public override fun visitReceiverParameterDescriptor(descriptor: ReceiverParameterDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        val node = createDocumentation(descriptor!!, data!!)
+        return node
+    }
+}
diff --git a/src/Kotlin/DocumentationContext.kt b/src/Kotlin/DocumentationContext.kt
new file mode 100644
index 00000000..1491aa2d
--- /dev/null
+++ b/src/Kotlin/DocumentationContext.kt
@@ -0,0 +1,47 @@
+package org.jetbrains.dokka
+
+import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor
+import org.jetbrains.jet.lang.resolve.BindingContext
+import org.jetbrains.jet.lang.resolve.scopes.JetScope
+import org.jetbrains.jet.lang.descriptors.ModuleDescriptor
+import org.jetbrains.jet.lang.resolve.name.FqName
+
+public class DocumentationContext(val bindingContext: BindingContext) {
+    val descriptorToNode = hashMapOf<DeclarationDescriptor, DocumentationNode>()
+    val nodeToDescriptor = hashMapOf<DocumentationNode, DeclarationDescriptor>()
+
+    fun register(descriptor: DeclarationDescriptor, node: DocumentationNode) {
+        descriptorToNode.put(descriptor, node)
+        nodeToDescriptor.put(node, descriptor)
+    }
+
+    fun getResolutionScope(node: DocumentationNode): JetScope {
+        val descriptor = nodeToDescriptor[node] ?: throw IllegalArgumentException("Node is not known to this context")
+        return bindingContext.getResolutionScope(descriptor)
+    }
+
+    fun parseDocumentation(descriptor: DeclarationDescriptor): Content {
+        val docText = bindingContext.getDocumentationElements(descriptor).map { it.extractText() }.join("\n")
+        val tree = MarkdownProcessor.parse(docText)
+        println(tree.toTestString())
+        val content = tree.toContent()
+        return content
+    }
+}
+
+fun BindingContext.createDocumentationModule(name: String,
+                                             module: ModuleDescriptor,
+                                             packages: Set<FqName>,
+                                             options: DocumentationOptions = DocumentationOptions()): DocumentationModule {
+    val documentationModule = DocumentationModule(name)
+    val context = DocumentationContext(this)
+    val visitor = DocumentationNodeBuilder(context)
+    for (packageName in packages) {
+        val pkg = module.getPackage(packageName)
+        pkg!!.accept(DocumentationBuildingVisitor(this, options, visitor), documentationModule)
+    }
+
+    // TODO: Uncomment for resolve verification
+    // checkResolveChildren(documentationModule)
+    return documentationModule
+}
diff --git a/src/Kotlin/DocumentationNodeBuilder.kt b/src/Kotlin/DocumentationNodeBuilder.kt
new file mode 100644
index 00000000..535f037f
--- /dev/null
+++ b/src/Kotlin/DocumentationNodeBuilder.kt
@@ -0,0 +1,177 @@
+package org.jetbrains.dokka
+
+import org.jetbrains.jet.lang.descriptors.impl.DeclarationDescriptorVisitorEmptyBodies
+import org.jetbrains.jet.lang.descriptors.MemberDescriptor
+import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor
+import org.jetbrains.jet.lang.types.JetType
+import org.jetbrains.jet.lang.descriptors.Named
+import org.jetbrains.jet.lang.descriptors.ReceiverParameterDescriptor
+import org.jetbrains.jet.lang.descriptors.ValueParameterDescriptor
+import org.jetbrains.jet.lang.descriptors.ClassDescriptor
+import org.jetbrains.jet.lang.descriptors.FunctionDescriptor
+import org.jetbrains.jet.lang.descriptors.TypeParameterDescriptor
+import org.jetbrains.jet.lang.descriptors.PropertyDescriptor
+import org.jetbrains.jet.lang.descriptors.ConstructorDescriptor
+import org.jetbrains.jet.lang.descriptors.PackageViewDescriptor
+import org.jetbrains.jet.lang.descriptors.PackageFragmentDescriptor
+import org.jetbrains.jet.lang.descriptors.ClassKind
+import org.jetbrains.jet.lang.types.lang.KotlinBuiltIns
+
+class DocumentationNodeBuilder(val context: DocumentationContext) : DeclarationDescriptorVisitorEmptyBodies<DocumentationNode, DocumentationNode>() {
+
+    fun reference(from: DocumentationNode, to: DocumentationNode, kind: DocumentationReference.Kind) {
+        from.addReferenceTo(to, kind)
+        if (kind == DocumentationReference.Kind.Link)
+            to.addReferenceTo(from, DocumentationReference.Kind.Link)
+        else
+            to.addReferenceTo(from, DocumentationReference.Kind.Owner)
+    }
+
+    fun addModality(descriptor: MemberDescriptor, data: DocumentationNode) {
+        val modifier = descriptor.getModality().name().toLowerCase()
+        val node = DocumentationNode(modifier, Content.Empty, DocumentationNode.Kind.Modifier)
+        reference(data, node, DocumentationReference.Kind.Detail)
+    }
+
+    fun addVisibility(descriptor: MemberDescriptor, data: DocumentationNode) {
+        val modifier = descriptor.getVisibility().toString()
+        val node = DocumentationNode(modifier, Content.Empty, DocumentationNode.Kind.Modifier)
+        reference(data, node, DocumentationReference.Kind.Detail)
+    }
+
+    fun addType(descriptor: DeclarationDescriptor, t: JetType?, data: DocumentationNode) {
+        if (t == null)
+            return
+        val typeConstructor = t.getConstructor()
+        val classifierDescriptor = typeConstructor.getDeclarationDescriptor()
+        val name = when (classifierDescriptor) {
+            is Named -> classifierDescriptor.getName().asString()
+            else -> "<anonymous>"
+        }
+        val node = DocumentationNode(name, Content.Empty, DocumentationNode.Kind.Type)
+        reference(data, node, DocumentationReference.Kind.Detail)
+
+        for (param in t.getArguments())
+            addType(descriptor, param.getType(), node)
+    }
+
+    override fun visitDeclarationDescriptor(descriptor: DeclarationDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val doc = context.parseDocumentation(descriptor)
+        val node = DocumentationNode(descriptor.getName().asString(), doc, DocumentationNode.Kind.Unknown)
+        reference(data!!, node, DocumentationReference.Kind.Link)
+        context.register(descriptor, node)
+        return node
+    }
+
+    override fun visitReceiverParameterDescriptor(descriptor: ReceiverParameterDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val node = DocumentationNode(descriptor.getName().asString(), Content.Empty, DocumentationNode.Kind.Receiver)
+        reference(data!!, node, DocumentationReference.Kind.Detail)
+
+        addType(descriptor, descriptor.getType(), node)
+
+        return node
+    }
+
+    override fun visitValueParameterDescriptor(descriptor: ValueParameterDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val doc = context.parseDocumentation(descriptor)
+        val node = DocumentationNode(descriptor.getName().asString(), doc, DocumentationNode.Kind.Parameter)
+        reference(data!!, node, DocumentationReference.Kind.Detail)
+
+        addType(descriptor, descriptor.getType(), node)
+
+        return node
+    }
+
+    override fun visitClassDescriptor(descriptor: ClassDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val doc = context.parseDocumentation(descriptor)
+        val node = DocumentationNode(descriptor.getName().asString(), doc, when (descriptor.getKind()) {
+            ClassKind.OBJECT -> org.jetbrains.dokka.DocumentationNode.Kind.Object
+            ClassKind.CLASS_OBJECT -> org.jetbrains.dokka.DocumentationNode.Kind.Object
+            ClassKind.TRAIT -> org.jetbrains.dokka.DocumentationNode.Kind.Interface
+            ClassKind.ENUM_CLASS -> org.jetbrains.dokka.DocumentationNode.Kind.Enum
+            ClassKind.ENUM_ENTRY -> org.jetbrains.dokka.DocumentationNode.Kind.EnumItem
+            else -> DocumentationNode.Kind.Class
+        })
+        reference(data!!, node, DocumentationReference.Kind.Member)
+        addModality(descriptor, node)
+        addVisibility(descriptor, node)
+        context.register(descriptor, node)
+        return node
+    }
+
+    override fun visitFunctionDescriptor(descriptor: FunctionDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val doc = context.parseDocumentation(descriptor)
+        val node = DocumentationNode(descriptor.getName().asString(), doc, DocumentationNode.Kind.Function)
+        reference(data!!, node, DocumentationReference.Kind.Member)
+
+        addType(descriptor, descriptor.getReturnType(), node)
+        addModality(descriptor, node)
+        addVisibility(descriptor, node)
+        context.register(descriptor, node)
+        return node
+    }
+
+    override fun visitTypeParameterDescriptor(descriptor: TypeParameterDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val doc = context.parseDocumentation(descriptor)
+        val node = DocumentationNode(descriptor.getName().asString(), doc, DocumentationNode.Kind.TypeParameter)
+        reference(data!!, node, DocumentationReference.Kind.Detail)
+        val builtIns = KotlinBuiltIns.getInstance()
+        for (constraint in descriptor.getUpperBounds()) {
+            if (constraint == builtIns.getDefaultBound())
+                continue
+            val constraintNode = DocumentationNode(constraint.toString(), Content.Empty, DocumentationNode.Kind.UpperBound)
+            reference(node, constraintNode, DocumentationReference.Kind.Detail)
+        }
+        for (constraint in descriptor.getLowerBounds()) {
+            if (builtIns.isNothing(constraint))
+                continue
+            val constraintNode = DocumentationNode(constraint.toString(), Content.Empty, DocumentationNode.Kind.LowerBound)
+            reference(node, constraintNode, DocumentationReference.Kind.Detail)
+        }
+        return node
+    }
+
+    override fun visitPropertyDescriptor(descriptor: PropertyDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val doc = context.parseDocumentation(descriptor)
+        val node = DocumentationNode(descriptor.getName().asString(), doc, DocumentationNode.Kind.Property)
+        reference(data!!, node, DocumentationReference.Kind.Member)
+
+        addType(descriptor, descriptor.getType(), node)
+        addModality(descriptor, node)
+        addVisibility(descriptor, node)
+        context.register(descriptor, node)
+        return node
+    }
+
+    override fun visitConstructorDescriptor(descriptor: ConstructorDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val doc = context.parseDocumentation(descriptor)
+        val node = DocumentationNode(descriptor.getName().asString(), doc, DocumentationNode.Kind.Constructor)
+        reference(data!!, node, DocumentationReference.Kind.Member)
+
+        addVisibility(descriptor, node)
+        context.register(descriptor, node)
+        return node
+    }
+
+    override fun visitPackageViewDescriptor(descriptor: PackageViewDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val node = DocumentationNode(descriptor.getFqName().asString(), Content.Empty, DocumentationNode.Kind.Package)
+        reference(data!!, node, DocumentationReference.Kind.Member)
+        return node
+    }
+
+    override fun visitPackageFragmentDescriptor(descriptor: PackageFragmentDescriptor?, data: DocumentationNode?): DocumentationNode? {
+        descriptor!!
+        val node = DocumentationNode(descriptor.fqName.asString(), Content.Empty, DocumentationNode.Kind.Package)
+        reference(data!!, node, DocumentationReference.Kind.Member)
+        return node
+    }
+}
\ No newline at end of file