Working javadoc parsing

2 files changed, 467 insertions, 0 deletions

diff --git a/plugins/base/src/main/kotlin/translators/psi/DefaultPsiToDocumentableTranslator.kt b/plugins/base/src/main/kotlin/translators/psi/DefaultPsiToDocumentableTranslator.kt
new file mode 100644
index 00000000..3e95865e
--- /dev/null
+++ b/plugins/base/src/main/kotlin/translators/psi/DefaultPsiToDocumentableTranslator.kt
@@ -0,0 +1,318 @@
+package org.jetbrains.dokka.base.translators.psi
+
+import com.intellij.lang.jvm.JvmModifier
+import com.intellij.lang.jvm.types.JvmReferenceType
+import com.intellij.psi.*
+import org.jetbrains.dokka.links.Callable
+import org.jetbrains.dokka.links.DRI
+import org.jetbrains.dokka.links.JavaClassReference
+import org.jetbrains.dokka.links.withClass
+import org.jetbrains.dokka.model.*
+import org.jetbrains.dokka.model.Annotation
+import org.jetbrains.dokka.model.Enum
+import org.jetbrains.dokka.model.Function
+import org.jetbrains.dokka.model.properties.PropertyContainer
+import org.jetbrains.dokka.pages.PlatformData
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.transformers.psi.PsiToDocumentableTranslator
+import org.jetbrains.dokka.utilities.DokkaLogger
+import org.jetbrains.kotlin.descriptors.Visibilities
+import org.jetbrains.kotlin.load.java.JvmAbi
+import org.jetbrains.kotlin.load.java.propertyNameByGetMethodName
+import org.jetbrains.kotlin.load.java.propertyNamesBySetMethodName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.resolve.DescriptorUtils
+
+object DefaultPsiToDocumentableTranslator : PsiToDocumentableTranslator {
+
+    override fun invoke(
+        moduleName: String,
+        psiFiles: List<PsiJavaFile>,
+        platformData: PlatformData,
+        context: DokkaContext
+    ): Module {
+        val docParser =
+            DokkaPsiParser(
+                platformData,
+                context.logger
+            )
+        return Module(
+            moduleName,
+            psiFiles.map { psiFile ->
+                val dri = DRI(packageName = psiFile.packageName)
+                Package(
+                    dri,
+                    emptyList(),
+                    emptyList(),
+                    psiFile.classes.map { docParser.parseClasslike(it, dri) },
+                    emptyList(),
+                    PlatformDependent.empty(),
+                    listOf(platformData)
+                )
+            },
+            PlatformDependent.empty(),
+            listOf(platformData)
+        )
+    }
+
+    class DokkaPsiParser(
+        private val platformData: PlatformData,
+        logger: DokkaLogger
+    ) {
+
+        private val javadocParser: JavaDocumentationParser = JavadocParser(logger)
+
+        private fun PsiModifierListOwner.getVisibility() = modifierList?.children?.toList()?.let { ml ->
+            when {
+                ml.any { it.text == PsiKeyword.PUBLIC } -> JavaVisibility.Public
+                ml.any { it.text == PsiKeyword.PROTECTED } -> JavaVisibility.Protected
+                ml.any { it.text == PsiKeyword.PRIVATE } -> JavaVisibility.Private
+                else -> JavaVisibility.Default
+            }
+        } ?: JavaVisibility.Default
+
+        private val PsiMethod.hash: Int
+            get() = "$returnType $name$parameterList".hashCode()
+
+        private val PsiClassType.shouldBeIgnored: Boolean
+            get() = isClass("java.lang.Enum") || isClass("java.lang.Object")
+
+        private fun PsiClassType.isClass(qName: String): Boolean {
+            val shortName = qName.substringAfterLast('.')
+            if (className == shortName) {
+                val psiClass = resolve()
+                return psiClass?.qualifiedName == qName
+            }
+            return false
+        }
+
+        private fun <T> T.toPlatformDependant() =
+            PlatformDependent(mapOf(platformData to this))
+
+        fun parseClasslike(psi: PsiClass, parent: DRI): Classlike = with(psi) {
+            val dri = parent.withClass(name.toString())
+            val ancestorsSet = hashSetOf<DRI>()
+            val superMethodsKeys = hashSetOf<Int>()
+            val superMethods = mutableListOf<PsiMethod>()
+            methods.forEach { superMethodsKeys.add(it.hash) }
+            fun addAncestors(element: PsiClass) {
+                ancestorsSet.add(element.toDRI())
+                element.interfaces.forEach(::addAncestors)
+                element.superClass?.let(::addAncestors)
+            }
+
+            fun parseSupertypes(superTypes: Array<PsiClassType>) {
+                superTypes.forEach { type ->
+                    (type as? PsiClassType)?.takeUnless { type.shouldBeIgnored }?.resolve()?.let {
+                        it.methods.forEach { method ->
+                            val hash = method.hash
+                            if (!method.isConstructor && !superMethodsKeys.contains(hash) &&
+                                method.getVisibility() != Visibilities.PRIVATE
+                            ) {
+                                superMethodsKeys.add(hash)
+                                superMethods.add(method)
+                            }
+                        }
+                        addAncestors(it)
+                        parseSupertypes(it.superTypes)
+                    }
+                }
+            }
+            parseSupertypes(superTypes)
+            val (regularFunctions, accessors) = splitFunctionsAndAccessors()
+            val documentation = javadocParser.parseDocumentation(this).toPlatformDependant()
+            val allFunctions = regularFunctions.mapNotNull { if (!it.isConstructor) parseFunction(it, dri) else null } +
+                    superMethods.map { parseFunction(it, dri, isInherited = true) }
+            val source = PsiDocumentableSource(this).toPlatformDependant()
+            val classlikes = innerClasses.map { parseClasslike(it, dri) }
+            val visibility = getVisibility().toPlatformDependant()
+            val ancestors = ancestorsSet.toList().toPlatformDependant()
+            return when {
+                isAnnotationType ->
+                    Annotation(
+                        name.orEmpty(),
+                        dri,
+                        documentation,
+                        source,
+                        allFunctions,
+                        fields.mapNotNull { parseField(it, dri, accessors[it].orEmpty()) },
+                        classlikes,
+                        visibility,
+                        null,
+                        constructors.map { parseFunction(it, dri, true) },
+                        listOf(platformData)
+                    )
+                isEnum -> Enum(
+                    dri,
+                    name.orEmpty(),
+                    fields.filterIsInstance<PsiEnumConstant>().map { entry ->
+                        EnumEntry(
+                            dri.withClass("$name.${entry.name}"),
+                            entry.name.orEmpty(),
+                            javadocParser.parseDocumentation(entry).toPlatformDependant(),
+                            emptyList(),
+                            emptyList(),
+                            emptyList(),
+                            listOf(platformData)
+                        )
+                    },
+                    documentation,
+                    source,
+                    allFunctions,
+                    fields.filter { it !is PsiEnumConstant }.map { parseField(it, dri, accessors[it].orEmpty()) },
+                    classlikes,
+                    visibility,
+                    null,
+                    constructors.map { parseFunction(it, dri, true) },
+                    ancestors,
+                    listOf(platformData)
+                )
+                isInterface -> Interface(
+                    dri,
+                    name.orEmpty(),
+                    documentation,
+                    source,
+                    allFunctions,
+                    fields.mapNotNull { parseField(it, dri, accessors[it].orEmpty()) },
+                    classlikes,
+                    visibility,
+                    null,
+                    mapTypeParameters(dri),
+                    ancestors,
+                    listOf(platformData)
+                )
+                else -> Class(
+                    dri,
+                    name.orEmpty(),
+                    constructors.map { parseFunction(it, dri, true) },
+                    allFunctions,
+                    fields.mapNotNull { parseField(it, dri, accessors[it].orEmpty()) },
+                    classlikes,
+                    source,
+                    visibility,
+                    null,
+                    mapTypeParameters(dri),
+                    ancestors,
+                    documentation,
+                    getModifier(),
+                    listOf(platformData)
+                )
+            }
+        }
+
+        private fun parseFunction(
+            psi: PsiMethod,
+            parent: DRI,
+            isConstructor: Boolean = false,
+            isInherited: Boolean = false
+        ): Function {
+            val dri = parent.copy(
+                callable = Callable(
+                    psi.name,
+                    JavaClassReference(psi.containingClass?.name.orEmpty()),
+                    psi.parameterList.parameters.map { parameter ->
+                        JavaClassReference(parameter.type.canonicalText)
+                    })
+            )
+            return Function(
+                dri,
+                if (isConstructor) "<init>" else psi.name,
+                isConstructor,
+                psi.parameterList.parameters.mapIndexed { index, psiParameter ->
+                    Parameter(
+                        dri.copy(target = index + 1),
+                        psiParameter.name,
+                        javadocParser.parseDocumentation(psiParameter).toPlatformDependant(),
+                        JavaTypeWrapper(psiParameter.type),
+                        listOf(platformData)
+                    )
+                },
+                javadocParser.parseDocumentation(psi).toPlatformDependant(),
+                PsiDocumentableSource(psi).toPlatformDependant(),
+                psi.getVisibility().toPlatformDependant(),
+                psi.returnType?.let { JavaTypeWrapper(type = it) } ?: JavaTypeWrapper.VOID,
+                psi.mapTypeParameters(dri),
+                null,
+                psi.getModifier(),
+                listOf(platformData),
+                PropertyContainer.empty<Function>() + InheritedFunction(
+                    isInherited
+                )
+
+            )
+        }
+
+        private fun PsiModifierListOwner.getModifier() = when {
+            hasModifier(JvmModifier.ABSTRACT) -> WithAbstraction.Modifier.Abstract
+            hasModifier(JvmModifier.FINAL) -> WithAbstraction.Modifier.Final
+            else -> null
+        }
+
+        private fun PsiTypeParameterListOwner.mapTypeParameters(dri: DRI): List<TypeParameter> {
+            fun mapProjections(bounds: Array<JvmReferenceType>) =
+                if (bounds.isEmpty()) listOf(Projection.Star) else bounds.mapNotNull {
+                    (it as PsiClassType).let { classType ->
+                        Projection.Nullable(Projection.TypeConstructor(classType.resolve()!!.toDRI(), emptyList()))
+                    }
+                }
+            return typeParameters.mapIndexed { index, type ->
+                TypeParameter(
+                    dri.copy(genericTarget = index),
+                    type.name.orEmpty(),
+                    javadocParser.parseDocumentation(type).toPlatformDependant(),
+                    mapProjections(type.bounds),
+                    listOf(platformData)
+                )
+            }
+        }
+
+        private fun PsiQualifiedNamedElement.toDRI() =
+            DRI(qualifiedName.orEmpty().substringBeforeLast('.', ""), name)
+
+        private fun PsiMethod.getPropertyNameForFunction() =
+            getAnnotation(DescriptorUtils.JVM_NAME.asString())?.findAttributeValue("name")?.text
+                ?: when {
+                    JvmAbi.isGetterName(name) -> propertyNameByGetMethodName(Name.identifier(name))?.asString()
+                    JvmAbi.isSetterName(name) -> propertyNamesBySetMethodName(Name.identifier(name)).firstOrNull()?.asString()
+                    else -> null
+                }
+
+        private fun PsiClass.splitFunctionsAndAccessors(): Pair<MutableList<PsiMethod>, MutableMap<PsiField, MutableList<PsiMethod>>> {
+            val fieldNames = fields.map { it.name to it }.toMap()
+            val accessors = mutableMapOf<PsiField, MutableList<PsiMethod>>()
+            val regularMethods = mutableListOf<PsiMethod>()
+            methods.forEach { method ->
+                val field = method.getPropertyNameForFunction()?.let { name -> fieldNames[name] }
+                if (field != null) {
+                    accessors.getOrPut(field, ::mutableListOf).add(method)
+                } else {
+                    regularMethods.add(method)
+                }
+            }
+            return regularMethods to accessors
+        }
+
+        private fun parseField(psi: PsiField, parent: DRI, accessors: List<PsiMethod>): Property {
+            val dri = parent.copy(
+                callable = Callable(
+                    psi.name!!, // TODO: Investigate if this is indeed nullable
+                    JavaClassReference(psi.containingClass?.name.orEmpty()),
+                    emptyList()
+                )
+            )
+            return Property(
+                dri,
+                psi.name!!, // TODO: Investigate if this is indeed nullable
+                javadocParser.parseDocumentation(psi).toPlatformDependant(),
+                PsiDocumentableSource(psi).toPlatformDependant(),
+                psi.getVisibility().toPlatformDependant(),
+                JavaTypeWrapper(psi.type),
+                null,
+                accessors.firstOrNull { it.hasParameters() }?.let { parseFunction(it, parent) },
+                accessors.firstOrNull { it.returnType == psi.type }?.let { parseFunction(it, parent) },
+                psi.getModifier(),
+                listOf(platformData)
+            )
+        }
+    }
+}
diff --git a/plugins/base/src/main/kotlin/translators/psi/JavadocParser.kt b/plugins/base/src/main/kotlin/translators/psi/JavadocParser.kt
new file mode 100644
index 00000000..5b9af028
--- /dev/null
+++ b/plugins/base/src/main/kotlin/translators/psi/JavadocParser.kt
@@ -0,0 +1,149 @@
+package org.jetbrains.dokka.base.translators.psi
+
+import com.intellij.psi.*
+import com.intellij.psi.impl.source.javadoc.PsiDocParamRef
+import com.intellij.psi.impl.source.tree.JavaDocElementType
+import com.intellij.psi.impl.source.tree.LeafPsiElement
+import com.intellij.psi.javadoc.*
+import com.intellij.psi.util.PsiTreeUtil
+import org.jetbrains.dokka.links.DRI
+import org.jetbrains.dokka.model.doc.*
+import org.jetbrains.dokka.model.doc.Deprecated
+import org.jetbrains.dokka.utilities.DokkaLogger
+import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Element
+import org.jsoup.nodes.Node
+import org.jsoup.nodes.TextNode
+
+interface JavaDocumentationParser {
+    fun parseDocumentation(element: PsiNamedElement): DocumentationNode
+}
+
+class JavadocParser(
+    private val logger: DokkaLogger // TODO: Add logging
+) : JavaDocumentationParser {
+
+    override fun parseDocumentation(element: PsiNamedElement): DocumentationNode {
+        val docComment = (element as? PsiDocCommentOwner)?.docComment ?: return DocumentationNode(emptyList())
+        val nodes = mutableListOf<TagWrapper>()
+        docComment.getDescription()?.let { nodes.add(it) }
+        nodes.addAll(docComment.tags.mapNotNull { tag ->
+            when (tag.name) {
+                "param" -> Param(P(convertJavadocElements(tag.dataElements.toList())), tag.text)
+                "throws" -> Throws(P(convertJavadocElements(tag.dataElements.toList())), tag.text)
+                "return" -> Return(P(convertJavadocElements(tag.dataElements.toList())))
+                "author" -> Author(P(convertJavadocElements(tag.dataElements.toList())))
+                "see" -> See(P(getSeeTagElementContent(tag)), tag.referenceElement()?.text.orEmpty())
+                "deprecated" -> Deprecated(P(convertJavadocElements(tag.dataElements.toList())))
+                else -> null
+            }
+        })
+        return DocumentationNode(nodes)
+    }
+
+    private fun getSeeTagElementContent(tag: PsiDocTag): List<DocTag> =
+        listOfNotNull(tag.referenceElement()?.toDocumentationLink())
+
+    private fun PsiDocComment.getDescription(): Description? =
+        convertJavadocElements(descriptionElements.dropWhile {
+            it.text.trim().isEmpty()
+        }).takeIf { it.isNotEmpty() }?.let { list ->
+            Description(P(list))
+        }
+
+    private fun convertJavadocElements(elements: Iterable<PsiElement>): List<DocTag> =
+        elements.mapNotNull {
+            when (it) {
+                is PsiReference -> convertJavadocElements(it.children.toList())
+                is PsiInlineDocTag -> listOfNotNull(convertInlineDocTag(it))
+                is PsiDocParamRef -> listOfNotNull(it.toDocumentationLink())
+                is PsiDocTagValue,
+                is PsiWhiteSpace -> listOfNotNull(Text(it.text))
+                is LeafPsiElement -> Jsoup.parse(it.text).body().childNodes().mapNotNull { convertHtmlNode(it) }
+                else -> null
+            }
+        }.flatten()
+
+    private fun convertHtmlNode(node: Node, insidePre: Boolean = false): DocTag? = when (node) {
+        is TextNode -> Text(body = if (insidePre) node.wholeText else node.text())
+        is Element -> createBlock(node)
+        else -> null
+    }
+
+    private fun createBlock(element: Element): DocTag {
+        val children = element.childNodes().mapNotNull { convertHtmlNode(it) }
+        return when (element.tagName()) {
+            "p" -> P(children)
+            "b" -> B(children)
+            "strong" -> Strong(children)
+            "i" -> I(children)
+            "em" -> Em(children)
+            "code" -> Code(children)
+            "pre" -> Pre(children)
+            "ul" -> Ul(children)
+            "ol" -> Ol(children)
+            "li" -> Li(children)
+            //"a" -> createLink(element, children) // TODO: add proper inline link handling
+            "br" -> Br
+            else -> Text(body = element.ownText())
+        }
+    }
+/*
+
+    private fun createLink(element: Element, children: List<DocTag>): DocTag {
+        return when {
+            element.hasAttr("docref") -> {
+                A(children, params = mapOf("docref" to element.attr("docref")))
+            }
+            element.hasAttr("href") -> {
+                val href = element.attr("href")
+
+                val uri = try {
+                    A(children, params = mapOf("href" to href))
+                } catch (_: Exception) {
+                    null
+                }
+
+                if (uri?.isAbsolute == false) {
+                    ContentLocalLink(href)
+                } else {
+                    ContentExternalLink(href)
+                }
+            }
+            element.hasAttr("name") -> {
+                ContentBookmark(element.attr("name"))
+            }
+            else -> Text()
+        }
+    }*/
+
+    private fun PsiElement.toDocumentationLink(labelElement: PsiElement? = null) =
+        reference?.resolve()?.let {
+            val dri = DRI.from(it)
+            val label = labelElement ?: children.firstOrNull { it is PsiDocToken && it.text.isNotBlank() } ?: this
+            DocumentationLink(dri, convertJavadocElements(listOfNotNull(label)))
+        }
+
+    private fun convertInlineDocTag(tag: PsiInlineDocTag) = when (tag.name) {
+        "link", "linkplain" -> {
+            tag.referenceElement()?.toDocumentationLink(tag.dataElements.firstIsInstanceOrNull<PsiDocToken>())
+        }
+        "code", "literal" -> {
+            Code(listOf(Text(tag.text)))
+        }
+        else -> Text(tag.text)
+    }
+
+    private fun PsiDocTag.referenceElement(): PsiElement? =
+        linkElement()?.let {
+            if (it.node.elementType == JavaDocElementType.DOC_REFERENCE_HOLDER) {
+                PsiTreeUtil.findChildOfType(it, PsiJavaCodeReferenceElement::class.java)
+            } else {
+                it
+            }
+        }
+
+    private fun PsiDocTag.linkElement(): PsiElement? =
+        valueElement ?: dataElements.firstOrNull { it !is PsiWhiteSpace }
+}