Decompose Kotlin/Java analysis (#3034)

* Extract analysis into separate modules

66 files changed, 134 insertions, 5162 deletions

diff --git a/plugins/base/src/main/kotlin/DokkaBase.kt b/plugins/base/src/main/kotlin/DokkaBase.kt
index 8fc46870..6b000ac6 100644
--- a/plugins/base/src/main/kotlin/DokkaBase.kt
+++ b/plugins/base/src/main/kotlin/DokkaBase.kt
@@ -3,14 +3,14 @@
 package org.jetbrains.dokka.base
 
 import org.jetbrains.dokka.CoreExtensions
-import org.jetbrains.dokka.analysis.KotlinAnalysis
-import org.jetbrains.dokka.analysis.ProjectKotlinAnalysis
+import org.jetbrains.dokka.base.generation.SingleModuleGeneration
 import org.jetbrains.dokka.base.renderers.*
 import org.jetbrains.dokka.base.renderers.html.*
 import org.jetbrains.dokka.base.renderers.html.command.consumers.PathToRootConsumer
+import org.jetbrains.dokka.base.renderers.html.command.consumers.ReplaceVersionsConsumer
 import org.jetbrains.dokka.base.renderers.html.command.consumers.ResolveLinkConsumer
-import org.jetbrains.dokka.base.resolvers.external.ExternalLocationProviderFactory
 import org.jetbrains.dokka.base.resolvers.external.DefaultExternalLocationProviderFactory
+import org.jetbrains.dokka.base.resolvers.external.ExternalLocationProviderFactory
 import org.jetbrains.dokka.base.resolvers.external.javadoc.JavadocExternalLocationProviderFactory
 import org.jetbrains.dokka.base.resolvers.local.DokkaLocationProviderFactory
 import org.jetbrains.dokka.base.resolvers.local.LocationProviderFactory
@@ -23,24 +23,13 @@ import org.jetbrains.dokka.base.transformers.pages.annotations.SinceKotlinTransf
 import org.jetbrains.dokka.base.transformers.pages.comments.CommentsToContentConverter
 import org.jetbrains.dokka.base.transformers.pages.comments.DocTagToContentConverter
 import org.jetbrains.dokka.base.transformers.pages.merger.*
-import org.jetbrains.dokka.base.transformers.pages.samples.DefaultSamplesTransformer
 import org.jetbrains.dokka.base.transformers.pages.sourcelinks.SourceLinksTransformer
-import org.jetbrains.dokka.base.translators.descriptors.DefaultDescriptorToDocumentableTranslator
-import org.jetbrains.dokka.base.translators.documentables.DefaultDocumentableToPageTranslator
-import org.jetbrains.dokka.base.translators.psi.DefaultPsiToDocumentableTranslator
-import org.jetbrains.dokka.base.generation.SingleModuleGeneration
-import org.jetbrains.dokka.base.renderers.html.command.consumers.ReplaceVersionsConsumer
 import org.jetbrains.dokka.base.transformers.pages.tags.CustomTagContentProvider
 import org.jetbrains.dokka.base.transformers.pages.tags.SinceKotlinTagContentProvider
-import org.jetbrains.dokka.base.translators.descriptors.DefaultExternalDocumentablesProvider
-import org.jetbrains.dokka.base.translators.descriptors.ExternalClasslikesTranslator
-import org.jetbrains.dokka.base.translators.descriptors.ExternalDocumentablesProvider
-import org.jetbrains.dokka.base.utils.NoopIntellijLoggerFactory
+import org.jetbrains.dokka.base.translators.documentables.DefaultDocumentableToPageTranslator
 import org.jetbrains.dokka.plugability.DokkaPlugin
 import org.jetbrains.dokka.plugability.DokkaPluginApiPreview
 import org.jetbrains.dokka.plugability.PluginApiPreviewAcknowledgement
-import org.jetbrains.dokka.plugability.querySingle
-import org.jetbrains.dokka.renderers.PostAction
 import org.jetbrains.dokka.transformers.documentation.PreMergeDocumentableTransformer
 import org.jetbrains.dokka.transformers.pages.PageTransformer
 
@@ -55,25 +44,16 @@ class DokkaBase : DokkaPlugin() {
     val externalLocationProviderFactory by extensionPoint<ExternalLocationProviderFactory>()
     val outputWriter by extensionPoint<OutputWriter>()
     val htmlPreprocessors by extensionPoint<PageTransformer>()
-    val kotlinAnalysis by extensionPoint<KotlinAnalysis>()
+
     @Deprecated("It is not used anymore")
     val tabSortingStrategy by extensionPoint<TabSortingStrategy>()
     val immediateHtmlCommandConsumer by extensionPoint<ImmediateHtmlCommandConsumer>()
-    val externalDocumentablesProvider by extensionPoint<ExternalDocumentablesProvider>()
-    val externalClasslikesTranslator by extensionPoint<ExternalClasslikesTranslator>()
+
 
     val singleGeneration by extending {
         CoreExtensions.generation providing ::SingleModuleGeneration
     }
 
-    val descriptorToDocumentableTranslator by extending {
-        CoreExtensions.sourceToDocumentableTranslator providing ::DefaultDescriptorToDocumentableTranslator
-    }
-
-    val psiToDocumentableTranslator by extending {
-        CoreExtensions.sourceToDocumentableTranslator providing ::DefaultPsiToDocumentableTranslator
-    }
-
     val documentableMerger by extending {
         CoreExtensions.documentableMerger providing ::DefaultDocumentableMerger
     }
@@ -168,7 +148,9 @@ class DokkaBase : DokkaPlugin() {
     }
 
     val pageMerger by extending {
-        CoreExtensions.pageTransformer providing ::PageMerger
+        CoreExtensions.pageTransformer providing ::PageMerger order {
+            // TODO [beresnev] make last() or at least after samples transformer
+        }
     }
 
     val sourceSetMerger by extending {
@@ -189,15 +171,6 @@ class DokkaBase : DokkaPlugin() {
         CoreExtensions.renderer providing ::HtmlRenderer
     }
 
-    val defaultKotlinAnalysis by extending {
-        kotlinAnalysis providing { ctx ->
-            ProjectKotlinAnalysis(
-                sourceSets = ctx.configuration.sourceSets,
-                logger = ctx.logger
-            )
-        }
-    }
-
     val locationProvider by extending {
         locationProviderFactory providing ::DokkaLocationProviderFactory
     }
@@ -218,12 +191,6 @@ class DokkaBase : DokkaPlugin() {
         htmlPreprocessors with RootCreator applyIf { !delayTemplateSubstitution }
     }
 
-    val defaultSamplesTransformer by extending {
-        CoreExtensions.pageTransformer providing ::DefaultSamplesTransformer order {
-            before(pageMerger)
-        }
-    }
-
     val sourceLinksTransformer by extending {
         htmlPreprocessors providing ::SourceLinksTransformer order { after(rootCreator) }
     }
@@ -275,26 +242,6 @@ class DokkaBase : DokkaPlugin() {
         htmlPreprocessors providing ::SearchbarDataInstaller order { after(sourceLinksTransformer) }
     }
 
-    val defaultExternalDocumentablesProvider by extending {
-        externalDocumentablesProvider providing ::DefaultExternalDocumentablesProvider
-    }
-
-    val defaultExternalClasslikesTranslator by extending {
-        externalClasslikesTranslator providing ::DefaultDescriptorToDocumentableTranslator
-    }
-
-    internal val disposeKotlinAnalysisPostAction by extending {
-        CoreExtensions.postActions with PostAction { this@DokkaBase.querySingle { kotlinAnalysis }.close() }
-    }
-
-    private companion object {
-        init {
-            // Suppress messages emitted by the IntelliJ logger since
-            // there's not much the end user can do about it
-            com.intellij.openapi.diagnostic.Logger.setFactory(NoopIntellijLoggerFactory())
-        }
-    }
-
     @OptIn(DokkaPluginApiPreview::class)
     override fun pluginApiPreviewAcknowledgement(): PluginApiPreviewAcknowledgement =
         PluginApiPreviewAcknowledgement
diff --git a/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt b/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt
deleted file mode 100644
index d49e7c7a..00000000
--- a/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt
+++ /dev/null
@@ -1,604 +0,0 @@
-package org.jetbrains.dokka.base.parsers
-
-import com.intellij.psi.PsiElement
-import org.intellij.markdown.MarkdownElementTypes
-import org.intellij.markdown.MarkdownTokenTypes
-import org.intellij.markdown.ast.ASTNode
-import org.intellij.markdown.ast.CompositeASTNode
-import org.intellij.markdown.ast.LeafASTNode
-import org.intellij.markdown.ast.impl.ListItemCompositeNode
-import org.intellij.markdown.flavours.gfm.GFMElementTypes
-import org.intellij.markdown.flavours.gfm.GFMFlavourDescriptor
-import org.intellij.markdown.flavours.gfm.GFMTokenTypes
-import org.intellij.markdown.html.HtmlGenerator
-import org.jetbrains.dokka.base.parsers.factories.DocTagsFromIElementFactory
-import org.jetbrains.dokka.links.DRI
-import org.jetbrains.dokka.links.PointingToDeclaration
-import org.jetbrains.dokka.model.doc.*
-import org.jetbrains.dokka.model.doc.Suppress
-import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag
-import org.jetbrains.kotlin.kdoc.psi.impl.KDocSection
-import org.jetbrains.kotlin.kdoc.psi.impl.KDocTag
-import java.net.MalformedURLException
-import java.net.URL
-import org.intellij.markdown.parser.MarkdownParser as IntellijMarkdownParser
-
-open class MarkdownParser(
-    private val externalDri: (String) -> DRI?,
-    private val kdocLocation: String?,
-) : Parser() {
-
-    private lateinit var destinationLinksMap: Map<String, String>
-    private lateinit var text: String
-
-    override fun parseStringToDocNode(extractedString: String): DocTag {
-        val gfmFlavourDescriptor = GFMFlavourDescriptor()
-        val markdownAstRoot = IntellijMarkdownParser(gfmFlavourDescriptor).buildMarkdownTreeFromString(extractedString)
-        destinationLinksMap = getAllDestinationLinks(extractedString, markdownAstRoot).toMap()
-        text = extractedString
-
-        val parsed = visitNode(markdownAstRoot)
-        if (parsed.size == 1) {
-            return parsed.first()
-        }
-        return CustomDocTag(children = parsed, params = emptyMap(), name = "")
-    }
-
-    override fun preparse(text: String) = text.replace("\r\n", "\n").replace("\r", "\n")
-
-    override fun parseTagWithBody(tagName: String, content: String): TagWrapper =
-        when (tagName) {
-            "see" -> {
-                val referencedName = content.substringBefore(' ')
-                val dri = externalDri(referencedName)
-                See(
-                    parseStringToDocNode(content.substringAfter(' ')),
-                    dri?.fqDeclarationName() ?: referencedName,
-                    dri
-                )
-            }
-            "throws", "exception" -> {
-                val dri = externalDri(content.substringBefore(' '))
-                Throws(
-                    parseStringToDocNode(content.substringAfter(' ')),
-                    dri?.fqDeclarationName() ?: content.substringBefore(' '),
-                    dri
-                )
-            }
-            else -> super.parseTagWithBody(tagName, content)
-        }
-
-    private fun headersHandler(node: ASTNode) =
-        DocTagsFromIElementFactory.getInstance(
-            node.type,
-            visitNode(node.children.find { it.type == MarkdownTokenTypes.ATX_CONTENT }
-                ?: throw detailedException("Wrong AST Tree. Header does not contain expected content", node)
-            ).flatMap { it.children }
-        )
-
-    private fun horizontalRulesHandler() =
-        DocTagsFromIElementFactory.getInstance(MarkdownTokenTypes.HORIZONTAL_RULE)
-
-    private fun emphasisHandler(node: ASTNode) =
-        DocTagsFromIElementFactory.getInstance(
-            node.type,
-            children = node.children.evaluateChildrenWithDroppedEnclosingTokens(1)
-        )
-
-    private fun strongHandler(node: ASTNode) =
-        DocTagsFromIElementFactory.getInstance(
-            node.type,
-            children = node.children.evaluateChildrenWithDroppedEnclosingTokens(2)
-        )
-
-    private fun List<ASTNode>.evaluateChildrenWithDroppedEnclosingTokens(count: Int) =
-        drop(count).dropLast(count).evaluateChildren()
-
-    private fun blockquotesHandler(node: ASTNode) =
-        DocTagsFromIElementFactory.getInstance(
-            node.type, children = node.children
-                .filterIsInstance<CompositeASTNode>()
-                .evaluateChildren()
-        )
-
-    private fun listsHandler(node: ASTNode): List<DocTag> {
-
-        val children = node.children.filterIsInstance<ListItemCompositeNode>().flatMap {
-            if (it.children.last().type in listOf(
-                    MarkdownElementTypes.ORDERED_LIST,
-                    MarkdownElementTypes.UNORDERED_LIST
-                )
-            ) {
-                val nestedList = it.children.last()
-                (it.children as MutableList).removeAt(it.children.lastIndex)
-                listOf(it, nestedList)
-            } else
-                listOf(it)
-        }
-
-        return DocTagsFromIElementFactory.getInstance(
-            node.type,
-            children =
-            children
-                .flatMap {
-                    if (it.type == MarkdownElementTypes.LIST_ITEM)
-                        DocTagsFromIElementFactory.getInstance(
-                            it.type,
-                            children = it
-                                .children
-                                .filterIsInstance<CompositeASTNode>()
-                                .evaluateChildren()
-                        )
-                    else
-                        visitNode(it)
-                },
-            params =
-            if (node.type == MarkdownElementTypes.ORDERED_LIST) {
-                val listNumberNode = node.children.first().children.first()
-                mapOf(
-                    "start" to text.substring(
-                        listNumberNode.startOffset,
-                        listNumberNode.endOffset
-                    ).trim().dropLast(1)
-                )
-            } else
-                emptyMap()
-        )
-    }
-
-    private fun resolveDRI(mdLink: String): DRI? =
-        mdLink
-            .removePrefix("[")
-            .removeSuffix("]")
-            .let { link ->
-                try {
-                    URL(link)
-                    null
-                } catch (e: MalformedURLException) {
-                    externalDri(link)
-                }
-            }
-
-    private fun getAllDestinationLinks(text: String, node: ASTNode): List<Pair<String, String>> =
-        node.children
-            .filter { it.type == MarkdownElementTypes.LINK_DEFINITION }
-            .map {
-                text.substring(it.children[0].startOffset, it.children[0].endOffset).toLowerCase() to
-                        text.substring(it.children[2].startOffset, it.children[2].endOffset)
-            } +
-                node.children.filterIsInstance<CompositeASTNode>().flatMap { getAllDestinationLinks(text, it) }
-
-
-    private fun referenceLinksHandler(node: ASTNode): List<DocTag> {
-        val linkLabel = node.children.find { it.type == MarkdownElementTypes.LINK_LABEL }
-            ?: throw detailedException("Wrong AST Tree. Reference link does not contain link label", node)
-        val linkText = node.children.findLast { it.type == MarkdownElementTypes.LINK_TEXT } ?: linkLabel
-
-        val linkKey = text.substring(linkLabel.startOffset, linkLabel.endOffset)
-
-        val link = destinationLinksMap[linkKey.toLowerCase()] ?: linkKey
-
-        return linksHandler(linkText, link)
-    }
-
-    private fun inlineLinksHandler(node: ASTNode): List<DocTag> {
-        val linkText = node.children.find { it.type == MarkdownElementTypes.LINK_TEXT }
-            ?: throw detailedException("Wrong AST Tree. Inline link does not contain link text", node)
-        val linkDestination = node.children.find { it.type == MarkdownElementTypes.LINK_DESTINATION }
-        val linkTitle = node.children.find { it.type == MarkdownElementTypes.LINK_TITLE }
-
-        // Link destination may be ommited: https://github.github.com/gfm/#example-495
-        val link = linkDestination?.let { text.substring(it.startOffset, it.endOffset) }
-
-        return linksHandler(linkText, link, linkTitle)
-    }
-
-    private fun markdownFileHandler(node: ASTNode) =
-        DocTagsFromIElementFactory.getInstance(
-            node.type,
-            children = node.children
-