Restructure the project to utilize included builds (#3174)

* Refactor and simplify artifact publishing
* Update Gradle to 8.4
* Refactor and simplify convention plugins and build scripts

Fixes #3132

---------

Co-authored-by: Adam <897017+aSemy@users.noreply.github.com>
Co-authored-by: Oleg Yukhnevich <whyoleg@gmail.com>

7 files changed, 445 insertions, 0 deletions

diff --git a/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/AllModulesPageGeneration.kt b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/AllModulesPageGeneration.kt
new file mode 100644
index 00000000..11d2d32c
--- /dev/null
+++ b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/AllModulesPageGeneration.kt
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.allModulesPage
+
+import org.jetbrains.dokka.CoreExtensions
+import org.jetbrains.dokka.Timer
+import org.jetbrains.dokka.generation.Generation
+import org.jetbrains.dokka.pages.RootPageNode
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.plugability.plugin
+import org.jetbrains.dokka.plugability.query
+import org.jetbrains.dokka.plugability.querySingle
+import org.jetbrains.dokka.templates.TemplatingPlugin
+import org.jetbrains.dokka.templates.TemplatingResult
+import org.jetbrains.dokka.transformers.pages.CreationContext
+
+public class AllModulesPageGeneration(private val context: DokkaContext) : Generation {
+
+    private val allModulesPagePlugin by lazy { context.plugin<AllModulesPagePlugin>() }
+    private val templatingPlugin by lazy { context.plugin<TemplatingPlugin>() }
+
+    override fun Timer.generate() {
+        report("Processing submodules")
+        val generationContext = processSubmodules()
+
+        report("Creating all modules page")
+        val pages = createAllModulesPage(generationContext)
+
+        report("Transforming pages")
+        val transformedPages = transformAllModulesPage(pages)
+
+        report("Rendering")
+        render(transformedPages)
+
+        report("Processing multimodule")
+        processMultiModule(transformedPages)
+
+        report("Finish submodule processing")
+        finishProcessingSubmodules()
+
+        report("Running post-actions")
+        runPostActions()
+    }
+
+    override val generationName: String = "index page for project"
+
+    public fun createAllModulesPage(allModulesContext: DefaultAllModulesContext): RootPageNode =
+        allModulesPagePlugin.querySingle { allModulesPageCreator }.invoke(allModulesContext)
+
+    public fun transformAllModulesPage(pages: RootPageNode): RootPageNode =
+        allModulesPagePlugin.query { allModulesPageTransformer }.fold(pages) { acc, t -> t(acc) }
+
+    public fun render(transformedPages: RootPageNode) {
+        context.single(CoreExtensions.renderer).render(transformedPages)
+    }
+
+    public fun runPostActions() {
+        context[CoreExtensions.postActions].forEach { it() }
+    }
+
+    public fun processSubmodules(): DefaultAllModulesContext {
+        return templatingPlugin.querySingle { submoduleTemplateProcessor }
+            .process(context.configuration.modules)
+            .let { DefaultAllModulesContext(it) }
+    }
+
+    public fun processMultiModule(root: RootPageNode) {
+        templatingPlugin.querySingle { multimoduleTemplateProcessor }.process(root)
+    }
+
+    public fun finishProcessingSubmodules() {
+        templatingPlugin.query { templateProcessingStrategy }.forEach { it.finish(context.configuration.outputDir) }
+    }
+
+    public data class DefaultAllModulesContext(val nonEmptyModules: List<String>) : CreationContext {
+        public constructor(templating: TemplatingResult) : this(templating.modules)
+    }
+}
diff --git a/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/AllModulesPagePlugin.kt b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/AllModulesPagePlugin.kt
new file mode 100644
index 00000000..06202082
--- /dev/null
+++ b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/AllModulesPagePlugin.kt
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.allModulesPage
+
+import org.jetbrains.dokka.CoreExtensions
+import org.jetbrains.dokka.base.DokkaBase
+import org.jetbrains.dokka.base.resolvers.local.DokkaLocationProviderFactory
+import org.jetbrains.dokka.base.resolvers.local.LocationProviderFactory
+import org.jetbrains.dokka.generation.Generation
+import org.jetbrains.dokka.plugability.*
+import org.jetbrains.dokka.templates.CommandHandler
+import org.jetbrains.dokka.templates.TemplatingPlugin
+import org.jetbrains.dokka.transformers.pages.PageCreator
+import org.jetbrains.dokka.transformers.pages.PageTransformer
+
+public class AllModulesPagePlugin : DokkaPlugin() {
+
+    public val partialLocationProviderFactory: ExtensionPoint<LocationProviderFactory> by extensionPoint()
+    public val allModulesPageCreator: ExtensionPoint<PageCreator<AllModulesPageGeneration.DefaultAllModulesContext>> by extensionPoint()
+    public val allModulesPageTransformer: ExtensionPoint<PageTransformer> by extensionPoint()
+    public val externalModuleLinkResolver: ExtensionPoint<ExternalModuleLinkResolver> by extensionPoint()
+
+    public val allModulesPageCreators: Extension<PageCreator<AllModulesPageGeneration.DefaultAllModulesContext>, *, *> by extending {
+        allModulesPageCreator providing ::MultimodulePageCreator
+    }
+
+    private val dokkaBase: DokkaBase by lazy { plugin<DokkaBase>() }
+
+    public val multimoduleLocationProvider: Extension<LocationProviderFactory, *, *> by extending {
+        (dokkaBase.locationProviderFactory
+                providing MultimoduleLocationProvider::Factory
+                override plugin<DokkaBase>().locationProvider)
+    }
+
+    public val baseLocationProviderFactory: Extension<LocationProviderFactory, *, *> by extending {
+        partialLocationProviderFactory providing ::DokkaLocationProviderFactory
+    }
+
+    public val allModulesPageGeneration: Extension<Generation, *, *> by extending {
+        (CoreExtensions.generation
+                providing ::AllModulesPageGeneration
+                override dokkaBase.singleGeneration)
+    }
+
+    public val resolveLinkCommandHandler: Extension<CommandHandler, *, *> by extending {
+        plugin<TemplatingPlugin>().directiveBasedCommandHandlers providing ::ResolveLinkCommandHandler
+    }
+
+    public val multiModuleLinkResolver: Extension<ExternalModuleLinkResolver, *, *> by extending {
+        externalModuleLinkResolver providing ::DefaultExternalModuleLinkResolver
+    }
+
+    @OptIn(DokkaPluginApiPreview::class)
+    override fun pluginApiPreviewAcknowledgement(): PluginApiPreviewAcknowledgement =
+        PluginApiPreviewAcknowledgement
+}
diff --git a/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/ExternalModuleLinkResolver.kt b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/ExternalModuleLinkResolver.kt
new file mode 100644
index 00000000..da747bda
--- /dev/null
+++ b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/ExternalModuleLinkResolver.kt
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.allModulesPage
+
+import org.jetbrains.dokka.DokkaConfiguration.DokkaModuleDescription
+import org.jetbrains.dokka.base.DokkaBase
+import org.jetbrains.dokka.base.resolvers.shared.ExternalDocumentation
+import org.jetbrains.dokka.base.resolvers.shared.PackageList
+import org.jetbrains.dokka.base.resolvers.shared.PackageList.Companion.PACKAGE_LIST_NAME
+import org.jetbrains.dokka.links.DRI
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.plugability.plugin
+import org.jetbrains.dokka.plugability.query
+import java.io.File
+import java.net.URL
+
+public interface ExternalModuleLinkResolver {
+    public fun resolve(dri: DRI, fileContext: File): String?
+    public fun resolveLinkToModuleIndex(moduleName: String): String?
+}
+
+public class DefaultExternalModuleLinkResolver(
+    public val context: DokkaContext
+) : ExternalModuleLinkResolver {
+    private val elpFactory = context.plugin<DokkaBase>().query { externalLocationProviderFactory }
+    private val externalDocumentations by lazy(::setupExternalDocumentations)
+    private val elps by lazy {
+        elpFactory.flatMap { externalDocumentations.map { ed -> it.getExternalLocationProvider(ed) } }.filterNotNull()
+    }
+
+    private fun setupExternalDocumentations(): List<ExternalDocumentation> =
+        context.configuration.modules.mapNotNull { module ->
+            loadPackageListForModule(module)?.let { packageList ->
+                ExternalDocumentation(
+                    URL("file:/${module.relativePathToOutputDirectory.toRelativeOutputDir()}"),
+                    packageList
+                )
+            }
+        }
+
+
+    private fun File.toRelativeOutputDir(): File = if (isAbsolute) {
+        relativeToOrSelf(context.configuration.outputDir)
+    } else {
+        this
+    }
+
+    private fun loadPackageListForModule(module: DokkaModuleDescription) =
+        module.sourceOutputDirectory.walkTopDown().maxDepth(3).firstOrNull { it.name == PACKAGE_LIST_NAME }?.let {
+            PackageList.load(
+                URL("file:" + it.path),
+                8,
+                true
+            )
+        }
+
+    override fun resolve(dri: DRI, fileContext: File): String? {
+        val absoluteLink = elps.mapNotNull { it.resolve(dri) }.firstOrNull() ?: return null
+        val modulePath = context.configuration.outputDir.absolutePath.split(File.separator)
+        val contextPath = fileContext.absolutePath.split(File.separator)
+        val commonPathElements = modulePath.zip(contextPath)
+            .takeWhile { (a, b) -> a == b }.count()
+
+        return (List(contextPath.size - commonPathElements - 1) { ".." } + modulePath.drop(commonPathElements)).joinToString(
+            "/"
+        ) + absoluteLink.removePrefix("file:")
+    }
+
+    override fun resolveLinkToModuleIndex(moduleName: String): String? =
+        context.configuration.modules.firstOrNull { it.name == moduleName }
+            ?.let { module ->
+                val packageList = loadPackageListForModule(module)
+                val extension = packageList?.linkFormat?.linkExtension?.let { ".$it" }.orEmpty()
+                "${module.relativePathToOutputDirectory}/index$extension"
+            }
+
+}
diff --git a/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/MultimoduleLocationProvider.kt b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/MultimoduleLocationProvider.kt
new file mode 100644
index 00000000..b0fa13d0
--- /dev/null
+++ b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/MultimoduleLocationProvider.kt
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.allModulesPage
+
+import org.jetbrains.dokka.allModulesPage.MultimodulePageCreator.Companion.MULTIMODULE_PACKAGE_PLACEHOLDER
+import org.jetbrains.dokka.base.resolvers.local.DokkaBaseLocationProvider
+import org.jetbrains.dokka.base.resolvers.local.LocationProvider
+import org.jetbrains.dokka.base.resolvers.local.LocationProviderFactory
+import org.jetbrains.dokka.links.DRI
+import org.jetbrains.dokka.model.DisplaySourceSet
+import org.jetbrains.dokka.pages.ContentPage
+import org.jetbrains.dokka.pages.PageNode
+import org.jetbrains.dokka.pages.RootPageNode
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.plugability.plugin
+import org.jetbrains.dokka.plugability.querySingle
+
+public open class MultimoduleLocationProvider(
+    private val root: RootPageNode, dokkaContext: DokkaContext,
+    public val extension: String = ".html"
+) : DokkaBaseLocationProvider(root, dokkaContext) {
+
+    private val defaultLocationProvider =
+        dokkaContext.plugin<AllModulesPagePlugin>().querySingle { partialLocationProviderFactory }
+            .getLocationProvider(root)
+    private val externalModuleLinkResolver =
+        dokkaContext.plugin<AllModulesPagePlugin>().querySingle { externalModuleLinkResolver }
+
+    override fun resolve(dri: DRI, sourceSets: Set<DisplaySourceSet>, context: PageNode?): String? {
+        return if (dri == MultimodulePageCreator.MULTIMODULE_ROOT_DRI) {
+            pathToRoot(root) + "index"
+        } else {
+            dri.takeIf { it.packageName == MULTIMODULE_PACKAGE_PLACEHOLDER }
+                ?.classNames
+                ?.let(externalModuleLinkResolver::resolveLinkToModuleIndex)
+        }
+    }
+
+    override fun resolve(node: PageNode, context: PageNode?, skipExtension: Boolean): String? {
+        return if (node is ContentPage && MultimodulePageCreator.MULTIMODULE_ROOT_DRI in node.dri) {
+            pathToRoot(root) + "index" + if (!skipExtension) extension else ""
+        } else {
+            defaultLocationProvider.resolve(node, context, skipExtension)
+        }
+    }
+
+    override fun pathToRoot(from: PageNode): String = defaultLocationProvider.pathToRoot(from)
+
+    override fun ancestors(node: PageNode): List<PageNode> = listOf(root)
+
+    public class Factory(
+        private val context: DokkaContext
+    ) : LocationProviderFactory {
+        override fun getLocationProvider(pageNode: RootPageNode): LocationProvider =
+            MultimoduleLocationProvider(pageNode, context)
+    }
+}
diff --git a/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/MultimodulePageCreator.kt b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/MultimodulePageCreator.kt
new file mode 100644
index 00000000..7b832d21
--- /dev/null
+++ b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/MultimodulePageCreator.kt
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.allModulesPage
+
+import org.jetbrains.dokka.DokkaConfiguration.DokkaModuleDescription
+import org.jetbrains.dokka.DokkaConfiguration.DokkaSourceSet
+import org.jetbrains.dokka.analysis.markdown.jb.MarkdownParser
+import org.jetbrains.dokka.base.DokkaBase
+import org.jetbrains.dokka.base.resolvers.anchors.SymbolAnchorHint
+import org.jetbrains.dokka.base.transformers.pages.comments.DocTagToContentConverter
+import org.jetbrains.dokka.base.translators.documentables.PageContentBuilder
+import org.jetbrains.dokka.links.DRI
+import org.jetbrains.dokka.model.doc.DocTag
+import org.jetbrains.dokka.model.doc.DocumentationNode
+import org.jetbrains.dokka.model.doc.P
+import org.jetbrains.dokka.model.properties.PropertyContainer
+import org.jetbrains.dokka.pages.*
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.plugability.plugin
+import org.jetbrains.dokka.plugability.querySingle
+import org.jetbrains.dokka.transformers.pages.PageCreator
+import org.jetbrains.dokka.utilities.DokkaLogger
+import org.jetbrains.dokka.analysis.kotlin.internal.InternalKotlinAnalysisPlugin
+import java.io.File
+
+public class MultimodulePageCreator(
+    private val context: DokkaContext,
+) : PageCreator<AllModulesPageGeneration.DefaultAllModulesContext> {
+    private val commentsConverter by lazy { context.plugin<DokkaBase>().querySingle { commentsToContentConverter } }
+    private val signatureProvider by lazy { context.plugin<DokkaBase>().querySingle { signatureProvider } }
+    private val moduleDocumentationReader by lazy { context.plugin<InternalKotlinAnalysisPlugin>().querySingle { moduleAndPackageDocumentationReader } }
+
+    override fun invoke(creationContext: AllModulesPageGeneration.DefaultAllModulesContext): RootPageNode {
+        val modules = context.configuration.modules
+        val sourceSetData = emptySet<DokkaSourceSet>()
+        val builder = PageContentBuilder(commentsConverter, signatureProvider, context.logger)
+        val contentNode = builder.contentFor(
+            dri = DRI(MULTIMODULE_PACKAGE_PLACEHOLDER),
+            kind = ContentKind.Cover,
+            sourceSets = sourceSetData
+        ) {
+            getMultiModuleDocumentation(context.configuration.includes).takeIf { it.isNotEmpty() }?.let { nodes ->
+                group(kind = ContentKind.Cover) {
+                    nodes.forEach { node ->
+                        group {
+                            node.children.forEach { comment(it.root) }
+                        }
+                    }
+                }
+            }
+            header(2, "All modules:")
+            table(styles = setOf(MultimoduleTable)) {
+                header { group { text("Name") } }
+                modules.filter { it.name in creationContext.nonEmptyModules }.sortedBy { it.name }
+                    .forEach { module ->
+                        val displayedModuleDocumentation = getDisplayedModuleDocumentation(module)
+                        val dri = DRI(packageName = MULTIMODULE_PACKAGE_PLACEHOLDER, classNames = module.name)
+                        val dci = DCI(setOf(dri), ContentKind.Comment)
+                        val extraWithAnchor = PropertyContainer.withAll(SymbolAnchorHint(module.name, ContentKind.Main))
+                        row(setOf(dri), emptySet(), styles = emptySet(), extra = extraWithAnchor) {
+                            +linkNode(module.name, dri, DCI(setOf(dri), ContentKind.Main), extra = extraWithAnchor)
+                            +ContentGroup(
+                                children =
+                                if (displayedModuleDocumentation != null)
+                                    DocTagToContentConverter().buildContent(
+                                        displayedModuleDocumentation,
+                                        dci,
+                                        emptySet()
+                                    )
+                                else emptyList(),
+                                dci = dci,
+                                sourceSets = emptySet(),
+                                style = emptySet()
+                            )
+                        }
+                    }
+            }
+        }
+        return MultimoduleRootPageNode(
+            setOf(MULTIMODULE_ROOT_DRI),
+            contentNode
+        )
+    }
+
+    private fun getMultiModuleDocumentation(files: Set<File>): List<DocumentationNode> =
+        files.map { MarkdownParser({ null }, it.absolutePath).parse(it.readText()) }
+
+    private fun getDisplayedModuleDocumentation(module: DokkaModuleDescription): P? {
+        return moduleDocumentationReader.read(module)?.firstParagraph()
+    }
+
+    private fun DocumentationNode.firstParagraph(): P? =
+        this.children
+            .map { it.root }
+            .mapNotNull { it.firstParagraph() }
+            .firstOrNull()
+
+    /**
+     * @return The very first, most inner paragraph. If any [P] is wrapped inside another [P], the inner one
+     * is preferred.
+     */
+    private fun DocTag.firstParagraph(): P? {
+        val firstChildParagraph = children.mapNotNull { it.firstParagraph() }.firstOrNull()
+        return if (firstChildParagraph == null && this is P) this
+        else firstChildParagraph
+    }
+
+    public companion object {
+        public const val MULTIMODULE_PACKAGE_PLACEHOLDER: String = ".ext"
+        public val MULTIMODULE_ROOT_DRI: DRI =
+            DRI(packageName = MULTIMODULE_PACKAGE_PLACEHOLDER, classNames = "allModules")
+    }
+}
diff --git a/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/ResolveLinkCommandHandler.kt b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/ResolveLinkCommandHandler.kt
new file mode 100644
index 00000000..f6d34271
--- /dev/null
+++ b/dokka-subprojects/plugin-all-modules-page/src/main/kotlin/org/jetbrains/dokka/allModulesPage/ResolveLinkCommandHandler.kt
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.allModulesPage
+
+import org.jetbrains.dokka.base.templating.Command
+import org.jetbrains.dokka.base.templating.ResolveLinkCommand
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.plugability.plugin
+import org.jetbrains.dokka.plugability.querySingle
+import org.jetbrains.dokka.templates.CommandHandler
+import org.jsoup.nodes.Attributes
+import org.jsoup.nodes.Element
+import org.jsoup.parser.Tag
+import java.io.File
+
+public class ResolveLinkCommandHandler(context: DokkaContext) : CommandHandler {
+
+    private val externalModuleLinkResolver =
+        context.plugin<AllModulesPagePlugin>().querySingle { externalModuleLinkResolver }
+
+    override fun handleCommandAsTag(command: Command, body: Element, input: File, output: File) {
+        command as ResolveLinkCommand
+        val link = externalModuleLinkResolver.resolve(command.dri, output)
+        if (link == null) {
+            val children = body.childNodes().toList()
+            val attributes = Attributes().apply {
+                put("data-unresolved-link", command.dri.toString())
+            }
+            val el = Element(Tag.valueOf("span"), "", attributes).apply {
+                children.forEach { ch -> appendChild(ch) }
+            }
+            body.replaceWith(el)
+            return
+        }
+
+        val attributes = Attributes().apply {
+            put("href", link)
+        }
+        val children = body.childNodes().toList()
+        val el = Element(Tag.valueOf("a"), "", attributes).apply {
+            children.forEach { ch -> appendChild(ch) }
+        }
+        body.replaceWith(el)
+    }
+
+    override fun canHandle(command: Command): Boolean = command is ResolveLinkCommand
+}
diff --git a/dokka-subprojects/plugin-all-modules-page/src/main/resources/META-INF/services/org.jetbrains.dokka.plugability.DokkaPlugin b/dokka-subprojects/plugin-all-modules-page/src/main/resources/META-INF/services/org.jetbrains.dokka.plugability.DokkaPlugin
new file mode 100644
index 00000000..f50db659
--- /dev/null
+++ b/dokka-subprojects/plugin-all-modules-page/src/main/resources/META-INF/services/org.jetbrains.dokka.plugability.DokkaPlugin
@@ -0,0 +1,5 @@
+#
+# Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+#
+
+org.jetbrains.dokka.allModulesPage.AllModulesPagePlugin