Restructure the project to utilize included builds (#3174)

* Refactor and simplify artifact publishing
* Update Gradle to 8.4
* Refactor and simplify convention plugins and build scripts

Fixes #3132

---------

Co-authored-by: Adam <897017+aSemy@users.noreply.github.com>
Co-authored-by: Oleg Yukhnevich <whyoleg@gmail.com>

55 files changed, 4972 insertions, 0 deletions

diff --git a/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaArtifacts.kt b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaArtifacts.kt
new file mode 100644
index 00000000..241c0449
--- /dev/null
+++ b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaArtifacts.kt
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.gradle
+
+import org.gradle.api.Project
+import org.gradle.api.artifacts.Dependency
+import org.jetbrains.dokka.DokkaVersion
+
+internal val Project.dokkaArtifacts get() = DokkaArtifacts(this)
+
+internal class DokkaArtifacts(private val project: Project) {
+    private fun fromModuleName(name: String): Dependency =
+        project.dependencies.create("org.jetbrains.dokka:$name:${DokkaVersion.version}")
+
+    // TODO [beresnev] analysis switcher
+    val analysisKotlinDescriptors get() = fromModuleName("analysis-kotlin-descriptors")
+    val analysisKotlinSymbols get() = fromModuleName("analysis-kotlin-symbols")
+
+    val allModulesPage get() = fromModuleName("all-modules-page-plugin")
+    val dokkaCore get() = fromModuleName("dokka-core")
+    val dokkaBase get() = fromModuleName("dokka-base")
+    val javadocPlugin get() = fromModuleName("javadoc-plugin")
+    val gfmPlugin get() = fromModuleName("gfm-plugin")
+    val gfmTemplateProcessing get() = fromModuleName("gfm-template-processing-plugin")
+    val jekyllTemplateProcessing get() = fromModuleName("jekyll-template-processing-plugin")
+    val jekyllPlugin get() = fromModuleName("jekyll-plugin")
+}
diff --git a/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaMultiModuleFileLayout.kt b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaMultiModuleFileLayout.kt
new file mode 100644
index 00000000..b6120129
--- /dev/null
+++ b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaMultiModuleFileLayout.kt
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.gradle
+
+import org.gradle.api.file.Directory
+import org.gradle.api.provider.Provider
+import org.jetbrains.dokka.DokkaException
+import org.jetbrains.dokka.gradle.DokkaMultiModuleFileLayout.CompactInParent
+import org.jetbrains.dokka.gradle.DokkaMultiModuleFileLayout.NoCopy
+import java.io.File
+
+/**
+ * @see DokkaMultiModuleFileLayout.targetChildOutputDirectory
+ * @see NoCopy
+ * @see CompactInParent
+ */
+fun interface DokkaMultiModuleFileLayout {
+
+    /**
+     * @param parent: The [DokkaMultiModuleTask] that is initiating a composite documentation run
+     * @param child: Some child task registered in [parent]
+     * @return The target output directory of the [child] dokka task referenced by [parent]. This should
+     * be unique for all registered child tasks.
+     */
+    fun targetChildOutputDirectory(parent: DokkaMultiModuleTask, child: AbstractDokkaTask): Provider<Directory>
+
+    /**
+     * Will link to the original [AbstractDokkaTask.outputDirectory]. This requires no copying of the output files.
+     */
+    object NoCopy : DokkaMultiModuleFileLayout {
+        override fun targetChildOutputDirectory(
+            parent: DokkaMultiModuleTask,
+            child: AbstractDokkaTask
+        ): Provider<Directory> = child.outputDirectory
+    }
+
+    /**
+     * Will point to a subfolder inside the output directory of the parent.
+     * The subfolder will follow the structure of the gradle project structure
+     * e.g.
+     * :parentProject:firstAncestor:secondAncestor will be be resolved to
+     * {parent output directory}/firstAncestor/secondAncestor
+     */
+    object CompactInParent : DokkaMultiModuleFileLayout {
+        override fun targetChildOutputDirectory(
+            parent: DokkaMultiModuleTask,
+            child: AbstractDokkaTask
+        ): Provider<Directory> {
+            val relativeProjectPath = parent.project.relativeProjectPath(child.project.path)
+            val relativeFilePath = relativeProjectPath.replace(":", File.separator)
+            check(!File(relativeFilePath).isAbsolute) { "Unexpected absolute path $relativeFilePath" }
+            return parent.outputDirectory.dir(relativeFilePath)
+        }
+    }
+}
+
+internal fun DokkaMultiModuleTask.targetChildOutputDirectory(
+    child: AbstractDokkaTask
+): Provider<Directory> = fileLayout.get().targetChildOutputDirectory(this, child)
+
+
+internal fun DokkaMultiModuleTask.copyChildOutputDirectories() {
+    childDokkaTasks.forEach { child ->
+        this.copyChildOutputDirectory(child)
+    }
+}
+
+internal fun DokkaMultiModuleTask.copyChildOutputDirectory(child: AbstractDokkaTask) {
+    val targetChildOutputDirectory = project.file(fileLayout.get().targetChildOutputDirectory(this, child))
+    val sourceChildOutputDirectory = child.outputDirectory.asFile.get()
+
+    /* Pointing to the same directory -> No copy necessary */
+    if (sourceChildOutputDirectory.absoluteFile == targetChildOutputDirectory.absoluteFile) {
+        return
+    }
+
+    /* Cannot target *inside* the original folder */
+    if (targetChildOutputDirectory.absoluteFile.startsWith(sourceChildOutputDirectory.absoluteFile)) {
+        throw DokkaException(
+            "Cannot re-locate output directory into itself.\n" +
+                    "sourceChildOutputDirectory=${sourceChildOutputDirectory.path}\n" +
+                    "targetChildOutputDirectory=${targetChildOutputDirectory.path}"
+        )
+    }
+
+    /* Source output directory is empty -> No copy necessary */
+    if (!sourceChildOutputDirectory.exists()) {
+        return
+    }
+
+    sourceChildOutputDirectory.copyRecursively(targetChildOutputDirectory, overwrite = true)
+}
diff --git a/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaPlugin.kt b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaPlugin.kt
new file mode 100644
index 00000000..77fba8f2
--- /dev/null
+++ b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaPlugin.kt
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.gradle
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.artifacts.Dependency
+import org.gradle.kotlin.dsl.register
+import org.gradle.kotlin.dsl.withType
+import org.gradle.util.GradleVersion
+import org.jetbrains.dokka.DokkaDefaults
+
+open class DokkaPlugin : Plugin<Project> {
+    override fun apply(project: Project) {
+        if (GradleVersion.version(project.gradle.gradleVersion) < GradleVersion.version("5.6")) {
+            project.logger.warn("Dokka: Build is using unsupported gradle version, expected at least 5.6 but got ${project.gradle.gradleVersion}. This may result in strange errors")
+        }
+        if (project.shouldUseK2())
+            project.logger.warn(
+                "Dokka's K2 Analysis is being used. " +
+                        "It is still under active development and is thus experimental. " +
+                        "It can be the cause of failed builds or incorrectly generated documentation. " +
+                        "If you encounter an issue, please consider reporting it: https://github.com/Kotlin/dokka/issues"
+            )
+
+        project.setupDokkaTasks("dokkaHtml") {
+            description = "Generates documentation in 'html' format"
+        }
+
+        project.setupDokkaTasks(
+            name = "dokkaJavadoc",
+            multiModuleTaskSupported = false
+        ) {
+            plugins.dependencies.add(project.dokkaArtifacts.javadocPlugin)
+            description = "Generates documentation in 'javadoc' format"
+        }
+
+        project.setupDokkaTasks(
+            "dokkaGfm",
+            allModulesPageAndTemplateProcessing = project.dokkaArtifacts.gfmTemplateProcessing
+        ) {
+            plugins.dependencies.add(project.dokkaArtifacts.gfmPlugin)
+            description = "Generates documentation in GitHub flavored markdown format"
+        }
+
+        project.setupDokkaTasks(
+            "dokkaJekyll",
+            allModulesPageAndTemplateProcessing = project.dokkaArtifacts.jekyllTemplateProcessing
+        ) {
+            plugins.dependencies.add(project.dokkaArtifacts.jekyllPlugin)
+            description = "Generates documentation in Jekyll flavored markdown format"
+        }
+
+        project.configureEachAbstractDokkaTask()
+        project.configureEachDokkaMultiModuleTask()
+    }
+
+    /**
+     * Creates [DokkaTask], [DokkaMultiModuleTask] for the given
+     * name and configuration.
+     */
+    private fun Project.setupDokkaTasks(
+        name: String,
+        multiModuleTaskSupported: Boolean = true,
+        allModulesPageAndTemplateProcessing: Dependency = project.dokkaArtifacts.allModulesPage,
+        configuration: AbstractDokkaTask.() -> Unit = {}
+    ) {
+        project.maybeCreateDokkaPluginConfiguration(name)
+        project.maybeCreateDokkaRuntimeConfiguration(name)
+        project.tasks.register<DokkaTask>(name) {
+            configuration()
+        }
+
+        if (project.parent != null) {
+            val partialName = "${name}Partial"
+            project.maybeCreateDokkaPluginConfiguration(partialName)
+            project.maybeCreateDokkaRuntimeConfiguration(partialName)
+            project.tasks.register<DokkaTaskPartial>(partialName) {
+                configuration()
+            }
+        }
+
+        if (project.subprojects.isNotEmpty()) {
+            if (multiModuleTaskSupported) {
+                val multiModuleName = "${name}MultiModule"
+                project.maybeCreateDokkaPluginConfiguration(multiModuleName, setOf(allModulesPageAndTemplateProcessing))
+                project.maybeCreateDokkaRuntimeConfiguration(multiModuleName)
+
+                project.tasks.register<DokkaMultiModuleTask>(multiModuleName) {
+                    @Suppress("DEPRECATION")
+                    addSubprojectChildTasks("${name}Partial")
+                    configuration()
+                    description = "Runs all subprojects '$name' tasks and generates module navigation page"
+                }
+
+                project.tasks.register<DefaultTask>("${name}Multimodule") {
+                    group = "deprecated"
+                    description = "DEPRECATED: 'Multimodule' is deprecated. Use 'MultiModule' instead."
+                    dependsOn(multiModuleName)
+                    doLast {
+                        logger.warn("'Multimodule' is deprecated. Use 'MultiModule' instead")
+                    }
+                }
+            }
+
+            project.tasks.register<DokkaCollectorTask>("${name}Collector") {
+                @Suppress("DEPRECATION")
+                addSubprojectChildTasks(name)
+                description =
+                    "Generates documentation merging all subprojects '$name' tasks into one virtual module"
+            }
+        }
+    }
+
+    private fun Project.configureEachAbstractDokkaTask() {
+        tasks.withType<AbstractDokkaTask>().configureEach {
+            val formatClassifier = name.removePrefix("dokka").decapitalize()
+            outputDirectory.convention(project.layout.buildDirectory.dir("dokka/$formatClassifier"))
+            cacheRoot.convention(project.layout.dir(providers.provider { DokkaDefaults.cacheRoot }))
+        }
+    }
+
+    private fun Project.configureEachDokkaMultiModuleTask() {
+        tasks.withType<DokkaMultiModuleTask>().configureEach {
+            sourceChildOutputDirectories.from({ childDokkaTasks.map { it.outputDirectory } })
+        }
+    }
+}
diff --git a/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaProperty.kt b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaProperty.kt
new file mode 100644
index 00000000..6c6e967d
--- /dev/null
+++ b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaProperty.kt
@@ -0,0 +1,10 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.gradle
+
+import org.gradle.api.provider.Provider
+
+
+internal fun Provider<String>.getValidVersionOrNull() = orNull?.takeIf { it != "unspecified" }
diff --git a/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaSourceSetMapper.kt b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaSourceSetMapper.kt
new file mode 100644
index 00000000..c0112719
--- /dev/null
+++ b/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/DokkaSourceSetMapper.kt
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2014-2023 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package org.jetbrains.dokka.gradle
+
+import org.jetbrains.dokka.*
+import org.jetbrains.dokka.DokkaConfiguration.ExternalDocumentationLink
+import java.io.File
+
+internal fun GradleDokkaSourceSetBuilder.toDokkaSourceSetImpl(): DokkaSourceSetImpl = DokkaSourceSetImpl(
+    classpath = classpath.toList(),
+    displayName = displayNameOrDefault(),
+    sourceSetID = sourceSetID,
+    sourceRoots = sourceRoots.toSet(),
+    dependentSourceSets = dependentSourceSets.get().toSet(),
+    samples = samples.toSet(),
+    includes = includes.toSet(),
+    includeNonPublic = includeNonPublic.get(),
+    documentedVisibilities = documentedVisibilities.get(),
+    reportUndocumented = reportUndocumented.get(),
+    skipEmptyPackages = skipEmptyPackages.get(),
+    skipDeprecated = skipDeprecated.get(),
+    jdkVersion = jdkVersion.get(),
+    sourceLinks = sourceLinks.get().build().toSet(),
+    perPackageOptions = perPackageOptions.get().build(),
+    externalDocumentationLinks = externalDocumentationLinksWithDefaults(),
+    languageVersion = languageVersion.orNull,
+    apiVersion = apiVersion.orNull,
+    noStdlibLink = noStdlibLink.get(),
+    noJdkLink = noJdkLink.get(),
+    suppressedFiles = suppressedFilesWithDefaults(),
+    analysisPlatform = platform.get()
+)
+
+private fun GradleDokkaSourceSetBuilder.displayNameOrDefault(): String {
+    displayName.orNull?.let { return it }
+    if (name.endsWith("Main") && name != "Main") {
+        return name.removeSuffix("Main")
+    }
+
+    return name
+}
+
+private fun GradleDokkaSourceSetBuilder.externalDocumentationLinksWithDefaults(): Set<ExternalDocumentationLinkImpl> {
+    return externalDocumentationLinks.get().build()
+        .run {
+            if (noJdkLink.get()) this
+            else this + ExternalDocumentationLink.jdk(jdkVersion.get())
+        }
+        .run {
+            if (noStdlibLink.get()) this
+            else this + ExternalDocumentationLink.kotlinStdlib()
+        }
+        .run {
+            if (noAndroidSdkLink.get() || !project.isAndroidProject()) this
+            else this +
+                    ExternalDocumentationLink.androidSdk() +
+                    ExternalDocumentationLink.androidX()
+        }
+        .toSet()
+}
+
+private fun GradleDokkaSourceSetBuilder.suppressedFilesWithDefaults(): Set<File> {
+    val suppressedGeneratedFiles = if (suppressGeneratedFiles.get()) {
+        val generatedRoot = project.buildDir.resolve("generated").absoluteFile
+        sourceRoots
+            .filter { it.startsWith(generatedRoot) }
+            .flatMap { it.walk().toList() }
+            .toSet()
+    } else {
+        emptySet()
+    }
+
+    return suppressedFiles.toSet() + suppressedGeneratedFiles
+}
diff --git a/dokka-runners/runner-gradle-plugin-classic/src/main/kotlin/org/jetbrains/dokka/gradle/GradleDokkaSourceSetBuilder.kt b/dokka-runners/runner