Versioning (#1654)

* Adding versioning mechanism for multimodule

* Versioning improvement

* Refactor configuration, add ordering

* Fix integration tests

* Change packages, unignore test

Co-authored-by: Marcin Aman <marcin.aman@gmail.com>

50 files changed, 915 insertions, 292 deletions

diff --git a/plugins/all-modules-page/build.gradle.kts b/plugins/all-modules-page/build.gradle.kts
index c6e88574..9c5fe1c3 100644
--- a/plugins/all-modules-page/build.gradle.kts
+++ b/plugins/all-modules-page/build.gradle.kts
@@ -6,6 +6,8 @@ registerDokkaArtifactPublication("dokkaAllModulesPage") {
 
 dependencies {
     implementation(project(":plugins:base"))
+    implementation(project(":plugins:templating"))
+    implementation(project(":plugins:versioning"))
     testImplementation(project(":plugins:base"))
     testImplementation(project(":plugins:base:base-test-utils"))
     testImplementation(project(":plugins:gfm"))
@@ -13,10 +15,9 @@ dependencies {
 
     val coroutines_version: String by project
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutines_version")
-
-    implementation("org.jsoup:jsoup:1.12.1")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.11.1")
-
     val kotlinx_html_version: String by project
-    testImplementation("org.jetbrains.kotlinx:kotlinx-html-jvm:$kotlinx_html_version")
+    implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:$kotlinx_html_version")
+
+    implementation("org.jsoup:jsoup:1.12.1")
 }
\ No newline at end of file
diff --git a/plugins/all-modules-page/src/main/kotlin/AllModulesPageGeneration.kt b/plugins/all-modules-page/src/main/kotlin/AllModulesPageGeneration.kt
index 5ac854b4..0013feed 100644
--- a/plugins/all-modules-page/src/main/kotlin/AllModulesPageGeneration.kt
+++ b/plugins/all-modules-page/src/main/kotlin/AllModulesPageGeneration.kt
@@ -8,15 +8,23 @@ import org.jetbrains.dokka.plugability.DokkaContext
 import org.jetbrains.dokka.plugability.plugin
 import org.jetbrains.dokka.plugability.query
 import org.jetbrains.dokka.plugability.querySingle
+import org.jetbrains.dokka.templates.TemplatingPlugin
+import org.jetbrains.dokka.versioning.VersioningPlugin
 
 class AllModulesPageGeneration(private val context: DokkaContext) : Generation {
 
     private val allModulesPagePlugin by lazy { context.plugin<AllModulesPagePlugin>() }
+    private val templatingPlugin by lazy { context.plugin<TemplatingPlugin>() }
+    private val versioningPlugin by lazy { context.plugin<VersioningPlugin>() }
 
     override fun Timer.generate() {
+
         report("Creating all modules page")
         val pages = createAllModulesPage()
 
+        report("Copy previous documentation")
+        handlePreviousDocs()
+
         report("Transforming pages")
         val transformedPages = transformAllModulesPage(pages)
 
@@ -29,6 +37,8 @@ class AllModulesPageGeneration(private val context: DokkaContext) : Generation {
 
     override val generationName = "index page for project"
 
+    fun handlePreviousDocs() = versioningPlugin.querySingle { versioningHandler }.invoke()
+
     fun createAllModulesPage() = allModulesPagePlugin.querySingle { allModulesPageCreator }.invoke()
 
     fun transformAllModulesPage(pages: RootPageNode) =
@@ -39,5 +49,5 @@ class AllModulesPageGeneration(private val context: DokkaContext) : Generation {
     }
 
     fun processSubmodules() =
-        allModulesPagePlugin.querySingle { templateProcessor }.process()
+        templatingPlugin.querySingle { templateProcessor }.process()
 }
\ No newline at end of file
diff --git a/plugins/all-modules-page/src/main/kotlin/AllModulesPagePlugin.kt b/plugins/all-modules-page/src/main/kotlin/AllModulesPagePlugin.kt
index c99293ef..9f4b62ee 100644
--- a/plugins/all-modules-page/src/main/kotlin/AllModulesPagePlugin.kt
+++ b/plugins/all-modules-page/src/main/kotlin/AllModulesPagePlugin.kt
@@ -1,31 +1,29 @@
 package org.jetbrains.dokka.allModulesPage
 
 import org.jetbrains.dokka.CoreExtensions
-import org.jetbrains.dokka.allModulesPage.templates.*
 import org.jetbrains.dokka.base.DokkaBase
 import org.jetbrains.dokka.base.resolvers.local.DokkaLocationProviderFactory
 import org.jetbrains.dokka.base.resolvers.local.LocationProviderFactory
 import org.jetbrains.dokka.plugability.DokkaPlugin
+import org.jetbrains.dokka.templates.TemplatingPlugin
 import org.jetbrains.dokka.transformers.pages.PageCreator
 import org.jetbrains.dokka.transformers.pages.PageTransformer
 
 class AllModulesPagePlugin : DokkaPlugin() {
 
-    val templateProcessor by extensionPoint<TemplateProcessor>()
-    val templateProcessingStrategy by extensionPoint<TemplateProcessingStrategy>()
     val partialLocationProviderFactory by extensionPoint<LocationProviderFactory>()
     val allModulesPageCreator by extensionPoint<PageCreator>()
     val allModulesPageTransformer by extensionPoint<PageTransformer>()
     val externalModuleLinkResolver by extensionPoint<ExternalModuleLinkResolver>()
 
-    val substitutor by extensionPoint<Substitutor>()
-
     val allModulesPageCreators by extending {
         allModulesPageCreator providing ::MultimodulePageCreator
     }
 
+    val dokkaBase by lazy { plugin<DokkaBase>() }
+
     val multimoduleLocationProvider by extending {
-        (plugin<DokkaBase>().locationProviderFactory
+        (dokkaBase.locationProviderFactory
                 providing MultimoduleLocationProvider::Factory
                 override plugin<DokkaBase>().locationProvider)
     }
@@ -37,37 +35,11 @@ class AllModulesPagePlugin : DokkaPlugin() {
     val allModulesPageGeneration by extending {
         (CoreExtensions.generation
                 providing ::AllModulesPageGeneration
-                override plugin<DokkaBase>().singleGeneration)
-    }
-
-    val defaultTemplateProcessor by extending {
-        templateProcessor providing ::DefaultTemplateProcessor
-    }
-
-    val directiveBasedHtmlTemplateProcessingStrategy by extending {
-        templateProcessingStrategy providing ::DirectiveBasedHtmlTemplateProcessingStrategy order {
-            before(fallbackProcessingStrategy)
-        }
-    }
-
-    val fallbackProcessingStrategy by extending {
-        templateProcessingStrategy providing ::FallbackTemplateProcessingStrategy
-    }
-
-    val navigationSearchTemplateStrategy by extending {
-        templateProcessingStrategy providing ::NavigationSearchTemplateStrategy order {
-            before(fallbackProcessingStrategy)
-        }
-    }
-
-    val pagesSearchTemplateStrategy by extending {
-        templateProcessingStrategy providing ::PagesSearchTemplateStrategy order {
-            before(fallbackProcessingStrategy)
-        }
+                override dokkaBase.singleGeneration)
     }
 
-    val pathToRootSubstitutor by extending {
-        substitutor providing ::PathToRootSubstitutor
+    val resolveLinkCommandHandler by extending {
+        plugin<TemplatingPlugin>().directiveBasedCommandHandlers providing ::ResolveLinkCommandHandler
     }
 
     val multiModuleLinkResolver by extending {
diff --git a/plugins/all-modules-page/src/main/kotlin/templates/ExternalModuleLinkResolver.kt b/plugins/all-modules-page/src/main/kotlin/ExternalModuleLinkResolver.kt
index d0e787b6..74513957 100644
--- a/plugins/all-modules-page/src/main/kotlin/templates/ExternalModuleLinkResolver.kt
+++ b/plugins/all-modules-page/src/main/kotlin/ExternalModuleLinkResolver.kt
@@ -1,4 +1,4 @@
-package org.jetbrains.dokka.allModulesPage.templates
+package org.jetbrains.dokka.allModulesPage
 
 import org.jetbrains.dokka.DokkaConfiguration
 import org.jetbrains.dokka.base.DokkaBase
@@ -30,14 +30,22 @@ class DefaultExternalModuleLinkResolver(val context: DokkaContext) : ExternalMod
             context.configuration.modules.map(::loadPackageListForModule).toMap()
         return packageLists.mapNotNull { (module, packageList) ->
             packageList?.let {
-                ExternalDocumentation(
-                    URL("file:/${module.name}/${module.name}"),
-                    packageList
-                )
+                context.configuration.modules.find { it.name == module.name }?.let { m ->
+                    ExternalDocumentation(
+                        URL("file:/${m.relativePathToOutputDirectory.toRelativeOutputDir()}"),
+                        packageList
+                    )
+                }
             }
         }
     }
 
+    private fun File.toRelativeOutputDir(): File = if(isAbsolute) {
+        relativeToOrSelf(context.configuration.outputDir)
+    } else {
+        this
+    }
+
     private fun loadPackageListForModule(module: DokkaConfiguration.DokkaModuleDescription) =
         module.sourceOutputDirectory.resolve(File(identifierToFilename(module.name))).let {
             it to PackageList.load(
@@ -71,7 +79,7 @@ class DefaultExternalModuleLinkResolver(val context: DokkaContext) : ExternalMod
                     RecognizedLinkFormat.DokkaJekyll -> ".md"
                     else -> ""
                 }
-                "${module.relativePathToOutputDirectory}/${identifierToFilename(moduleName)}/index$extension"
+                "${module.relativePathToOutputDirectory}/index$extension"
             }
 
 }
diff --git a/plugins/all-modules-page/src/main/kotlin/MultimoduleLocationProvider.kt b/plugins/all-modules-page/src/main/kotlin/MultimoduleLocationProvider.kt
index 1dbbd386..c7c32d55 100644
--- a/plugins/all-modules-page/src/main/kotlin/MultimoduleLocationProvider.kt
+++ b/plugins/all-modules-page/src/main/kotlin/MultimoduleLocationProvider.kt
@@ -1,7 +1,6 @@
 package org.jetbrains.dokka.allModulesPage
 
 import org.jetbrains.dokka.allModulesPage.MultimodulePageCreator.Companion.MULTIMODULE_PACKAGE_PLACEHOLDER
-import org.jetbrains.dokka.base.DokkaBase
 import org.jetbrains.dokka.base.resolvers.local.DokkaBaseLocationProvider
 import org.jetbrains.dokka.base.resolvers.local.LocationProviderFactory
 import org.jetbrains.dokka.links.DRI
diff --git a/plugins/all-modules-page/src/main/kotlin/MultimodulePageCreator.kt b/plugins/all-modules-page/src/main/kotlin/MultimodulePageCreator.kt
index a333d7c4..458cf4c1 100644
--- a/plugins/all-modules-page/src/main/kotlin/MultimodulePageCreator.kt
+++ b/plugins/all-modules-page/src/main/kotlin/MultimodulePageCreator.kt
@@ -8,6 +8,7 @@ import org.jetbrains.dokka.base.parsers.moduleAndPackage.ModuleAndPackageDocumen
 import org.jetbrains.dokka.base.parsers.moduleAndPackage.parseModuleAndPackageDocumentation
 import org.jetbrains.dokka.base.parsers.moduleAndPackage.parseModuleAndPackageDocumentationFragments
 import org.jetbrains.dokka.base.resolvers.anchors.SymbolAnchorHint
+import org.jetbrains.dokka.base.templating.InsertTemplateExtra
 import org.jetbrains.dokka.base.transformers.pages.comments.DocTagToContentConverter
 import org.jetbrains.dokka.base.translators.documentables.PageContentBuilder
 import org.jetbrains.dokka.links.DRI
@@ -17,10 +18,14 @@ import org.jetbrains.dokka.model.doc.P
 import org.jetbrains.dokka.model.properties.PropertyContainer
 import org.jetbrains.dokka.pages.*
 import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.plugability.configuration
 import org.jetbrains.dokka.plugability.plugin
 import org.jetbrains.dokka.plugability.querySingle
 import org.jetbrains.dokka.transformers.pages.PageCreator
 import org.jetbrains.dokka.utilities.DokkaLogger
+import org.jetbrains.dokka.versioning.ReplaceVersionsCommand
+import org.jetbrains.dokka.versioning.VersioningConfiguration
+import org.jetbrains.dokka.versioning.VersioningPlugin
 
 class MultimodulePageCreator(
     private val context: DokkaContext,
@@ -39,6 +44,11 @@ class MultimodulePageCreator(
             kind = ContentKind.Cover,
             sourceSets = sourceSetData
         ) {
+            /* The line below checks if there is a provided configuration for versioning.
+             If not, we are skipping the template for inserting versions navigation */
+            configuration<VersioningPlugin, VersioningConfiguration>(context)?.let {
+                group(extra = PropertyContainer.withAll(InsertTemplateExtra(ReplaceVersionsCommand))) { }
+            }
             header(2, "All modules:")
             table(styles = setOf(MultimoduleTable)) {
                 modules.forEach { module ->
diff --git a/plugins/all-modules-page/src/main/kotlin/ResolveLinkCommandHandler.kt b/plugins/all-modules-page/src/main/kotlin/ResolveLinkCommandHandler.kt
new file mode 100644
index 00000000..e881a5ab
--- /dev/null
+++ b/plugins/all-modules-page/src/main/kotlin/ResolveLinkCommandHandler.kt
@@ -0,0 +1,45 @@
+package org.jetbrains.dokka.allModulesPage
+
+import org.jetbrains.dokka.base.templating.Command
+import org.jetbrains.dokka.base.templating.ResolveLinkCommand
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.plugability.plugin
+import org.jetbrains.dokka.plugability.querySingle
+import org.jetbrains.dokka.templates.CommandHandler
+import org.jsoup.nodes.Attributes
+import org.jsoup.nodes.Element
+import org.jsoup.parser.Tag
+import java.io.File
+
+class ResolveLinkCommandHandler(context: DokkaContext) : CommandHandler {
+
+    private val externalModuleLinkResolver =
+        context.plugin<AllModulesPagePlugin>().querySingle { externalModuleLinkResolver }
+
+    override fun handleCommand(element: Element, command: Command, input: File, output: File) {
+        command as ResolveLinkCommand
+        val link = externalModuleLinkResolver.resolve(command.dri, output)
+        if (link == null) {
+            val children = element.childNodes().toList()
+            val attributes = Attributes().apply {
+                put("data-unresolved-link", command.dri.toString())
+            }
+            val el = Element(Tag.valueOf("span"), "", attributes).apply {
+                children.forEach { ch -> appendChild(ch) }
+            }
+            element.replaceWith(el)
+            return
+        }
+
+        val attributes = Attributes().apply {
+            put("href", link)
+        }
+        val children = element.childNodes().toList()
+        val el = Element(Tag.valueOf("a"), "", attributes).apply {
+            children.forEach { ch -> appendChild(ch) }
+        }
+        element.replaceWith(el)
+    }
+
+    override fun canHandle(command: Command): Boolean = command is ResolveLinkCommand
+}
\ No newline at end of file
diff --git a/plugins/all-modules-page/src/main/kotlin/templates/DirectiveBasedTemplateProcessing.kt b/plugins/all-modules-page/src/main/kotlin/templates/DirectiveBasedTemplateProcessing.kt
deleted file mode 100644
index 2b065731..00000000
--- a/plugins/all-modules-page/src/main/kotlin/templates/DirectiveBasedTemplateProcessing.kt
+++ /dev/null
@@ -1,141 +0,0 @@
-package org.jetbrains.dokka.allModulesPage.templates
-
-import kotlinx.coroutines.Dispatchers.IO
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.withContext
-import org.jetbrains.dokka.allModulesPage.AllModulesPagePlugin
-import org.jetbrains.dokka.base.templating.*
-import org.jetbrains.dokka.plugability.DokkaContext
-import org.jetbrains.dokka.plugability.plugin
-import org.jetbrains.dokka.plugability.query
-import org.jetbrains.dokka.plugability.querySingle
-import org.jsoup.Jsoup
-import org.jsoup.nodes.*
-import org.jsoup.parser.Tag
-import java.io.File
-import java.nio.file.Files
-import java.util.concurrent.ConcurrentHashMap
-
-class DirectiveBasedHtmlTemplateProcessingStrategy(private val context: DokkaContext) : TemplateProcessingStrategy {
-    private val navigationFragments = ConcurrentHashMap<String, Element>()
-
-    private val substitutors = context.plugin<AllModulesPagePlugin>().query { substitutor }
-    private val externalModuleLinkResolver = context.plugin<AllModulesPagePlugin>().querySingle { externalModuleLinkResolver }
-
-    override suspend fun process(input: File, output: File): Boolean = coroutineScope {
-        if (input.extension == "html") {
-            launch {
-                val document = withContext(IO) { Jsoup.parse(input, "UTF-8") }
-                document.outputSettings().indentAmount(0).prettyPrint(false)
-                document.select("dokka-template-command").forEach {
-                    when (val command = parseJson<Command>(it.attr("data"))) {
-                        is ResolveLinkCommand -> resolveLink(it, command, output)
-                        is AddToNavigationCommand -> navigationFragments[command.moduleName] = it
-                        is SubstitutionCommand -> substitute(it, TemplatingContext(input, output, it, command))
-                        else -> context.logger.warn("Unknown templating command $command")
-                    }
-                }
-                withContext(IO) { Files.write(output.toPath(), listOf(document.outerHtml())) }
-            }
-            true
-        } else false
-    }
-
-    private fun substitute(element: Element, commandContext: TemplatingContext<SubstitutionCommand>) {
-        val regex = commandContext.command.pattern.toRegex()
-        element.children().forEach { it.traverseToSubstitute(regex, commandContext) }
-
-        val childrenCopy = element.children().toList()
-        val position = element.elementSiblingIndex()
-        val parent = element.parent()
-        element.remove()
-
-        parent.insertChildren(position, childrenCopy)
-    }
-
-    private fun Node.traverseToSubstitute(regex: Regex, commandContext: TemplatingContext<SubstitutionCommand>) {
-        when (this) {
-            is TextNode -> replaceWith(TextNode(wholeText.substitute(regex, commandContext)))
-            is DataNode -> replaceWith(DataNode(wholeData.substitute(regex, commandContext)))
-            is Element -> {
-                attributes().forEach { attr(it.key, it.value.substitute(regex, commandContext)) }
-                childNodes().forEach { it.traverseToSubstitute(regex, commandContext) }
-            }
-        }
-    }
-
-    private fun String.substitute(regex: Regex, commandContext: TemplatingContext<SubstitutionCommand>) = buildString {
-        var lastOffset = 0
-        regex.findAll(this@substitute).forEach { match ->
-            append(this@substitute, lastOffset, match.range.first)
-            append(findSubstitution(commandContext, match))
-            lastOffset = match.range.last + 1
-        }
-        append(this@substitute, lastOffset, this@substitute.length)
-    }
-
-    private fun findSubstitution(commandContext: TemplatingContext<SubstitutionCommand>, match: MatchResult): String =
-        substitutors.asSequence().mapNotNull { it.trySubstitute(commandContext, match) }.firstOrNull() ?: match.value
-
-    override suspend fun finish(output: File) {
-        if (navigationFragments.isNotEmpty()) {
-            val attributes = Attributes().apply {
-                put("class", "sideMenu")
-            }
-            val node = Element(Tag.valueOf("div"), "", attributes)
-            navigationFragments.entries.sortedBy { it.key }.forEach { (moduleName, command) ->
-                command.select("a").forEach { a ->
-                    a.attr("href")?.also { a.attr("href", "${moduleName}/${it}") }
-                }
-                command.childNodes().toList().forEachIndexed { index, child ->
-                    if (index == 0) {
-                        child.attr("id", "$moduleName-nav-submenu")
-                    }
-                    node.appendChild(child)
-                }
-            }
-
-            withContext(IO) {
-                Files.write(output.resolve("navigation.html").toPath(), listOf(node.outerHtml()))
-            }
-
-            node.select("a").forEach { a ->
-                a.attr("href")?.also { a.attr("href", "../${it}") }
-            }
-            navigationFragments.keys.forEach {
-                withContext(IO) {
-                    Files.write(
-                        output.resolve(it).resolve("navigation.html").toPath(),
-                        listOf(node.outerHtml())
-                    )
-                }
-            }
-        }
-    }
-
-    private fun resolveLink(it: Element, command: ResolveLinkCommand, fileContext: File) {
-
-        val link = externalModuleLinkResolver.resolve(command.dri, fileContext)
-        if (link == null) {
-            val children = it.childNodes().toList()
-            val attributes = Attributes().apply {
-                put("data-unresolved-link", command.dri.toString())
-            }
-            val element = Element(Tag.valueOf("span"), "", attributes).apply {
-                children.forEach { ch -> appendChild(ch) }
-            }
-            it.replaceWith(element)
-            return