Flatten multi-module structure (#1980)

* Add support for multimodule package lists

* Merge package-lists in multi-module generation

* Remove double-wrapping of modules in multi-module generation

* Handle empty modules in package lists

10 files changed, 94 insertions, 32 deletions

diff --git a/plugins/templating/src/main/kotlin/templates/DirectiveBasedTemplateProcessing.kt b/plugins/templating/src/main/kotlin/templates/DirectiveBasedTemplateProcessing.kt
index c3b9aa53..2b4951a1 100644
--- a/plugins/templating/src/main/kotlin/templates/DirectiveBasedTemplateProcessing.kt
+++ b/plugins/templating/src/main/kotlin/templates/DirectiveBasedTemplateProcessing.kt
@@ -1,5 +1,6 @@
 package org.jetbrains.dokka.templates
 
+import org.jetbrains.dokka.DokkaConfiguration
 import org.jetbrains.dokka.base.templating.Command
 import org.jetbrains.dokka.base.templating.parseJson
 import org.jetbrains.dokka.plugability.DokkaContext
@@ -15,7 +16,7 @@ class DirectiveBasedHtmlTemplateProcessingStrategy(private val context: DokkaCon
     private val directiveBasedCommandHandlers =
         context.plugin<TemplatingPlugin>().query { directiveBasedCommandHandlers }
 
-    override fun process(input: File, output: File): Boolean =
+    override fun process(input: File, output: File, moduleContext: DokkaConfiguration.DokkaModuleDescription?): Boolean =
         if (input.isFile && input.extension == "html") {
             val document = Jsoup.parse(input, "UTF-8")
             document.outputSettings().indentAmount(0).prettyPrint(false)
diff --git a/plugins/templating/src/main/kotlin/templates/FallbackTemplateProcessingStrategy.kt b/plugins/templating/src/main/kotlin/templates/FallbackTemplateProcessingStrategy.kt
index 4e88c318..fdd9059d 100644
--- a/plugins/templating/src/main/kotlin/templates/FallbackTemplateProcessingStrategy.kt
+++ b/plugins/templating/src/main/kotlin/templates/FallbackTemplateProcessingStrategy.kt
@@ -1,13 +1,13 @@
 package org.jetbrains.dokka.templates
 
+import org.jetbrains.dokka.DokkaConfiguration
 import org.jetbrains.dokka.plugability.DokkaContext
 import java.io.File
-import java.nio.file.Files
 
-class FallbackTemplateProcessingStrategy(dokkaContext: DokkaContext) : TemplateProcessingStrategy {
+class FallbackTemplateProcessingStrategy : TemplateProcessingStrategy {
 
-    override fun process(input: File, output: File): Boolean {
-        if(input != output) input.copyTo(output, overwrite = true)
+    override fun process(input: File, output: File, moduleContext: DokkaConfiguration.DokkaModuleDescription?): Boolean {
+        if (input != output) input.copyTo(output, overwrite = true)
         return true
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/templating/src/main/kotlin/templates/JsonElementBasedTemplateProcessingStrategy.kt b/plugins/templating/src/main/kotlin/templates/JsonElementBasedTemplateProcessingStrategy.kt
index dec37799..9e064a6a 100644
--- a/plugins/templating/src/main/kotlin/templates/JsonElementBasedTemplateProcessingStrategy.kt
+++ b/plugins/templating/src/main/kotlin/templates/JsonElementBasedTemplateProcessingStrategy.kt
@@ -1,5 +1,6 @@
 package org.jetbrains.dokka.allModulesPage.templates
 
+import org.jetbrains.dokka.DokkaConfiguration.DokkaModuleDescription
 import org.jetbrains.dokka.base.renderers.html.SearchRecord
 import org.jetbrains.dokka.base.templating.AddToSearch
 import org.jetbrains.dokka.base.templating.parseJson
@@ -18,11 +19,11 @@ abstract class BaseJsonNavigationTemplateProcessingStrategy(val context: DokkaCo
     open fun canProcess(file: File): Boolean =
         file.extension == "json" && file.nameWithoutExtension == navigationFileNameWithoutExtension
 
-    override fun process(input: File, output: File): Boolean {
+    override fun process(input: File, output: File, moduleContext: DokkaModuleDescription?): Boolean {
         val canProcess = canProcess(input)
         if (canProcess) {
             runCatching { parseJson<AddToSearch>(input.readText()) }.getOrNull()?.let { command ->
-                context.configuration.modules.find { it.name == command.moduleName }?.relativePathToOutputDirectory
+                moduleContext?.relativePathToOutputDirectory
                     ?.relativeToOrSelf(context.configuration.outputDir)
                     ?.let { key ->
                         fragments[key.toString()] = command.elements
@@ -43,7 +44,7 @@ abstract class BaseJsonNavigationTemplateProcessingStrategy(val context: DokkaCo
     }
 
     private fun fallbackToCopy(input: File, output: File) {
-        context.logger.warn("Falling back to just copying file for ${input.name} even thought it should process it")
+        context.logger.warn("Falling back to just copying ${input.name} file even though it should have been processed")
         input.copyTo(output)
     }
 
@@ -62,4 +63,4 @@ class PagesSearchTemplateStrategy(val dokkaContext: DokkaContext) :
     BaseJsonNavigationTemplateProcessingStrategy(dokkaContext) {
     override val navigationFileNameWithoutExtension: String = "pages"
     override val path: String = "scripts"
-}
\ No newline at end of file
+}
diff --git a/plugins/templating/src/main/kotlin/templates/PackageListProcessingStrategy.kt b/plugins/templating/src/main/kotlin/templates/PackageListProcessingStrategy.kt
new file mode 100644
index 00000000..07238ea7
--- /dev/null
+++ b/plugins/templating/src/main/kotlin/templates/PackageListProcessingStrategy.kt
@@ -0,0 +1,51 @@
+package org.jetbrains.dokka.allModulesPage.templates
+
+import org.jetbrains.dokka.DokkaConfiguration.DokkaModuleDescription
+import org.jetbrains.dokka.base.renderers.PackageListService
+import org.jetbrains.dokka.base.resolvers.local.DokkaLocationProvider
+import org.jetbrains.dokka.base.resolvers.shared.PackageList
+import org.jetbrains.dokka.base.resolvers.shared.PackageList.Companion.PACKAGE_LIST_NAME
+import org.jetbrains.dokka.plugability.DokkaContext
+import org.jetbrains.dokka.templates.TemplateProcessingStrategy
+import java.io.File
+
+class PackageListProcessingStrategy(val context: DokkaContext) : TemplateProcessingStrategy {
+    private val fragments = mutableSetOf<PackageList>()
+
+    private fun canProcess(file: File, moduleContext: DokkaModuleDescription?): Boolean =
+            file.extension.isBlank() && file.nameWithoutExtension == PACKAGE_LIST_NAME && moduleContext != null
+
+    override fun process(input: File, output: File, moduleContext: DokkaModuleDescription?): Boolean {
+        val canProcess = canProcess(input, moduleContext)
+        if (canProcess) {
+            val packageList = PackageList.load(input.toURI().toURL(), 8, true)
+            val moduleFilename = moduleContext?.name?.let { "$it/" }
+            packageList?.copy(
+                    modules = mapOf(moduleContext?.name.orEmpty() to packageList.modules.getOrDefault(PackageList.SINGLE_MODULE_NAME, emptySet())),
+                    locations = packageList.locations.entries.associate { it.key to "$moduleFilename${it.value}" }
+            )?.let { fragments.add(it) } ?: fallbackToCopy(input, output)
+        }
+        return canProcess
+    }
+
+    override fun finish(output: File) {
+        if (fragments.isNotEmpty()) {
+            val linkFormat = fragments.first().linkFormat
+
+            if (!fragments.all { it.linkFormat == linkFormat }) {
+                context.logger.error("Link format is inconsistent between modules: " + fragments.joinToString { it.linkFormat.formatName } )
+            }
+
+            val locations: Map<String, String> = fragments.map { it.locations }.fold(emptyMap()) { acc, el -> acc + el }
+            val modules: Map<String, Set<String>> = fragments.map { it.modules }.fold(emptyMap()) { acc, el -> acc + el }
+            val mergedPackageList = PackageListService.renderPackageList(locations, modules, linkFormat.formatName, linkFormat.linkExtension)
+            output.mkdirs()
+            output.resolve(PACKAGE_LIST_NAME).writeText(mergedPackageList)
+        }
+    }
+
+    private fun fallbackToCopy(input: File, output: File) {
+        context.logger.warn("Falling back to just copying ${input.name} file even though it should have been processed")
+        input.copyTo(output)
+    }
+}
diff --git a/plugins/templating/src/main/kotlin/templates/SourcesetDependencyProcessingStrategy.kt b/plugins/templating/src/main/kotlin/templates/SourcesetDependencyProcessingStrategy.kt
index 11bbc39a..ddebe160 100644
--- a/plugins/templating/src/main/kotlin/templates/SourcesetDependencyProcessingStrategy.kt
+++ b/plugins/templating/src/main/kotlin/templates/SourcesetDependencyProcessingStrategy.kt
@@ -1,5 +1,6 @@
 package templates
 
+import org.jetbrains.dokka.DokkaConfiguration
 import org.jetbrains.dokka.base.templating.AddToSourcesetDependencies
 import org.jetbrains.dokka.base.templating.parseJson
 import org.jetbrains.dokka.base.templating.toJsonString
@@ -23,11 +24,11 @@ class SourcesetDependencyProcessingStrategy(val context: DokkaContext) : Templat
         }
     }
 
-    override fun process(input: File, output: File): Boolean =
+    override fun process(input: File, output: File, moduleContext: DokkaConfiguration.DokkaModuleDescription?): Boolean =
         input.takeIf { it.name == fileName }
             ?.runCatching { parseJson<AddToSourcesetDependencies>(input.readText()) }
             ?.getOrNull()
             ?.also { (moduleName, content) ->
                 fragments += (moduleName to content)
             } != null
-}
\ No newline at end of file
+}
diff --git a/plugins/templating/src/main/kotlin/templates/TemplateProcessor.kt b/plugins/templating/src/main/kotlin/templates/TemplateProcessor.kt
index c67654ec..5f36530b 100644
--- a/plugins/templating/src/main/kotlin/templates/TemplateProcessor.kt
+++ b/plugins/templating/src/main/kotlin/templates/TemplateProcessor.kt
@@ -1,7 +1,9 @@
 package org.jetbrains.dokka.templates
 
-import kotlinx.coroutines.*
-import org.jetbrains.dokka.DokkaConfiguration
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.runBlocking
+import org.jetbrains.dokka.DokkaConfiguration.DokkaModuleDescription
 import org.jetbrains.dokka.base.DokkaBase
 import org.jetbrains.dokka.base.templating.Command
 import org.jetbrains.dokka.model.withDescendants
@@ -16,7 +18,7 @@ import java.io.File
 interface TemplateProcessor
 
 interface SubmoduleTemplateProcessor : TemplateProcessor {
-    fun process(modules: List<DokkaConfiguration.DokkaModuleDescription>): TemplatingResult
+    fun process(modules: List<DokkaModuleDescription>): TemplatingResult
 }
 
 interface MultiModuleTemplateProcessor : TemplateProcessor {
@@ -24,7 +26,7 @@ interface MultiModuleTemplateProcessor : TemplateProcessor {
 }
 
 interface TemplateProcessingStrategy {
-    fun process(input: File, output: File): Boolean
+    fun process(input: File, output: File, moduleContext: DokkaModuleDescription?): Boolean
     fun finish(output: File) {}
 }
 
@@ -36,18 +38,18 @@ class DefaultSubmoduleTemplateProcessor(
         context.plugin<TemplatingPlugin>().query { templateProcessingStrategy }
 
     private val configuredModulesPaths =
-        context.configuration.modules.map { it.sourceOutputDirectory.absolutePath to it.name }.toMap()
+            context.configuration.modules.associate { it.sourceOutputDirectory.absolutePath to it.name }
 
-    override fun process(modules: List<DokkaConfiguration.DokkaModuleDescription>) =
+    override fun process(modules: List<DokkaModuleDescription>) =
         runBlocking(Dispatchers.Default) {
             coroutineScope {
                 modules.fold(TemplatingResult()) { acc, module ->
-                    acc + module.sourceOutputDirectory.visit(context.configuration.outputDir.resolve(module.relativePathToOutputDirectory))
+                    acc + module.sourceOutputDirectory.visit(context.configuration.outputDir.resolve(module.relativePathToOutputDirectory), module)
                 }
             }
         }
 
-    private suspend fun File.visit(target: File, acc: TemplatingResult = TemplatingResult()): TemplatingResult =
+    private suspend fun File.visit(target: File, module: DokkaModuleDescription, acc: TemplatingResult = TemplatingResult()): TemplatingResult =
         coroutineScope {
             val source = this@visit
             if (source.isDirectory) {
@@ -59,17 +61,17 @@ class DefaultSubmoduleTemplateProcessor(
                     ?: acc
 
                 files.fold(accWithSelf) { acc, path ->
-                    source.resolve(path).visit(target.resolve(path), acc)
+                    source.resolve(path).visit(target.resolve(path), module, acc)
                 }
             } else {
-                strategies.first { it.process(source, target) }
+                strategies.first { it.process(source, target, module) }
                 acc
             }
         }
 }
 
 class DefaultMultiModuleTemplateProcessor(
-    context: DokkaContext,
+    val context: DokkaContext,
 ) : MultiModuleTemplateProcessor {
     private val strategies: List<TemplateProcessingStrategy> =
         context.plugin<TemplatingPlugin>().query { templateProcessingStrategy }
@@ -78,10 +80,9 @@ class DefaultMultiModuleTemplateProcessor(
 
     override fun process(generatedPagesTree: RootPageNode) {
         val locationProvider = locationProviderFactory.getLocationProvider(generatedPagesTree)
-        generatedPagesTree.withDescendants().mapNotNull { locationProvider.resolve(it)?.let { File(it) } }
-            .forEach { location -> strategies.first { it.process(location, location) } }
+        generatedPagesTree.withDescendants().mapNotNull { pageNode -> locationProvider.resolve(pageNode)?.let { File(it) } }
+            .forEach { location -> strategies.first { it.process(location, location, null) } }
     }
-
 }
 
 data class TemplatingContext<out T : Command>(
@@ -93,4 +94,4 @@ data class TemplatingContext<out T : Command>(
 
 data class TemplatingResult(val modules: List<String> = emptyList()) {
     operator fun plus(rhs: TemplatingResult): TemplatingResult = TemplatingResult((modules + rhs.modules).distinct())
-}
\ No newline at end of file
+}
diff --git a/plugins/templating/src/main/kotlin/templates/TemplatingPlugin.kt b/plugins/templating/src/main/kotlin/templates/TemplatingPlugin.kt
index 562b12c6..0842bb2b 100644
--- a/plugins/templating/src/main/kotlin/templates/TemplatingPlugin.kt
+++ b/plugins/templating/src/main/kotlin/templates/TemplatingPlugin.kt
@@ -1,6 +1,7 @@
 package org.jetbrains.dokka.templates
 
 import org.jetbrains.dokka.allModulesPage.templates.NavigationSearchTemplateStrategy
+import org.jetbrains.dokka.allModulesPage.templates.PackageListProcessingStrategy
 import org.jetbrains.dokka.allModulesPage.templates.PagesSearchTemplateStrategy
 import org.jetbrains.dokka.plugability.DokkaPlugin
 import templates.SourcesetDependencyProcessingStrategy
@@ -45,8 +46,14 @@ class TemplatingPlugin : DokkaPlugin() {
         }
     }
 
+    val packageListProcessingStrategy by extending {
+        templateProcessingStrategy providing ::PackageListProcessingStrategy order {
+            before(fallbackProcessingStrategy)
+        }
+    }
+
     val fallbackProcessingStrategy by extending {
-        templateProcessingStrategy providing ::FallbackTemplateProcessingStrategy
+        templateProcessingStrategy with FallbackTemplateProcessingStrategy()
     }
 
     val pathToRootSubstitutor by extending {
@@ -59,4 +66,4 @@ class TemplatingPlugin : DokkaPlugin() {
     val substitutionCommandHandler by extending {
         directiveBasedCommandHandlers providing ::SubstitutionCommandHandler
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/templating/src/test/kotlin/templates/AddToNavigationCommandResolutionTest.kt b/plugins/templating/src/test/kotlin/templates/AddToNavigationCommandResolutionTest.kt
index a1e11dfe..fb229643 100644
--- a/plugins/templating/src/test/kotlin/templates/AddToNavigationCommandResolutionTest.kt
+++ b/plugins/templating/src/test/kotlin/templates/AddToNavigationCommandResolutionTest.kt
@@ -134,4 +134,4 @@ class AddToNavigationCommandResolutionTest : TemplatingAbstractTest() {
     }
 
     private data class ModuleWithPrefix(val moduleName: String, val prefix: String? = null)
-}
\ No newline at end of file
+}
diff --git a/plugins/templating/src/test/kotlin/templates/AddToSearchCommandResolutionTest.kt b/plugins/templating/src/test/kotlin/templates/AddToSearchCommandResolutionTest.kt
index bb4fb1c4..aba668f7 100644
--- a/plugins/templating/src/test/kotlin/templates/AddToSearchCommandResolutionTest.kt
+++ b/plugins/templating/src/test/kotlin/templates/AddToSearchCommandResolutionTest.kt
@@ -80,4 +80,4 @@ class AddToSearchCommandResolutionTest : TemplatingAbstractTest() {
 
         return listOf(module1Navigation, module2Navigation)
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/templating/src/test/kotlin/templates/SubstitutionCommandResolutionTest.kt b/plugins/templating/src/test/kotlin/templates/SubstitutionCommandResolutionTest.kt
index 9d211876..ce2a8afd 100644
--- a/plugins/templating/src/test/kotlin/templates/SubstitutionCommandResolutionTest.kt
+++ b/plugins/templating/src/test/kotlin/templates/SubstitutionCommandResolutionTest.kt
@@ -66,4 +66,4 @@ class SubstitutionCommandResolutionTest : TemplatingAbstractTest() {
         return module1Content
     }
 
-}
\ No newline at end of file
+}