Flatten multi-module structure (#1980)

* Add support for multimodule package lists

* Merge package-lists in multi-module generation

* Remove double-wrapping of modules in multi-module generation

* Handle empty modules in package lists

4 files changed, 79 insertions, 33 deletions

diff --git a/plugins/base/src/main/kotlin/resolvers/external/DefaultExternalLocationProvider.kt b/plugins/base/src/main/kotlin/resolvers/external/DefaultExternalLocationProvider.kt
index 09eb7cc4..fc7f57f4 100644
--- a/plugins/base/src/main/kotlin/resolvers/external/DefaultExternalLocationProvider.kt
+++ b/plugins/base/src/main/kotlin/resolvers/external/DefaultExternalLocationProvider.kt
@@ -22,11 +22,21 @@ open class DefaultExternalLocationProvider(
     }
 
     protected open fun DRI.constructPath(): String {
-        val classNamesChecked = classNames ?: return "$docURL${packageName ?: ""}/index$extension"
+        val modulePart = packageName?.let { packageName ->
+            externalDocumentation.packageList.moduleFor(packageName)?.let {
+                if (it.isNotBlank())
+                    "$it/"
+                else
+                    ""
+            }
+        }.orEmpty()
+
+        val docWithModule = docURL + modulePart
+        val classNamesChecked = classNames ?: return "$docWithModule${packageName ?: ""}/index$extension"
         val classLink = (listOfNotNull(packageName) + classNamesChecked.split('.'))
             .joinToString("/", transform = ::identifierToFilename)
 
         val fileName = callable?.let { identifierToFilename(it.name) } ?: "index"
-        return "$docURL$classLink/$fileName$extension"
+        return "$docWithModule$classLink/$fileName$extension"
     }
 }
diff --git a/plugins/base/src/main/kotlin/resolvers/external/javadoc/JavadocExternalLocationProvider.kt b/plugins/base/src/main/kotlin/resolvers/external/javadoc/JavadocExternalLocationProvider.kt
index b0398cd7..f1a32cb4 100644
--- a/plugins/base/src/main/kotlin/resolvers/external/javadoc/JavadocExternalLocationProvider.kt
+++ b/plugins/base/src/main/kotlin/resolvers/external/javadoc/JavadocExternalLocationProvider.kt
@@ -8,22 +8,33 @@ import org.jetbrains.dokka.plugability.DokkaContext
 import org.jetbrains.dokka.utilities.htmlEscape
 
 open class JavadocExternalLocationProvider(
-    externalDocumentation: ExternalDocumentation,
-    val brackets: String,
-    val separator: String,
-    dokkaContext: DokkaContext
+        externalDocumentation: ExternalDocumentation,
+        val brackets: String,
+        val separator: String,
+        dokkaContext: DokkaContext
 ) : DefaultExternalLocationProvider(externalDocumentation, ".html", dokkaContext) {
 
     override fun DRI.constructPath(): String {
         val packageLink = packageName?.replace(".", "/")
+        val modulePart = packageName?.let { packageName ->
+            externalDocumentation.packageList.moduleFor(packageName)?.let {
+                if (it.isNotBlank())
+                    "$it/"
+                else
+                    ""
+            }
+        }.orEmpty()
+
+        val docWithModule = docURL + modulePart
+
         if (classNames == null) {
-            return "$docURL$packageLink/package-summary$extension".htmlEscape()
+            return "$docWithModule$packageLink/package-summary$extension".htmlEscape()
         }
         val classLink =
-            if (packageLink == null) "${classNames}$extension" else "$packageLink/${classNames}$extension"
-        val callableChecked = callable ?: return "$docURL$classLink".htmlEscape()
+                if (packageLink == null) "${classNames}$extension" else "$packageLink/${classNames}$extension"
+        val callableChecked = callable ?: return "$docWithModule$classLink".htmlEscape()
 
-        return ("$docURL$classLink#" + anchorPart(callableChecked)).htmlEscape()
+        return ("$docWithModule$classLink#" + anchorPart(callableChecked)).htmlEscape()
     }
 
     protected open fun anchorPart(callable: Callable) = callable.name +
diff --git a/plugins/base/src/main/kotlin/resolvers/local/DefaultLocationProvider.kt b/plugins/base/src/main/kotlin/resolvers/local/DefaultLocationProvider.kt
index 87683414..3647bfa7 100644
--- a/plugins/base/src/main/kotlin/resolvers/local/DefaultLocationProvider.kt
+++ b/plugins/base/src/main/kotlin/resolvers/local/DefaultLocationProvider.kt
@@ -23,22 +23,20 @@ abstract class DefaultLocationProvider(
         dokkaContext.plugin<DokkaBase>().query { externalLocationProviderFactory }
 
     protected val externalLocationProviders: Map<ExternalDocumentation, ExternalLocationProvider?> = dokkaContext
-        .configuration
-        .sourceSets
-        .flatMap { sourceSet ->
-            sourceSet.externalDocumentationLinks.map {
-                PackageList.load(it.packageListUrl, sourceSet.jdkVersion, dokkaContext.configuration.offlineMode)
-                    ?.let { packageList -> ExternalDocumentation(it.url, packageList) }
+            .configuration
+            .sourceSets
+            .flatMap { sourceSet ->
+                sourceSet.externalDocumentationLinks.map {
+                    PackageList.load(it.packageListUrl, sourceSet.jdkVersion, dokkaContext.configuration.offlineMode)
+                            ?.let { packageList -> ExternalDocumentation(it.url, packageList) }
+                }
+            }
+            .filterNotNull().associateWith { extDocInfo ->
+                externalLocationProviderFactories
+                    .mapNotNull { it.getExternalLocationProvider(extDocInfo) }
+                    .firstOrNull()
+                    ?: run { dokkaContext.logger.error("No ExternalLocationProvider for '${extDocInfo.packageList.url}' found"); null }
             }
-        }
-        .filterNotNull()
-        .map { extDocInfo ->
-            val externalLocationProvider = (externalLocationProviderFactories.asSequence()
-                .mapNotNull { it.getExternalLocationProvider(extDocInfo) }.firstOrNull()
-                ?: run { dokkaContext.logger.error("No ExternalLocationProvider for '${extDocInfo.packageList.url}' found"); null })
-            extDocInfo to externalLocationProvider
-        }
-        .toMap()
 
     protected val packagesIndex: Map<String, ExternalLocationProvider?> =
         externalLocationProviders
diff --git a/plugins/base/src/main/kotlin/resolvers/shared/PackageList.kt b/plugins/base/src/main/kotlin/resolvers/shared/PackageList.kt
index a06365eb..469904bd 100644
--- a/plugins/base/src/main/kotlin/resolvers/shared/PackageList.kt
+++ b/plugins/base/src/main/kotlin/resolvers/shared/PackageList.kt
@@ -1,20 +1,33 @@
 package org.jetbrains.dokka.base.resolvers.shared
 
-import org.jetbrains.dokka.base.renderers.PackageListService
 import java.net.URL
 
+typealias Module = String
+
 data class PackageList(
     val linkFormat: RecognizedLinkFormat,
-    val packages: Set<String>,
+    val modules: Map<Module, Set<String>>,
     val locations: Map<String, String>,
     val url: URL
 ) {
+    val packages: Set<String>
+        get() = modules.values.flatten().toSet()
+
+    fun moduleFor(packageName: String) = modules.asSequence()
+            .filter { it.value.contains(packageName) }
+            .firstOrNull()?.key
+
     companion object {
+        const val PACKAGE_LIST_NAME = "package-list"
+        const val MODULE_DELIMITER = "module:"
+        const val DOKKA_PARAM_PREFIX = "\$dokka"
+        const val SINGLE_MODULE_NAME = ""
+
         fun load(url: URL, jdkVersion: Int, offlineMode: Boolean = false): PackageList? {
             if (offlineMode && url.protocol.toLowerCase() != "file")
                 return null
 
-            val packageListStream = kotlin.runCatching { url.readContent() }.onFailure {
+            val packageListStream = runCatching { url.readContent() }.onFailure {
                 println("Failed to download package-list from $url, this might suggest that remote resource is not available," +
                         " module is empty or dokka output got corrupted")
                 return null
@@ -22,22 +35,36 @@ data class PackageList(
 
             val (params, packages) = packageListStream
                 .bufferedReader()
-                .useLines { lines -> lines.partition { it.startsWith(PackageListService.DOKKA_PARAM_PREFIX) } }
+                .useLines { lines -> lines.partition { it.startsWith(DOKKA_PARAM_PREFIX) } }
 
             val paramsMap = splitParams(params)
             val format = linkFormat(paramsMap["format"]?.singleOrNull(), jdkVersion)
             val locations = splitLocations(paramsMap["location"].orEmpty()).filterKeys(String::isNotEmpty)
 
-            return PackageList(format, packages.filter(String::isNotBlank).toSet(), locations, url)
+            val modulesMap = splitPackages(packages)
+            return PackageList(format, modulesMap, locations, url)
         }
 
         private fun splitParams(params: List<String>) = params.asSequence()
-            .map { it.removePrefix("${PackageListService.DOKKA_PARAM_PREFIX}.").split(":", limit = 2) }
+            .map { it.removePrefix("$DOKKA_PARAM_PREFIX.").split(":", limit = 2) }
             .groupBy({ (key, _) -> key }, { (_, value) -> value })
 
         private fun splitLocations(locations: List<String>) = locations.map { it.split("\u001f", limit = 2) }
-            .map { (key, value) -> key to value }
-            .toMap()
+                .associate { (key, value) -> key to value }
+
+        private fun splitPackages(packages: List<String>): Map<Module, Set<String>> =
+                packages.fold(("" to mutableMapOf<Module, Set<String>>())) { (lastModule, acc), el ->
+                    val currentModule : String
+                    when {
+                        el.startsWith(MODULE_DELIMITER) -> currentModule = el.substringAfter(MODULE_DELIMITER)
+                        el.isNotBlank() -> {
+                            currentModule = lastModule
+                            acc[currentModule] = acc.getOrDefault(lastModule, emptySet()) + el
+                        }
+                        else -> currentModule = lastModule
+                    }
+                    currentModule to acc
+                }.second
 
         private fun linkFormat(formatName: String?, jdkVersion: Int) =
             formatName?.let { RecognizedLinkFormat.fromString(it) }