diff options
Diffstat (limited to 'plugins/base/src/main/kotlin')
4 files changed, 115 insertions, 55 deletions
diff --git a/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt b/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt index 34dceeea..4787c091 100644 --- a/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt +++ b/plugins/base/src/main/kotlin/parsers/MarkdownParser.kt @@ -12,24 +12,8 @@ import org.intellij.markdown.flavours.gfm.GFMFlavourDescriptor import org.intellij.markdown.flavours.gfm.GFMTokenTypes import org.jetbrains.dokka.base.parsers.factories.DocTagsFromIElementFactory import org.jetbrains.dokka.links.DRI -import org.jetbrains.dokka.model.doc.Author -import org.jetbrains.dokka.model.doc.Constructor -import org.jetbrains.dokka.model.doc.CustomTagWrapper -import org.jetbrains.dokka.model.doc.Description -import org.jetbrains.dokka.model.doc.DocTag -import org.jetbrains.dokka.model.doc.DocumentationLink -import org.jetbrains.dokka.model.doc.DocumentationNode -import org.jetbrains.dokka.model.doc.Param -import org.jetbrains.dokka.model.doc.Property -import org.jetbrains.dokka.model.doc.Receiver -import org.jetbrains.dokka.model.doc.Return -import org.jetbrains.dokka.model.doc.Sample -import org.jetbrains.dokka.model.doc.See -import org.jetbrains.dokka.model.doc.Since +import org.jetbrains.dokka.model.doc.* import org.jetbrains.dokka.model.doc.Suppress -import org.jetbrains.dokka.model.doc.TagWrapper -import org.jetbrains.dokka.model.doc.Text -import org.jetbrains.dokka.model.doc.Throws import org.jetbrains.kotlin.kdoc.parser.KDocKnownTag import org.jetbrains.kotlin.kdoc.psi.impl.KDocSection import org.jetbrains.kotlin.kdoc.psi.impl.KDocTag @@ -50,7 +34,12 @@ open class MarkdownParser( val markdownAstRoot = IntellijMarkdownParser(gfmFlavourDescriptor).buildMarkdownTreeFromString(extractedString) destinationLinksMap = getAllDestinationLinks(extractedString, markdownAstRoot).toMap() text = extractedString - return visitNode(markdownAstRoot) + + val parsed = visitNode(markdownAstRoot) + if (parsed.size == 1) { + return parsed.first() + } + return CustomDocTag(children = parsed, params = emptyMap(), name = "") } override fun preparse(text: String) = text.replace("\r\n", "\n").replace("\r", "\n") @@ -82,7 +71,7 @@ open class MarkdownParser( node.type, visitNode(node.children.find { it.type == MarkdownTokenTypes.ATX_CONTENT } ?: throw detailedException("Wrong AST Tree. Header does not contain expected content", node) - ).children + ).flatMap { it.children } ) private fun horizontalRulesHandler(node: ASTNode) = @@ -110,7 +99,7 @@ open class MarkdownParser( .evaluateChildren() ) - private fun listsHandler(node: ASTNode): DocTag { + private fun listsHandler(node: ASTNode): List<DocTag> { val children = node.children.filterIsInstance<ListItemCompositeNode>().flatMap { if (it.children.last().type in listOf( @@ -129,7 +118,7 @@ open class MarkdownParser( node.type, children = children - .map { + .flatMap { if (it.type == MarkdownElementTypes.LIST_ITEM) DocTagsFromIElementFactory.getInstance( it.type, @@ -178,7 +167,7 @@ open class MarkdownParser( node.children.filterIsInstance<CompositeASTNode>().flatMap { getAllDestinationLinks(text, it) } - private fun referenceLinksHandler(node: ASTNode): DocTag { + private fun referenceLinksHandler(node: ASTNode): List<DocTag> { val linkLabel = node.children.find { it.type == MarkdownElementTypes.LINK_LABEL } ?: throw detailedException("Wrong AST Tree. Reference link does not contain link label", node) val linkText = node.children.findLast { it.type == MarkdownElementTypes.LINK_TEXT } ?: linkLabel @@ -190,7 +179,7 @@ open class MarkdownParser( return linksHandler(linkText, link) } - private fun inlineLinksHandler(node: ASTNode): DocTag { + private fun inlineLinksHandler(node: ASTNode): List<DocTag> { val linkText = node.children.find { it.type == MarkdownElementTypes.LINK_TEXT } ?: throw detailedException("Wrong AST Tree. Inline link does not contain link text", node) val linkDestination = node.children.find { it.type == MarkdownElementTypes.LINK_DESTINATION } @@ -208,13 +197,13 @@ open class MarkdownParser( children = node.children.evaluateChildren() ) - private fun autoLinksHandler(node: ASTNode): DocTag { + private fun autoLinksHandler(node: ASTNode): List<DocTag> { val link = text.substring(node.startOffset + 1, node.endOffset - 1) return linksHandler(node, link) } - private fun linksHandler(linkText: ASTNode, link: String?, linkTitle: ASTNode? = null): DocTag { + private fun linksHandler(linkText: ASTNode, link: String?, linkTitle: ASTNode? = null): List<DocTag> { val dri: DRI? = link?.let { resolveDRI(it) } val linkOrEmpty = link ?: "" val linkTextString = @@ -247,9 +236,10 @@ open class MarkdownParser( body = text.substring(node.startOffset, node.endOffset) ) - private fun textHandler(node: ASTNode) = DocTagsFromIElementFactory.getInstance( + private fun textHandler(node: ASTNode, keepAllFormatting: Boolean) = DocTagsFromIElementFactory.getInstance( MarkdownTokenTypes.TEXT, - body = text.substring(node.startOffset, node.endOffset).transform() + body = text.substring(node.startOffset, node.endOffset).transform(), + keepFormatting = keepAllFormatting ) private fun strikeThroughHandler(node: ASTNode) = DocTagsFromIElementFactory.getInstance( @@ -284,7 +274,7 @@ open class MarkdownParser( false } - private fun imagesHandler(node: ASTNode): DocTag = + private fun imagesHandler(node: ASTNode): List<DocTag> = with(node.children.last().children) { val destination = find { it.type == MarkdownElementTypes.LINK_DESTINATION } val description = find { it.type == MarkdownElementTypes.LINK_TEXT } @@ -304,7 +294,7 @@ open class MarkdownParser( } - private fun rawHtmlHandler(node: ASTNode): DocTag = + private fun rawHtmlHandler(node: ASTNode): List<DocTag> = DocTagsFromIElementFactory.getInstance( node.type, body = text.substring(node.startOffset, node.endOffset) @@ -313,12 +303,9 @@ open class MarkdownParser( private fun codeSpansHandler(node: ASTNode) = DocTagsFromIElementFactory.getInstance( node.type, - children = listOf( - DocTagsFromIElementFactory.getInstance( - MarkdownTokenTypes.TEXT, - body = text.substring(node.startOffset + 1, node.endOffset - 1).replace('\n', ' ').trimIndent() - ) - + children = DocTagsFromIElementFactory.getInstance( + MarkdownTokenTypes.TEXT, + body = text.substring(node.startOffset + 1, node.endOffset - 1).replace('\n', ' ').trimIndent() ) ) @@ -334,7 +321,7 @@ open class MarkdownParser( LeafASTNode(MarkdownTokenTypes.HARD_LINE_BREAK, 0, 0) else it - }.evaluateChildren(), + }.evaluateChildren(keepAllFormatting = true), params = node .children .find { it.type == MarkdownTokenTypes.FENCE_LANG } @@ -343,7 +330,7 @@ open class MarkdownParser( ) private fun codeBlocksHandler(node: ASTNode) = - DocTagsFromIElementFactory.getInstance(node.type, children = node.children.mergeLeafASTNodes().map { + DocTagsFromIElementFactory.getInstance(node.type, children = node.children.mergeLeafASTNodes().flatMap { DocTagsFromIElementFactory.getInstance( MarkdownTokenTypes.TEXT, body = text.substring(it.startOffset, it.endOffset) @@ -356,7 +343,7 @@ open class MarkdownParser( children = node.children.evaluateChildren() ) - private fun visitNode(node: ASTNode): DocTag = + private fun visitNode(node: ASTNode, keepAllFormatting: Boolean = false): List<DocTag> = when (node.type) { MarkdownElementTypes.ATX_1, MarkdownElementTypes.ATX_2, @@ -389,7 +376,7 @@ open class MarkdownParser( MarkdownTokenTypes.CODE_FENCE_CONTENT, MarkdownTokenTypes.CODE_LINE, -> codeLineHandler(node) - MarkdownTokenTypes.TEXT -> textHandler(node) + MarkdownTokenTypes.TEXT -> textHandler(node, keepAllFormatting) MarkdownElementTypes.MARKDOWN_FILE -> markdownFileHandler(node) GFMElementTypes.STRIKETHROUGH -> strikeThroughHandler(node) GFMElementTypes.TABLE -> tableHandler(node) @@ -402,18 +389,22 @@ open class MarkdownParser( private fun List<ASTNode>.filterTabSeparators() = this.filterNot { it.type == GFMTokenTypes.TABLE_SEPARATOR } - private fun List<ASTNode>.evaluateChildren(): List<DocTag> = - this.removeUselessTokens().swapImagesThatShouldBeLinks().mergeLeafASTNodes().map { visitNode(it) } + private fun List<ASTNode>.evaluateChildren(keepAllFormatting: Boolean = false): List<DocTag> = + this.removeUselessTokens().swapImagesThatShouldBeLinks(keepAllFormatting).mergeLeafASTNodes().flatMap { visitNode(it, keepAllFormatting) } - private fun List<ASTNode>.swapImagesThatShouldBeLinks(): List<ASTNode> = - flatMap { node -> - if (node.type == MarkdownElementTypes.IMAGE - && node.children.firstOrNull()?.let { it is LeafASTNode && it.type.name == "!" } == true - && node.children.lastOrNull()?.type == MarkdownElementTypes.SHORT_REFERENCE_LINK - ) { - node.children - } else { - listOf(node) + private fun List<ASTNode>.swapImagesThatShouldBeLinks(keepAllFormatting: Boolean): List<ASTNode> = + if (keepAllFormatting) { + this + } else { + flatMap { node -> + if (node.type == MarkdownElementTypes.IMAGE + && node.children.firstOrNull()?.let { it is LeafASTNode && it.type.name == "!" } == true + && node.children.lastOrNull()?.type == MarkdownElementTypes.SHORT_REFERENCE_LINK + ) { + node.children + } else { + listOf(node) + } } } diff --git a/plugins/base/src/main/kotlin/parsers/factories/DocTagsFromIElementFactory.kt b/plugins/base/src/main/kotlin/parsers/factories/DocTagsFromIElementFactory.kt index 9ee11732..a3cbcc2e 100644 --- a/plugins/base/src/main/kotlin/parsers/factories/DocTagsFromIElementFactory.kt +++ b/plugins/base/src/main/kotlin/parsers/factories/DocTagsFromIElementFactory.kt @@ -6,12 +6,15 @@ import org.intellij.markdown.MarkdownElementTypes import org.intellij.markdown.MarkdownTokenTypes import org.intellij.markdown.flavours.gfm.GFMElementTypes import org.intellij.markdown.flavours.gfm.GFMTokenTypes +import org.jetbrains.dokka.base.translators.parseWithNormalisedSpaces import org.jetbrains.dokka.links.DRI import org.jetbrains.dokka.model.doc.DocTag.Companion.contentTypeParam import java.lang.NullPointerException object DocTagsFromIElementFactory { - fun getInstance(type: IElementType, children: List<DocTag> = emptyList(), params: Map<String, String> = emptyMap(), body: String? = null, dri: DRI? = null) = + + @Suppress("IMPLICIT_CAST_TO_ANY") + fun getInstance(type: IElementType, children: List<DocTag> = emptyList(), params: Map<String, String> = emptyMap(), body: String? = null, dri: DRI? = null, keepFormatting: Boolean = false) = when(type) { MarkdownElementTypes.SHORT_REFERENCE_LINK, MarkdownElementTypes.FULL_REFERENCE_LINK, @@ -33,7 +36,11 @@ object DocTagsFromIElementFactory { MarkdownElementTypes.ORDERED_LIST -> Ol(children, params) MarkdownElementTypes.UNORDERED_LIST -> Ul(children, params) MarkdownElementTypes.PARAGRAPH -> P(children, params) - MarkdownTokenTypes.TEXT -> Text(body ?: throw NullPointerException("Text body should be at least empty string passed to DocNodes factory!"), children, params ) + MarkdownTokenTypes.TEXT -> if (keepFormatting) Text( + body.orEmpty(), + children, + params + ) else body?.parseWithNormalisedSpaces(renderWhiteCharactersAsSpaces = false).orEmpty() MarkdownTokenTypes.HORIZONTAL_RULE -> HorizontalRule MarkdownTokenTypes.HARD_LINE_BREAK -> Br GFMElementTypes.STRIKETHROUGH -> Strikethrough(children, params) @@ -46,5 +53,10 @@ object DocTagsFromIElementFactory { MarkdownTokenTypes.HTML_TAG, MarkdownTokenTypes.HTML_BLOCK_CONTENT -> Text(body.orEmpty(), params = params + contentTypeParam("html")) else -> CustomDocTag(children, params, type.name) + }.let { + when (it) { + is List<*> -> it as List<DocTag> + else -> listOf(it as DocTag) + } } } diff --git a/plugins/base/src/main/kotlin/translators/parseWithNormalisedSpaces.kt b/plugins/base/src/main/kotlin/translators/parseWithNormalisedSpaces.kt new file mode 100644 index 00000000..4bb60f1a --- /dev/null +++ b/plugins/base/src/main/kotlin/translators/parseWithNormalisedSpaces.kt @@ -0,0 +1,50 @@ +package org.jetbrains.dokka.base.translators + +import org.intellij.markdown.lexer.Compat.codePointToString +import org.intellij.markdown.lexer.Compat.forEachCodePoint +import org.jetbrains.dokka.model.doc.DocTag +import org.jetbrains.dokka.model.doc.DocTag.Companion.contentTypeParam +import org.jetbrains.dokka.model.doc.Text +import org.jsoup.Jsoup +import org.jsoup.internal.StringUtil +import org.jsoup.nodes.Entities + +internal fun String.parseHtmlEncodedWithNormalisedSpaces( + renderWhiteCharactersAsSpaces: Boolean +): List<DocTag> { + val accum = StringBuilder() + val tags = mutableListOf<DocTag>() + var lastWasWhite = false + + forEachCodePoint { c -> + if (renderWhiteCharactersAsSpaces && StringUtil.isWhitespace(c)) { + if (!lastWasWhite) { + accum.append(' ') + lastWasWhite = true + } + } else if (codePointToString(c).let { it != Entities.escape(it) }) { + accum.toString().takeIf { it.isNotBlank() }?.let { tags.add(Text(it)) } + accum.delete(0, accum.length) + + accum.appendCodePoint(c) + tags.add(Text(accum.toString(), params = contentTypeParam("html"))) + accum.delete(0, accum.length) + } else if (!StringUtil.isInvisibleChar(c)) { + accum.appendCodePoint(c) + lastWasWhite = false + } + } + accum.toString().takeIf { it.isNotBlank() }?.let { tags.add(Text(it)) } + return tags +} + +/** + * Parses string into [Text] doc tags that can have either value of the string or html-encoded value with content-type=html parameter. + * Content type is added when dealing with html entries like ` ` + */ +internal fun String.parseWithNormalisedSpaces( + renderWhiteCharactersAsSpaces: Boolean +): List<DocTag> = + //parsing it using jsoup is required to get codePoints, otherwise they are interpreted separately, as chars + //But we dont need to do it for java as it is already parsed with jsoup + Jsoup.parseBodyFragment(this).body().wholeText().parseHtmlEncodedWithNormalisedSpaces(renderWhiteCharactersAsSpaces)
\ No newline at end of file diff --git a/plugins/base/src/main/kotlin/translators/psi/parsers/JavadocParser.kt b/plugins/base/src/main/kotlin/translators/psi/parsers/JavadocParser.kt index 53424ef9..ce022dd7 100644 --- a/plugins/base/src/main/kotlin/translators/psi/parsers/JavadocParser.kt +++ b/plugins/base/src/main/kotlin/translators/psi/parsers/JavadocParser.kt @@ -8,12 +8,14 @@ import com.intellij.psi.impl.source.tree.LazyParseablePsiElement import com.intellij.psi.impl.source.tree.LeafPsiElement import com.intellij.psi.javadoc.* import org.intellij.markdown.MarkdownElementTypes +import org.intellij.markdown.lexer.Compat.forEachCodePoint import org.jetbrains.dokka.analysis.DokkaResolutionFacade import org.jetbrains.dokka.analysis.from import org.jetbrains.dokka.base.parsers.MarkdownParser +import org.jetbrains.dokka.base.translators.parseHtmlEncodedWithNormalisedSpaces +import org.jetbrains.dokka.base.translators.parseWithNormalisedSpaces import org.jetbrains.dokka.links.DRI import org.jetbrains.dokka.model.doc.* -import org.jetbrains.dokka.model.doc.Deprecated import org.jetbrains.dokka.utilities.DokkaLogger import org.jetbrains.dokka.utilities.enumValueOrNull import org.jetbrains.kotlin.idea.kdoc.resolveKDocLink @@ -22,7 +24,9 @@ import org.jetbrains.kotlin.idea.util.CommentSaver.Companion.tokenType import org.jetbrains.kotlin.psi.psiUtil.getNextSiblingIgnoringWhitespace import org.jetbrains.kotlin.psi.psiUtil.siblings import org.jsoup.Jsoup +import org.jsoup.internal.StringUtil import org.jsoup.nodes.Element +import org.jsoup.nodes.Entities import org.jsoup.nodes.Node import org.jsoup.nodes.TextNode import java.util.* @@ -402,8 +406,11 @@ class JavadocParser( } private fun convertHtmlNode(node: Node, insidePre: Boolean = false): List<DocTag> = when (node) { - is TextNode -> (if (insidePre) node.wholeText else node.text() - .takeIf { it.isNotBlank() })?.let { listOf(Text(body = it)) }.orEmpty() + is TextNode -> (if (insidePre) { + node.wholeText.takeIf { it.isNotBlank() }?.let { listOf(Text(body = it)) } + } else { + node.wholeText.parseHtmlEncodedWithNormalisedSpaces(renderWhiteCharactersAsSpaces = true) + }).orEmpty() is Element -> createBlock(node) else -> emptyList() } |