diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/Kotlin/ContentBuilder.kt | 81 | ||||
-rw-r--r-- | src/Kotlin/DocumentationBuilder.kt | 19 | ||||
-rw-r--r-- | src/Markdown/GeneratedParserUtilBase.java | 1031 | ||||
-rw-r--r-- | src/Markdown/MarkdownProcessor.kt | 156 | ||||
-rw-r--r-- | src/Markdown/MarkdownTokenType.kt | 6 | ||||
-rw-r--r-- | src/Markdown/markdown.bnf | 98 | ||||
-rw-r--r-- | src/Markdown/markdown.leg | 781 | ||||
-rw-r--r-- | src/main.kt | 3 |
8 files changed, 117 insertions, 2058 deletions
diff --git a/src/Kotlin/ContentBuilder.kt b/src/Kotlin/ContentBuilder.kt index 635fc74f..8079fb4c 100644 --- a/src/Kotlin/ContentBuilder.kt +++ b/src/Kotlin/ContentBuilder.kt @@ -1,23 +1,22 @@ package org.jetbrains.dokka -import org.jetbrains.markdown.MarkdownElementTypes import java.util.ArrayDeque -import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor -import org.jetbrains.jet.lang.resolve.name.Name -import org.jetbrains.jet.lang.resolve.DescriptorToSourceUtils -import org.jetbrains.jet.lang.resolve.scopes.JetScope -import org.jetbrains.jet.lang.resolve.name.FqName +import org.jetbrains.jet.lang.descriptors.* +import org.jetbrains.jet.lang.resolve.* +import org.jetbrains.jet.lang.resolve.scopes.* +import org.jetbrains.jet.lang.resolve.name.* +import net.nicoulaj.idea.markdown.lang.* -public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: DeclarationDescriptor): Content { +public fun DocumentationBuilder.buildContent(tree: MarkdownNode, descriptor: DeclarationDescriptor): Content { val nodeStack = ArrayDeque<ContentNode>() nodeStack.push(Content()) - tree.visit {(node, text, processChildren) -> + tree.visit {(node, processChildren) -> val parent = nodeStack.peek()!! - val nodeType = node.getTokenType() - val nodeText = tree.getNodeText(node) + val nodeType = node.type + val nodeText = tree.text when (nodeType) { - MarkdownElementTypes.BULLET_LIST -> { + MarkdownElementTypes.UNORDERED_LIST -> { nodeStack.push(ContentList()) processChildren() parent.append(nodeStack.pop()) @@ -27,9 +26,7 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: Dec processChildren() parent.append(nodeStack.pop()) } - MarkdownElementTypes.HORIZONTAL_RULE -> { - } - MarkdownElementTypes.LIST_BLOCK -> { + MarkdownElementTypes.LIST_ITEM -> { nodeStack.push(ContentBlock()) processChildren() parent.append(nodeStack.pop()) @@ -44,50 +41,42 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: Dec processChildren() parent.append(nodeStack.pop()) } - MarkdownElementTypes.CODE -> { + MarkdownTokenTypes.CODE -> { nodeStack.push(ContentCode()) processChildren() parent.append(nodeStack.pop()) } - MarkdownElementTypes.ANONYMOUS_SECTION -> { - nodeStack.push(ContentSection("")) - processChildren() - parent.append(nodeStack.pop()) - } - MarkdownElementTypes.DIRECTIVE -> { - val name = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_NAME)?.let { tree.getNodeText(it) } ?: "" - val params = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_PARAMS)?.let { tree.getNodeText(it) } ?: "" - when (name) { - "code" -> parent.append(functionBody(descriptor, params)) - } - } - MarkdownElementTypes.NAMED_SECTION -> { - val label = tree.findChildByType(node, MarkdownElementTypes.SECTION_NAME)?.let { tree.getNodeText(it) } ?: "" - nodeStack.push(ContentSection(label)) - processChildren() - parent.append(nodeStack.pop()) - } - MarkdownElementTypes.LINK -> { - val target = tree.findChildByType(node, MarkdownElementTypes.TARGET)?.let { tree.getNodeText(it) } ?: "" - val href = tree.findChildByType(node, MarkdownElementTypes.HREF)?.let { tree.getNodeText(it) } + /* MarkdownElementTypes.ANONYMOUS_SECTION -> { + nodeStack.push(ContentSection("")) + processChildren() + parent.append(nodeStack.pop()) + } + MarkdownElementTypes.DIRECTIVE -> { + val name = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_NAME)?.let { tree.getNodeText(it) } ?: "" + val params = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_PARAMS)?.let { tree.getNodeText(it) } ?: "" + when (name) { + "code" -> parent.append(functionBody(descriptor, params)) + } + } + MarkdownElementTypes.NAMED_SECTION -> { + val label = tree.findChildByType(node, MarkdownElementTypes.SECTION_NAME)?.let { tree.getNodeText(it) } ?: "" + nodeStack.push(ContentSection(label)) + processChildren() + parent.append(nodeStack.pop()) + }*/ + MarkdownElementTypes.INLINE_LINK -> { + val target = node.child(MarkdownElementTypes.LINK_TITLE)?.let { it.text } ?: "" + val href = node.child(MarkdownElementTypes.LINK_DESTINATION)?.let { it.text } val link = if (href != null) ContentExternalLink(href) else ContentExternalLink(target) link.append(ContentText(target)) parent.append(link) } - MarkdownElementTypes.PLAIN_TEXT -> { + MarkdownTokenTypes.TEXT -> { nodeStack.push(ContentText(nodeText)) processChildren() parent.append(nodeStack.pop()) } - MarkdownElementTypes.END_LINE -> { - nodeStack.push(ContentText(nodeText)) - processChildren() - parent.append(nodeStack.pop()) - } - MarkdownElementTypes.BLANK_LINE -> { - processChildren() - } - MarkdownElementTypes.PARA -> { + MarkdownElementTypes.PARAGRAPH -> { nodeStack.push(ContentParagraph()) processChildren() parent.append(nodeStack.pop()) diff --git a/src/Kotlin/DocumentationBuilder.kt b/src/Kotlin/DocumentationBuilder.kt index 0acff1dc..d8be9d5a 100644 --- a/src/Kotlin/DocumentationBuilder.kt +++ b/src/Kotlin/DocumentationBuilder.kt @@ -1,17 +1,12 @@ package org.jetbrains.dokka import org.jetbrains.jet.lang.descriptors.* -import org.jetbrains.dokka.DocumentationNode.Kind -import org.jetbrains.jet.lang.types.TypeProjection -import org.jetbrains.jet.lang.types.Variance -import org.jetbrains.jet.lang.types.JetType -import org.jetbrains.jet.lang.types.lang.KotlinBuiltIns -import org.jetbrains.jet.lang.resolve.BindingContext -import org.jetbrains.jet.lang.resolve.name.Name -import org.jetbrains.jet.lang.resolve.scopes.JetScope -import org.jetbrains.jet.lang.psi.JetFile -import org.jetbrains.jet.lang.resolve.name.FqName -import org.jetbrains.jet.lang.resolve.lazy.ResolveSession +import org.jetbrains.dokka.DocumentationNode.* +import org.jetbrains.jet.lang.types.* +import org.jetbrains.jet.lang.types.lang.* +import org.jetbrains.jet.lang.resolve.scopes.* +import org.jetbrains.jet.lang.resolve.name.* +import org.jetbrains.jet.lang.resolve.lazy.* public data class DocumentationOptions(val includeNonPublic: Boolean = false) @@ -24,7 +19,7 @@ class DocumentationBuilder(val session: ResolveSession, val options: Documentati fun parseDocumentation(descriptor: DeclarationDescriptor): Content { val docText = descriptor.getDocumentationElements().map { it.extractText() }.join("\n") - val tree = MarkdownProcessor.parse(docText) + val tree = parseMarkdown(docText) //println(tree.toTestString()) val content = buildContent(tree, descriptor) return content diff --git a/src/Markdown/GeneratedParserUtilBase.java b/src/Markdown/GeneratedParserUtilBase.java deleted file mode 100644 index 9dd999b5..00000000 --- a/src/Markdown/GeneratedParserUtilBase.java +++ /dev/null @@ -1,1031 +0,0 @@ -/* - * Copyright 2011-2014 Gregory Shrago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.dokka.Markdown; - -import com.intellij.lang.*; -import com.intellij.lang.impl.PsiBuilderAdapter; -import com.intellij.lang.impl.PsiBuilderImpl; -import com.intellij.lexer.Lexer; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.Comparing; -import com.intellij.openapi.util.Key; -import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.text.StringHash; -import com.intellij.openapi.util.text.StringUtil; -import com.intellij.psi.PsiFile; -import com.intellij.psi.PsiReference; -import com.intellij.psi.TokenType; -import com.intellij.psi.impl.source.resolve.FileContextUtil; -import com.intellij.psi.impl.source.tree.CompositePsiElement; -import com.intellij.psi.tree.ICompositeElementType; -import com.intellij.psi.tree.IElementType; -import com.intellij.psi.tree.TokenSet; -import com.intellij.util.Function; -import com.intellij.util.PairProcessor; -import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.containers.LimitedPool; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedList; - -/** - * @author gregsh - */ -public class GeneratedParserUtilBase { - - private static final Logger LOG = Logger.getInstance("org.intellij.grammar.parser.GeneratedParserUtilBase"); - - private static final int MAX_RECURSION_LEVEL = 1000; - private static final int MAX_VARIANTS_SIZE = 10000; - private static final int MAX_VARIANTS_TO_DISPLAY = 50; - - private static final int INITIAL_VARIANTS_SIZE = 1000; - private static final int VARIANTS_POOL_SIZE = 10000; - private static final int FRAMES_POOL_SIZE = 500; - - public static final IElementType DUMMY_BLOCK = new DummyBlockElementType(); - - public interface Parser { - boolean parse(PsiBuilder builder, int level); - } - - public static final Parser TOKEN_ADVANCER = new Parser() { - @Override - public boolean parse(PsiBuilder builder, int level) { - if (builder.eof()) return false; - builder.advanceLexer(); - return true; - } - }; - - public static final Parser TRUE_CONDITION = new Parser() { - @Override - public boolean parse(PsiBuilder builder, int level) { - return true; - } - }; - - public static boolean eof(PsiBuilder builder_, int level_) { - return builder_.eof(); - } - - public static int current_position_(PsiBuilder builder_) { - return builder_.rawTokenIndex(); - } - - public static boolean recursion_guard_(PsiBuilder builder_, int level_, String funcName_) { - if (level_ > MAX_RECURSION_LEVEL) { - builder_.error("Maximum recursion level (" + MAX_RECURSION_LEVEL + ") reached in '" + funcName_ + "'"); - return false; - } - return true; - } - - public static boolean empty_element_parsed_guard_(PsiBuilder builder_, String funcName_, int prev_position_) { - if (prev_position_ == current_position_(builder_)) { - builder_.error("Empty element parsed in '" + funcName_ + "' at offset " + builder_.getCurrentOffset()); - return false; - } - return true; - } - - public static boolean invalid_left_marker_guard_(PsiBuilder builder_, PsiBuilder.Marker marker_, String funcName_) { - //builder_.error("Invalid left marker encountered in " + funcName_ +" at offset " + builder_.getCurrentOffset()); - boolean goodMarker = marker_ != null; // && ((LighterASTNode)marker_).getTokenType() != TokenType.ERROR_ELEMENT; - if (!goodMarker) return false; - ErrorState state = ErrorState.get(builder_); - - return !state.frameStack.isEmpty(); - } - - public static TokenSet create_token_set_(IElementType... tokenTypes_) { - return TokenSet.create(tokenTypes_); - } - - private static boolean consumeTokens(PsiBuilder builder_, boolean smart, int pin, IElementType... tokens) { - ErrorState state = ErrorState.get(builder_); - if (state.completionState != null && state.predicateCount == 0) { - addCompletionVariant(builder_, state.completionState, tokens); - } - // suppress single token completion - CompletionState completionState = state.completionState; - state.completionState = null; - boolean result_ = true; - boolean pinned_ = false; - for (int i = 0, tokensLength = tokens.length; i < tokensLength; i++) { - if (pin > 0 && i == pin) pinned_ = result_; - if (result_ || pinned_) { - boolean fast = smart && i == 0; - if (!(fast ? consumeTokenFast(builder_, tokens[i]) : consumeToken(builder_, tokens[i]))) { - result_ = false; - if (pin < 0 || pinned_) report_error_(builder_, state, false); - } - } - } - state.completionState = completionState; - return pinned_ || result_; - } - - public static boolean consumeTokens(PsiBuilder builder_, int pin_, IElementType... token) { - return consumeTokens(builder_, false, pin_, token); - } - - public static boolean consumeTokensSmart(PsiBuilder builder_, int pin_, IElementType... token) { - return consumeTokens(builder_, true, pin_, token); - } - - public static boolean parseTokens(PsiBuilder builder_, int pin_, IElementType... tokens) { - return parseTokens(builder_, false, pin_, tokens); - } - - public static boolean parseTokensSmart(PsiBuilder builder_, int pin_, IElementType... tokens) { - return parseTokens(builder_, true, pin_, tokens); - } - - public static boolean parseTokens(PsiBuilder builder_, boolean smart, int pin_, IElementType... tokens) { - PsiBuilder.Marker marker_ = builder_.mark(); - boolean result_ = consumeTokens(builder_, smart, pin_, tokens); - if (!result_) { - marker_.rollbackTo(); - } - else { - marker_.drop(); - } - return result_; - } - - public static boolean consumeTokenSmart(PsiBuilder builder_, IElementType token) { - addCompletionVariantSmart(builder_, token); - return consumeTokenFast(builder_, token); - } - - public static boolean consumeTokenSmart(PsiBuilder builder_, String token) { - addCompletionVariantSmart(builder_, token); - return consumeTokenFast(builder_, token); - } - - public static boolean consumeToken(PsiBuilder builder_, IElementType token) { - addVariantSmart(builder_, token, true); - if (nextTokenIsFast(builder_, token)) { - builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeTokenFast(PsiBuilder builder_, IElementType token) { - if (nextTokenIsFast(builder_, token)) { - builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeToken(PsiBuilder builder_, String text) { - return consumeToken(builder_, text, ErrorState.get(builder_).caseSensitive); - } - - public static boolean consumeToken(PsiBuilder builder_, String text, boolean caseSensitive) { - addVariantSmart(builder_, text, true); - int count = nextTokenIsFast(builder_, text, caseSensitive); - if (count > 0) { - while (count-- > 0) builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeTokenFast(PsiBuilder builder_, String text) { - int count = nextTokenIsFast(builder_, text, ErrorState.get(builder_).caseSensitive); - if (count > 0) { - while (count-- > 0) builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType token) { - return builder_.getTokenType() == token; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType... tokens) { - IElementType tokenType = builder_.getTokenType(); - for (IElementType token : tokens) { - if (token == tokenType) return true; - } - return false; - } - - public static boolean nextTokenIs(PsiBuilder builder_, String frameName, IElementType... tokens) { - ErrorState state = ErrorState.get(builder_); - if (state.completionState != null) return true; - boolean track = !state.suppressErrors && state.predicateCount < 2 && state.predicateSign; - if (!track) return nextTokenIsFast(builder_, tokens); - IElementType tokenType = builder_.getTokenType(); - if (StringUtil.isNotEmpty(frameName)) { - addVariantInner(state, builder_.rawTokenIndex(), frameName); - } - else { - for (IElementType token : tokens) { - addVariant(builder_, state, token); - } - } - if (tokenType == null) return false; - for (IElementType token : tokens) { - if (tokenType == token) return true; - } - return false; - } - - public static boolean nextTokenIs(PsiBuilder builder_, IElementType token) { - if (!addVariantSmart(builder_, token, false)) return true; - return nextTokenIsFast(builder_, token); - } - - public static boolean nextTokenIs(PsiBuilder builder_, String tokenText) { - if (!addVariantSmart(builder_, tokenText, false)) return true; - return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, String tokenText) { - return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0; - } - - public static int nextTokenIsFast(PsiBuilder builder_, String tokenText, boolean caseSensitive) { - CharSequence sequence = builder_.getOriginalText(); - int offset = builder_.getCurrentOffset(); - int endOffset = offset + tokenText.length(); - CharSequence subSequence = sequence.subSequence(offset, Math.min(endOffset, sequence.length())); - - if (!Comparing.equal(subSequence, tokenText, caseSensitive)) return 0; - - int count = 0; - while (true) { - int nextOffset = builder_.rawTokenTypeStart(++count); - if (nextOffset > endOffset) { - return -count; - } - else if (nextOffset == endOffset) { - break; - } - } - return count; - } - - private static void addCompletionVariantSmart(PsiBuilder builder_, Object token) { - ErrorState state = ErrorState.get(builder_); - CompletionState completionState = state.completionState; - if (completionState != null && state.predicateCount == 0) { - addCompletionVariant(builder_, completionState, token); - } - } - - private static boolean addVariantSmart(PsiBuilder builder_, Object token, boolean force) { - ErrorState state = ErrorState.get(builder_); - // skip FIRST check in completion mode - if (state.completionState != null && !force) return false; - builder_.eof(); - if (!state.suppressErrors && state.predicateCount < 2) { - addVariant(builder_, state, token); - } - return true; - } - - public static void addVariant(PsiBuilder builder_, String text) { - addVariant(builder_, ErrorState.get(builder_), text); - } - - private static void addVariant(PsiBuilder builder_, ErrorState state, Object o) { - builder_.eof(); // skip whitespaces - addVariantInner(state, builder_.rawTokenIndex(), o); - - CompletionState completionState = state.completionState; - if (completionState != null && state.predicateSign) { - addCompletionVariant(builder_, completionState, o); - } - } - - private static void addVariantInner(ErrorState state, int pos, Object o) { - Variant variant = state.VARIANTS.alloc().init(pos, o); - if (state.predicateSign) { - state.variants.add(variant); - if (state.lastExpectedVariantPos < variant.position) { - state.lastExpectedVariantPos = variant.position; - } - } - else { - state.unexpected.add(variant); - } - } - - private static void addCompletionVariant(@NotNull PsiBuilder builder_, @NotNull CompletionState completionState, Object o) { - int offset = builder_.getCurrentOffset(); - if (!builder_.eof() && offset == builder_.rawTokenTypeStart(1)) return; // suppress for zero-length tokens - - boolean add = false; - int diff = completionState.offset - offset; - String text = completionState.convertItem(o); - int length = text == null? 0 : text.length(); - if (length == 0) return; - if (diff == 0) { - add = true; - } - else if (diff > 0 && diff <= length) { - CharSequence fragment = builder_.getOriginalText().subSequence(offset, completionState.offset); - add = completionState.prefixMatches(fragment.toString(), text); - } - else if (diff < 0) { - for (int i=-1; ; i--) { - IElementType type = builder_.rawLookup(i); - int tokenStart = builder_.rawTokenTypeStart(i); - if (isWhitespaceOrComment(builder_, type)) { - diff = completionState.offset - tokenStart; - } - else if (type != null && tokenStart < completionState.offset) { - CharSequence fragment = builder_.getOriginalText().subSequence(tokenStart, completionState.offset); - if (completionState.prefixMatches(fragment.toString(), text)) { - diff = completionState.offset - tokenStart; - } - break; - } - else break; - } - add = diff >= 0 && diff < length; - } - add = add && length > 1 && !(text.charAt(0) == '<' && text.charAt(length - 1) == '>') && - !(text.charAt(0) == '\'' && text.charAt(length - 1) == '\'' && length < 5); - if (add) { - completionState.addItem(builder_, text); - } - } - - public static boolean isWhitespaceOrComment(@NotNull PsiBuilder builder_, @Nullable IElementType type) { - return ((PsiBuilderImpl)((Builder)builder_).getDelegate()).whitespaceOrComment(type); - } - - // here's the new section API for compact parsers & less IntelliJ platform API exposure - public static final int _NONE_ = 0x0; - public static final int _COLLAPSE_ = 0x1; - public static final int _LEFT_ = 0x2; - public static final int _LEFT_INNER_ = 0x4; - public static final int _AND_ = 0x8; - public static final int _NOT_ = 0x10; - - // simple enter/exit methods pair that doesn't require frame object - public static PsiBuilder.Marker enter_section_(PsiBuilder builder_) { - return builder_.mark(); - } - - public static void exit_section_(PsiBuilder builder_, - PsiBuilder.Marker marker, - @Nullable IElementType elementType, - boolean result) { - close_marker_impl_(ErrorState.get(builder_).frameStack.peekLast(), marker, elementType, result); - } - - // complex enter/exit methods pair with frame object - public static PsiBuilder.Marker enter_section_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) { - PsiBuilder.Marker marker = builder_.mark(); - enter_section_impl_(builder_, level, modifiers, frameName); - return marker; - } - - private static void enter_section_impl_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) { - ErrorState state = ErrorState.get(builder_); - Frame frame = state.FRAMES.alloc().init(builder_, state, level, modifiers, frameName); - Frame prevFrame = state.frameStack.peekLast(); - if (prevFrame != null && prevFrame.errorReportedAt > frame.position) { - // report error for previous unsuccessful frame - reportError(builder_, state, frame, true, false); - } - if (((frame.modifiers & _LEFT_) | (frame.modifiers & _LEFT_INNER_)) != 0) { - PsiBuilder.Marker left = (PsiBuilder.Marker)builder_.getLatestDoneMarker(); - if (invalid_left_marker_guard_(builder_, left, frameName)) { - frame.leftMarker = left; - } - } - state.frameStack.add(frame); - if ((modifiers & _AND_) != 0) { - if (state.predicateCount == 0 && !state.predicateSign) { - throw new AssertionError("Incorrect false predicate sign"); - } - state.predicateCount++; - } - else if ((modifiers & _NOT_) != 0) { - if (state.predicateCount == 0) { - state.predicateSign = false; - } - else { - state.predicateSign = !state.predicateSign; - } - state.predicateCount++; - } - } - - public static void exit_section_(PsiBuilder builder_, - int level, - PsiBuilder.Marker marker, - @Nullable IElementType elementType, - boolean result, - boolean pinned, - @Nullable Parser eatMore) { - ErrorState state = ErrorState.get(builder_); - - Frame frame = state.frameStack.pollLast(); - if (frame == null || level != frame.level) { - LOG.error("Unbalanced error section: got " + frame + ", expected level " + level); - if (frame != null) state.FRAMES.recycle(frame); - close_marker_impl_(frame, marker, elementType, result); - return; - } - - if (((frame.modifiers & _AND_) | (frame.modifiers & _NOT_)) != 0) { - close_marker_impl_(frame, marker, null, false); - state.predicateCount--; - if ((frame.modifiers & _NOT_) != 0) state.predicateSign = !state.predicateSign; - state.FRAMES.recycle(frame); - return; - } - exit_section_impl_(state, frame, builder_, marker, elementType, result, pinned); - - int initialPos = builder_.rawTokenIndex(); - boolean willFail = !result && !pinned; - if (willFail && initialPos == frame.position && state.lastExpectedVariantPos == frame.position && - frame.name != null && state.variants.size() - frame.variantCount > 1) { - state.clearVariants(true, frame.variantCount); - addVariantInner(state, initialPos, frame.name); - } - int lastErrorPos = getLastVariantPos(state, initialPos); - if (!state.suppressErrors && eatMore != null) { - state.suppressErrors = true; - final boolean eatMoreFlagOnce = !builder_.eof() && eatMore.parse(builder_, frame.level + 1); - boolean eatMoreFlag = eatMoreFlagOnce || !result && frame.position == initialPos && lastErrorPos > frame.position; - - PsiBuilderImpl.ProductionMarker latestDoneMarker = - (pinned || result) && (state.altMode || elementType != null) && - eatMoreFlagOnce ? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null; - PsiBuilder.Marker extensionMarker = null; - IElementType extensionTokenType = null; - // whitespace prefix makes the very first frame offset bigger than marker start offset which is always 0 - if (latestDoneMarker instanceof PsiBuilder.Marker && - frame.position >= latestDoneMarker.getStartIndex() && - frame.position <= latestDoneMarker.getEndIndex()) { - extensionMarker = ((PsiBuilder.Marker)latestDoneMarker).precede(); - extensionTokenType = latestDoneMarker.getTokenType(); - ((PsiBuilder.Marker)latestDoneMarker).drop(); - } - // advance to the last error pos - // skip tokens until lastErrorPos. parseAsTree might look better here... - int parenCount = 0; - while ((eatMoreFlag || parenCount > 0) && builder_.rawTokenIndex() < lastErrorPos) { - builder_.advanceLexer(); - eatMoreFlag = eatMore.parse(builder_, frame.level + 1); - } - boolean errorReported = frame.errorReportedAt == initialPos || !result && frame.errorReportedAt >= frame.position; - if (errorReported) { - if (eatMoreFlag) { - builder_.advanceLexer(); - parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); - } - } - else if (eatMoreFlag) { - errorReported = reportError(builder_, state, frame, true, true); - parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); - } - else if (eatMoreFlagOnce || (!result && frame.position != builder_.rawTokenIndex()) || frame.errorReportedAt > initialPos) { - errorReported = reportError(builder_, state, frame, true, false); - } - if (extensionMarker != null) { - extensionMarker.done(extensionTokenType); - } - state.suppressErrors = false; - if (errorReported || result) { - state.clearVariants(true, 0); - state.clearVariants(false, 0); - state.lastExpectedVariantPos = -1; - } - } - else if (!result && pinned && frame.errorReportedAt < 0) { - // do not report if there are errors beyond current position - if (lastErrorPos == initialPos) { - // do not force, inner recoverRoot might have skipped some tokens - reportError(builder_, state, frame, false, false); - } - else if (lastErrorPos > initialPos) { - // set error pos here as if it is reported for future reference - frame.errorReportedAt = lastErrorPos; - } - } - // propagate errorReportedAt up the stack to avoid duplicate reporting - Frame prevFrame = willFail && eatMore == null ? null : state.frameStack.peekLast(); - if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) { - prevFrame.errorReportedAt = frame.errorReportedAt; - } - state.FRAMES.recycle(frame); - } - - private static void exit_section_impl_(ErrorState state, - Frame frame, - PsiBuilder builder_, - PsiBuilder.Marker marker, - IElementType elementType, - boolean result, - boolean pinned) { - if (elementType != null && marker != null) { - if ((frame.modifiers & _COLLAPSE_) != 0) { - PsiBuilderImpl.ProductionMarker last = result || pinned? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null; - if (last != null && last.getStartIndex() == frame.position && - state.typeExtends(last.getTokenType(), elementType)) { - IElementType resultType = last.getTokenType(); - ((PsiBuilder.Marker)last).drop(); - marker.done(resultType); - return; - } - } - if (result || pinned) { - if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) { - marker.done(elementType); - frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType()); - frame.leftMarker.drop(); - } - else if ((frame.modifiers & _LEFT_) != 0 && frame.leftMarker != null) { - marker.drop(); - frame.leftMarker.precede().done(elementType); - } - else { - if (frame.level == 0) builder_.eof(); // skip whitespaces - marker.done(elementType); - } - } - else { - close_marker_impl_(frame, marker, null, false); - } - } - else if (result || pinned) { - if (marker != null) marker.drop(); - if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) { - frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType()); - frame.leftMarker.drop(); - } - } - else { - close_marker_impl_(frame, marker, null, false); - } - } - - private static void close_marker_impl_(Frame frame, PsiBuilder.Marker marker, IElementType elementType, boolean result) { - if (marker == null) return; - if (result) { - if (elementType != null) { - marker.done(elementType); - } - else { - marker.drop(); - } - } - else { - if (frame != null) { - int position = ((PsiBuilderImpl.ProductionMarker)marker).getStartIndex(); - if (frame.errorReportedAt > position) { - frame.errorReportedAt = frame.errorReportedAtPrev; - } - } - marker.rollbackTo(); - } - } - - public static boolean report_error_(PsiBuilder builder_, boolean result_) { - if (!result_) report_error_(builder_, ErrorState.get(builder_), false); - return result_; - } - - public static void report_error_(PsiBuilder builder_, ErrorState state, boolean advance) { - Frame frame = state.frameStack.isEmpty()? null : state.frameStack.getLast(); - if (frame == null) { - LOG.error("unbalanced enter/exit section call: got null"); - return; - } - int position = builder_.rawTokenIndex(); - if (frame.errorReportedAt < position && getLastVariantPos(state, position + 1) <= position) { - reportError(builder_, state, frame, true, advance); - } - } - - private static int getLastVariantPos(ErrorState state, int defValue) { - return state.lastExpectedVariantPos < 0? defValue : state.lastExpectedVariantPos; - } - - private static boolean reportError(PsiBuilder builder_, - ErrorState state, - Frame frame, - boolean force, - boolean advance) { - String expectedText = state.getExpectedText(builder_); - boolean notEmpty = StringUtil.isNotEmpty(expectedText); - if (force || notEmpty || advance) { - String gotText = builder_.eof()? "unexpected end of file" : - notEmpty? "got '" + builder_.getTokenText() +"'" : - "'" + builder_.getTokenText() +"' unexpected"; - String message = expectedText + gotText; - if (advance) { - PsiBuilder.Marker mark = builder_.mark(); - builder_.advanceLexer(); - mark.error(message); - } - else { - builder_.error(message); - } - builder_.eof(); // skip whitespaces - frame.errorReportedAt = builder_.rawTokenIndex(); - return true; - } - return false; - } - - - public static final Key<CompletionState> COMPLETION_STATE_KEY = Key.create("COMPLETION_STATE_KEY"); - - public static class CompletionState implements Function<Object, String> { - public final int offset; - public final Collection<String> items = ContainerUtil.newTroveSet(); - - public CompletionState(int offset_) { - offset = offset_; - } - - @Nullable - public String convertItem(Object o) { - return o instanceof Object[] ? StringUtil.join((Object[]) o, this, " ") : o.toString(); - } - - @Override - public String fun(Object o) { - return o.toString(); - } - - public void addItem(@NotNull PsiBuilder builder, @NotNull String text) { - items.add(text); - } - - public boolean prefixMatches(@NotNull String prefix, @NotNull String variant) { - return StringUtil.startsWithIgnoreCase(variant, prefix); - } - } - - public static class Builder extends PsiBuilderAdapter { - public final ErrorState state; - public final PsiParser parser; - - public Builder(PsiBuilder builder_, ErrorState state_, PsiParser parser_) { - super(builder_); - state = state_; - parser = parser_; - } - - public Lexer getLexer() { - return ((PsiBuilderImpl)myDelegate).getLexer(); - } - } - - public static PsiBuilder adapt_builder_(IElementType root, PsiBuilder builder, PsiParser parser) { - return adapt_builder_(root, builder, parser, null); - } - - public static PsiBuilder adapt_builder_(IElementType root, PsiBuilder builder, PsiParser parser, TokenSet[] extendsSets) { - ErrorState state = new ErrorState(); - ErrorState.initState(state, builder, root, extendsSets); - return new Builder(builder, state, parser); - } - - public static class ErrorState { - TokenSet[] extendsSets; - public PairProcessor<IElementType, IElementType> altExtendsChecker; - - int predicateCount; - boolean predicateSign = true; - boolean suppressErrors; - public final LinkedList<Frame> frameStack = new LinkedList<Frame>(); - public CompletionState completionState; - - private boolean caseSensitive; - public boolean altMode; - - int lastExpectedVariantPos = -1; - MyList<Variant> variants = new MyList<Variant>(INITIAL_VARIANTS_SIZE); - MyList<Variant> unexpected = new MyList<Variant>(INITIAL_VARIANTS_SIZE / 10); - - final LimitedPool<Variant> VARIANTS = new LimitedPool<Variant>(VARIANTS_POOL_SIZE, new LimitedPool.ObjectFactory<Variant>() { - @Override - public Variant create() { - return new Variant(); - } - - @Override - public void cleanup(final Variant o) { - } - }); - final LimitedPool<Frame> FRAMES = new LimitedPool<Frame>(FRAMES_POOL_SIZE, new LimitedPool.ObjectFactory<Frame>() { - @Override - public Frame create() { - return new Frame(); - } - - @Override - public void cleanup(final Frame o) { - } - }); - - public static ErrorState get(PsiBuilder builder) { - return ((Builder)builder).state; - } - - public static void initState(ErrorState state, PsiBuilder builder, IElementType root, TokenSet[] extendsSets) { - state.extendsSets = extendsSets; - PsiFile file = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY); - state.completionState = file == null? null: file.getUserData(COMPLETION_STATE_KEY); - Language language = file == null? root.getLanguage() : file.getLanguage(); - state.caseSensitive = language.isCaseSensitive(); - } - - public String getExpectedText(PsiBuilder builder_) { - int position = builder_.rawTokenIndex(); - StringBuilder sb = new StringBuilder(); - if (addExpected(sb, position, true)) { - sb.append(" expected, "); - } - else if (addExpected(sb, position, false)) sb.append(" unexpected, "); - return sb.toString(); - } - - private boolean addExpected(StringBuilder sb, int position, boolean expected) { - MyList<Variant> list = expected ? variants : unexpected; - String[] strings = new String[list.size()]; - long[] hashes = new long[strings.length]; - Arrays.fill(strings, ""); - int count = 0; - loop: for (Variant variant : list) { - if (position == variant.position) { - String text = variant.object.toString(); - long hash = StringHash.calc(text); - for (int i=0; i<count; i++) { - if (hashes[i] == hash) continue loop; - } - hashes[count] = hash; - strings[count] = text; - count++; - } - } - Arrays.sort(strings); - count = 0; - for (String s : strings) { - if (s.length() == 0) continue; - if (count++ > 0) { - if (count > MAX_VARIANTS_TO_DISPLAY) { - sb.append(" and ..."); - break; - } - else { - sb.append(", "); - } - } - char c = s.charAt(0); - String displayText = c == '<' || StringUtil.isJavaIdentifierStart(c) ? s : '\'' + s + '\''; - sb.append(displayText); - } - if (count > 1 && count < MAX_VARIANTS_TO_DISPLAY) { - int idx = sb.lastIndexOf(", "); - sb.replace(idx, idx + 1, " or"); - } - return count > 0; - } - - public void clearVariants(boolean expected, int start) { - MyList<Variant> list = expected? variants : unexpected; - if (start < 0 || start >= list.size()) return; - for (int i = start, len = list.size(); i < len; i ++) { - VARIANTS.recycle(list.get(i)); - } - list.setSize(start); - } - - boolean typeExtends(IElementType child_, IElementType parent_) { - if (child_ == parent_) return true; - if (extendsSets != null) { - for (TokenSet set : extendsSets) { - if (set.contains(child_) && set.contains(parent_)) return true; - } - } - return altExtendsChecker != null && altExtendsChecker.process(child_, parent_); - } - } - - public static class Frame { - public int offset; - public int position; - public int level; - public int modifiers; - public String name; - public int variantCount; - public int errorReportedAt; - public int errorReportedAtPrev; - public PsiBuilder.Marker leftMarker; - - public Frame() { - } - - public Frame init(PsiBuilder builder_, ErrorState state, int level_, int modifiers_, String name_) { - offset = builder_.getCurrentOffset(); - position = builder_.rawTokenIndex(); - level = level_; - modifiers = modifiers_; - name = name_; - variantCount = state.variants.size(); - errorReportedAt = -1; - - Frame prev = state.frameStack.peekLast(); - errorReportedAtPrev = prev == null? -1 : prev.errorReportedAt; - leftMarker = null; - return this; - } - - @Override - public String toString() { - String mod = modifiers == _NONE_ ? "_NONE_, " : - ((modifiers & _COLLAPSE_) != 0? "_CAN_COLLAPSE_, ": "") + - ((modifiers & _LEFT_) != 0? "_LEFT_, ": "") + - ((modifiers & _LEFT_INNER_) != 0? "_LEFT_INNER_, ": "") + - ((modifiers & _AND_) != 0? "_AND_, ": "") + - ((modifiers & _NOT_) != 0? "_NOT_, ": ""); - return String.format("{%s:%s:%d, %d, %s%s}", offset, position, level, errorReportedAt, mod, name); - } - } - - - private static class Variant { - int position; - Object object; - - public Variant init(int pos, Object o) { - position = pos; - object = o; - return this; - } - - @Override - public String toString() { - return "<" + position + ", " + object + ">"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Variant variant = (Variant)o; - - if (position != variant.position) return false; - if (!this.object.equals(variant.object)) return false; - - return true; - } - - @Override - public int hashCode() { - int result = position; - result = 31 * result + object.hashCode(); - return result; - } - } - - - private static final int MAX_CHILDREN_IN_TREE = 10; - public static boolean parseAsTree(ErrorState state, final PsiBuilder builder_, int level, final IElementType chunkType, - boolean checkBraces, final Parser parser, final Parser eatMoreCondition) { - final LinkedList<Pair<PsiBuilder.Marker, PsiBuilder.Marker>> parenList = new LinkedList<Pair<PsiBuilder.Marker, PsiBuilder.Marker>>(); - final LinkedList<Pair<PsiBuilder.Marker, Integer>> siblingList = new LinkedList<Pair<PsiBuilder.Marker, Integer>>(); - PsiBuilder.Marker marker = null; - - final Runnable checkSiblingsRunnable = new Runnable() { - @Override - public void run() { - main: - while (!siblingList.isEmpty()) { - final Pair<PsiBuilder.Marker, PsiBuilder.Marker> parenPair = parenList.peek(); - final int rating = siblingList.getFirst().second; - int count = 0; - for (Pair<PsiBuilder.Marker, Integer> pair : siblingList) { - if (pair.second != rating || parenPair != null && pair.first == parenPair.second) break main; - if (++count >= MAX_CHILDREN_IN_TREE) { - final PsiBuilder.Marker parentMarker = pair.first.precede(); - while (count-- > 0) { - siblingList.removeFirst(); - } - parentMarker.done(chunkType); - siblingList.addFirst(Pair.create(parentMarker, rating + 1)); - continue main; - } - } - break; - } - } - }; - int totalCount = 0; - int tokenCount = 0; - while (true) { - final IElementType tokenType = builder_.getTokenType(); - if (marker == null) { - marker = builder_.mark(); - } - final boolean result = (!parenList.isEmpty() || eatMoreCondition.parse(builder_, level + 1)) && parser.parse(builder_, level + 1); - if (result) { - tokenCount++; - totalCount++; - } - if (!result) { - break; - } - - if (tokenCount >= MAX_CHILDREN_IN_TREE && marker != null) { - marker.done(chunkType); - siblingList.addFirst(Pair.create(marker, 1)); - checkSiblingsRunnable.run(); - marker = null; - tokenCount = 0; - } - } - if (marker != null) { - marker.drop(); - } - for (Pair<PsiBuilder.Marker, PsiBuilder.Marker> pair : parenList) { - pair.first.drop(); - } - return totalCount != 0; - } - - private static class DummyBlockElementType extends IElementType implements ICompositeElementType{ - DummyBlockElementType() { - super("DUMMY_BLOCK", Language.ANY); - } - - @NotNull - @Override - public ASTNode createCompositeNode() { - return new DummyBlock(); - } - } - - public static class DummyBlock extends CompositePsiElement { - DummyBlock() { - super(DUMMY_BLOCK); - } - - @NotNull - @Override - public PsiReference[] getReferences() { - return PsiReference.EMPTY_ARRAY; - } - - @NotNull - @Override - public Language getLanguage() { - return getParent().getLanguage(); - } - } - - private static class MyList<E> extends ArrayList<E> { - MyList(int initialCapacity) { - super(initialCapacity); - } - - protected void setSize(int fromIndex) { - removeRange(fromIndex, size()); - } - - @Override - public boolean add(E e) { - int size = size(); - if (size >= MAX_VARIANTS_SIZE) { - removeRange(MAX_VARIANTS_SIZE / 4, size - MAX_VARIANTS_SIZE / 4); - } - return super.add(e); - } - } -} diff --git a/src/Markdown/MarkdownProcessor.kt b/src/Markdown/MarkdownProcessor.kt index 040f2ad5..b5e18f92 100644 --- a/src/Markdown/MarkdownProcessor.kt +++ b/src/Markdown/MarkdownProcessor.kt @@ -1,80 +1,34 @@ package org.jetbrains.dokka -import org.jetbrains.markdown.* -import com.intellij.lang.impl.PsiBuilderImpl -import com.intellij.psi.tree.TokenSet -import com.intellij.lang.Language -import com.intellij.psi.tree.IFileElementType -import com.intellij.lang.LighterASTNode -import com.intellij.util.diff.FlyweightCapableTreeStructure -import com.intellij.openapi.util.Ref -import org.jetbrains.markdown.lexer.MarkdownLexer -import com.intellij.psi.tree.IElementType +import net.nicoulaj.idea.markdown.lang.ast.* +import net.nicoulaj.idea.markdown.lang.parser.dialects.commonmark.* +import net.nicoulaj.idea.markdown.lang.parser.* +import net.nicoulaj.idea.markdown.lang.* -public object MarkdownProcessor { - val EXPR_LANGUAGE = object : Language("MARKDOWN") {} - val DOCUMENT = IFileElementType("DOCUMENT", EXPR_LANGUAGE); - - public fun parse(markdown: String): MarkdownTree { - val parser = MarkdownParser() - val builder = PsiBuilderImpl(null, null, TokenSet.EMPTY, TokenSet.EMPTY, MarkdownLexer(), null, markdown, null, null) - parser.parse_only_(DOCUMENT, builder) - val light = builder.getLightTree()!! - return MarkdownTree(markdown, light) - } +class MarkdownNode(val node: ASTNode, val markdown: String) { + val children: List<MarkdownNode> get() = node.children.map { MarkdownNode(it, markdown) } + val endOffset: Int get() = node.endOffset + val startOffset: Int get() = node.startOffset + val type: IElementType get() = node.type + val text: String get() = markdown.substring(startOffset, endOffset) + fun child(type: IElementType): MarkdownNode? = children.firstOrNull { it.type == type } } -public class MarkdownTree(private val text: String, private val structure: FlyweightCapableTreeStructure<LighterASTNode>) { - fun visit(action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) { - visit(structure.getRoot(), action) - } - - fun findChildByType(node: LighterASTNode, findType: IElementType) : LighterASTNode? { - val ref: Ref<Array<LighterASTNode>?> = Ref.create<Array<LighterASTNode>?>() - val count = structure.getChildren(node, ref) - val children = ref.get() - if (children != null) { - for (index in 0..count - 1) { - val child = children[index] - val nodeType = child.getTokenType() - if (nodeType == findType) - return child - val nestedChild = findChildByType(child, findType) - if (nestedChild != null) - return nestedChild - } - } - return null - } - - fun getNodeText(node: LighterASTNode) : String { - return text.substring(node.getStartOffset(), node.getEndOffset()) - } - - fun visit(node: LighterASTNode, action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) { - action(node, text) { - val ref : Ref<Array<LighterASTNode>?> = Ref.create<Array<LighterASTNode>?>() - val count = structure.getChildren(node, ref) - val children = ref.get() - if (children != null) { - for (index in 0..count - 1) { - val child = children[index] - visit(child, action) - } - } +fun MarkdownNode.visit(action: (MarkdownNode, () -> Unit) -> Unit) { + action(this) { + for (child in children) { + child.visit(action) } } - } -public fun MarkdownTree.toTestString(): String { +public fun MarkdownNode.toTestString(): String { val sb = StringBuilder() var level = 0 - visit {(node, text, visitChildren) -> - val nodeText = text.substring(node.getStartOffset(), node.getEndOffset()) + visit {(node, visitChildren) -> sb.append(" ".repeat(level * 2)) - sb.append(node.getTokenType().toString()) - sb.append(":" + nodeText.replace("\n", "\u23CE")) + sb.append(node.type.toString()) + sb.append(":" + node.text.replace("\n", "\u23CE")) sb.appendln() level++ visitChildren() @@ -83,26 +37,23 @@ public fun MarkdownTree.toTestString(): String { return sb.toString() } -public fun MarkdownTree.toHtml(): String { +public fun MarkdownNode.toHtml(): String { val sb = StringBuilder() - visit {(node, text, processChildren) -> - val nodeType = node.getTokenType() - val nodeText = text.substring(node.getStartOffset(), node.getEndOffset()) + visit {(node, processChildren) -> + val nodeType = node.type + val nodeText = node.text when (nodeType) { - MarkdownElementTypes.BULLET_LIST -> { + MarkdownElementTypes.UNORDERED_LIST -> { sb.appendln("<ul>") processChildren() sb.appendln("</ul>") } - MarkdownElementTypes.HORIZONTAL_RULE -> { - sb.appendln("<hr/>") - } MarkdownElementTypes.ORDERED_LIST -> { sb.appendln("<ol>") processChildren() sb.appendln("</ol>") } - MarkdownElementTypes.LIST_BLOCK -> { + MarkdownElementTypes.LIST_ITEM -> { sb.append("<li>") processChildren() sb.appendln("</li>") @@ -117,20 +68,54 @@ public fun MarkdownTree.toHtml(): String { processChildren() sb.append("</strong>") } - MarkdownElementTypes.PLAIN_TEXT -> { - sb.append(nodeText) + MarkdownElementTypes.ATX_1 -> { + sb.append("<h1>") + processChildren() + sb.append("</h1>") } - MarkdownElementTypes.END_LINE -> { - sb.appendln() + MarkdownElementTypes.ATX_2 -> { + sb.append("<h2>") + processChildren() + sb.append("</h2>") } - MarkdownElementTypes.BLANK_LINE -> { - sb.appendln() + MarkdownElementTypes.ATX_3 -> { + sb.append("<h3>") + processChildren() + sb.append("</h3>") + } + MarkdownElementTypes.ATX_4 -> { + sb.append("<h4>") + processChildren() + sb.append("</h4>") + } + MarkdownElementTypes.ATX_5 -> { + sb.append("<h5>") + processChildren() + sb.append("</h5>") + } + MarkdownElementTypes.ATX_6 -> { + sb.append("<h6>") + processChildren() + sb.append("</h6>") } - MarkdownElementTypes.PARA -> { + MarkdownElementTypes.BLOCK_QUOTE -> { + sb.append("<blockquote>") + processChildren() + sb.append("</blockquote>") + } + MarkdownElementTypes.PARAGRAPH -> { sb.append("<p>") processChildren() sb.appendln("</p>") } + MarkdownTokenTypes.CODE -> { + sb.append("<pre><code>") + sb.append(nodeText) + sb.append("</code><pre>") + } + MarkdownTokenTypes.TEXT -> { + sb.append(nodeText) + } else -> { processChildren() } @@ -139,9 +124,16 @@ public fun MarkdownTree.toHtml(): String { return sb.toString() } +fun parseMarkdown(markdown: String): MarkdownNode { + if (markdown.isEmpty()) + return MarkdownNode(LeafASTNode(MarkdownElementTypes.MARKDOWN_FILE, 0, 0), markdown) + return MarkdownNode(MarkdownParser(CommonMarkMarkerProcessor()).buildMarkdownTreeFromString(markdown), markdown) +} fun markdownToHtml(markdown: String): String { - val markdownTree = MarkdownProcessor.parse(markdown) + + val tree = MarkdownParser(CommonMarkMarkerProcessor()).buildMarkdownTreeFromString(markdown) + val markdownTree = MarkdownNode(tree, markdown) val ast = markdownTree.toTestString() return markdownTree.toHtml() } diff --git a/src/Markdown/MarkdownTokenType.kt b/src/Markdown/MarkdownTokenType.kt deleted file mode 100644 index 293228c3..00000000 --- a/src/Markdown/MarkdownTokenType.kt +++ /dev/null @@ -1,6 +0,0 @@ -package org.jetbrains.dokka.Markdown - -import com.intellij.psi.tree.IElementType - -public class MarkdownTokenType(debugName: String) : IElementType(debugName, null) { -}
\ No newline at end of file diff --git a/src/Markdown/markdown.bnf b/src/Markdown/markdown.bnf deleted file mode 100644 index 2e4979ec..00000000 --- a/src/Markdown/markdown.bnf +++ /dev/null @@ -1,98 +0,0 @@ -{ - psiPackage = 'org.jetbrains.markdown' - psiImplPackage = 'org.jetbrains.markdown.impl' - - parserClass="org.jetbrains.markdown.MarkdownParser" - parserUtilClass="org.jetbrains.dokka.Markdown.GeneratedParserUtilBase" - elementTypeHolderClass = 'org.jetbrains.markdown.MarkdownElementTypes' - - tokenTypeClass = 'org.jetbrains.dokka.Markdown.MarkdownTokenType' - - generatePsi = false - - tokens=[ - BOM - EOL - EOP - Number - SPECIAL - Word - ] - -} - -Document ::= BOM? Whitespace* AnonymousSection? (Whitespace* NamedSection)* - -AnonymousSection ::= SectionBody -NamedSection ::= SectionHeader SectionBody -private SectionHeader ::= '$' SectionName OptionalSpace ':' OptionalSpace -SectionName ::= SectionNameStart | '{' OptionalSpace SectionNameStart (Space+ Word)* OptionalSpace '}' -private SectionNameStart ::= '$'? Word -SectionBody::= Block* - -Unused ::= Special -BlankLine ::= OptionalSpace EOL - -Whitespace ::= Space | EOL | EOP -private OptionalSpace ::= Space* -private NonindentSpace ::= (" " | " " | " ")? - -EndLine ::= TerminalEndline | NormalEndline -private NormalEndline ::= EOL !BlankLine -private TerminalEndline ::= OptionalSpace <<eof>> -private Indent ::= "\t" | " " - -// ---- BLOCKS ---- -Block ::= BlankLine* ( - OrderedList - | BulletList - | HorizontalRule - | Directive - | Para - ) - -Para ::= Inlines (EOP | Space* <<eof>>)? - -HorizontalRule ::= NonindentSpace - ( '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')* - | '-' OptionalSpace '-' OptionalSpace '-' (OptionalSpace '-')* - | '_' OptionalSpace '_' OptionalSpace '_' (OptionalSpace '_')*) - OptionalSpace EOL BlankLine+ - -Directive ::= '{' DirectiveName Space+ DirectiveParams '}' -DirectiveName ::= Word -DirectiveParams ::= PlainText - -Bullet ::= !HorizontalRule NonindentSpace ('+' | '*' | '-') Space+ -Enumerator ::= NonindentSpace Number '.' Space+ - -BulletList ::= &Bullet List -OrderedList ::= &Enumerator List - -private List ::= (ListItem BlankLine*)+ -ListItem ::= (Bullet | Enumerator) ListBlock ( ListContinuationBlock )* - -ListBlock ::= !BlankLine Inlines ( ListBlockLine )* -ListBlockLine ::= !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Inlines - -ListContinuationBlock ::= BlankLine* (Indent ListBlock)+ - - -// ---- INLINES ---- -private Inlines ::= (!EndLine Inline | EndLine &Inline )+ EndLine? -Inline ::= Strong | Emph | Code | Link | PlainText -PlainText ::= (Word | Number | Space | ':')+ - -Emph ::= EmphStar | EmphUnderscore -private EmphStar ::= '*' !Whitespace (!'*' Inline)+ '*' -private EmphUnderscore ::= '_' !Whitespace (!'_' Inline)+ '_' - -Code ::= '`' !Whitespace (!'`' Inline)+ '`' - -Strong ::= StrongStar | StrongUnderscore -StrongStar ::= '**' !Whitespace (!'**' Inline)+ '**' -StrongUnderscore ::= '__' !Whitespace (!'__' Inline)+ '__' - -Link ::= '[' Target ']' ('(' Href ')')? -Target ::= Word+ -Href ::= (Word | Number | ':' | '/')+
\ No newline at end of file diff --git a/src/Markdown/markdown.leg b/src/Markdown/markdown.leg deleted file mode 100644 index ea8bc522..00000000 --- a/src/Markdown/markdown.leg +++ /dev/null @@ -1,781 +0,0 @@ -%{ -/********************************************************************** - - markdown_parser.leg - markdown parser in C using a PEG grammar. - (c) 2008 John MacFarlane (jgm at berkeley dot edu). - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License or the MIT - license. See LICENSE for details. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - ***********************************************************************/ - -#include <stdbool.h> -#include <assert.h> -#include "markdown_peg.h" -#include "utility_functions.h" - - - -/********************************************************************** - - Definitions for leg parser generator. - YY_INPUT is the function the parser calls to get new input. - We take all new input from (static) charbuf. - - ***********************************************************************/ - - - -# define YYSTYPE element * -#ifdef __DEBUG__ -# define YY_DEBUG 1 -#endif - -#define YY_INPUT(buf, result, max_size) \ -{ \ - int yyc; \ - if (charbuf && *charbuf != '\0') { \ - yyc= *charbuf++; \ - } else { \ - yyc= EOF; \ - } \ - result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ -} - -#define YY_RULE(T) T - - -/********************************************************************** - - PEG grammar and parser actions for markdown syntax. - - ***********************************************************************/ - -%} - -Doc = BOM? a:StartList ( Block { a = cons($$, a); } )* - { parse_result = reverse(a); } - -Block = BlankLine* - ( BlockQuote - | Verbatim - | Note - | Reference - | HorizontalRule - | Heading - | OrderedList - | BulletList - | HtmlBlock - | StyleBlock - | Para - | Plain ) - -Para = NonindentSpace a:Inlines BlankLine+ - { $$ = a; $$->key = PARA; } - -Plain = a:Inlines - { $$ = a; $$->key = PLAIN; } - -AtxInline = !Newline !(Sp '#'* Sp Newline) Inline - -AtxStart = < ( "######" | "#####" | "####" | "###" | "##" | "#" ) > - { $$ = mk_element(H1 + (strlen(yytext) - 1)); } - -AtxHeading = s:AtxStart Sp a:StartList ( AtxInline { a = cons($$, a); } )+ (Sp '#'* Sp)? Newline - { $$ = mk_list(s->key, a); - free(s); } - -SetextHeading = SetextHeading1 | SetextHeading2 - -SetextBottom1 = '='+ Newline - -SetextBottom2 = '-'+ Newline - -SetextHeading1 = &(RawLine SetextBottom1) - a:StartList ( !Endline Inline { a = cons($$, a); } )+ Sp Newline - SetextBottom1 { $$ = mk_list(H1, a); } - -SetextHeading2 = &(RawLine SetextBottom2) - a:StartList ( !Endline Inline { a = cons($$, a); } )+ Sp Newline - SetextBottom2 { $$ = mk_list(H2, a); } - -Heading = SetextHeading | AtxHeading - -BlockQuote = a:BlockQuoteRaw - { $$ = mk_element(BLOCKQUOTE); - $$->children = a; - } - -BlockQuoteRaw = a:StartList - (( '>' ' '? Line { a = cons($$, a); } ) - ( !'>' !BlankLine Line { a = cons($$, a); } )* - ( BlankLine { a = cons(mk_str("\n"), a); } )* - )+ - { $$ = mk_str_from_list(a, true); - $$->key = RAW; - } - -NonblankIndentedLine = !BlankLine IndentedLine - -VerbatimChunk = a:StartList - ( BlankLine { a = cons(mk_str("\n"), a); } )* - ( NonblankIndentedLine { a = cons($$, a); } )+ - { $$ = mk_str_from_list(a, false); } - -Verbatim = a:StartList ( VerbatimChunk { a = cons($$, a); } )+ - { $$ = mk_str_from_list(a, false); - $$->key = VERBATIM; } - -HorizontalRule = NonindentSpace - ( '*' Sp '*' Sp '*' (Sp '*')* - | '-' Sp '-' Sp '-' (Sp '-')* - | '_' Sp '_' Sp '_' (Sp '_')*) - Sp Newline BlankLine+ - { $$ = mk_element(HRULE); } - -Bullet = !HorizontalRule NonindentSpace ('+' | '*' | '-') Spacechar+ - -BulletList = &Bullet (ListTight | ListLoose) - { $$->key = BULLETLIST; } - -ListTight = a:StartList - ( ListItemTight { a = cons($$, a); } )+ - BlankLine* !(Bullet | Enumerator) - { $$ = mk_list(LIST, a); } - -ListLoose = a:StartList - ( b:ListItem BlankLine* - { element *li; - li = b->children; - li->contents.str = realloc(li->contents.str, strlen(li->contents.str) + 3); - strcat(li->contents.str, "\n\n"); /* In loose list, \n\n added to end of each element */ - a = cons(b, a); - } )+ - { $$ = mk_list(LIST, a); } - -ListItem = ( Bullet | Enumerator ) - a:StartList - ListBlock { a = cons($$, a); } - ( ListContinuationBlock { a = cons($$, a); } )* - { element *raw; - raw = mk_str_from_list(a, false); - raw->key = RAW; - $$ = mk_element(LISTITEM); - $$->children = raw; - } - -ListItemTight = - ( Bullet | Enumerator ) - a:StartList - ListBlock { a = cons($$, a); } - ( !BlankLine - ListContinuationBlock { a = cons($$, a); } )* - !ListContinuationBlock - { element *raw; - raw = mk_str_from_list(a, false); - raw->key = RAW; - $$ = mk_element(LISTITEM); - $$->children = raw; - } - -ListBlock = a:StartList - !BlankLine Line { a = cons($$, a); } - ( ListBlockLine { a = cons($$, a); } )* - { $$ = mk_str_from_list(a, false); } - -ListContinuationBlock = a:StartList - ( < BlankLine* > - { if (strlen(yytext) == 0) - a = cons(mk_str("\001"), a); /* block separator */ - else - a = cons(mk_str(yytext), a); } ) - ( Indent ListBlock { a = cons($$, a); } )+ - { $$ = mk_str_from_list(a, false); } - -Enumerator = NonindentSpace [0-9]+ '.' Spacechar+ - -OrderedList = &Enumerator (ListTight | ListLoose) - { $$->key = ORDEREDLIST; } - -ListBlockLine = !BlankLine - !( Indent? (Bullet | Enumerator) ) - !HorizontalRule - OptionallyIndentedLine - -# Parsers for different kinds of block-level HTML content. -# This is repetitive due to constraints of PEG grammar. - -HtmlBlockOpenAddress = '<' Spnl ("address" | "ADDRESS") Spnl HtmlAttribute* '>' -HtmlBlockCloseAddress = '<' Spnl '/' ("address" | "ADDRESS") Spnl '>' -HtmlBlockAddress = HtmlBlockOpenAddress (HtmlBlockAddress | !HtmlBlockCloseAddress .)* HtmlBlockCloseAddress - -HtmlBlockOpenBlockquote = '<' Spnl ("blockquote" | "BLOCKQUOTE") Spnl HtmlAttribute* '>' -HtmlBlockCloseBlockquote = '<' Spnl '/' ("blockquote" | "BLOCKQUOTE") Spnl '>' -HtmlBlockBlockquote = HtmlBlockOpenBlockquote (HtmlBlockBlockquote | !HtmlBlockCloseBlockquote .)* HtmlBlockCloseBlockquote - -HtmlBlockOpenCenter = '<' Spnl ("center" | "CENTER") Spnl HtmlAttribute* '>' -HtmlBlockCloseCenter = '<' Spnl '/' ("center" | "CENTER") Spnl '>' -HtmlBlockCenter = HtmlBlockOpenCenter (HtmlBlockCenter | !HtmlBlockCloseCenter .)* HtmlBlockCloseCenter - -HtmlBlockOpenDir = '<' Spnl ("dir" | "DIR") Spnl HtmlAttribute* '>' -HtmlBlockCloseDir = '<' Spnl '/' ("dir" | "DIR") Spnl '>' -HtmlBlockDir = HtmlBlockOpenDir (HtmlBlockDir | !HtmlBlockCloseDir .)* HtmlBlockCloseDir - -HtmlBlockOpenDiv = '<' Spnl ("div" | "DIV") Spnl HtmlAttribute* '>' -HtmlBlockCloseDiv = '<' Spnl '/' ("div" | "DIV") Spnl '>' -HtmlBlockDiv = HtmlBlockOpenDiv (HtmlBlockDiv | !HtmlBlockCloseDiv .)* HtmlBlockCloseDiv - -HtmlBlockOpenDl = '<' Spnl ("dl" | "DL") Spnl HtmlAttribute* '>' -HtmlBlockCloseDl = '<' Spnl '/' ("dl" | "DL") Spnl '>' -HtmlBlockDl = HtmlBlockOpenDl (HtmlBlockDl | !HtmlBlockCloseDl .)* HtmlBlockCloseDl - -HtmlBlockOpenFieldset = '<' Spnl ("fieldset" | "FIELDSET") Spnl HtmlAttribute* '>' -HtmlBlockCloseFieldset = '<' Spnl '/' ("fieldset" | "FIELDSET") Spnl '>' -HtmlBlockFieldset = HtmlBlockOpenFieldset (HtmlBlockFieldset | !HtmlBlockCloseFieldset .)* HtmlBlockCloseFieldset - -HtmlBlockOpenForm = '<' Spnl ("form" | "FORM") Spnl HtmlAttribute* '>' -HtmlBlockCloseForm = '<' Spnl '/' ("form" | "FORM") Spnl '>' -HtmlBlockForm = HtmlBlockOpenForm (HtmlBlockForm | !HtmlBlockCloseForm .)* HtmlBlockCloseForm - -HtmlBlockOpenH1 = '<' Spnl ("h1" | "H1") Spnl HtmlAttribute* '>' -HtmlBlockCloseH1 = '<' Spnl '/' ("h1" | "H1") Spnl '>' -HtmlBlockH1 = HtmlBlockOpenH1 (HtmlBlockH1 | !HtmlBlockCloseH1 .)* HtmlBlockCloseH1 - -HtmlBlockOpenH2 = '<' Spnl ("h2" | "H2") Spnl HtmlAttribute* '>' -HtmlBlockCloseH2 = '<' Spnl '/' ("h2" | "H2") Spnl '>' -HtmlBlockH2 = HtmlBlockOpenH2 (HtmlBlockH2 | !HtmlBlockCloseH2 .)* HtmlBlockCloseH2 - -HtmlBlockOpenH3 = '<' Spnl ("h3" | "H3") Spnl HtmlAttribute* '>' -HtmlBlockCloseH3 = '<' Spnl '/' ("h3" | "H3") Spnl '>' -HtmlBlockH3 = HtmlBlockOpenH3 (HtmlBlockH3 | !HtmlBlockCloseH3 .)* HtmlBlockCloseH3 - -HtmlBlockOpenH4 = '<' Spnl ("h4" | "H4") Spnl HtmlAttribute* '>' -HtmlBlockCloseH4 = '<' Spnl '/' ("h4" | "H4") Spnl '>' -HtmlBlockH4 = HtmlBlockOpenH4 (HtmlBlockH4 | !HtmlBlockCloseH4 .)* HtmlBlockCloseH4 - -HtmlBlockOpenH5 = '<' Spnl ("h5" | "H5") Spnl HtmlAttribute* '>' -HtmlBlockCloseH5 = '<' Spnl '/' ("h5" | "H5") Spnl '>' -HtmlBlockH5 = HtmlBlockOpenH5 (HtmlBlockH5 | !HtmlBlockCloseH5 .)* HtmlBlockCloseH5 - -HtmlBlockOpenH6 = '<' Spnl ("h6" | "H6") Spnl HtmlAttribute* '>' -HtmlBlockCloseH6 = '<' Spnl '/' ("h6" | "H6") Spnl '>' -HtmlBlockH6 = HtmlBlockOpenH6 (HtmlBlockH6 | !HtmlBlockCloseH6 .)* HtmlBlockCloseH6 - -HtmlBlockOpenMenu = '<' Spnl ("menu" | "MENU") Spnl HtmlAttribute* '>' -HtmlBlockCloseMenu = '<' Spnl '/' ("menu" | "MENU") Spnl '>' -HtmlBlockMenu = HtmlBlockOpenMenu (HtmlBlockMenu | !HtmlBlockCloseMenu .)* HtmlBlockCloseMenu - -HtmlBlockOpenNoframes = '<' Spnl ("noframes" | "NOFRAMES") Spnl HtmlAttribute* '>' -HtmlBlockCloseNoframes = '<' Spnl '/' ("noframes" | "NOFRAMES") Spnl '>' -HtmlBlockNoframes = HtmlBlockOpenNoframes (HtmlBlockNoframes | !HtmlBlockCloseNoframes .)* HtmlBlockCloseNoframes - -HtmlBlockOpenNoscript = '<' Spnl ("noscript" | "NOSCRIPT") Spnl HtmlAttribute* '>' -HtmlBlockCloseNoscript = '<' Spnl '/' ("noscript" | "NOSCRIPT") Spnl '>' -HtmlBlockNoscript = HtmlBlockOpenNoscript (HtmlBlockNoscript | !HtmlBlockCloseNoscript .)* HtmlBlockCloseNoscript - -HtmlBlockOpenOl = '<' Spnl ("ol" | "OL") Spnl HtmlAttribute* '>' -HtmlBlockCloseOl = '<' Spnl '/' ("ol" | "OL") Spnl '>' -HtmlBlockOl = HtmlBlockOpenOl (HtmlBlockOl | !HtmlBlockCloseOl .)* HtmlBlockCloseOl - -HtmlBlockOpenP = '<' Spnl ("p" | "P") Spnl HtmlAttribute* '>' -HtmlBlockCloseP = '<' Spnl '/' ("p" | "P") Spnl '>' -HtmlBlockP = HtmlBlockOpenP (HtmlBlockP | !HtmlBlockCloseP .)* HtmlBlockCloseP - -HtmlBlockOpenPre = '<' Spnl ("pre" | "PRE") Spnl HtmlAttribute* '>' -HtmlBlockClosePre = '<' Spnl '/' ("pre" | "PRE") Spnl '>' -HtmlBlockPre = HtmlBlockOpenPre (HtmlBlockPre | !HtmlBlockClosePre .)* HtmlBlockClosePre - -HtmlBlockOpenTable = '<' Spnl ("table" | "TABLE") Spnl HtmlAttribute* '>' -HtmlBlockCloseTable = '<' Spnl '/' ("table" | "TABLE") Spnl '>' -HtmlBlockTable = HtmlBlockOpenTable (HtmlBlockTable | !HtmlBlockCloseTable .)* HtmlBlockCloseTable - -HtmlBlockOpenUl = '<' Spnl ("ul" | "UL") Spnl HtmlAttribute* '>' -HtmlBlockCloseUl = '<' Spnl '/' ("ul" | "UL") Spnl '>' -HtmlBlockUl = HtmlBlockOpenUl (HtmlBlockUl | !HtmlBlockCloseUl .)* HtmlBlockCloseUl - -HtmlBlockOpenDd = '<' Spnl ("dd" | "DD") Spnl HtmlAttribute* '>' -HtmlBlockCloseDd = '<' Spnl '/' ("dd" | "DD") Spnl '>' -HtmlBlockDd = HtmlBlockOpenDd (HtmlBlockDd | !HtmlBlockCloseDd .)* HtmlBlockCloseDd - -HtmlBlockOpenDt = '<' Spnl ("dt" | "DT") Spnl HtmlAttribute* '>' -HtmlBlockCloseDt = '<' Spnl '/' ("dt" | "DT") Spnl '>' -HtmlBlockDt = HtmlBlockOpenDt (HtmlBlockDt | !HtmlBlockCloseDt .)* HtmlBlockCloseDt - -HtmlBlockOpenFrameset = '<' Spnl ("frameset" | "FRAMESET") Spnl HtmlAttribute* '>' -HtmlBlockCloseFrameset = '<' Spnl '/' ("frameset" | "FRAMESET") Spnl '>' -HtmlBlockFrameset = HtmlBlockOpenFrameset (HtmlBlockFrameset | !HtmlBlockCloseFrameset .)* HtmlBlockCloseFrameset - -HtmlBlockOpenLi = '<' Spnl ("li" | "LI") Spnl HtmlAttribute* '>' -HtmlBlockCloseLi = '<' Spnl '/' ("li" | "LI") Spnl '>' -HtmlBlockLi = HtmlBlockOpenLi (HtmlBlockLi | !HtmlBlockCloseLi .)* HtmlBlockCloseLi - -HtmlBlockOpenTbody = '<' Spnl ("tbody" | "TBODY") Spnl HtmlAttribute* '>' -HtmlBlockCloseTbody = '<' Spnl '/' ("tbody" | "TBODY") Spnl '>' -HtmlBlockTbody = HtmlBlockOpenTbody (HtmlBlockTbody | !HtmlBlockCloseTbody .)* HtmlBlockCloseTbody - -HtmlBlockOpenTd = '<' Spnl ("td" | "TD") Spnl HtmlAttribute* '>' -HtmlBlockCloseTd = '<' Spnl '/' ("td" | "TD") Spnl '>' -HtmlBlockTd = HtmlBlockOpenTd (HtmlBlockTd | !HtmlBlockCloseTd .)* HtmlBlockCloseTd - -HtmlBlockOpenTfoot = '<' Spnl ("tfoot" | "TFOOT") Spnl HtmlAttribute* '>' -HtmlBlockCloseTfoot = '<' Spnl '/' ("tfoot" | "TFOOT") Spnl '>' -HtmlBlockTfoot = HtmlBlockOpenTfoot (HtmlBlockTfoot | !HtmlBlockCloseTfoot .)* HtmlBlockCloseTfoot - -HtmlBlockOpenTh = '<' Spnl ("th" | "TH") Spnl HtmlAttribute* '>' -HtmlBlockCloseTh = '<' Spnl '/' ("th" | "TH") Spnl '>' -HtmlBlockTh = HtmlBlockOpenTh (HtmlBlockTh | !HtmlBlockCloseTh .)* HtmlBlockCloseTh - -HtmlBlockOpenThead = '<' Spnl ("thead" | "THEAD") Spnl HtmlAttribute* '>' -HtmlBlockCloseThead = '<' Spnl '/' ("thead" | "THEAD") Spnl '>' -HtmlBlockThead = HtmlBlockOpenThead (HtmlBlockThead | !HtmlBlockCloseThead .)* HtmlBlockCloseThead - -HtmlBlockOpenTr = '<' Spnl ("tr" | "TR") Spnl HtmlAttribute* '>' -HtmlBlockCloseTr = '<' Spnl '/' ("tr" | "TR") Spnl '>' -HtmlBlockTr = HtmlBlockOpenTr (HtmlBlockTr | !HtmlBlockCloseTr .)* HtmlBlockCloseTr - -HtmlBlockOpenScript = '<' Spnl ("script" | "SCRIPT") Spnl HtmlAttribute* '>' -HtmlBlockCloseScript = '<' Spnl '/' ("script" | "SCRIPT") Spnl '>' -HtmlBlockScript = HtmlBlockOpenScript (!HtmlBlockCloseScript .)* HtmlBlockCloseScript - -HtmlBlockOpenHead = '<' Spnl ("head" | "HEAD") Spnl HtmlAttribute* '>' -HtmlBlockCloseHead = '<' Spnl '/' ("head" | "HEAD") Spnl '>' -HtmlBlockHead = HtmlBlockOpenHead (!HtmlBlockCloseHead .)* HtmlBlockCloseHead - -HtmlBlockInTags = HtmlBlockAddress - | HtmlBlockBlockquote - | HtmlBlockCenter - | HtmlBlockDir - | HtmlBlockDiv - | HtmlBlockDl - | HtmlBlockFieldset - | HtmlBlockForm - | HtmlBlockH1 - | HtmlBlockH2 - | HtmlBlockH3 - | HtmlBlockH4 - | HtmlBlockH5 - | HtmlBlockH6 - | HtmlBlockMenu - | HtmlBlockNoframes - | HtmlBlockNoscript - | HtmlBlockOl - | HtmlBlockP - | HtmlBlockPre - | HtmlBlockTable - | HtmlBlockUl - | HtmlBlockDd - | HtmlBlockDt - | HtmlBlockFrameset - | HtmlBlockLi - | HtmlBlockTbody - | HtmlBlockTd - | HtmlBlockTfoot - | HtmlBlockTh - | HtmlBlockThead - | HtmlBlockTr - | HtmlBlockScript - | HtmlBlockHead - -HtmlBlock = < ( HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing ) > - BlankLine+ - { if (extension(EXT_FILTER_HTML)) { - $$ = mk_list(LIST, NULL); - } else { - $$ = mk_str(yytext); - $$->key = HTMLBLOCK; - } - } - -HtmlBlockSelfClosing = '<' Spnl HtmlBlockType Spnl HtmlAttribute* '/' Spnl '>' - -HtmlBlockType = "address" | "blockquote" | "center" | "dir" | "div" | "dl" | "fieldset" | "form" | "h1" | "h2" | "h3" | - "h4" | "h5" | "h6" | "hr" | "isindex" | "menu" | "noframes" | "noscript" | "ol" | "p" | "pre" | "table" | - "ul" | "dd" | "dt" | "frameset" | "li" | "tbody" | "td" | "tfoot" | "th" | "thead" | "tr" | "script" | - "ADDRESS" | "BLOCKQUOTE" | "CENTER" | "DIR" | "DIV" | "DL" | "FIELDSET" | "FORM" | "H1" | "H2" | "H3" | - "H4" | "H5" | "H6" | "HR" | "ISINDEX" | "MENU" | "NOFRAMES" | "NOSCRIPT" | "OL" | "P" | "PRE" | "TABLE" | - "UL" | "DD" | "DT" | "FRAMESET" | "LI" | "TBODY" | "TD" | "TFOOT" | "TH" | "THEAD" | "TR" | "SCRIPT" - -StyleOpen = '<' Spnl ("style" | "STYLE") Spnl HtmlAttribute* '>' -StyleClose = '<' Spnl '/' ("style" | "STYLE") Spnl '>' -InStyleTags = StyleOpen (!StyleClose .)* StyleClose -StyleBlock = < InStyleTags > - BlankLine* - { if (extension(EXT_FILTER_STYLES)) { - $$ = mk_list(LIST, NULL); - } else { - $$ = mk_str(yytext); - $$->key = HTMLBLOCK; - } - } - -Inlines = a:StartList ( !Endline Inline { a = cons($$, a); } - | c:Endline &Inline { a = cons(c, a); } )+ Endline? - { $$ = mk_list(LIST, a); } - -Inline = Str - | Endline - | UlOrStarLine - | Space - | Strong - | Emph - | Strike - | Image - | Link - | NoteReference - | InlineNote - | Code - | RawHtml - | Entity - | EscapedChar - | Smart - | Symbol - -Space = Spacechar+ - { $$ = mk_str(" "); - $$->key = SPACE; } - -Str = a:StartList < NormalChar+ > { a = cons(mk_str(yytext), a); } - ( StrChunk { a = cons($$, a); } )* - { if (a->next == NULL) { $$ = a; } else { $$ = mk_list(LIST, a); } } - -StrChunk = < (NormalChar | '_'+ &Alphanumeric)+ > { $$ = mk_str(yytext); } | - AposChunk - -AposChunk = &{ extension(EXT_SMART) } '\'' &Alphanumeric - { $$ = mk_element(APOSTROPHE); } - -EscapedChar = '\\' !Newline < [-\\`|*_{}[\]()#+.!><] > - { $$ = mk_str(yytext); } - -Entity = ( HexEntity | DecEntity | CharEntity ) - { $$ = mk_str(yytext); $$->key = HTML; } - -Endline = LineBreak | TerminalEndline | NormalEndline - -NormalEndline = Sp Newline !BlankLine !'>' !AtxStart - !(Line ('='+ | '-'+) Newline) - { $$ = mk_str("\n"); - $$->key = SPACE; } - -TerminalEndline = Sp Newline Eof - { $$ = NULL; } - -LineBreak = " " NormalEndline - { $$ = mk_element(LINEBREAK); } - -Symbol = < SpecialChar > - { $$ = mk_str(yytext); } - -# This keeps the parser from getting bogged down on long strings of '*' or '_', -# or strings of '*' or '_' with space on each side: -UlOrStarLine = (UlLine | StarLine) { $$ = mk_str(yytext); } -StarLine = < "****" '*'* > | < Spacechar '*'+ &Spacechar > -UlLine = < "____" '_'* > | < Spacechar '_'+ &Spacechar > - -Emph = EmphStar | EmphUl - -Whitespace = Spacechar | Newline - -EmphStar = '*' !Whitespace - a:StartList - ( !'*' b:Inline { a = cons(b, a); } - | b:StrongStar { a = cons(b, a); } - )+ - '*' - { $$ = mk_list(EMPH, a); } - -EmphUl = '_' !Whitespace - a:StartList - ( !'_' b:Inline { a = cons(b, a); } - | b:StrongUl { a = cons(b, a); } - )+ - '_' - { $$ = mk_list(EMPH, a); } - -Strong = StrongStar | StrongUl - -StrongStar = "**" !Whitespace - a:StartList - ( !"**" b:Inline { a = cons(b, a); })+ - "**" - { $$ = mk_list(STRONG, a); } - -StrongUl = "__" !Whitespace - a:StartList - ( !"__" b:Inline { a = cons(b, a); })+ - "__" - { $$ = mk_list(STRONG, a); } - -Strike = &{ extension(EXT_STRIKE) } - "~~" !Whitespace - a:StartList - ( !"~~" b:Inline { a = cons(b, a); })+ - "~~" - { $$ = mk_list(STRIKE, a); } - -Image = '!' ( ExplicitLink | ReferenceLink ) - { if ($$->key == LINK) { - $$->key = IMAGE; - } else { - element *result; - result = $$; - $$->children = cons(mk_str("!"), result->children); - } } - -Link = ExplicitLink | ReferenceLink | AutoLink - -ReferenceLink = ReferenceLinkDouble | ReferenceLinkSingle - -ReferenceLinkDouble = a:Label < Spnl > !"[]" b:Label - { link match; - if (find_reference(&match, b->children)) { - $$ = mk_link(a->children, match.url, match.title); - free(a); - free_element_list(b); - } else { - element *result; - result = mk_element(LIST); - result->children = cons(mk_str("["), cons(a, cons(mk_str("]"), cons(mk_str(yytext), - cons(mk_str("["), cons(b, mk_str("]"))))))); - $$ = result; - } - } - -ReferenceLinkSingle = a:Label < (Spnl "[]")? > - { link match; - if (find_reference(&match, a->children)) { - $$ = mk_link(a->children, match.url, match.title); - free(a); - } - else { - element *result; - result = mk_element(LIST); - result->children = cons(mk_str("["), cons(a, cons(mk_str("]"), mk_str(yytext)))); - $$ = result; - } - } - -ExplicitLink = l:Label '(' Sp s:Source Spnl t:Title Sp ')' - { $$ = mk_link(l->children, s->contents.str, t->contents.str); - free_element(s); - free_element(t); - free(l); } - -Source = ( '<' < SourceContents > '>' | < SourceContents > ) - { $$ = mk_str(yytext); } - -SourceContents = ( ( !'(' !')' !'>' Nonspacechar )+ | '(' SourceContents ')')* - -Title = ( TitleSingle | TitleDouble | < "" > ) - { $$ = mk_str(yytext); } - -TitleSingle = '\'' < ( !( '\'' Sp ( ')' | Newline ) ) . )* > '\'' - -TitleDouble = '"' < ( !( '"' Sp ( ')' | Newline ) ) . )* > '"' - -AutoLink = AutoLinkUrl | AutoLinkEmail - -AutoLinkUrl = '<' < [A-Za-z]+ "://" ( !Newline !'>' . )+ > '>' - { $$ = mk_link(mk_str(yytext), yytext, ""); } - -AutoLinkEmail = '<' ( "mailto:" )? < [-A-Za-z0-9+_./!%~$]+ '@' ( !Newline !'>' . )+ > '>' - { char *mailto = malloc(strlen(yytext) + 8); - sprintf(mailto, "mailto:%s", yytext); - $$ = mk_link(mk_str(yytext), mailto, ""); - free(mailto); - } - -Reference = NonindentSpace !"[]" l:Label ':' Spnl s:RefSrc t:RefTitle BlankLine+ - { $$ = mk_link(l->children, s->contents.str, t->contents.str); - free_element(s); - free_element(t); - free(l); - $$->key = REFERENCE; } - -Label = '[' ( !'^' &{ extension(EXT_NOTES) } | &. &{ !extension(EXT_NOTES) } ) - a:StartList - ( !']' Inline { a = cons($$, a); } )* - ']' - { $$ = mk_list(LIST, a); } - -RefSrc = < Nonspacechar+ > - { $$ = mk_str(yytext); - $$->key = HTML; } - -RefTitle = ( RefTitleSingle | RefTitleDouble | RefTitleParens | EmptyTitle ) - { $$ = mk_str(yytext); } - -EmptyTitle = < "" > - -RefTitleSingle = Spnl '\'' < ( !( '\'' Sp Newline | Newline ) . )* > '\'' - -RefTitleDouble = Spnl '"' < ( !('"' Sp Newline | Newline) . )* > '"' - -RefTitleParens = Spnl '(' < ( !(')' Sp Newline | Newline) . )* > ')' - -References = a:StartList - ( b:Reference { a = cons(b, a); } | SkipBlock )* - { references = reverse(a); } - -Ticks1 = "`" !'`' -Ticks2 = "``" !'`' -Ticks3 = "```" !'`' -Ticks4 = "````" !'`' -Ticks5 = "`````" !'`' - -Code = ( Ticks1 Sp < ( ( !'`' Nonspacechar )+ | !Ticks1 '`'+ | !( Sp Ticks1 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks1 - | Ticks2 Sp < ( ( !'`' Nonspacechar )+ | !Ticks2 '`'+ | !( Sp Ticks2 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks2 - | Ticks3 Sp < ( ( !'`' Nonspacechar )+ | !Ticks3 '`'+ | !( Sp Ticks3 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks3 - | Ticks4 Sp < ( ( !'`' Nonspacechar )+ | !Ticks4 '`'+ | !( Sp Ticks4 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks4 - | Ticks5 Sp < ( ( !'`' Nonspacechar )+ | !Ticks5 '`'+ | !( Sp Ticks5 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks5 - ) - { $$ = mk_str(yytext); $$->key = CODE; } - -RawHtml = < (HtmlComment | HtmlBlockScript | HtmlTag) > - { if (extension(EXT_FILTER_HTML)) { - $$ = mk_list(LIST, NULL); - } else { - $$ = mk_str(yytext); - $$->key = HTML; - } - } - -BlankLine = Sp Newline - -Quoted = '"' (!'"' .)* '"' | '\'' (!'\'' .)* '\'' -HtmlAttribute = (AlphanumericAscii | '-')+ Spnl ('=' Spnl (Quoted | (!'>' Nonspacechar)+))? Spnl -HtmlComment = "<!--" (!"-->" .)* "-->" -HtmlTag = '<' Spnl '/'? AlphanumericAscii+ Spnl HtmlAttribute* '/'? Spnl '>' -Eof = !. -Spacechar = ' ' | '\t' -Nonspacechar = !Spacechar !Newline . -Newline = '\n' | '\r' '\n'? -Sp = Spacechar* -Spnl = Sp (Newline Sp)? -SpecialChar = '~' | '*' | '_' | '`' | '&' | '[' | ']' | '(' | ')' | '<' | '!' | '#' | '\\' | '\'' | '"' | ExtendedSpecialChar -NormalChar = !( SpecialChar | Spacechar | Newline ) . -Alphanumeric = [0-9A-Za-z] | '\200' | '\201' | '\202' | '\203' | '\204' | '\205' | '\206' | '\207' | '\210' | '\211' | '\212' | '\213' | '\214' | '\215' | '\216' | '\217' | '\220' | '\221' | '\222' | '\223' | '\224' | '\225' | '\226' | '\227' | '\230' | '\231' | '\232' | '\233' | '\234' | '\235' | '\236' | '\237' | '\240' | '\241' | '\242' | '\243' | '\244' | '\245' | '\246' | '\247' | '\250' | '\251' | '\252' | '\253' | '\254' | '\255' | '\256' | '\257' | '\260' | '\261' | '\262' | '\263' | '\264' | '\265' | '\266' | '\267' | '\270' | '\271' | '\272' | '\273' | '\274' | '\275' | '\276' | '\277' | '\300' | '\301' | '\302' | '\303' | '\304' | '\305' | '\306' | '\307' | '\310' | '\311' | '\312' | '\313' | '\314' | '\315' | '\316' | '\317' | '\320' | '\321' | '\322' | '\323' | '\324' | '\325' | '\326' | '\327' | '\330' | '\331' | '\332' | '\333' | '\334' | '\335' | '\336' | '\337' | '\340' | '\341' | '\342' | '\343' | '\344' | '\345' | '\346' | '\347' | '\350' | '\351' | '\352' | '\353' | '\354' | '\355' | '\356' | '\357' | '\360' | '\361' | '\362' | '\363' | '\364' | '\365' | '\366' | '\367' | '\370' | '\371' | '\372' | '\373' | '\374' | '\375' | '\376' | '\377' -AlphanumericAscii = [A-Za-z0-9] -Digit = [0-9] -BOM = "\357\273\277" - -HexEntity = < '&' '#' [Xx] [0-9a-fA-F]+ ';' > -DecEntity = < '&' '#' [0-9]+ > ';' > -CharEntity = < '&' [A-Za-z0-9]+ ';' > - -NonindentSpace = " " | " " | " " | "" -Indent = "\t" | " " -IndentedLine = Indent Line -OptionallyIndentedLine = Indent? Line - -# StartList starts a list data structure that can be added to with cons: -StartList = &. - { $$ = NULL; } - -Line = RawLine - { $$ = mk_str(yytext); } -RawLine = ( < (!'\r' !'\n' .)* Newline > | < .+ > Eof ) - -SkipBlock = HtmlBlock - | ( !'#' !SetextBottom1 !SetextBottom2 !BlankLine RawLine )+ BlankLine* - | BlankLine+ - | RawLine - -# Syntax extensions - -ExtendedSpecialChar = &{ extension(EXT_SMART) } ('.' | '-' | '\'' | '"') - | &{ extension(EXT_NOTES) } ( '^' ) - -Smart = &{ extension(EXT_SMART) } - ( Ellipsis | Dash | SingleQuoted | DoubleQuoted | Apostrophe ) - -Apostrophe = '\'' - { $$ = mk_element(APOSTROPHE); } - -Ellipsis = ("..." | ". . .") - { $$ = mk_element(ELLIPSIS); } - -Dash = EmDash | EnDash - -EnDash = '-' &Digit - { $$ = mk_element(ENDASH); } - -EmDash = ("---" | "--") - { $$ = mk_element(EMDASH); } - -SingleQuoteStart = '\'' !(Spacechar | Newline) - -SingleQuoteEnd = '\'' !Alphanumeric - -SingleQuoted = SingleQuoteStart - a:StartList - ( !SingleQuoteEnd b:Inline { a = cons(b, a); } )+ - SingleQuoteEnd - { $$ = mk_list(SINGLEQUOTED, a); } - -DoubleQuoteStart = '"' - -DoubleQuoteEnd = '"' - -DoubleQuoted = DoubleQuoteStart - a:StartList - ( !DoubleQuoteEnd b:Inline { a = cons(b, a); } )+ - DoubleQuoteEnd - { $$ = mk_list(DOUBLEQUOTED, a); } - -NoteReference = &{ extension(EXT_NOTES) } - ref:RawNoteReference - { element *match; - if (find_note(&match, ref->contents.str)) { - $$ = mk_element(NOTE); - assert(match->children != NULL); - $$->children = match->children; - $$->contents.str = 0; - } else { - char *s; - s = malloc(strlen(ref->contents.str) + 4); - sprintf(s, "[^%s]", ref->contents.str); - $$ = mk_str(s); - free(s); - } - } - -RawNoteReference = "[^" < ( !Newline !']' . )+ > ']' - { $$ = mk_str(yytext); } - -Note = &{ extension(EXT_NOTES) } - NonindentSpace ref:RawNoteReference ':' Sp - a:StartList - ( RawNoteBlock { a = cons($$, a); } ) - ( &Indent RawNoteBlock { a = cons($$, a); } )* - { $$ = mk_list(NOTE, a); - $$->contents.str = strdup(ref->contents.str); - } - -InlineNote = &{ extension(EXT_NOTES) } - "^[" - a:StartList - ( !']' Inline { a = cons($$, a); } )+ - ']' - { $$ = mk_list(NOTE, a); - $$->contents.str = 0; } - -Notes = a:StartList - ( b:Note { a = cons(b, a); } | SkipBlock )* - { notes = reverse(a); } - -RawNoteBlock = a:StartList - ( !BlankLine OptionallyIndentedLine { a = cons($$, a); } )+ - ( < BlankLine* > { a = cons(mk_str(yytext), a); } ) - { $$ = mk_str_from_list(a, true); - $$->key = RAW; - } - -%% - diff --git a/src/main.kt b/src/main.kt index 6136300f..4e627376 100644 --- a/src/main.kt +++ b/src/main.kt @@ -8,7 +8,6 @@ import org.jetbrains.jet.utils.PathUtil import java.io.File import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor import org.jetbrains.jet.lang.resolve.name.FqName -import java.lang.reflect.Constructor class DokkaArguments { Argument(value = "src", description = "Source file or directory (allows many paths separated by the system path separator)") @@ -86,7 +85,7 @@ public fun main(args: Array<String>) { val file = File(include) if (file.exists()) { val text = file.readText() - val tree = MarkdownProcessor.parse(text) + val tree = parseMarkdown(text) val content = buildContent(tree, session.getPackageFragment(FqName.ROOT)) moduleContent.children.addAll(content.children) } else { |