From c9ca0d86e810ac7240b249b145810f6862a58e93 Mon Sep 17 00:00:00 2001 From: Ilya Ryzhenkov Date: Mon, 15 Dec 2014 20:54:16 +0300 Subject: Migrate to non-PsiBuilder fully-featured markdown parser. --- src/Markdown/GeneratedParserUtilBase.java | 1031 ----------------------------- src/Markdown/MarkdownProcessor.kt | 156 +++-- src/Markdown/MarkdownTokenType.kt | 6 - src/Markdown/markdown.bnf | 98 --- src/Markdown/markdown.leg | 781 ---------------------- 5 files changed, 74 insertions(+), 1998 deletions(-) delete mode 100644 src/Markdown/GeneratedParserUtilBase.java delete mode 100644 src/Markdown/MarkdownTokenType.kt delete mode 100644 src/Markdown/markdown.bnf delete mode 100644 src/Markdown/markdown.leg (limited to 'src/Markdown') diff --git a/src/Markdown/GeneratedParserUtilBase.java b/src/Markdown/GeneratedParserUtilBase.java deleted file mode 100644 index 9dd999b5..00000000 --- a/src/Markdown/GeneratedParserUtilBase.java +++ /dev/null @@ -1,1031 +0,0 @@ -/* - * Copyright 2011-2014 Gregory Shrago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.dokka.Markdown; - -import com.intellij.lang.*; -import com.intellij.lang.impl.PsiBuilderAdapter; -import com.intellij.lang.impl.PsiBuilderImpl; -import com.intellij.lexer.Lexer; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.Comparing; -import com.intellij.openapi.util.Key; -import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.text.StringHash; -import com.intellij.openapi.util.text.StringUtil; -import com.intellij.psi.PsiFile; -import com.intellij.psi.PsiReference; -import com.intellij.psi.TokenType; -import com.intellij.psi.impl.source.resolve.FileContextUtil; -import com.intellij.psi.impl.source.tree.CompositePsiElement; -import com.intellij.psi.tree.ICompositeElementType; -import com.intellij.psi.tree.IElementType; -import com.intellij.psi.tree.TokenSet; -import com.intellij.util.Function; -import com.intellij.util.PairProcessor; -import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.containers.LimitedPool; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedList; - -/** - * @author gregsh - */ -public class GeneratedParserUtilBase { - - private static final Logger LOG = Logger.getInstance("org.intellij.grammar.parser.GeneratedParserUtilBase"); - - private static final int MAX_RECURSION_LEVEL = 1000; - private static final int MAX_VARIANTS_SIZE = 10000; - private static final int MAX_VARIANTS_TO_DISPLAY = 50; - - private static final int INITIAL_VARIANTS_SIZE = 1000; - private static final int VARIANTS_POOL_SIZE = 10000; - private static final int FRAMES_POOL_SIZE = 500; - - public static final IElementType DUMMY_BLOCK = new DummyBlockElementType(); - - public interface Parser { - boolean parse(PsiBuilder builder, int level); - } - - public static final Parser TOKEN_ADVANCER = new Parser() { - @Override - public boolean parse(PsiBuilder builder, int level) { - if (builder.eof()) return false; - builder.advanceLexer(); - return true; - } - }; - - public static final Parser TRUE_CONDITION = new Parser() { - @Override - public boolean parse(PsiBuilder builder, int level) { - return true; - } - }; - - public static boolean eof(PsiBuilder builder_, int level_) { - return builder_.eof(); - } - - public static int current_position_(PsiBuilder builder_) { - return builder_.rawTokenIndex(); - } - - public static boolean recursion_guard_(PsiBuilder builder_, int level_, String funcName_) { - if (level_ > MAX_RECURSION_LEVEL) { - builder_.error("Maximum recursion level (" + MAX_RECURSION_LEVEL + ") reached in '" + funcName_ + "'"); - return false; - } - return true; - } - - public static boolean empty_element_parsed_guard_(PsiBuilder builder_, String funcName_, int prev_position_) { - if (prev_position_ == current_position_(builder_)) { - builder_.error("Empty element parsed in '" + funcName_ + "' at offset " + builder_.getCurrentOffset()); - return false; - } - return true; - } - - public static boolean invalid_left_marker_guard_(PsiBuilder builder_, PsiBuilder.Marker marker_, String funcName_) { - //builder_.error("Invalid left marker encountered in " + funcName_ +" at offset " + builder_.getCurrentOffset()); - boolean goodMarker = marker_ != null; // && ((LighterASTNode)marker_).getTokenType() != TokenType.ERROR_ELEMENT; - if (!goodMarker) return false; - ErrorState state = ErrorState.get(builder_); - - return !state.frameStack.isEmpty(); - } - - public static TokenSet create_token_set_(IElementType... tokenTypes_) { - return TokenSet.create(tokenTypes_); - } - - private static boolean consumeTokens(PsiBuilder builder_, boolean smart, int pin, IElementType... tokens) { - ErrorState state = ErrorState.get(builder_); - if (state.completionState != null && state.predicateCount == 0) { - addCompletionVariant(builder_, state.completionState, tokens); - } - // suppress single token completion - CompletionState completionState = state.completionState; - state.completionState = null; - boolean result_ = true; - boolean pinned_ = false; - for (int i = 0, tokensLength = tokens.length; i < tokensLength; i++) { - if (pin > 0 && i == pin) pinned_ = result_; - if (result_ || pinned_) { - boolean fast = smart && i == 0; - if (!(fast ? consumeTokenFast(builder_, tokens[i]) : consumeToken(builder_, tokens[i]))) { - result_ = false; - if (pin < 0 || pinned_) report_error_(builder_, state, false); - } - } - } - state.completionState = completionState; - return pinned_ || result_; - } - - public static boolean consumeTokens(PsiBuilder builder_, int pin_, IElementType... token) { - return consumeTokens(builder_, false, pin_, token); - } - - public static boolean consumeTokensSmart(PsiBuilder builder_, int pin_, IElementType... token) { - return consumeTokens(builder_, true, pin_, token); - } - - public static boolean parseTokens(PsiBuilder builder_, int pin_, IElementType... tokens) { - return parseTokens(builder_, false, pin_, tokens); - } - - public static boolean parseTokensSmart(PsiBuilder builder_, int pin_, IElementType... tokens) { - return parseTokens(builder_, true, pin_, tokens); - } - - public static boolean parseTokens(PsiBuilder builder_, boolean smart, int pin_, IElementType... tokens) { - PsiBuilder.Marker marker_ = builder_.mark(); - boolean result_ = consumeTokens(builder_, smart, pin_, tokens); - if (!result_) { - marker_.rollbackTo(); - } - else { - marker_.drop(); - } - return result_; - } - - public static boolean consumeTokenSmart(PsiBuilder builder_, IElementType token) { - addCompletionVariantSmart(builder_, token); - return consumeTokenFast(builder_, token); - } - - public static boolean consumeTokenSmart(PsiBuilder builder_, String token) { - addCompletionVariantSmart(builder_, token); - return consumeTokenFast(builder_, token); - } - - public static boolean consumeToken(PsiBuilder builder_, IElementType token) { - addVariantSmart(builder_, token, true); - if (nextTokenIsFast(builder_, token)) { - builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeTokenFast(PsiBuilder builder_, IElementType token) { - if (nextTokenIsFast(builder_, token)) { - builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeToken(PsiBuilder builder_, String text) { - return consumeToken(builder_, text, ErrorState.get(builder_).caseSensitive); - } - - public static boolean consumeToken(PsiBuilder builder_, String text, boolean caseSensitive) { - addVariantSmart(builder_, text, true); - int count = nextTokenIsFast(builder_, text, caseSensitive); - if (count > 0) { - while (count-- > 0) builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeTokenFast(PsiBuilder builder_, String text) { - int count = nextTokenIsFast(builder_, text, ErrorState.get(builder_).caseSensitive); - if (count > 0) { - while (count-- > 0) builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType token) { - return builder_.getTokenType() == token; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType... tokens) { - IElementType tokenType = builder_.getTokenType(); - for (IElementType token : tokens) { - if (token == tokenType) return true; - } - return false; - } - - public static boolean nextTokenIs(PsiBuilder builder_, String frameName, IElementType... tokens) { - ErrorState state = ErrorState.get(builder_); - if (state.completionState != null) return true; - boolean track = !state.suppressErrors && state.predicateCount < 2 && state.predicateSign; - if (!track) return nextTokenIsFast(builder_, tokens); - IElementType tokenType = builder_.getTokenType(); - if (StringUtil.isNotEmpty(frameName)) { - addVariantInner(state, builder_.rawTokenIndex(), frameName); - } - else { - for (IElementType token : tokens) { - addVariant(builder_, state, token); - } - } - if (tokenType == null) return false; - for (IElementType token : tokens) { - if (tokenType == token) return true; - } - return false; - } - - public static boolean nextTokenIs(PsiBuilder builder_, IElementType token) { - if (!addVariantSmart(builder_, token, false)) return true; - return nextTokenIsFast(builder_, token); - } - - public static boolean nextTokenIs(PsiBuilder builder_, String tokenText) { - if (!addVariantSmart(builder_, tokenText, false)) return true; - return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, String tokenText) { - return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0; - } - - public static int nextTokenIsFast(PsiBuilder builder_, String tokenText, boolean caseSensitive) { - CharSequence sequence = builder_.getOriginalText(); - int offset = builder_.getCurrentOffset(); - int endOffset = offset + tokenText.length(); - CharSequence subSequence = sequence.subSequence(offset, Math.min(endOffset, sequence.length())); - - if (!Comparing.equal(subSequence, tokenText, caseSensitive)) return 0; - - int count = 0; - while (true) { - int nextOffset = builder_.rawTokenTypeStart(++count); - if (nextOffset > endOffset) { - return -count; - } - else if (nextOffset == endOffset) { - break; - } - } - return count; - } - - private static void addCompletionVariantSmart(PsiBuilder builder_, Object token) { - ErrorState state = ErrorState.get(builder_); - CompletionState completionState = state.completionState; - if (completionState != null && state.predicateCount == 0) { - addCompletionVariant(builder_, completionState, token); - } - } - - private static boolean addVariantSmart(PsiBuilder builder_, Object token, boolean force) { - ErrorState state = ErrorState.get(builder_); - // skip FIRST check in completion mode - if (state.completionState != null && !force) return false; - builder_.eof(); - if (!state.suppressErrors && state.predicateCount < 2) { - addVariant(builder_, state, token); - } - return true; - } - - public static void addVariant(PsiBuilder builder_, String text) { - addVariant(builder_, ErrorState.get(builder_), text); - } - - private static void addVariant(PsiBuilder builder_, ErrorState state, Object o) { - builder_.eof(); // skip whitespaces - addVariantInner(state, builder_.rawTokenIndex(), o); - - CompletionState completionState = state.completionState; - if (completionState != null && state.predicateSign) { - addCompletionVariant(builder_, completionState, o); - } - } - - private static void addVariantInner(ErrorState state, int pos, Object o) { - Variant variant = state.VARIANTS.alloc().init(pos, o); - if (state.predicateSign) { - state.variants.add(variant); - if (state.lastExpectedVariantPos < variant.position) { - state.lastExpectedVariantPos = variant.position; - } - } - else { - state.unexpected.add(variant); - } - } - - private static void addCompletionVariant(@NotNull PsiBuilder builder_, @NotNull CompletionState completionState, Object o) { - int offset = builder_.getCurrentOffset(); - if (!builder_.eof() && offset == builder_.rawTokenTypeStart(1)) return; // suppress for zero-length tokens - - boolean add = false; - int diff = completionState.offset - offset; - String text = completionState.convertItem(o); - int length = text == null? 0 : text.length(); - if (length == 0) return; - if (diff == 0) { - add = true; - } - else if (diff > 0 && diff <= length) { - CharSequence fragment = builder_.getOriginalText().subSequence(offset, completionState.offset); - add = completionState.prefixMatches(fragment.toString(), text); - } - else if (diff < 0) { - for (int i=-1; ; i--) { - IElementType type = builder_.rawLookup(i); - int tokenStart = builder_.rawTokenTypeStart(i); - if (isWhitespaceOrComment(builder_, type)) { - diff = completionState.offset - tokenStart; - } - else if (type != null && tokenStart < completionState.offset) { - CharSequence fragment = builder_.getOriginalText().subSequence(tokenStart, completionState.offset); - if (completionState.prefixMatches(fragment.toString(), text)) { - diff = completionState.offset - tokenStart; - } - break; - } - else break; - } - add = diff >= 0 && diff < length; - } - add = add && length > 1 && !(text.charAt(0) == '<' && text.charAt(length - 1) == '>') && - !(text.charAt(0) == '\'' && text.charAt(length - 1) == '\'' && length < 5); - if (add) { - completionState.addItem(builder_, text); - } - } - - public static boolean isWhitespaceOrComment(@NotNull PsiBuilder builder_, @Nullable IElementType type) { - return ((PsiBuilderImpl)((Builder)builder_).getDelegate()).whitespaceOrComment(type); - } - - // here's the new section API for compact parsers & less IntelliJ platform API exposure - public static final int _NONE_ = 0x0; - public static final int _COLLAPSE_ = 0x1; - public static final int _LEFT_ = 0x2; - public static final int _LEFT_INNER_ = 0x4; - public static final int _AND_ = 0x8; - public static final int _NOT_ = 0x10; - - // simple enter/exit methods pair that doesn't require frame object - public static PsiBuilder.Marker enter_section_(PsiBuilder builder_) { - return builder_.mark(); - } - - public static void exit_section_(PsiBuilder builder_, - PsiBuilder.Marker marker, - @Nullable IElementType elementType, - boolean result) { - close_marker_impl_(ErrorState.get(builder_).frameStack.peekLast(), marker, elementType, result); - } - - // complex enter/exit methods pair with frame object - public static PsiBuilder.Marker enter_section_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) { - PsiBuilder.Marker marker = builder_.mark(); - enter_section_impl_(builder_, level, modifiers, frameName); - return marker; - } - - private static void enter_section_impl_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) { - ErrorState state = ErrorState.get(builder_); - Frame frame = state.FRAMES.alloc().init(builder_, state, level, modifiers, frameName); - Frame prevFrame = state.frameStack.peekLast(); - if (prevFrame != null && prevFrame.errorReportedAt > frame.position) { - // report error for previous unsuccessful frame - reportError(builder_, state, frame, true, false); - } - if (((frame.modifiers & _LEFT_) | (frame.modifiers & _LEFT_INNER_)) != 0) { - PsiBuilder.Marker left = (PsiBuilder.Marker)builder_.getLatestDoneMarker(); - if (invalid_left_marker_guard_(builder_, left, frameName)) { - frame.leftMarker = left; - } - } - state.frameStack.add(frame); - if ((modifiers & _AND_) != 0) { - if (state.predicateCount == 0 && !state.predicateSign) { - throw new AssertionError("Incorrect false predicate sign"); - } - state.predicateCount++; - } - else if ((modifiers & _NOT_) != 0) { - if (state.predicateCount == 0) { - state.predicateSign = false; - } - else { - state.predicateSign = !state.predicateSign; - } - state.predicateCount++; - } - } - - public static void exit_section_(PsiBuilder builder_, - int level, - PsiBuilder.Marker marker, - @Nullable IElementType elementType, - boolean result, - boolean pinned, - @Nullable Parser eatMore) { - ErrorState state = ErrorState.get(builder_); - - Frame frame = state.frameStack.pollLast(); - if (frame == null || level != frame.level) { - LOG.error("Unbalanced error section: got " + frame + ", expected level " + level); - if (frame != null) state.FRAMES.recycle(frame); - close_marker_impl_(frame, marker, elementType, result); - return; - } - - if (((frame.modifiers & _AND_) | (frame.modifiers & _NOT_)) != 0) { - close_marker_impl_(frame, marker, null, false); - state.predicateCount--; - if ((frame.modifiers & _NOT_) != 0) state.predicateSign = !state.predicateSign; - state.FRAMES.recycle(frame); - return; - } - exit_section_impl_(state, frame, builder_, marker, elementType, result, pinned); - - int initialPos = builder_.rawTokenIndex(); - boolean willFail = !result && !pinned; - if (willFail && initialPos == frame.position && state.lastExpectedVariantPos == frame.position && - frame.name != null && state.variants.size() - frame.variantCount > 1) { - state.clearVariants(true, frame.variantCount); - addVariantInner(state, initialPos, frame.name); - } - int lastErrorPos = getLastVariantPos(state, initialPos); - if (!state.suppressErrors && eatMore != null) { - state.suppressErrors = true; - final boolean eatMoreFlagOnce = !builder_.eof() && eatMore.parse(builder_, frame.level + 1); - boolean eatMoreFlag = eatMoreFlagOnce || !result && frame.position == initialPos && lastErrorPos > frame.position; - - PsiBuilderImpl.ProductionMarker latestDoneMarker = - (pinned || result) && (state.altMode || elementType != null) && - eatMoreFlagOnce ? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null; - PsiBuilder.Marker extensionMarker = null; - IElementType extensionTokenType = null; - // whitespace prefix makes the very first frame offset bigger than marker start offset which is always 0 - if (latestDoneMarker instanceof PsiBuilder.Marker && - frame.position >= latestDoneMarker.getStartIndex() && - frame.position <= latestDoneMarker.getEndIndex()) { - extensionMarker = ((PsiBuilder.Marker)latestDoneMarker).precede(); - extensionTokenType = latestDoneMarker.getTokenType(); - ((PsiBuilder.Marker)latestDoneMarker).drop(); - } - // advance to the last error pos - // skip tokens until lastErrorPos. parseAsTree might look better here... - int parenCount = 0; - while ((eatMoreFlag || parenCount > 0) && builder_.rawTokenIndex() < lastErrorPos) { - builder_.advanceLexer(); - eatMoreFlag = eatMore.parse(builder_, frame.level + 1); - } - boolean errorReported = frame.errorReportedAt == initialPos || !result && frame.errorReportedAt >= frame.position; - if (errorReported) { - if (eatMoreFlag) { - builder_.advanceLexer(); - parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); - } - } - else if (eatMoreFlag) { - errorReported = reportError(builder_, state, frame, true, true); - parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); - } - else if (eatMoreFlagOnce || (!result && frame.position != builder_.rawTokenIndex()) || frame.errorReportedAt > initialPos) { - errorReported = reportError(builder_, state, frame, true, false); - } - if (extensionMarker != null) { - extensionMarker.done(extensionTokenType); - } - state.suppressErrors = false; - if (errorReported || result) { - state.clearVariants(true, 0); - state.clearVariants(false, 0); - state.lastExpectedVariantPos = -1; - } - } - else if (!result && pinned && frame.errorReportedAt < 0) { - // do not report if there are errors beyond current position - if (lastErrorPos == initialPos) { - // do not force, inner recoverRoot might have skipped some tokens - reportError(builder_, state, frame, false, false); - } - else if (lastErrorPos > initialPos) { - // set error pos here as if it is reported for future reference - frame.errorReportedAt = lastErrorPos; - } - } - // propagate errorReportedAt up the stack to avoid duplicate reporting - Frame prevFrame = willFail && eatMore == null ? null : state.frameStack.peekLast(); - if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) { - prevFrame.errorReportedAt = frame.errorReportedAt; - } - state.FRAMES.recycle(frame); - } - - private static void exit_section_impl_(ErrorState state, - Frame frame, - PsiBuilder builder_, - PsiBuilder.Marker marker, - IElementType elementType, - boolean result, - boolean pinned) { - if (elementType != null && marker != null) { - if ((frame.modifiers & _COLLAPSE_) != 0) { - PsiBuilderImpl.ProductionMarker last = result || pinned? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null; - if (last != null && last.getStartIndex() == frame.position && - state.typeExtends(last.getTokenType(), elementType)) { - IElementType resultType = last.getTokenType(); - ((PsiBuilder.Marker)last).drop(); - marker.done(resultType); - return; - } - } - if (result || pinned) { - if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) { - marker.done(elementType); - frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType()); - frame.leftMarker.drop(); - } - else if ((frame.modifiers & _LEFT_) != 0 && frame.leftMarker != null) { - marker.drop(); - frame.leftMarker.precede().done(elementType); - } - else { - if (frame.level == 0) builder_.eof(); // skip whitespaces - marker.done(elementType); - } - } - else { - close_marker_impl_(frame, marker, null, false); - } - } - else if (result || pinned) { - if (marker != null) marker.drop(); - if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) { - frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType()); - frame.leftMarker.drop(); - } - } - else { - close_marker_impl_(frame, marker, null, false); - } - } - - private static void close_marker_impl_(Frame frame, PsiBuilder.Marker marker, IElementType elementType, boolean result) { - if (marker == null) return; - if (result) { - if (elementType != null) { - marker.done(elementType); - } - else { - marker.drop(); - } - } - else { - if (frame != null) { - int position = ((PsiBuilderImpl.ProductionMarker)marker).getStartIndex(); - if (frame.errorReportedAt > position) { - frame.errorReportedAt = frame.errorReportedAtPrev; - } - } - marker.rollbackTo(); - } - } - - public static boolean report_error_(PsiBuilder builder_, boolean result_) { - if (!result_) report_error_(builder_, ErrorState.get(builder_), false); - return result_; - } - - public static void report_error_(PsiBuilder builder_, ErrorState state, boolean advance) { - Frame frame = state.frameStack.isEmpty()? null : state.frameStack.getLast(); - if (frame == null) { - LOG.error("unbalanced enter/exit section call: got null"); - return; - } - int position = builder_.rawTokenIndex(); - if (frame.errorReportedAt < position && getLastVariantPos(state, position + 1) <= position) { - reportError(builder_, state, frame, true, advance); - } - } - - private static int getLastVariantPos(ErrorState state, int defValue) { - return state.lastExpectedVariantPos < 0? defValue : state.lastExpectedVariantPos; - } - - private static boolean reportError(PsiBuilder builder_, - ErrorState state, - Frame frame, - boolean force, - boolean advance) { - String expectedText = state.getExpectedText(builder_); - boolean notEmpty = StringUtil.isNotEmpty(expectedText); - if (force || notEmpty || advance) { - String gotText = builder_.eof()? "unexpected end of file" : - notEmpty? "got '" + builder_.getTokenText() +"'" : - "'" + builder_.getTokenText() +"' unexpected"; - String message = expectedText + gotText; - if (advance) { - PsiBuilder.Marker mark = builder_.mark(); - builder_.advanceLexer(); - mark.error(message); - } - else { - builder_.error(message); - } - builder_.eof(); // skip whitespaces - frame.errorReportedAt = builder_.rawTokenIndex(); - return true; - } - return false; - } - - - public static final Key COMPLETION_STATE_KEY = Key.create("COMPLETION_STATE_KEY"); - - public static class CompletionState implements Function { - public final int offset; - public final Collection items = ContainerUtil.newTroveSet(); - - public CompletionState(int offset_) { - offset = offset_; - } - - @Nullable - public String convertItem(Object o) { - return o instanceof Object[] ? StringUtil.join((Object[]) o, this, " ") : o.toString(); - } - - @Override - public String fun(Object o) { - return o.toString(); - } - - public void addItem(@NotNull PsiBuilder builder, @NotNull String text) { - items.add(text); - } - - public boolean prefixMatches(@NotNull String prefix, @NotNull String variant) { - return StringUtil.startsWithIgnoreCase(variant, prefix); - } - } - - public static class Builder extends PsiBuilderAdapter { - public final ErrorState state; - public final PsiParser parser; - - public Builder(PsiBuilder builder_, ErrorState state_, PsiParser parser_) { - super(builder_); - state = state_; - parser = parser_; - } - - public Lexer getLexer() { - return ((PsiBuilderImpl)myDelegate).getLexer(); - } - } - - public static PsiBuilder adapt_builder_(IElementType root, PsiBuilder builder, PsiParser parser) { - return adapt_builder_(root, builder, parser, null); - } - - public static PsiBuilder adapt_builder_(IElementType root, PsiBuilder builder, PsiParser parser, TokenSet[] extendsSets) { - ErrorState state = new ErrorState(); - ErrorState.initState(state, builder, root, extendsSets); - return new Builder(builder, state, parser); - } - - public static class ErrorState { - TokenSet[] extendsSets; - public PairProcessor altExtendsChecker; - - int predicateCount; - boolean predicateSign = true; - boolean suppressErrors; - public final LinkedList frameStack = new LinkedList(); - public CompletionState completionState; - - private boolean caseSensitive; - public boolean altMode; - - int lastExpectedVariantPos = -1; - MyList variants = new MyList(INITIAL_VARIANTS_SIZE); - MyList unexpected = new MyList(INITIAL_VARIANTS_SIZE / 10); - - final LimitedPool VARIANTS = new LimitedPool(VARIANTS_POOL_SIZE, new LimitedPool.ObjectFactory() { - @Override - public Variant create() { - return new Variant(); - } - - @Override - public void cleanup(final Variant o) { - } - }); - final LimitedPool FRAMES = new LimitedPool(FRAMES_POOL_SIZE, new LimitedPool.ObjectFactory() { - @Override - public Frame create() { - return new Frame(); - } - - @Override - public void cleanup(final Frame o) { - } - }); - - public static ErrorState get(PsiBuilder builder) { - return ((Builder)builder).state; - } - - public static void initState(ErrorState state, PsiBuilder builder, IElementType root, TokenSet[] extendsSets) { - state.extendsSets = extendsSets; - PsiFile file = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY); - state.completionState = file == null? null: file.getUserData(COMPLETION_STATE_KEY); - Language language = file == null? root.getLanguage() : file.getLanguage(); - state.caseSensitive = language.isCaseSensitive(); - } - - public String getExpectedText(PsiBuilder builder_) { - int position = builder_.rawTokenIndex(); - StringBuilder sb = new StringBuilder(); - if (addExpected(sb, position, true)) { - sb.append(" expected, "); - } - else if (addExpected(sb, position, false)) sb.append(" unexpected, "); - return sb.toString(); - } - - private boolean addExpected(StringBuilder sb, int position, boolean expected) { - MyList list = expected ? variants : unexpected; - String[] strings = new String[list.size()]; - long[] hashes = new long[strings.length]; - Arrays.fill(strings, ""); - int count = 0; - loop: for (Variant variant : list) { - if (position == variant.position) { - String text = variant.object.toString(); - long hash = StringHash.calc(text); - for (int i=0; i 0) { - if (count > MAX_VARIANTS_TO_DISPLAY) { - sb.append(" and ..."); - break; - } - else { - sb.append(", "); - } - } - char c = s.charAt(0); - String displayText = c == '<' || StringUtil.isJavaIdentifierStart(c) ? s : '\'' + s + '\''; - sb.append(displayText); - } - if (count > 1 && count < MAX_VARIANTS_TO_DISPLAY) { - int idx = sb.lastIndexOf(", "); - sb.replace(idx, idx + 1, " or"); - } - return count > 0; - } - - public void clearVariants(boolean expected, int start) { - MyList list = expected? variants : unexpected; - if (start < 0 || start >= list.size()) return; - for (int i = start, len = list.size(); i < len; i ++) { - VARIANTS.recycle(list.get(i)); - } - list.setSize(start); - } - - boolean typeExtends(IElementType child_, IElementType parent_) { - if (child_ == parent_) return true; - if (extendsSets != null) { - for (TokenSet set : extendsSets) { - if (set.contains(child_) && set.contains(parent_)) return true; - } - } - return altExtendsChecker != null && altExtendsChecker.process(child_, parent_); - } - } - - public static class Frame { - public int offset; - public int position; - public int level; - public int modifiers; - public String name; - public int variantCount; - public int errorReportedAt; - public int errorReportedAtPrev; - public PsiBuilder.Marker leftMarker; - - public Frame() { - } - - public Frame init(PsiBuilder builder_, ErrorState state, int level_, int modifiers_, String name_) { - offset = builder_.getCurrentOffset(); - position = builder_.rawTokenIndex(); - level = level_; - modifiers = modifiers_; - name = name_; - variantCount = state.variants.size(); - errorReportedAt = -1; - - Frame prev = state.frameStack.peekLast(); - errorReportedAtPrev = prev == null? -1 : prev.errorReportedAt; - leftMarker = null; - return this; - } - - @Override - public String toString() { - String mod = modifiers == _NONE_ ? "_NONE_, " : - ((modifiers & _COLLAPSE_) != 0? "_CAN_COLLAPSE_, ": "") + - ((modifiers & _LEFT_) != 0? "_LEFT_, ": "") + - ((modifiers & _LEFT_INNER_) != 0? "_LEFT_INNER_, ": "") + - ((modifiers & _AND_) != 0? "_AND_, ": "") + - ((modifiers & _NOT_) != 0? "_NOT_, ": ""); - return String.format("{%s:%s:%d, %d, %s%s}", offset, position, level, errorReportedAt, mod, name); - } - } - - - private static class Variant { - int position; - Object object; - - public Variant init(int pos, Object o) { - position = pos; - object = o; - return this; - } - - @Override - public String toString() { - return "<" + position + ", " + object + ">"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Variant variant = (Variant)o; - - if (position != variant.position) return false; - if (!this.object.equals(variant.object)) return false; - - return true; - } - - @Override - public int hashCode() { - int result = position; - result = 31 * result + object.hashCode(); - return result; - } - } - - - private static final int MAX_CHILDREN_IN_TREE = 10; - public static boolean parseAsTree(ErrorState state, final PsiBuilder builder_, int level, final IElementType chunkType, - boolean checkBraces, final Parser parser, final Parser eatMoreCondition) { - final LinkedList> parenList = new LinkedList>(); - final LinkedList> siblingList = new LinkedList>(); - PsiBuilder.Marker marker = null; - - final Runnable checkSiblingsRunnable = new Runnable() { - @Override - public void run() { - main: - while (!siblingList.isEmpty()) { - final Pair parenPair = parenList.peek(); - final int rating = siblingList.getFirst().second; - int count = 0; - for (Pair pair : siblingList) { - if (pair.second != rating || parenPair != null && pair.first == parenPair.second) break main; - if (++count >= MAX_CHILDREN_IN_TREE) { - final PsiBuilder.Marker parentMarker = pair.first.precede(); - while (count-- > 0) { - siblingList.removeFirst(); - } - parentMarker.done(chunkType); - siblingList.addFirst(Pair.create(parentMarker, rating + 1)); - continue main; - } - } - break; - } - } - }; - int totalCount = 0; - int tokenCount = 0; - while (true) { - final IElementType tokenType = builder_.getTokenType(); - if (marker == null) { - marker = builder_.mark(); - } - final boolean result = (!parenList.isEmpty() || eatMoreCondition.parse(builder_, level + 1)) && parser.parse(builder_, level + 1); - if (result) { - tokenCount++; - totalCount++; - } - if (!result) { - break; - } - - if (tokenCount >= MAX_CHILDREN_IN_TREE && marker != null) { - marker.done(chunkType); - siblingList.addFirst(Pair.create(marker, 1)); - checkSiblingsRunnable.run(); - marker = null; - tokenCount = 0; - } - } - if (marker != null) { - marker.drop(); - } - for (Pair pair : parenList) { - pair.first.drop(); - } - return totalCount != 0; - } - - private static class DummyBlockElementType extends IElementType implements ICompositeElementType{ - DummyBlockElementType() { - super("DUMMY_BLOCK", Language.ANY); - } - - @NotNull - @Override - public ASTNode createCompositeNode() { - return new DummyBlock(); - } - } - - public static class DummyBlock extends CompositePsiElement { - DummyBlock() { - super(DUMMY_BLOCK); - } - - @NotNull - @Override - public PsiReference[] getReferences() { - return PsiReference.EMPTY_ARRAY; - } - - @NotNull - @Override - public Language getLanguage() { - return getParent().getLanguage(); - } - } - - private static class MyList extends ArrayList { - MyList(int initialCapacity) { - super(initialCapacity); - } - - protected void setSize(int fromIndex) { - removeRange(fromIndex, size()); - } - - @Override - public boolean add(E e) { - int size = size(); - if (size >= MAX_VARIANTS_SIZE) { - removeRange(MAX_VARIANTS_SIZE / 4, size - MAX_VARIANTS_SIZE / 4); - } - return super.add(e); - } - } -} diff --git a/src/Markdown/MarkdownProcessor.kt b/src/Markdown/MarkdownProcessor.kt index 040f2ad5..b5e18f92 100644 --- a/src/Markdown/MarkdownProcessor.kt +++ b/src/Markdown/MarkdownProcessor.kt @@ -1,80 +1,34 @@ package org.jetbrains.dokka -import org.jetbrains.markdown.* -import com.intellij.lang.impl.PsiBuilderImpl -import com.intellij.psi.tree.TokenSet -import com.intellij.lang.Language -import com.intellij.psi.tree.IFileElementType -import com.intellij.lang.LighterASTNode -import com.intellij.util.diff.FlyweightCapableTreeStructure -import com.intellij.openapi.util.Ref -import org.jetbrains.markdown.lexer.MarkdownLexer -import com.intellij.psi.tree.IElementType +import net.nicoulaj.idea.markdown.lang.ast.* +import net.nicoulaj.idea.markdown.lang.parser.dialects.commonmark.* +import net.nicoulaj.idea.markdown.lang.parser.* +import net.nicoulaj.idea.markdown.lang.* -public object MarkdownProcessor { - val EXPR_LANGUAGE = object : Language("MARKDOWN") {} - val DOCUMENT = IFileElementType("DOCUMENT", EXPR_LANGUAGE); - - public fun parse(markdown: String): MarkdownTree { - val parser = MarkdownParser() - val builder = PsiBuilderImpl(null, null, TokenSet.EMPTY, TokenSet.EMPTY, MarkdownLexer(), null, markdown, null, null) - parser.parse_only_(DOCUMENT, builder) - val light = builder.getLightTree()!! - return MarkdownTree(markdown, light) - } +class MarkdownNode(val node: ASTNode, val markdown: String) { + val children: List get() = node.children.map { MarkdownNode(it, markdown) } + val endOffset: Int get() = node.endOffset + val startOffset: Int get() = node.startOffset + val type: IElementType get() = node.type + val text: String get() = markdown.substring(startOffset, endOffset) + fun child(type: IElementType): MarkdownNode? = children.firstOrNull { it.type == type } } -public class MarkdownTree(private val text: String, private val structure: FlyweightCapableTreeStructure) { - fun visit(action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) { - visit(structure.getRoot(), action) - } - - fun findChildByType(node: LighterASTNode, findType: IElementType) : LighterASTNode? { - val ref: Ref?> = Ref.create?>() - val count = structure.getChildren(node, ref) - val children = ref.get() - if (children != null) { - for (index in 0..count - 1) { - val child = children[index] - val nodeType = child.getTokenType() - if (nodeType == findType) - return child - val nestedChild = findChildByType(child, findType) - if (nestedChild != null) - return nestedChild - } - } - return null - } - - fun getNodeText(node: LighterASTNode) : String { - return text.substring(node.getStartOffset(), node.getEndOffset()) - } - - fun visit(node: LighterASTNode, action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) { - action(node, text) { - val ref : Ref?> = Ref.create?>() - val count = structure.getChildren(node, ref) - val children = ref.get() - if (children != null) { - for (index in 0..count - 1) { - val child = children[index] - visit(child, action) - } - } +fun MarkdownNode.visit(action: (MarkdownNode, () -> Unit) -> Unit) { + action(this) { + for (child in children) { + child.visit(action) } } - } -public fun MarkdownTree.toTestString(): String { +public fun MarkdownNode.toTestString(): String { val sb = StringBuilder() var level = 0 - visit {(node, text, visitChildren) -> - val nodeText = text.substring(node.getStartOffset(), node.getEndOffset()) + visit {(node, visitChildren) -> sb.append(" ".repeat(level * 2)) - sb.append(node.getTokenType().toString()) - sb.append(":" + nodeText.replace("\n", "\u23CE")) + sb.append(node.type.toString()) + sb.append(":" + node.text.replace("\n", "\u23CE")) sb.appendln() level++ visitChildren() @@ -83,26 +37,23 @@ public fun MarkdownTree.toTestString(): String { return sb.toString() } -public fun MarkdownTree.toHtml(): String { +public fun MarkdownNode.toHtml(): String { val sb = StringBuilder() - visit {(node, text, processChildren) -> - val nodeType = node.getTokenType() - val nodeText = text.substring(node.getStartOffset(), node.getEndOffset()) + visit {(node, processChildren) -> + val nodeType = node.type + val nodeText = node.text when (nodeType) { - MarkdownElementTypes.BULLET_LIST -> { + MarkdownElementTypes.UNORDERED_LIST -> { sb.appendln("
    ") processChildren() sb.appendln("
") } - MarkdownElementTypes.HORIZONTAL_RULE -> { - sb.appendln("
") - } MarkdownElementTypes.ORDERED_LIST -> { sb.appendln("
    ") processChildren() sb.appendln("
") } - MarkdownElementTypes.LIST_BLOCK -> { + MarkdownElementTypes.LIST_ITEM -> { sb.append("
  • ") processChildren() sb.appendln("
    • ") @@ -117,20 +68,54 @@ public fun MarkdownTree.toHtml(): String { processChildren() sb.append("") } - MarkdownElementTypes.PLAIN_TEXT -> { - sb.append(nodeText) + MarkdownElementTypes.ATX_1 -> { + sb.append("

    ") + processChildren() + sb.append("

    ") } - MarkdownElementTypes.END_LINE -> { - sb.appendln() + MarkdownElementTypes.ATX_2 -> { + sb.append("

    ") + processChildren() + sb.append("

    ") } - MarkdownElementTypes.BLANK_LINE -> { - sb.appendln() + MarkdownElementTypes.ATX_3 -> { + sb.append("

    ") + processChildren() + sb.append("

    ") + } + MarkdownElementTypes.ATX_4 -> { + sb.append("

    ") + processChildren() + sb.append("

    ") + } + MarkdownElementTypes.ATX_5 -> { + sb.append("
    ") + processChildren() + sb.append("
    ") + } + MarkdownElementTypes.ATX_6 -> { + sb.append("
    ") + processChildren() + sb.append("
    ") } - MarkdownElementTypes.PARA -> { + MarkdownElementTypes.BLOCK_QUOTE -> { + sb.append("
    ") + processChildren() + sb.append("
    ") + } + MarkdownElementTypes.PARAGRAPH -> { sb.append("

    ") processChildren() sb.appendln("

    ") } + MarkdownTokenTypes.CODE -> { + sb.append("
    ")
+                sb.append(nodeText)
+                sb.append("
    ")
+            }
+            MarkdownTokenTypes.TEXT -> {
+                sb.append(nodeText)
+            }
             else -> {
                 processChildren()
             }
@@ -139,9 +124,16 @@ public fun MarkdownTree.toHtml(): String {
     return sb.toString()
 }
 
+fun parseMarkdown(markdown: String): MarkdownNode {
+    if (markdown.isEmpty())
+        return MarkdownNode(LeafASTNode(MarkdownElementTypes.MARKDOWN_FILE, 0, 0), markdown)
+    return MarkdownNode(MarkdownParser(CommonMarkMarkerProcessor()).buildMarkdownTreeFromString(markdown), markdown)
+}
 
 fun markdownToHtml(markdown: String): String {
-    val markdownTree = MarkdownProcessor.parse(markdown)
+
+    val tree = MarkdownParser(CommonMarkMarkerProcessor()).buildMarkdownTreeFromString(markdown)
+    val markdownTree = MarkdownNode(tree, markdown)
     val ast = markdownTree.toTestString()
     return markdownTree.toHtml()
 }
diff --git a/src/Markdown/MarkdownTokenType.kt b/src/Markdown/MarkdownTokenType.kt
deleted file mode 100644
index 293228c3..00000000
--- a/src/Markdown/MarkdownTokenType.kt
+++ /dev/null
@@ -1,6 +0,0 @@
-package org.jetbrains.dokka.Markdown
-
-import com.intellij.psi.tree.IElementType
-
-public class MarkdownTokenType(debugName: String) : IElementType(debugName, null) {
-}
\ No newline at end of file
diff --git a/src/Markdown/markdown.bnf b/src/Markdown/markdown.bnf
deleted file mode 100644
index 2e4979ec..00000000
--- a/src/Markdown/markdown.bnf
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  psiPackage = 'org.jetbrains.markdown'
-  psiImplPackage = 'org.jetbrains.markdown.impl'
-
-  parserClass="org.jetbrains.markdown.MarkdownParser"
-  parserUtilClass="org.jetbrains.dokka.Markdown.GeneratedParserUtilBase"
-  elementTypeHolderClass = 'org.jetbrains.markdown.MarkdownElementTypes'
-
-  tokenTypeClass = 'org.jetbrains.dokka.Markdown.MarkdownTokenType'
-
-  generatePsi = false
-
-  tokens=[
-    BOM
-    EOL
-    EOP
-    Number
-    SPECIAL
-    Word
-  ]
-
-}
-
-Document ::= BOM? Whitespace* AnonymousSection? (Whitespace* NamedSection)*
-
-AnonymousSection ::= SectionBody
-NamedSection ::= SectionHeader SectionBody
-private SectionHeader ::= '$' SectionName OptionalSpace ':' OptionalSpace
-SectionName ::= SectionNameStart | '{' OptionalSpace SectionNameStart (Space+ Word)* OptionalSpace '}'
-private SectionNameStart ::= '$'? Word
-SectionBody::= Block*
-
-Unused ::= Special
-BlankLine ::= OptionalSpace EOL
-
-Whitespace ::= Space | EOL | EOP
-private OptionalSpace ::= Space*
-private NonindentSpace ::= ("   " | "  " | " ")?
-
-EndLine ::= TerminalEndline | NormalEndline
-private NormalEndline ::=   EOL !BlankLine
-private TerminalEndline ::= OptionalSpace <>
-private Indent ::=            "\t" | "    "
-
-// ---- BLOCKS ----
-Block ::= BlankLine* (
-              OrderedList
-            | BulletList
-            | HorizontalRule
-            | Directive
-            | Para
-            )
-
-Para ::= Inlines (EOP | Space* <>)?
-
-HorizontalRule ::= NonindentSpace
-                 ( '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')*
-                 | '-' OptionalSpace '-' OptionalSpace '-' (OptionalSpace '-')*
-                 | '_' OptionalSpace '_' OptionalSpace '_' (OptionalSpace '_')*)
-                 OptionalSpace EOL BlankLine+
-
-Directive ::= '{' DirectiveName Space+ DirectiveParams '}'
-DirectiveName ::= Word
-DirectiveParams ::= PlainText
-
-Bullet ::= !HorizontalRule NonindentSpace ('+' | '*' | '-') Space+
-Enumerator ::= NonindentSpace Number '.' Space+
-
-BulletList ::= &Bullet List
-OrderedList ::= &Enumerator List
-
-private List ::=  (ListItem BlankLine*)+
-ListItem      ::= (Bullet | Enumerator) ListBlock ( ListContinuationBlock )*
-
-ListBlock ::= !BlankLine Inlines ( ListBlockLine )*
-ListBlockLine ::= !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Inlines
-
-ListContinuationBlock ::= BlankLine* (Indent ListBlock)+
-
-
-// ---- INLINES ----
-private Inlines ::= (!EndLine Inline | EndLine &Inline )+ EndLine?
-Inline  ::= Strong | Emph | Code | Link | PlainText
-PlainText ::= (Word | Number | Space | ':')+
-
-Emph ::= EmphStar | EmphUnderscore
-private EmphStar ::= '*' !Whitespace (!'*' Inline)+ '*'
-private EmphUnderscore ::= '_' !Whitespace (!'_' Inline)+ '_'
-
-Code ::= '`' !Whitespace (!'`' Inline)+ '`'
-
-Strong ::= StrongStar | StrongUnderscore
-StrongStar ::= '**' !Whitespace (!'**' Inline)+ '**'
-StrongUnderscore ::= '__' !Whitespace (!'__' Inline)+ '__'
-
-Link ::=  '[' Target ']' ('(' Href ')')?
-Target ::= Word+
-Href ::= (Word | Number | ':' | '/')+
\ No newline at end of file
diff --git a/src/Markdown/markdown.leg b/src/Markdown/markdown.leg
deleted file mode 100644
index ea8bc522..00000000
--- a/src/Markdown/markdown.leg
+++ /dev/null
@@ -1,781 +0,0 @@
-%{
-/**********************************************************************
-
-  markdown_parser.leg - markdown parser in C using a PEG grammar.
-  (c) 2008 John MacFarlane (jgm at berkeley dot edu).
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License or the MIT
-  license.  See LICENSE for details.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
- ***********************************************************************/
-
-#include 
-#include 
-#include "markdown_peg.h"
-#include "utility_functions.h"
-
-
-
-/**********************************************************************
-
-  Definitions for leg parser generator.
-  YY_INPUT is the function the parser calls to get new input.
-  We take all new input from (static) charbuf.
-
- ***********************************************************************/
-
-
-
-# define YYSTYPE element *
-#ifdef __DEBUG__
-# define YY_DEBUG 1
-#endif
-
-#define YY_INPUT(buf, result, max_size)              \
-{                                                    \
-    int yyc;                                         \
-    if (charbuf && *charbuf != '\0') {               \
-        yyc= *charbuf++;                             \
-    } else {                                         \
-        yyc= EOF;                                    \
-    }                                                \
-    result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1);     \
-}
-
-#define YY_RULE(T)	T
-
-
-/**********************************************************************
-
-  PEG grammar and parser actions for markdown syntax.
-
- ***********************************************************************/
-
-%}
-
-Doc =       BOM? a:StartList ( Block { a = cons($$, a); } )*
-            { parse_result = reverse(a); }
-
-Block =     BlankLine*
-            ( BlockQuote
-            | Verbatim
-            | Note
-            | Reference
-            | HorizontalRule
-            | Heading
-            | OrderedList
-            | BulletList
-            | HtmlBlock
-            | StyleBlock
-            | Para
-            | Plain )
-
-Para =      NonindentSpace a:Inlines BlankLine+
-            { $$ = a; $$->key = PARA; }
-
-Plain =     a:Inlines
-            { $$ = a; $$->key = PLAIN; }
-
-AtxInline = !Newline !(Sp '#'* Sp Newline) Inline
-
-AtxStart =  < ( "######" | "#####" | "####" | "###" | "##" | "#" ) >
-            { $$ = mk_element(H1 + (strlen(yytext) - 1)); }
-
-AtxHeading = s:AtxStart Sp a:StartList ( AtxInline { a = cons($$, a); } )+ (Sp '#'* Sp)?  Newline
-            { $$ = mk_list(s->key, a);
-              free(s); }
-
-SetextHeading = SetextHeading1 | SetextHeading2
-
-SetextBottom1 = '='+ Newline
-
-SetextBottom2 = '-'+ Newline
-
-SetextHeading1 =  &(RawLine SetextBottom1)
-                  a:StartList ( !Endline Inline { a = cons($$, a); } )+ Sp Newline
-                  SetextBottom1 { $$ = mk_list(H1, a); }
-
-SetextHeading2 =  &(RawLine SetextBottom2)
-                  a:StartList ( !Endline Inline { a = cons($$, a); } )+ Sp Newline
-                  SetextBottom2 { $$ = mk_list(H2, a); }
-
-Heading = SetextHeading | AtxHeading
-
-BlockQuote = a:BlockQuoteRaw
-             {  $$ = mk_element(BLOCKQUOTE);
-                $$->children = a;
-             }
-
-BlockQuoteRaw =  a:StartList
-                 (( '>' ' '? Line { a = cons($$, a); } )
-                  ( !'>' !BlankLine Line { a = cons($$, a); } )*
-                  ( BlankLine { a = cons(mk_str("\n"), a); } )*
-                 )+
-                 {   $$ = mk_str_from_list(a, true);
-                     $$->key = RAW;
-                 }
-
-NonblankIndentedLine = !BlankLine IndentedLine
-
-VerbatimChunk = a:StartList
-                ( BlankLine { a = cons(mk_str("\n"), a); } )*
-                ( NonblankIndentedLine { a = cons($$, a); } )+
-                { $$ = mk_str_from_list(a, false); }
-
-Verbatim =     a:StartList ( VerbatimChunk { a = cons($$, a); } )+
-               { $$ = mk_str_from_list(a, false);
-                 $$->key = VERBATIM; }
-
-HorizontalRule = NonindentSpace
-                 ( '*' Sp '*' Sp '*' (Sp '*')*
-                 | '-' Sp '-' Sp '-' (Sp '-')*
-                 | '_' Sp '_' Sp '_' (Sp '_')*)
-                 Sp Newline BlankLine+
-                 { $$ = mk_element(HRULE); }
-
-Bullet = !HorizontalRule NonindentSpace ('+' | '*' | '-') Spacechar+
-
-BulletList = &Bullet (ListTight | ListLoose)
-             { $$->key = BULLETLIST; }
-
-ListTight = a:StartList
-            ( ListItemTight { a = cons($$, a); } )+
-            BlankLine* !(Bullet | Enumerator)
-            { $$ = mk_list(LIST, a); }
-
-ListLoose = a:StartList
-            ( b:ListItem BlankLine*
-              {   element *li;
-                  li = b->children;
-                  li->contents.str = realloc(li->contents.str, strlen(li->contents.str) + 3);
-                  strcat(li->contents.str, "\n\n");  /* In loose list, \n\n added to end of each element */
-                  a = cons(b, a);
-              } )+
-            { $$ = mk_list(LIST, a); }
-
-ListItem =  ( Bullet | Enumerator )
-            a:StartList
-            ListBlock { a = cons($$, a); }
-            ( ListContinuationBlock { a = cons($$, a); } )*
-            {  element *raw;
-               raw = mk_str_from_list(a, false);
-               raw->key = RAW;
-               $$ = mk_element(LISTITEM);
-               $$->children = raw;
-            }
-
-ListItemTight =
-            ( Bullet | Enumerator )
-            a:StartList
-            ListBlock { a = cons($$, a); }
-            ( !BlankLine
-              ListContinuationBlock { a = cons($$, a); } )*
-            !ListContinuationBlock
-            {  element *raw;
-               raw = mk_str_from_list(a, false);
-               raw->key = RAW;
-               $$ = mk_element(LISTITEM);
-               $$->children = raw;
-            }
-
-ListBlock = a:StartList
-            !BlankLine Line { a = cons($$, a); }
-            ( ListBlockLine { a = cons($$, a); } )*
-            { $$ = mk_str_from_list(a, false); }
-
-ListContinuationBlock = a:StartList
-                        ( < BlankLine* >
-                          {   if (strlen(yytext) == 0)
-                                   a = cons(mk_str("\001"), a); /* block separator */
-                              else
-                                   a = cons(mk_str(yytext), a); } )
-                        ( Indent ListBlock { a = cons($$, a); } )+
-                        {  $$ = mk_str_from_list(a, false); }
-
-Enumerator = NonindentSpace [0-9]+ '.' Spacechar+
-
-OrderedList = &Enumerator (ListTight | ListLoose)
-              { $$->key = ORDEREDLIST; }
-
-ListBlockLine = !BlankLine
-                !( Indent? (Bullet | Enumerator) )
-                !HorizontalRule
-                OptionallyIndentedLine
-
-# Parsers for different kinds of block-level HTML content.
-# This is repetitive due to constraints of PEG grammar.
-
-HtmlBlockOpenAddress = '<' Spnl ("address" | "ADDRESS") Spnl HtmlAttribute* '>'
-HtmlBlockCloseAddress = '<' Spnl '/' ("address" | "ADDRESS") Spnl '>'
-HtmlBlockAddress = HtmlBlockOpenAddress (HtmlBlockAddress | !HtmlBlockCloseAddress .)* HtmlBlockCloseAddress
-
-HtmlBlockOpenBlockquote = '<' Spnl ("blockquote" | "BLOCKQUOTE") Spnl HtmlAttribute* '>'
-HtmlBlockCloseBlockquote = '<' Spnl '/' ("blockquote" | "BLOCKQUOTE") Spnl '>'
-HtmlBlockBlockquote = HtmlBlockOpenBlockquote (HtmlBlockBlockquote | !HtmlBlockCloseBlockquote .)* HtmlBlockCloseBlockquote
-
-HtmlBlockOpenCenter = '<' Spnl ("center" | "CENTER") Spnl HtmlAttribute* '>'
-HtmlBlockCloseCenter = '<' Spnl '/' ("center" | "CENTER") Spnl '>'
-HtmlBlockCenter = HtmlBlockOpenCenter (HtmlBlockCenter | !HtmlBlockCloseCenter .)* HtmlBlockCloseCenter
-
-HtmlBlockOpenDir = '<' Spnl ("dir" | "DIR") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDir = '<' Spnl '/' ("dir" | "DIR") Spnl '>'
-HtmlBlockDir = HtmlBlockOpenDir (HtmlBlockDir | !HtmlBlockCloseDir .)* HtmlBlockCloseDir
-
-HtmlBlockOpenDiv = '<' Spnl ("div" | "DIV") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDiv = '<' Spnl '/' ("div" | "DIV") Spnl '>'
-HtmlBlockDiv = HtmlBlockOpenDiv (HtmlBlockDiv | !HtmlBlockCloseDiv .)* HtmlBlockCloseDiv
-
-HtmlBlockOpenDl = '<' Spnl ("dl" | "DL") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDl = '<' Spnl '/' ("dl" | "DL") Spnl '>'
-HtmlBlockDl = HtmlBlockOpenDl (HtmlBlockDl | !HtmlBlockCloseDl .)* HtmlBlockCloseDl
-
-HtmlBlockOpenFieldset = '<' Spnl ("fieldset" | "FIELDSET") Spnl HtmlAttribute* '>'
-HtmlBlockCloseFieldset = '<' Spnl '/' ("fieldset" | "FIELDSET") Spnl '>'
-HtmlBlockFieldset = HtmlBlockOpenFieldset (HtmlBlockFieldset | !HtmlBlockCloseFieldset .)* HtmlBlockCloseFieldset
-
-HtmlBlockOpenForm = '<' Spnl ("form" | "FORM") Spnl HtmlAttribute* '>'
-HtmlBlockCloseForm = '<' Spnl '/' ("form" | "FORM") Spnl '>'
-HtmlBlockForm = HtmlBlockOpenForm (HtmlBlockForm | !HtmlBlockCloseForm .)* HtmlBlockCloseForm
-
-HtmlBlockOpenH1 = '<' Spnl ("h1" | "H1") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH1 = '<' Spnl '/' ("h1" | "H1") Spnl '>'
-HtmlBlockH1 = HtmlBlockOpenH1 (HtmlBlockH1 | !HtmlBlockCloseH1 .)* HtmlBlockCloseH1
-
-HtmlBlockOpenH2 = '<' Spnl ("h2" | "H2") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH2 = '<' Spnl '/' ("h2" | "H2") Spnl '>'
-HtmlBlockH2 = HtmlBlockOpenH2 (HtmlBlockH2 | !HtmlBlockCloseH2 .)* HtmlBlockCloseH2
-
-HtmlBlockOpenH3 = '<' Spnl ("h3" | "H3") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH3 = '<' Spnl '/' ("h3" | "H3") Spnl '>'
-HtmlBlockH3 = HtmlBlockOpenH3 (HtmlBlockH3 | !HtmlBlockCloseH3 .)* HtmlBlockCloseH3
-
-HtmlBlockOpenH4 = '<' Spnl ("h4" | "H4") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH4 = '<' Spnl '/' ("h4" | "H4") Spnl '>'
-HtmlBlockH4 = HtmlBlockOpenH4 (HtmlBlockH4 | !HtmlBlockCloseH4 .)* HtmlBlockCloseH4
-
-HtmlBlockOpenH5 = '<' Spnl ("h5" | "H5") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH5 = '<' Spnl '/' ("h5" | "H5") Spnl '>'
-HtmlBlockH5 = HtmlBlockOpenH5 (HtmlBlockH5 | !HtmlBlockCloseH5 .)* HtmlBlockCloseH5
-
-HtmlBlockOpenH6 = '<' Spnl ("h6" | "H6") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH6 = '<' Spnl '/' ("h6" | "H6") Spnl '>'
-HtmlBlockH6 = HtmlBlockOpenH6 (HtmlBlockH6 | !HtmlBlockCloseH6 .)* HtmlBlockCloseH6
-
-HtmlBlockOpenMenu = '<' Spnl ("menu" | "MENU") Spnl HtmlAttribute* '>'
-HtmlBlockCloseMenu = '<' Spnl '/' ("menu" | "MENU") Spnl '>'
-HtmlBlockMenu = HtmlBlockOpenMenu (HtmlBlockMenu | !HtmlBlockCloseMenu .)* HtmlBlockCloseMenu
-
-HtmlBlockOpenNoframes = '<' Spnl ("noframes" | "NOFRAMES") Spnl HtmlAttribute* '>'
-HtmlBlockCloseNoframes = '<' Spnl '/' ("noframes" | "NOFRAMES") Spnl '>'
-HtmlBlockNoframes = HtmlBlockOpenNoframes (HtmlBlockNoframes | !HtmlBlockCloseNoframes .)* HtmlBlockCloseNoframes
-
-HtmlBlockOpenNoscript = '<' Spnl ("noscript" | "NOSCRIPT") Spnl HtmlAttribute* '>'
-HtmlBlockCloseNoscript = '<' Spnl '/' ("noscript" | "NOSCRIPT") Spnl '>'
-HtmlBlockNoscript = HtmlBlockOpenNoscript (HtmlBlockNoscript | !HtmlBlockCloseNoscript .)* HtmlBlockCloseNoscript
-
-HtmlBlockOpenOl = '<' Spnl ("ol" | "OL") Spnl HtmlAttribute* '>'
-HtmlBlockCloseOl = '<' Spnl '/' ("ol" | "OL") Spnl '>'
-HtmlBlockOl = HtmlBlockOpenOl (HtmlBlockOl | !HtmlBlockCloseOl .)* HtmlBlockCloseOl
-
-HtmlBlockOpenP = '<' Spnl ("p" | "P") Spnl HtmlAttribute* '>'
-HtmlBlockCloseP = '<' Spnl '/' ("p" | "P") Spnl '>'
-HtmlBlockP = HtmlBlockOpenP (HtmlBlockP | !HtmlBlockCloseP .)* HtmlBlockCloseP
-
-HtmlBlockOpenPre = '<' Spnl ("pre" | "PRE") Spnl HtmlAttribute* '>'
-HtmlBlockClosePre = '<' Spnl '/' ("pre" | "PRE") Spnl '>'
-HtmlBlockPre = HtmlBlockOpenPre (HtmlBlockPre | !HtmlBlockClosePre .)* HtmlBlockClosePre
-
-HtmlBlockOpenTable = '<' Spnl ("table" | "TABLE") Spnl HtmlAttribute* '>'
-HtmlBlockCloseTable = '<' Spnl '/' ("table" | "TABLE") Spnl '>'
-HtmlBlockTable = HtmlBlockOpenTable (HtmlBlockTable | !HtmlBlockCloseTable .)* HtmlBlockCloseTable
-
-HtmlBlockOpenUl = '<' Spnl ("ul" | "UL") Spnl HtmlAttribute* '>'
-HtmlBlockCloseUl = '<' Spnl '/' ("ul" | "UL") Spnl '>'
-HtmlBlockUl = HtmlBlockOpenUl (HtmlBlockUl | !HtmlBlockCloseUl .)* HtmlBlockCloseUl
-
-HtmlBlockOpenDd = '<' Spnl ("dd" | "DD") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDd = '<' Spnl '/' ("dd" | "DD") Spnl '>'
-HtmlBlockDd = HtmlBlockOpenDd (HtmlBlockDd | !HtmlBlockCloseDd .)* HtmlBlockCloseDd
-
-HtmlBlockOpenDt = '<' Spnl ("dt" | "DT") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDt = '<' Spnl '/' ("dt" | "DT") Spnl '>'
-HtmlBlockDt = HtmlBlockOpenDt (HtmlBlockDt | !HtmlBlockCloseDt .)* HtmlBlockCloseDt
-
-HtmlBlockOpenFrameset = '<' Spnl ("frameset" | "FRAMESET") Spnl HtmlAttribute* '>'
-HtmlBlockCloseFrameset = '<' Spnl '/' ("frameset" | "FRAMESET") Spnl '>'
-HtmlBlockFrameset = HtmlBlockOpenFrameset (HtmlBlockFrameset | !HtmlBlockCloseFrameset .)* HtmlBlockCloseFrameset
-
-HtmlBlockOpenLi = '<' Spnl ("li" | "LI") Spnl HtmlAttribute* '>'
-HtmlBlockCloseLi = '<' Spnl '/' ("li" | "LI") Spnl '>'
-HtmlBlockLi = HtmlBlockOpenLi (HtmlBlockLi | !HtmlBlockCloseLi .)* HtmlBlockCloseLi
-
-HtmlBlockOpenTbody = '<' Spnl ("tbody" | "TBODY") Spnl HtmlAttribute* '>'
-HtmlBlockCloseTbody = '<' Spnl '/' ("tbody" | "TBODY") Spnl '>'
-HtmlBlockTbody = HtmlBlockOpenTbody (HtmlBlockTbody | !HtmlBlockCloseTbody .)* HtmlBlockCloseTbody
-
-HtmlBlockOpenTd = '<' Spnl ("td" | "TD") Spnl HtmlAttribute* '>'