diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/Formats/StructuredFormatService.kt | 6 | ||||
| -rw-r--r-- | src/Formats/TextFormatService.kt | 4 | ||||
| -rw-r--r-- | src/Markdown/GeneratedParserUtilBase.java | 1031 | ||||
| -rw-r--r-- | src/Markdown/MarkdownLexer.java | 10 | ||||
| -rw-r--r-- | src/Markdown/MarkdownProcessor.kt | 56 | ||||
| -rw-r--r-- | src/Markdown/MarkdownTokenType.kt | 6 | ||||
| -rw-r--r-- | src/Markdown/_MarkdownLexer.flex | 40 | ||||
| -rw-r--r-- | src/Markdown/markdown.bnf | 83 | ||||
| -rw-r--r-- | src/Markdown/markdown.leg | 781 | ||||
| -rw-r--r-- | src/Model/DocumentationContent.kt | 74 | ||||
| -rw-r--r-- | src/Model/DocumentationNodeBuilder.kt | 2 | ||||
| -rw-r--r-- | src/Processing/CrossReferences.kt | 12 | ||||
| -rw-r--r-- | src/RichContent/RichString.kt | 3 | ||||
| -rw-r--r-- | src/main.kt | 15 |
14 files changed, 2083 insertions, 40 deletions
diff --git a/src/Formats/StructuredFormatService.kt b/src/Formats/StructuredFormatService.kt index 339ccf73..0c58f553 100644 --- a/src/Formats/StructuredFormatService.kt +++ b/src/Formats/StructuredFormatService.kt @@ -63,8 +63,10 @@ public abstract class StructuredFormatService(val locationService: LocationServi } appendLine(to, formatText(node.doc.description)) appendLine(to) - for (section in node.doc.sections) { - appendLine(to, formatBold(formatText(section.label))) + for ((label, section) in node.doc.sections) { + if (label.startsWith("$")) + continue + appendLine(to, formatBold(formatText(label))) appendLine(to, formatText(section.text)) appendLine(to) } diff --git a/src/Formats/TextFormatService.kt b/src/Formats/TextFormatService.kt index 29f01a74..77a0bb65 100644 --- a/src/Formats/TextFormatService.kt +++ b/src/Formats/TextFormatService.kt @@ -12,8 +12,8 @@ public class TextFormatService(val signatureGenerator: LanguageService) : Format for (n in 0..node.doc.summary.length()) append("=") - for (section in node.doc.sections) { - appendln(section.label) + for ((label,section) in node.doc.sections) { + appendln(label) appendln(section.text) } } diff --git a/src/Markdown/GeneratedParserUtilBase.java b/src/Markdown/GeneratedParserUtilBase.java new file mode 100644 index 00000000..9dd999b5 --- /dev/null +++ b/src/Markdown/GeneratedParserUtilBase.java @@ -0,0 +1,1031 @@ +/* + * Copyright 2011-2014 Gregory Shrago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.dokka.Markdown; + +import com.intellij.lang.*; +import com.intellij.lang.impl.PsiBuilderAdapter; +import com.intellij.lang.impl.PsiBuilderImpl; +import com.intellij.lexer.Lexer; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.Comparing; +import com.intellij.openapi.util.Key; +import com.intellij.openapi.util.Pair; +import com.intellij.openapi.util.text.StringHash; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.psi.PsiFile; +import com.intellij.psi.PsiReference; +import com.intellij.psi.TokenType; +import com.intellij.psi.impl.source.resolve.FileContextUtil; +import com.intellij.psi.impl.source.tree.CompositePsiElement; +import com.intellij.psi.tree.ICompositeElementType; +import com.intellij.psi.tree.IElementType; +import com.intellij.psi.tree.TokenSet; +import com.intellij.util.Function; +import com.intellij.util.PairProcessor; +import com.intellij.util.containers.ContainerUtil; +import com.intellij.util.containers.LimitedPool; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedList; + +/** + * @author gregsh + */ +public class GeneratedParserUtilBase { + + private static final Logger LOG = Logger.getInstance("org.intellij.grammar.parser.GeneratedParserUtilBase"); + + private static final int MAX_RECURSION_LEVEL = 1000; + private static final int MAX_VARIANTS_SIZE = 10000; + private static final int MAX_VARIANTS_TO_DISPLAY = 50; + + private static final int INITIAL_VARIANTS_SIZE = 1000; + private static final int VARIANTS_POOL_SIZE = 10000; + private static final int FRAMES_POOL_SIZE = 500; + + public static final IElementType DUMMY_BLOCK = new DummyBlockElementType(); + + public interface Parser { + boolean parse(PsiBuilder builder, int level); + } + + public static final Parser TOKEN_ADVANCER = new Parser() { + @Override + public boolean parse(PsiBuilder builder, int level) { + if (builder.eof()) return false; + builder.advanceLexer(); + return true; + } + }; + + public static final Parser TRUE_CONDITION = new Parser() { + @Override + public boolean parse(PsiBuilder builder, int level) { + return true; + } + }; + + public static boolean eof(PsiBuilder builder_, int level_) { + return builder_.eof(); + } + + public static int current_position_(PsiBuilder builder_) { + return builder_.rawTokenIndex(); + } + + public static boolean recursion_guard_(PsiBuilder builder_, int level_, String funcName_) { + if (level_ > MAX_RECURSION_LEVEL) { + builder_.error("Maximum recursion level (" + MAX_RECURSION_LEVEL + ") reached in '" + funcName_ + "'"); + return false; + } + return true; + } + + public static boolean empty_element_parsed_guard_(PsiBuilder builder_, String funcName_, int prev_position_) { + if (prev_position_ == current_position_(builder_)) { + builder_.error("Empty element parsed in '" + funcName_ + "' at offset " + builder_.getCurrentOffset()); + return false; + } + return true; + } + + public static boolean invalid_left_marker_guard_(PsiBuilder builder_, PsiBuilder.Marker marker_, String funcName_) { + //builder_.error("Invalid left marker encountered in " + funcName_ +" at offset " + builder_.getCurrentOffset()); + boolean goodMarker = marker_ != null; // && ((LighterASTNode)marker_).getTokenType() != TokenType.ERROR_ELEMENT; + if (!goodMarker) return false; + ErrorState state = ErrorState.get(builder_); + + return !state.frameStack.isEmpty(); + } + + public static TokenSet create_token_set_(IElementType... tokenTypes_) { + return TokenSet.create(tokenTypes_); + } + + private static boolean consumeTokens(PsiBuilder builder_, boolean smart, int pin, IElementType... tokens) { + ErrorState state = ErrorState.get(builder_); + if (state.completionState != null && state.predicateCount == 0) { + addCompletionVariant(builder_, state.completionState, tokens); + } + // suppress single token completion + CompletionState completionState = state.completionState; + state.completionState = null; + boolean result_ = true; + boolean pinned_ = false; + for (int i = 0, tokensLength = tokens.length; i < tokensLength; i++) { + if (pin > 0 && i == pin) pinned_ = result_; + if (result_ || pinned_) { + boolean fast = smart && i == 0; + if (!(fast ? consumeTokenFast(builder_, tokens[i]) : consumeToken(builder_, tokens[i]))) { + result_ = false; + if (pin < 0 || pinned_) report_error_(builder_, state, false); + } + } + } + state.completionState = completionState; + return pinned_ || result_; + } + + public static boolean consumeTokens(PsiBuilder builder_, int pin_, IElementType... token) { + return consumeTokens(builder_, false, pin_, token); + } + + public static boolean consumeTokensSmart(PsiBuilder builder_, int pin_, IElementType... token) { + return consumeTokens(builder_, true, pin_, token); + } + + public static boolean parseTokens(PsiBuilder builder_, int pin_, IElementType... tokens) { + return parseTokens(builder_, false, pin_, tokens); + } + + public static boolean parseTokensSmart(PsiBuilder builder_, int pin_, IElementType... tokens) { + return parseTokens(builder_, true, pin_, tokens); + } + + public static boolean parseTokens(PsiBuilder builder_, boolean smart, int pin_, IElementType... tokens) { + PsiBuilder.Marker marker_ = builder_.mark(); + boolean result_ = consumeTokens(builder_, smart, pin_, tokens); + if (!result_) { + marker_.rollbackTo(); + } + else { + marker_.drop(); + } + return result_; + } + + public static boolean consumeTokenSmart(PsiBuilder builder_, IElementType token) { + addCompletionVariantSmart(builder_, token); + return consumeTokenFast(builder_, token); + } + + public static boolean consumeTokenSmart(PsiBuilder builder_, String token) { + addCompletionVariantSmart(builder_, token); + return consumeTokenFast(builder_, token); + } + + public static boolean consumeToken(PsiBuilder builder_, IElementType token) { + addVariantSmart(builder_, token, true); + if (nextTokenIsFast(builder_, token)) { + builder_.advanceLexer(); + return true; + } + return false; + } + + public static boolean consumeTokenFast(PsiBuilder builder_, IElementType token) { + if (nextTokenIsFast(builder_, token)) { + builder_.advanceLexer(); + return true; + } + return false; + } + + public static boolean consumeToken(PsiBuilder builder_, String text) { + return consumeToken(builder_, text, ErrorState.get(builder_).caseSensitive); + } + + public static boolean consumeToken(PsiBuilder builder_, String text, boolean caseSensitive) { + addVariantSmart(builder_, text, true); + int count = nextTokenIsFast(builder_, text, caseSensitive); + if (count > 0) { + while (count-- > 0) builder_.advanceLexer(); + return true; + } + return false; + } + + public static boolean consumeTokenFast(PsiBuilder builder_, String text) { + int count = nextTokenIsFast(builder_, text, ErrorState.get(builder_).caseSensitive); + if (count > 0) { + while (count-- > 0) builder_.advanceLexer(); + return true; + } + return false; + } + + public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType token) { + return builder_.getTokenType() == token; + } + + public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType... tokens) { + IElementType tokenType = builder_.getTokenType(); + for (IElementType token : tokens) { + if (token == tokenType) return true; + } + return false; + } + + public static boolean nextTokenIs(PsiBuilder builder_, String frameName, IElementType... tokens) { + ErrorState state = ErrorState.get(builder_); + if (state.completionState != null) return true; + boolean track = !state.suppressErrors && state.predicateCount < 2 && state.predicateSign; + if (!track) return nextTokenIsFast(builder_, tokens); + IElementType tokenType = builder_.getTokenType(); + if (StringUtil.isNotEmpty(frameName)) { + addVariantInner(state, builder_.rawTokenIndex(), frameName); + } + else { + for (IElementType token : tokens) { + addVariant(builder_, state, token); + } + } + if (tokenType == null) return false; + for (IElementType token : tokens) { + if (tokenType == token) return true; + } + return false; + } + + public static boolean nextTokenIs(PsiBuilder builder_, IElementType token) { + if (!addVariantSmart(builder_, token, false)) return true; + return nextTokenIsFast(builder_, token); + } + + public static boolean nextTokenIs(PsiBuilder builder_, String tokenText) { + if (!addVariantSmart(builder_, tokenText, false)) return true; + return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0; + } + + public static boolean nextTokenIsFast(PsiBuilder builder_, String tokenText) { + return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0; + } + + public static int nextTokenIsFast(PsiBuilder builder_, String tokenText, boolean caseSensitive) { + CharSequence sequence = builder_.getOriginalText(); + int offset = builder_.getCurrentOffset(); + int endOffset = offset + tokenText.length(); + CharSequence subSequence = sequence.subSequence(offset, Math.min(endOffset, sequence.length())); + + if (!Comparing.equal(subSequence, tokenText, caseSensitive)) return 0; + + int count = 0; + while (true) { + int nextOffset = builder_.rawTokenTypeStart(++count); + if (nextOffset > endOffset) { + return -count; + } + else if (nextOffset == endOffset) { + break; + } + } + return count; + } + + private static void addCompletionVariantSmart(PsiBuilder builder_, Object token) { + ErrorState state = ErrorState.get(builder_); + CompletionState completionState = state.completionState; + if (completionState != null && state.predicateCount == 0) { + addCompletionVariant(builder_, completionState, token); + } + } + + private static boolean addVariantSmart(PsiBuilder builder_, Object token, boolean force) { + ErrorState state = ErrorState.get(builder_); + // skip FIRST check in completion mode + if (state.completionState != null && !force) return false; + builder_.eof(); + if (!state.suppressErrors && state.predicateCount < 2) { + addVariant(builder_, state, token); + } + return true; + } + + public static void addVariant(PsiBuilder builder_, String text) { + addVariant(builder_, ErrorState.get(builder_), text); + } + + private static void addVariant(PsiBuilder builder_, ErrorState state, Object o) { + builder_.eof(); // skip whitespaces + addVariantInner(state, builder_.rawTokenIndex(), o); + + CompletionState completionState = state.completionState; + if (completionState != null && state.predicateSign) { + addCompletionVariant(builder_, completionState, o); + } + } + + private static void addVariantInner(ErrorState state, int pos, Object o) { + Variant variant = state.VARIANTS.alloc().init(pos, o); + if (state.predicateSign) { + state.variants.add(variant); + if (state.lastExpectedVariantPos < variant.position) { + state.lastExpectedVariantPos = variant.position; + } + } + else { + state.unexpected.add(variant); + } + } + + private static void addCompletionVariant(@NotNull PsiBuilder builder_, @NotNull CompletionState completionState, Object o) { + int offset = builder_.getCurrentOffset(); + if (!builder_.eof() && offset == builder_.rawTokenTypeStart(1)) return; // suppress for zero-length tokens + + boolean add = false; + int diff = completionState.offset - offset; + String text = completionState.convertItem(o); + int length = text == null? 0 : text.length(); + if (length == 0) return; + if (diff == 0) { + add = true; + } + else if (diff > 0 && diff <= length) { + CharSequence fragment = builder_.getOriginalText().subSequence(offset, completionState.offset); + add = completionState.prefixMatches(fragment.toString(), text); + } + else if (diff < 0) { + for (int i=-1; ; i--) { + IElementType type = builder_.rawLookup(i); + int tokenStart = builder_.rawTokenTypeStart(i); + if (isWhitespaceOrComment(builder_, type)) { + diff = completionState.offset - tokenStart; + } + else if (type != null && tokenStart < completionState.offset) { + CharSequence fragment = builder_.getOriginalText().subSequence(tokenStart, completionState.offset); + if (completionState.prefixMatches(fragment.toString(), text)) { + diff = completionState.offset - tokenStart; + } + break; + } + else break; + } + add = diff >= 0 && diff < length; + } + add = add && length > 1 && !(text.charAt(0) == '<' && text.charAt(length - 1) == '>') && + !(text.charAt(0) == '\'' && text.charAt(length - 1) == '\'' && length < 5); + if (add) { + completionState.addItem(builder_, text); + } + } + + public static boolean isWhitespaceOrComment(@NotNull PsiBuilder builder_, @Nullable IElementType type) { + return ((PsiBuilderImpl)((Builder)builder_).getDelegate()).whitespaceOrComment(type); + } + + // here's the new section API for compact parsers & less IntelliJ platform API exposure + public static final int _NONE_ = 0x0; + public static final int _COLLAPSE_ = 0x1; + public static final int _LEFT_ = 0x2; + public static final int _LEFT_INNER_ = 0x4; + public static final int _AND_ = 0x8; + public static final int _NOT_ = 0x10; + + // simple enter/exit methods pair that doesn't require frame object + public static PsiBuilder.Marker enter_section_(PsiBuilder builder_) { + return builder_.mark(); + } + + public static void exit_section_(PsiBuilder builder_, + PsiBuilder.Marker marker, + @Nullable IElementType elementType, + boolean result) { + close_marker_impl_(ErrorState.get(builder_).frameStack.peekLast(), marker, elementType, result); + } + + // complex enter/exit methods pair with frame object + public static PsiBuilder.Marker enter_section_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) { + PsiBuilder.Marker marker = builder_.mark(); + enter_section_impl_(builder_, level, modifiers, frameName); + return marker; + } + + private static void enter_section_impl_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) { + ErrorState state = ErrorState.get(builder_); + Frame frame = state.FRAMES.alloc().init(builder_, state, level, modifiers, frameName); + Frame prevFrame = state.frameStack.peekLast(); + if (prevFrame != null && prevFrame.errorReportedAt > frame.position) { + // report error for previous unsuccessful frame + reportError(builder_, state, frame, true, false); + } + if (((frame.modifiers & _LEFT_) | (frame.modifiers & _LEFT_INNER_)) != 0) { + PsiBuilder.Marker left = (PsiBuilder.Marker)builder_.getLatestDoneMarker(); + if (invalid_left_marker_guard_(builder_, left, frameName)) { + frame.leftMarker = left; + } + } + state.frameStack.add(frame); + if ((modifiers & _AND_) != 0) { + if (state.predicateCount == 0 && !state.predicateSign) { + throw new AssertionError("Incorrect false predicate sign"); + } + state.predicateCount++; + } + else if ((modifiers & _NOT_) != 0) { + if (state.predicateCount == 0) { + state.predicateSign = false; + } + else { + state.predicateSign = !state.predicateSign; + } + state.predicateCount++; + } + } + + public static void exit_section_(PsiBuilder builder_, + int level, + PsiBuilder.Marker marker, + @Nullable IElementType elementType, + boolean result, + boolean pinned, + @Nullable Parser eatMore) { + ErrorState state = ErrorState.get(builder_); + + Frame frame = state.frameStack.pollLast(); + if (frame == null || level != frame.level) { + LOG.error("Unbalanced error section: got " + frame + ", expected level " + level); + if (frame != null) state.FRAMES.recycle(frame); + close_marker_impl_(frame, marker, elementType, result); + return; + } + + if (((frame.modifiers & _AND_) | (frame.modifiers & _NOT_)) != 0) { + close_marker_impl_(frame, marker, null, false); + state.predicateCount--; + if ((frame.modifiers & _NOT_) != 0) state.predicateSign = !state.predicateSign; + state.FRAMES.recycle(frame); + return; + } + exit_section_impl_(state, frame, builder_, marker, elementType, result, pinned); + + int initialPos = builder_.rawTokenIndex(); + boolean willFail = !result && !pinned; + if (willFail && initialPos == frame.position && state.lastExpectedVariantPos == frame.position && + frame.name != null && state.variants.size() - frame.variantCount > 1) { + state.clearVariants(true, frame.variantCount); + addVariantInner(state, initialPos, frame.name); + } + int lastErrorPos = getLastVariantPos(state, initialPos); + if (!state.suppressErrors && eatMore != null) { + state.suppressErrors = true; + final boolean eatMoreFlagOnce = !builder_.eof() && eatMore.parse(builder_, frame.level + 1); + boolean eatMoreFlag = eatMoreFlagOnce || !result && frame.position == initialPos && lastErrorPos > frame.position; + + PsiBuilderImpl.ProductionMarker latestDoneMarker = + (pinned || result) && (state.altMode || elementType != null) && + eatMoreFlagOnce ? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null; + PsiBuilder.Marker extensionMarker = null; + IElementType extensionTokenType = null; + // whitespace prefix makes the very first frame offset bigger than marker start offset which is always 0 + if (latestDoneMarker instanceof PsiBuilder.Marker && + frame.position >= latestDoneMarker.getStartIndex() && + frame.position <= latestDoneMarker.getEndIndex()) { + extensionMarker = ((PsiBuilder.Marker)latestDoneMarker).precede(); + extensionTokenType = latestDoneMarker.getTokenType(); + ((PsiBuilder.Marker)latestDoneMarker).drop(); + } + // advance to the last error pos + // skip tokens until lastErrorPos. parseAsTree might look better here... + int parenCount = 0; + while ((eatMoreFlag || parenCount > 0) && builder_.rawTokenIndex() < lastErrorPos) { + builder_.advanceLexer(); + eatMoreFlag = eatMore.parse(builder_, frame.level + 1); + } + boolean errorReported = frame.errorReportedAt == initialPos || !result && frame.errorReportedAt >= frame.position; + if (errorReported) { + if (eatMoreFlag) { + builder_.advanceLexer(); + parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); + } + } + else if (eatMoreFlag) { + errorReported = reportError(builder_, state, frame, true, true); + parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); + } + else if (eatMoreFlagOnce || (!result && frame.position != builder_.rawTokenIndex()) || frame.errorReportedAt > initialPos) { + errorReported = reportError(builder_, state, frame, true, false); + } + if (extensionMarker != null) { + extensionMarker.done(extensionTokenType); + } + state.suppressErrors = false; + if (errorReported || result) { + state.clearVariants(true, 0); + state.clearVariants(false, 0); + state.lastExpectedVariantPos = -1; + } + } + else if (!result && pinned && frame.errorReportedAt < 0) { + // do not report if there are errors beyond current position + if (lastErrorPos == initialPos) { + // do not force, inner recoverRoot might have skipped some tokens + reportError(builder_, state, frame, false, false); + } + else if (lastErrorPos > initialPos) { + // set error pos here as if it is reported for future reference + frame.errorReportedAt = lastErrorPos; + } + } + // propagate errorReportedAt up the stack to avoid duplicate reporting + Frame prevFrame = willFail && eatMore == null ? null : state.frameStack.peekLast(); + if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) { + prevFrame.errorReportedAt = frame.errorReportedAt; + } + state.FRAMES.recycle(frame); + } + + private static void exit_section_impl_(ErrorState state, + Frame frame, + PsiBuilder builder_, + PsiBuilder.Marker marker, + IElementType elementType, + boolean result, + boolean pinned) { + if (elementType != null && marker != null) { + if ((frame.modifiers & _COLLAPSE_) != 0) { + PsiBuilderImpl.ProductionMarker last = result || pinned? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null; + if (last != null && last.getStartIndex() == frame.position && + state.typeExtends(last.getTokenType(), elementType)) { + IElementType resultType = last.getTokenType(); + ((PsiBuilder.Marker)last).drop(); + marker.done(resultType); + return; + } + } + if (result || pinned) { + if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) { + marker.done(elementType); + frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType()); + frame.leftMarker.drop(); + } + else if ((frame.modifiers & _LEFT_) != 0 && frame.leftMarker != null) { + marker.drop(); + frame.leftMarker.precede().done(elementType); + } + else { + if (frame.level == 0) builder_.eof(); // skip whitespaces + marker.done(elementType); + } + } + else { + close_marker_impl_(frame, marker, null, false); + } + } + else if (result || pinned) { + if (marker != null) marker.drop(); + if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) { + frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType()); + frame.leftMarker.drop(); + } + } + else { + close_marker_impl_(frame, marker, null, false); + } + } + + private static void close_marker_impl_(Frame frame, PsiBuilder.Marker marker, IElementType elementType, boolean result) { + if (marker == null) return; + if (result) { + if (elementType != null) { + marker.done(elementType); + } + else { + marker.drop(); + } + } + else { + if (frame != null) { + int position = ((PsiBuilderImpl.ProductionMarker)marker).getStartIndex(); + if (frame.errorReportedAt > position) { + frame.errorReportedAt = frame.errorReportedAtPrev; + } + } + marker.rollbackTo(); + } + } + + public static boolean report_error_(PsiBuilder builder_, boolean result_) { + if (!result_) report_error_(builder_, ErrorState.get(builder_), false); + return result_; + } + + public static void report_error_(PsiBuilder builder_, ErrorState state, boolean advance) { + Frame frame = state.frameStack.isEmpty()? null : state.frameStack.getLast(); + if (frame == null) { + LOG.error("unbalanced enter/exit section call: got null"); + return; + } + int position = builder_.rawTokenIndex(); + if (frame.errorReportedAt < position && getLastVariantPos(state, position + 1) <= position) { + reportError(builder_, state, frame, true, advance); + } + } + + private static int getLastVariantPos(ErrorState state, int defValue) { + return state.lastExpectedVariantPos < 0? defValue : state.lastExpectedVariantPos; + } + + private static boolean reportError(PsiBuilder builder_, + ErrorState state, + Frame frame, + boolean force, + boolean advance) { + String expectedText = state.getExpectedText(builder_); + boolean notEmpty = StringUtil.isNotEmpty(expectedText); + if (force || notEmpty || advance) { + String gotText = builder_.eof()? "unexpected end of file" : + notEmpty? "got '" + builder_.getTokenText() +"'" : + "'" + builder_.getTokenText() +"' unexpected"; + String message = expectedText + gotText; + if (advance) { + PsiBuilder.Marker mark = builder_.mark(); + builder_.advanceLexer(); + mark.error(message); + } + else { + builder_.error(message); + } + builder_.eof(); // skip whitespaces + frame.errorReportedAt = builder_.rawTokenIndex(); + return true; + } + return false; + } + + + public static final Key<CompletionState> COMPLETION_STATE_KEY = Key.create("COMPLETION_STATE_KEY"); + + public static class CompletionState implements Function<Object, String> { + public final int offset; + public final Collection<String> items = ContainerUtil.newTroveSet(); + + public CompletionState(int offset_) { + offset = offset_; + } + + @Nullable + public String convertItem(Object o) { + return o instanceof Object[] ? StringUtil.join((Object[]) o, this, " ") : o.toString(); + } + + @Override + public String fun(Object o) { + return o.toString(); + } + + public void addItem(@NotNull PsiBuilder builder, @NotNull String text) { + items.add(text); + } + + public boolean prefixMatches(@NotNull String prefix, @NotNull String variant) { + return StringUtil.startsWithIgnoreCase(variant, prefix); + } + } + + public static class Builder extends PsiBuilderAdapter { + public final ErrorState state; + public final PsiParser parser; + + public Builder(PsiBuilder builder_, ErrorState state_, PsiParser parser_) { + super(builder_); + state = state_; + parser = parser_; + } + + public Lexer getLexer() { + return ((PsiBuilderImpl)myDelegate).getLexer(); + } + } + + public static PsiBuilder adapt_builder_(IElementType root, PsiBuilder builder, PsiParser parser) { + return adapt_builder_(root, builder, parser, null); + } + + public static PsiBuilder adapt_builder_(IElementType root, PsiBuilder builder, PsiParser parser, TokenSet[] extendsSets) { + ErrorState state = new ErrorState(); + ErrorState.initState(state, builder, root, extendsSets); + return new Builder(builder, state, parser); + } + + public static class ErrorState { + TokenSet[] extendsSets; + public PairProcessor<IElementType, IElementType> altExtendsChecker; + + int predicateCount; + boolean predicateSign = true; + boolean suppressErrors; + public final LinkedList<Frame> frameStack = new LinkedList<Frame>(); + public CompletionState completionState; + + private boolean caseSensitive; + public boolean altMode; + + int lastExpectedVariantPos = -1; + MyList<Variant> variants = new MyList<Variant>(INITIAL_VARIANTS_SIZE); + MyList<Variant> unexpected = new MyList<Variant>(INITIAL_VARIANTS_SIZE / 10); + + final LimitedPool<Variant> VARIANTS = new LimitedPool<Variant>(VARIANTS_POOL_SIZE, new LimitedPool.ObjectFactory<Variant>() { + @Override + public Variant create() { + return new Variant(); + } + + @Override + public void cleanup(final Variant o) { + } + }); + final LimitedPool<Frame> FRAMES = new LimitedPool<Frame>(FRAMES_POOL_SIZE, new LimitedPool.ObjectFactory<Frame>() { + @Override + public Frame create() { + return new Frame(); + } + + @Override + public void cleanup(final Frame o) { + } + }); + + public static ErrorState get(PsiBuilder builder) { + return ((Builder)builder).state; + } + + public static void initState(ErrorState state, PsiBuilder builder, IElementType root, TokenSet[] extendsSets) { + state.extendsSets = extendsSets; + PsiFile file = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY); + state.completionState = file == null? null: file.getUserData(COMPLETION_STATE_KEY); + Language language = file == null? root.getLanguage() : file.getLanguage(); + state.caseSensitive = language.isCaseSensitive(); + } + + public String getExpectedText(PsiBuilder builder_) { + int position = builder_.rawTokenIndex(); + StringBuilder sb = new StringBuilder(); + if (addExpected(sb, position, true)) { + sb.append(" expected, "); + } + else if (addExpected(sb, position, false)) sb.append(" unexpected, "); + return sb.toString(); + } + + private boolean addExpected(StringBuilder sb, int position, boolean expected) { + MyList<Variant> list = expected ? variants : unexpected; + String[] strings = new String[list.size()]; + long[] hashes = new long[strings.length]; + Arrays.fill(strings, ""); + int count = 0; + loop: for (Variant variant : list) { + if (position == variant.position) { + String text = variant.object.toString(); + long hash = StringHash.calc(text); + for (int i=0; i<count; i++) { + if (hashes[i] == hash) continue loop; + } + hashes[count] = hash; + strings[count] = text; + count++; + } + } + Arrays.sort(strings); + count = 0; + for (String s : strings) { + if (s.length() == 0) continue; + if (count++ > 0) { + if (count > MAX_VARIANTS_TO_DISPLAY) { + sb.append(" and ..."); + break; + } + else { + sb.append(", "); + } + } + char c = s.charAt(0); + String displayText = c == '<' || StringUtil.isJavaIdentifierStart(c) ? s : '\'' + s + '\''; + sb.append(displayText); + } + if (count > 1 && count < MAX_VARIANTS_TO_DISPLAY) { + int idx = sb.lastIndexOf(", "); + sb.replace(idx, idx + 1, " or"); + } + return count > 0; + } + + public void clearVariants(boolean expected, int start) { + MyList<Variant> list = expected? variants : unexpected; + if (start < 0 || start >= list.size()) return; + for (int i = start, len = list.size(); i < len; i ++) { + VARIANTS.recycle(list.get(i)); + } + list.setSize(start); + } + + boolean typeExtends(IElementType child_, IElementType parent_) { + if (child_ == parent_) return true; + if (extendsSets != null) { + for (TokenSet set : extendsSets) { + if (set.contains(child_) && set.contains(parent_)) return true; + } + } + return altExtendsChecker != null && altExtendsChecker.process(child_, parent_); + } + } + + public static class Frame { + public int offset; + public int position; + public int level; + public int modifiers; + public String name; + public int variantCount; + public int errorReportedAt; + public int errorReportedAtPrev; + public PsiBuilder.Marker leftMarker; + + public Frame() { + } + + public Frame init(PsiBuilder builder_, ErrorState state, int level_, int modifiers_, String name_) { + offset = builder_.getCurrentOffset(); + position = builder_.rawTokenIndex(); + level = level_; + modifiers = modifiers_; + name = name_; + variantCount = state.variants.size(); + errorReportedAt = -1; + + Frame prev = state.frameStack.peekLast(); + errorReportedAtPrev = prev == null? -1 : prev.errorReportedAt; + leftMarker = null; + return this; + } + + @Override + public String toString() { + String mod = modifiers == _NONE_ ? "_NONE_, " : + ((modifiers & _COLLAPSE_) != 0? "_CAN_COLLAPSE_, ": "") + + ((modifiers & _LEFT_) != 0? "_LEFT_, ": "") + + ((modifiers & _LEFT_INNER_) != 0? "_LEFT_INNER_, ": "") + + ((modifiers & _AND_) != 0? "_AND_, ": "") + + ((modifiers & _NOT_) != 0? "_NOT_, ": ""); + return String.format("{%s:%s:%d, %d, %s%s}", offset, position, level, errorReportedAt, mod, name); + } + } + + + private static class Variant { + int position; + Object object; + + public Variant init(int pos, Object o) { + position = pos; + object = o; + return this; + } + + @Override + public String toString() { + return "<" + position + ", " + object + ">"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Variant variant = (Variant)o; + + if (position != variant.position) return false; + if (!this.object.equals(variant.object)) return false; + + return true; + } + + @Override + public int hashCode() { + int result = position; + result = 31 * result + object.hashCode(); + return result; + } + } + + + private static final int MAX_CHILDREN_IN_TREE = 10; + public static boolean parseAsTree(ErrorState state, final PsiBuilder builder_, int level, final IElementType chunkType, + boolean checkBraces, final Parser parser, final Parser eatMoreCondition) { + final LinkedList<Pair<PsiBuilder.Marker, PsiBuilder.Marker>> parenList = new LinkedList<Pair<PsiBuilder.Marker, PsiBuilder.Marker>>(); + final LinkedList<Pair<PsiBuilder.Marker, Integer>> siblingList = new LinkedList<Pair<PsiBuilder.Marker, Integer>>(); + PsiBuilder.Marker marker = null; + + final Runnable checkSiblingsRunnable = new Runnable() { + @Override + public void run() { + main: + while (!siblingList.isEmpty()) { + final Pair<PsiBuilder.Marker, PsiBuilder.Marker> parenPair = parenList.peek(); + final int rating = siblingList.getFirst().second; + int count = 0; + for (Pair<PsiBuilder.Marker, Integer> pair : siblingList) { + if (pair.second != rating || parenPair != null && pair.first == parenPair.second) break main; + if (++count >= MAX_CHILDREN_IN_TREE) { + final PsiBuilder.Marker parentMarker = pair.first.precede(); + while (count-- > 0) { + siblingList.removeFirst(); + } + parentMarker.done(chunkType); + siblingList.addFirst(Pair.create(parentMarker, rating + 1)); + continue main; + } + } + break; + } + } + }; + int totalCount = 0; + int tokenCount = 0; + while (true) { + final IElementType tokenType = builder_.getTokenType(); + if (marker == null) { + marker = builder_.mark(); + } + final boolean result = (!parenList.isEmpty() || eatMoreCondition.parse(builder_, level + 1)) && parser.parse(builder_, level + 1); + if (result) { + tokenCount++; + totalCount++; + } + if (!result) { + break; + } + + if (tokenCount >= MAX_CHILDREN_IN_TREE && marker != null) { + marker.done(chunkType); + siblingList.addFirst(Pair.create(marker, 1)); + checkSiblingsRunnable.run(); + marker = null; + tokenCount = 0; + } + } + if (marker != null) { + marker.drop(); + } + for (Pair<PsiBuilder.Marker, PsiBuilder.Marker> pair : parenList) { + pair.first.drop(); + } + return totalCount != 0; + } + + private static class DummyBlockElementType extends IElementType implements ICompositeElementType{ + DummyBlockElementType() { + super("DUMMY_BLOCK", Language.ANY); + } + + @NotNull + @Override + public ASTNode createCompositeNode() { + return new DummyBlock(); + } + } + + public static class DummyBlock extends CompositePsiElement { + DummyBlock() { + super(DUMMY_BLOCK); + } + + @NotNull + @Override + public PsiReference[] getReferences() { + return PsiReference.EMPTY_ARRAY; + } + + @NotNull + @Override + public Language getLanguage() { + return getParent().getLanguage(); + } + } + + private static class MyList<E> extends ArrayList<E> { + MyList(int initialCapacity) { + super(initialCapacity); + } + + protected void setSize(int fromIndex) { + removeRange(fromIndex, size()); + } + + @Override + public boolean add(E e) { + int size = size(); + if (size >= MAX_VARIANTS_SIZE) { + removeRange(MAX_VARIANTS_SIZE / 4, size - MAX_VARIANTS_SIZE / 4); + } + return super.add(e); + } + } +} diff --git a/src/Markdown/MarkdownLexer.java b/src/Markdown/MarkdownLexer.java new file mode 100644 index 00000000..3486d795 --- /dev/null +++ b/src/Markdown/MarkdownLexer.java @@ -0,0 +1,10 @@ +package org.jetbrains.dokka.Markdown; + +import com.intellij.lexer.FlexAdapter; +import org.jetbrains.markdown.impl._MarkdownLexer; + +public class MarkdownLexer extends FlexAdapter { + public MarkdownLexer() { + super(new _MarkdownLexer()); + } +} diff --git a/src/Markdown/MarkdownProcessor.kt b/src/Markdown/MarkdownProcessor.kt new file mode 100644 index 00000000..9843eb68 --- /dev/null +++ b/src/Markdown/MarkdownProcessor.kt @@ -0,0 +1,56 @@ +package org.jetbrains.dokka + +import org.jetbrains.markdown.* +import com.intellij.lang.impl.PsiBuilderImpl +import com.intellij.psi.tree.TokenSet +import com.intellij.lang.Language +import com.intellij.psi.tree.IFileElementType +import com.intellij.lang.LighterASTNode +import com.intellij.util.diff.FlyweightCapableTreeStructure +import com.intellij.openapi.util.Ref +import org.jetbrains.dokka.Markdown.MarkdownLexer + +public class MarkdownProcessor { + class object { + val EXPR_LANGUAGE = object : Language("MARKDOWN") {} + val DOCUMENT = IFileElementType("DOCUMENT", EXPR_LANGUAGE); + } + + public fun parse(markdown: String): MarkdownTree { + val parser = MarkdownParser() + val builder = PsiBuilderImpl(null, null, TokenSet.EMPTY, TokenSet.EMPTY, MarkdownLexer(), null, markdown, null, null) + parser.parse_only_(DOCUMENT, builder) + val light = builder.getLightTree()!! + return MarkdownTree(markdown, light) + } +} + +public class MarkdownTree(private val text: String, private val structure: FlyweightCapableTreeStructure<LighterASTNode>) { + public fun dump(): String { + val sb = StringBuilder() + visit(sb, "", structure.getRoot(), structure, text) + return sb.toString() + } +} + +fun markdownToHtml(markdown : String) : String { + return MarkdownProcessor().parse(markdown).dump() +} + + +fun visit(sb: StringBuilder, indent: String, node: LighterASTNode, structure: FlyweightCapableTreeStructure<LighterASTNode>, markdown: String) { + sb.append(indent) + sb.append(node.getTokenType().toString()) + val nodeText = markdown.substring(node.getStartOffset(), node.getEndOffset()) + sb.append(":" + nodeText.replace("\n","\u23CE")) + sb.appendln() + val ref = Ref.create<Array<LighterASTNode>?>() + val count = structure.getChildren(node, ref) + val children = ref.get() + if (children == null) + return + for (index in 0..count - 1) { + val child = children[index] + visit(sb, indent + " ", child, structure, markdown) + } +}
\ No newline at end of file diff --git a/src/Markdown/MarkdownTokenType.kt b/src/Markdown/MarkdownTokenType.kt new file mode 100644 index 00000000..293228c3 --- /dev/null +++ b/src/Markdown/MarkdownTokenType.kt @@ -0,0 +1,6 @@ +package org.jetbrains.dokka.Markdown + +import com.intellij.psi.tree.IElementType + +public class MarkdownTokenType(debugName: String) : IElementType(debugName, null) { +}
\ No newline at end of file diff --git a/src/Markdown/_MarkdownLexer.flex b/src/Markdown/_MarkdownLexer.flex new file mode 100644 index 00000000..9c76da8f --- /dev/null +++ b/src/Markdown/_MarkdownLexer.flex @@ -0,0 +1,40 @@ +package org.jetbrains.markdown.impl; + +import com.intellij.lexer.*; +import com.intellij.psi.tree.IElementType; +import static org.jetbrains.markdown.MarkdownElementTypes.*; + +%% + +%{ + public _MarkdownLexer() { + this((java.io.Reader)null); + } +%} + +%public +%class _MarkdownLexer +%implements FlexLexer +%function advance +%type IElementType +%unicode + +Newline="\r"|"\n"|"\r\n" +Spacechar=[\ \t\f] +NUMBER=[0-9]+(\.[0-9]*)? +STRING=[^~\*_`&\[\]()<!#\\ \t\n\r]+ +ANYCHAR=. +Line=!'\r' !'\n' .* {Newline} + +%% +<YYINITIAL> { + {Spacechar} { return SPACECHAR; } + {Newline} { return NEWLINE; } + "\\357\\273\\277" { return BOM; } + + {NUMBER} { return NUMBER; } + {STRING} { return STRING; } + {ANYCHAR} { return ANYCHAR; } + + [^] { return com.intellij.psi.TokenType.BAD_CHARACTER; } +} diff --git a/src/Markdown/markdown.bnf b/src/Markdown/markdown.bnf new file mode 100644 index 00000000..b0a3ede6 --- /dev/null +++ b/src/Markdown/markdown.bnf @@ -0,0 +1,83 @@ +{ + psiPackage = 'org.jetbrains.markdown' + psiImplPackage = 'org.jetbrains.markdown.impl' + + parserClass="org.jetbrains.markdown.MarkdownParser" + parserUtilClass="org.jetbrains.dokka.Markdown.GeneratedParserUtilBase" + elementTypeHolderClass = 'org.jetbrains.markdown.MarkdownElementTypes' + + tokenTypeClass = 'org.jetbrains.dokka.Markdown.MarkdownTokenType' + + tokens=[ + LINE_WS='regexp:[\ \t\f]' + EOL='"\r"|"\n"|"\r\n"' + BOM = '\357\273\277' + number='regexp:\d+(\.\d*)?' + String='regexp:[^~\*_`&\[\]()<!#\\ \t\n\r]+' + AnyChar='regexp:.' + ] +} + +Document ::= BOM? ( Block )* + +OptionalSpace ::= Spacechar* +RequiredSpace ::= Spacechar+ +NonindentSpace ::= (" " | " " | " ")? + +BlankLine ::= OptionalSpace Newline + +Whitespace ::= Spacechar | Newline +EndLine ::= LineBreak | TerminalEndline | NormalEndline +NormalEndline ::= OptionalSpace Newline !BlankLine +TerminalEndline ::= OptionalSpace Newline <<eof>> +LineBreak ::= " " NormalEndline +Indent ::= "\t" | " " + +// ---- BLOCKS ---- +Block ::= BlankLine* ( + Para + | Plain + | OrderedList + | BulletList + ) + +Para ::= NonindentSpace Inlines (BlankLine+ | TerminalEndline) +Plain ::= Inlines + +HorizontalRule ::= NonindentSpace + ( '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')* + | '-' OptionalSpace '-' OptionalSpace '-' (OptionalSpace '-')* + | '_' OptionalSpace '_' OptionalSpace '_' (OptionalSpace '_')*) + OptionalSpace Newline BlankLine+ + +Bullet ::= !HorizontalRule NonindentSpace ('+' | '*' | '-') Spacechar+ +Enumerator ::= NonindentSpace number '.' Spacechar+ + +BulletList ::= &Bullet List +OrderedList ::= &Enumerator List + +List ::= (ListItem BlankLine*)+ +ListItem ::= (Bullet | Enumerator) ListBlock ( ListContinuationBlock )* + +ListBlock ::= !BlankLine Plain ( ListBlockLine )* +ListBlockLine ::= !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Plain + +ListContinuationBlock ::= BlankLine* (Indent ListBlock)+ + + +// ---- INLINES ---- +Inlines ::= (!EndLine Inline | EndLine &Inline )+ EndLine? +Inline ::= String | EndLine | RequiredSpace | Strong | Emph | Link + +Emph ::= EmphStar | EmphUnderscore +EmphStar ::= '*' !Whitespace (!'*' Inline)+ '*' +EmphUnderscore ::= '_' !Whitespace (!'_' Inline)+ '_' + +Strong ::= StrongStar | StrongUnderscore +StrongStar ::= '**' !Whitespace (!'**' Inline)+ '**' +StrongUnderscore ::= '__' !Whitespace (!'__' Inline)+ '__' + +Link ::= ReferenceLink +ReferenceLink ::= ReferenceLinkSingle +ReferenceLinkSingle ::= '[' Target ']' +Target ::= String
\ No newline at end of file diff --git a/src/Markdown/markdown.leg b/src/Markdown/markdown.leg new file mode 100644 index 00000000..ea8bc522 --- /dev/null +++ b/src/Markdown/markdown.leg @@ -0,0 +1,781 @@ +%{ +/********************************************************************** + + markdown_parser.leg - markdown parser in C using a PEG grammar. + (c) 2008 John MacFarlane (jgm at berkeley dot edu). + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License or the MIT + license. See LICENSE for details. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + ***********************************************************************/ + +#include <stdbool.h> +#include <assert.h> +#include "markdown_peg.h" +#include "utility_functions.h" + + + +/********************************************************************** + + Definitions for leg parser generator. + YY_INPUT is the function the parser calls to get new input. + We take all new input from (static) charbuf. + + ***********************************************************************/ + + + +# define YYSTYPE element * +#ifdef __DEBUG__ +# define YY_DEBUG 1 +#endif + +#define YY_INPUT(buf, result, max_size) \ +{ \ + int yyc; \ + if (charbuf && *charbuf != '\0') { \ + yyc= *charbuf++; \ + } else { \ + yyc= EOF; \ + } \ + result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ +} + +#define YY_RULE(T) T + + +/********************************************************************** + + PEG grammar and parser actions for markdown syntax. + + ***********************************************************************/ + +%} + +Doc = BOM? a:StartList ( Block { a = cons($$, a); } )* + { parse_result = reverse(a); } + +Block = BlankLine* + ( BlockQuote + | Verbatim + | Note + | Reference + | HorizontalRule + | Heading + | OrderedList + | BulletList + | HtmlBlock + | StyleBlock + | Para + | Plain ) + +Para = NonindentSpace a:Inlines BlankLine+ + { $$ = a; $$->key = PARA; } + +Plain = a:Inlines + { $$ = a; $$->key = PLAIN; } + +AtxInline = !Newline !(Sp '#'* Sp Newline) Inline + +AtxStart = < ( "######" | "#####" | "####" | "###" | "##" | "#" ) > + { $$ = mk_element(H1 + (strlen(yytext) - 1)); } + +AtxHeading = s:AtxStart Sp a:StartList ( AtxInline { a = cons($$, a); } )+ (Sp '#'* Sp)? Newline + { $$ = mk_list(s->key, a); + free(s); } + +SetextHeading = SetextHeading1 | SetextHeading2 + +SetextBottom1 = '='+ Newline + +SetextBottom2 = '-'+ Newline + +SetextHeading1 = &(RawLine SetextBottom1) + a:StartList ( !Endline Inline { a = cons($$, a); } )+ Sp Newline + SetextBottom1 { $$ = mk_list(H1, a); } + +SetextHeading2 = &(RawLine SetextBottom2) + a:StartList ( !Endline Inline { a = cons($$, a); } )+ Sp Newline + SetextBottom2 { $$ = mk_list(H2, a); } + +Heading = SetextHeading | AtxHeading + +BlockQuote = a:BlockQuoteRaw + { $$ = mk_element(BLOCKQUOTE); + $$->children = a; + } + +BlockQuoteRaw = a:StartList + (( '>' ' '? Line { a = cons($$, a); } ) + ( !'>' !BlankLine Line { a = cons($$, a); } )* + ( BlankLine { a = cons(mk_str("\n"), a); } )* + )+ + { $$ = mk_str_from_list(a, true); + $$->key = RAW; + } + +NonblankIndentedLine = !BlankLine IndentedLine + +VerbatimChunk = a:StartList + ( BlankLine { a = cons(mk_str("\n"), a); } )* + ( NonblankIndentedLine { a = cons($$, a); } )+ + { $$ = mk_str_from_list(a, false); } + +Verbatim = a:StartList ( VerbatimChunk { a = cons($$, a); } )+ + { $$ = mk_str_from_list(a, false); + $$->key = VERBATIM; } + +HorizontalRule = NonindentSpace + ( '*' Sp '*' Sp '*' (Sp '*')* + | '-' Sp '-' Sp '-' (Sp '-')* + | '_' Sp '_' Sp '_' (Sp '_')*) + Sp Newline BlankLine+ + { $$ = mk_element(HRULE); } + +Bullet = !HorizontalRule NonindentSpace ('+' | '*' | '-') Spacechar+ + +BulletList = &Bullet (ListTight | ListLoose) + { $$->key = BULLETLIST; } + +ListTight = a:StartList + ( ListItemTight { a = cons($$, a); } )+ + BlankLine* !(Bullet | Enumerator) + { $$ = mk_list(LIST, a); } + +ListLoose = a:StartList + ( b:ListItem BlankLine* + { element *li; + li = b->children; + li->contents.str = realloc(li->contents.str, strlen(li->contents.str) + 3); + strcat(li->contents.str, "\n\n"); /* In loose list, \n\n added to end of each element */ + a = cons(b, a); + } )+ + { $$ = mk_list(LIST, a); } + +ListItem = ( Bullet | Enumerator ) + a:StartList + ListBlock { a = cons($$, a); } + ( ListContinuationBlock { a = cons($$, a); } )* + { element *raw; + raw = mk_str_from_list(a, false); + raw->key = RAW; + $$ = mk_element(LISTITEM); + $$->children = raw; + } + +ListItemTight = + ( Bullet | Enumerator ) + a:StartList + ListBlock { a = cons($$, a); } + ( !BlankLine + ListContinuationBlock { a = cons($$, a); } )* + !ListContinuationBlock + { element *raw; + raw = mk_str_from_list(a, false); + raw->key = RAW; + $$ = mk_element(LISTITEM); + $$->children = raw; + } + +ListBlock = a:StartList + !BlankLine Line { a = cons($$, a); } + ( ListBlockLine { a = cons($$, a); } )* + { $$ = mk_str_from_list(a, false); } + +ListContinuationBlock = a:StartList + ( < BlankLine* > + { if (strlen(yytext) == 0) + a = cons(mk_str("\001"), a); /* block separator */ + else + a = cons(mk_str(yytext), a); } ) + ( Indent ListBlock { a = cons($$, a); } )+ + { $$ = mk_str_from_list(a, false); } + +Enumerator = NonindentSpace [0-9]+ '.' Spacechar+ + +OrderedList = &Enumerator (ListTight | ListLoose) + { $$->key = ORDEREDLIST; } + +ListBlockLine = !BlankLine + !( Indent? (Bullet | Enumerator) ) + !HorizontalRule + OptionallyIndentedLine + +# Parsers for different kinds of block-level HTML content. +# This is repetitive due to constraints of PEG grammar. + +HtmlBlockOpenAddress = '<' Spnl ("address" | "ADDRESS") Spnl HtmlAttribute* '>' +HtmlBlockCloseAddress = '<' Spnl '/' ("address" | "ADDRESS") Spnl '>' +HtmlBlockAddress = HtmlBlockOpenAddress (HtmlBlockAddress | !HtmlBlockCloseAddress .)* HtmlBlockCloseAddress + +HtmlBlockOpenBlockquote = '<' Spnl ("blockquote" | "BLOCKQUOTE") Spnl HtmlAttribute* '>' +HtmlBlockCloseBlockquote = '<' Spnl '/' ("blockquote" | "BLOCKQUOTE") Spnl '>' +HtmlBlockBlockquote = HtmlBlockOpenBlockquote (HtmlBlockBlockquote | !HtmlBlockCloseBlockquote .)* HtmlBlockCloseBlockquote + +HtmlBlockOpenCenter = '<' Spnl ("center" | "CENTER") Spnl HtmlAttribute* '>' +HtmlBlockCloseCenter = '<' Spnl '/' ("center" | "CENTER") Spnl '>' +HtmlBlockCenter = HtmlBlockOpenCenter (HtmlBlockCenter | !HtmlBlockCloseCenter .)* HtmlBlockCloseCenter + +HtmlBlockOpenDir = '<' Spnl ("dir" | "DIR") Spnl HtmlAttribute* '>' +HtmlBlockCloseDir = '<' Spnl '/' ("dir" | "DIR") Spnl '>' +HtmlBlockDir = HtmlBlockOpenDir (HtmlBlockDir | !HtmlBlockCloseDir .)* HtmlBlockCloseDir + +HtmlBlockOpenDiv = '<' Spnl ("div" | "DIV") Spnl HtmlAttribute* '>' +HtmlBlockCloseDiv = '<' Spnl '/' ("div" | "DIV") Spnl '>' +HtmlBlockDiv = HtmlBlockOpenDiv (HtmlBlockDiv | !HtmlBlockCloseDiv .)* HtmlBlockCloseDiv + +HtmlBlockOpenDl = '<' Spnl ("dl" | "DL") Spnl HtmlAttribute* '>' +HtmlBlockCloseDl = '<' Spnl '/' ("dl" | "DL") Spnl '>' +HtmlBlockDl = HtmlBlockOpenDl (HtmlBlockDl | !HtmlBlockCloseDl .)* HtmlBlockCloseDl + +HtmlBlockOpenFieldset = '<' Spnl ("fieldset" | "FIELDSET") Spnl HtmlAttribute* '>' +HtmlBlockCloseFieldset = '<' Spnl '/' ("fieldset" | "FIELDSET") Spnl '>' +HtmlBlockFieldset = HtmlBlockOpenFieldset (HtmlBlockFieldset | !HtmlBlockCloseFieldset .)* HtmlBlockCloseFieldset + +HtmlBlockOpenForm = '<' Spnl ("form" | "FORM") Spnl HtmlAttribute* '>' +HtmlBlockCloseForm = '<' Spnl '/' ("form" | "FORM") Spnl '>' +HtmlBlockForm = HtmlBlockOpenForm (HtmlBlockForm | !HtmlBlockCloseForm .)* HtmlBlockCloseForm + +HtmlBlockOpenH1 = '<' Spnl ("h1" | "H1") Spnl HtmlAttribute* '>' +HtmlBlockCloseH1 = '<' Spnl '/' ("h1" | "H1") Spnl '>' +HtmlBlockH1 = HtmlBlockOpenH1 (HtmlBlockH1 | !HtmlBlockCloseH1 .)* HtmlBlockCloseH1 + +HtmlBlockOpenH2 = '<' Spnl ("h2" | "H2") Spnl HtmlAttribute* '>' +HtmlBlockCloseH2 = '<' Spnl '/' ("h2" | "H2") Spnl '>' +HtmlBlockH2 = HtmlBlockOpenH2 (HtmlBlockH2 | !HtmlBlockCloseH2 .)* HtmlBlockCloseH2 + +HtmlBlockOpenH3 = '<' Spnl ("h3" | "H3") Spnl HtmlAttribute* '>' +HtmlBlockCloseH3 = '<' Spnl '/' ("h3" | "H3") Spnl '>' +HtmlBlockH3 = HtmlBlockOpenH3 (HtmlBlockH3 | !HtmlBlockCloseH3 .)* HtmlBlockCloseH3 + +HtmlBlockOpenH4 = '<' Spnl ("h4" | "H4") Spnl HtmlAttribute* '>' +HtmlBlockCloseH4 = '<' Spnl '/' ("h4" | "H4") Spnl '>' +HtmlBlockH4 = HtmlBlockOpenH4 (HtmlBlockH4 | !HtmlBlockCloseH4 .)* HtmlBlockCloseH4 + +HtmlBlockOpenH5 = '<' Spnl ("h5" | "H5") Spnl HtmlAttribute* '>' +HtmlBlockCloseH5 = '<' Spnl '/' ("h5" | "H5") Spnl '>' +HtmlBlockH5 = HtmlBlockOpenH5 (HtmlBlockH5 | !HtmlBlockCloseH5 .)* HtmlBlockCloseH5 + +HtmlBlockOpenH6 = '<' Spnl ("h6" | "H6") Spnl HtmlAttribute* '>' +HtmlBlockCloseH6 = '<' Spnl '/' ("h6" | "H6") Spnl '>' +HtmlBlockH6 = HtmlBlockOpenH6 (HtmlBlockH6 | !HtmlBlockCloseH6 .)* HtmlBlockCloseH6 + +HtmlBlockOpenMenu = '<' Spnl ("menu" | "MENU") Spnl HtmlAttribute* '>' +HtmlBlockCloseMenu = '<' Spnl '/' ("menu" | "MENU") Spnl '>' +HtmlBlockMenu = HtmlBlockOpenMenu (HtmlBlockMenu | !HtmlBlockCloseMenu .)* HtmlBlockCloseMenu + +HtmlBlockOpenNoframes = '<' Spnl ("noframes" | "NOFRAMES") Spnl HtmlAttribute* '>' +HtmlBlockCloseNoframes = '<' Spnl '/' ("noframes" | "NOFRAMES") Spnl '>' +HtmlBlockNoframes = HtmlBlockOpenNoframes (HtmlBlockNoframes | !HtmlBlockCloseNoframes .)* HtmlBlockCloseNoframes + +HtmlBlockOpenNoscript = '<' Spnl ("noscript" | "NOSCRIPT") Spnl HtmlAttribute* '>' +HtmlBlockCloseNoscript = '<' Spnl '/' ("noscript" | "NOSCRIPT") Spnl '>' +HtmlBlockNoscript = HtmlBlockOpenNoscript (HtmlBlockNoscript | !HtmlBlockCloseNoscript .)* HtmlBlockCloseNoscript + +HtmlBlockOpenOl = '<' Spnl ("ol" | "OL") Spnl HtmlAttribute* '>' +HtmlBlockCloseOl = '<' Spnl '/' ("ol" | "OL") Spnl '>' +HtmlBlockOl = HtmlBlockOpenOl (HtmlBlockOl | !HtmlBlockCloseOl .)* HtmlBlockCloseOl + +HtmlBlockOpenP = '<' Spnl ("p" | "P") Spnl HtmlAttribute* '>' +HtmlBlockCloseP = '<' Spnl '/' ("p" | "P") Spnl '>' +HtmlBlockP = HtmlBlockOpenP (HtmlBlockP | !HtmlBlockCloseP .)* HtmlBlockCloseP + +HtmlBlockOpenPre = '<' Spnl ("pre" | "PRE") Spnl HtmlAttribute* '>' +HtmlBlockClosePre = '<' Spnl '/' ("pre" | "PRE") Spnl '>' +HtmlBlockPre = HtmlBlockOpenPre (HtmlBlockPre | !HtmlBlockClosePre .)* HtmlBlockClosePre + +HtmlBlockOpenTable = '<' Spnl ("table" | "TABLE") Spnl HtmlAttribute* '>' +HtmlBlockCloseTable = '<' Spnl '/' ("table" | "TABLE") Spnl '>' +HtmlBlockTable = HtmlBlockOpenTable (HtmlBlockTable | !HtmlBlockCloseTable .)* HtmlBlockCloseTable + +HtmlBlockOpenUl = '<' Spnl ("ul" | "UL") Spnl HtmlAttribute* '>' +HtmlBlockCloseUl = '<' Spnl '/' ("ul" | "UL") Spnl '>' +HtmlBlockUl = HtmlBlockOpenUl (HtmlBlockUl | !HtmlBlockCloseUl .)* HtmlBlockCloseUl + +HtmlBlockOpenDd = '<' Spnl ("dd" | "DD") Spnl HtmlAttribute* '>' +HtmlBlockCloseDd = '<' Spnl '/' ("dd" | "DD") Spnl '>' +HtmlBlockDd = HtmlBlockOpenDd (HtmlBlockDd | !HtmlBlockCloseDd .)* HtmlBlockCloseDd + +HtmlBlockOpenDt = '<' Spnl ("dt" | "DT") Spnl HtmlAttribute* '>' +HtmlBlockCloseDt = '<' Spnl '/' ("dt" | "DT") Spnl '>' +HtmlBlockDt = HtmlBlockOpenDt (HtmlBlockDt | !HtmlBlockCloseDt .)* HtmlBlockCloseDt + +HtmlBlockOpenFrameset = '<' Spnl ("frameset" | "FRAMESET") Spnl HtmlAttribute* '>' +HtmlBlockCloseFrameset = '<' Spnl '/' ("frameset" | "FRAMESET") Spnl '>' +HtmlBlockFrameset = HtmlBlockOpenFrameset (HtmlBlockFrameset | !HtmlBlockCloseFrameset .)* HtmlBlockCloseFrameset + +HtmlBlockOpenLi = '<' Spnl ("li" | "LI") Spnl HtmlAttribute* '>' +HtmlBlockCloseLi = '<' Spnl '/' ("li" | "LI") Spnl '>' +HtmlBlockLi = HtmlBlockOpenLi (HtmlBlockLi | !HtmlBlockCloseLi .)* HtmlBlockCloseLi + +HtmlBlockOpenTbody = '<' Spnl ("tbody" | "TBODY") Spnl HtmlAttribute* '>' +HtmlBlockCloseTbody = '<' Spnl '/' ("tbody" | "TBODY") Spnl '>' +HtmlBlockTbody = HtmlBlockOpenTbody (HtmlBlockTbody | !HtmlBlockCloseTbody .)* HtmlBlockCloseTbody + +HtmlBlockOpenTd = '<' Spnl ("td" | "TD") Spnl HtmlAttribute* '>' +HtmlBlockCloseTd = '<' Spnl '/' ("td" | "TD") Spnl '>' +HtmlBlockTd = HtmlBlockOpenTd (HtmlBlockTd | !HtmlBlockCloseTd .)* HtmlBlockCloseTd + +HtmlBlockOpenTfoot = '<' Spnl ("tfoot" | "TFOOT") Spnl HtmlAttribute* '>' +HtmlBlockCloseTfoot = '<' Spnl '/' ("tfoot" | "TFOOT") Spnl '>' +HtmlBlockTfoot = HtmlBlockOpenTfoot (HtmlBlockTfoot | !HtmlBlockCloseTfoot .)* HtmlBlockCloseTfoot + +HtmlBlockOpenTh = '<' Spnl ("th" | "TH") Spnl HtmlAttribute* '>' +HtmlBlockCloseTh = '<' Spnl '/' ("th" | "TH") Spnl '>' +HtmlBlockTh = HtmlBlockOpenTh (HtmlBlockTh | !HtmlBlockCloseTh .)* HtmlBlockCloseTh + +HtmlBlockOpenThead = '<' Spnl ("thead" | "THEAD") Spnl HtmlAttribute* '>' +HtmlBlockCloseThead = '<' Spnl '/' ("thead" | "THEAD") Spnl '>' +HtmlBlockThead = HtmlBlockOpenThead (HtmlBlockThead | !HtmlBlockCloseThead .)* HtmlBlockCloseThead + +HtmlBlockOpenTr = '<' Spnl ("tr" | "TR") Spnl HtmlAttribute* '>' +HtmlBlockCloseTr = '<' Spnl '/' ("tr" | "TR") Spnl '>' +HtmlBlockTr = HtmlBlockOpenTr (HtmlBlockTr | !HtmlBlockCloseTr .)* HtmlBlockCloseTr + +HtmlBlockOpenScript = '<' Spnl ("script" | "SCRIPT") Spnl HtmlAttribute* '>' +HtmlBlockCloseScript = '<' Spnl '/' ("script" | "SCRIPT") Spnl '>' +HtmlBlockScript = HtmlBlockOpenScript (!HtmlBlockCloseScript .)* HtmlBlockCloseScript + +HtmlBlockOpenHead = '<' Spnl ("head" | "HEAD") Spnl HtmlAttribute* '>' +HtmlBlockCloseHead = '<' Spnl '/' ("head" | "HEAD") Spnl '>' +HtmlBlockHead = HtmlBlockOpenHead (!HtmlBlockCloseHead .)* HtmlBlockCloseHead + +HtmlBlockInTags = HtmlBlockAddress + | HtmlBlockBlockquote + | HtmlBlockCenter + | HtmlBlockDir + | HtmlBlockDiv + | HtmlBlockDl + | HtmlBlockFieldset + | HtmlBlockForm + | HtmlBlockH1 + | HtmlBlockH2 + | HtmlBlockH3 + | HtmlBlockH4 + | HtmlBlockH5 + | HtmlBlockH6 + | HtmlBlockMenu + | HtmlBlockNoframes + | HtmlBlockNoscript + | HtmlBlockOl + | HtmlBlockP + | HtmlBlockPre + | HtmlBlockTable + | HtmlBlockUl + | HtmlBlockDd + | HtmlBlockDt + | HtmlBlockFrameset + | HtmlBlockLi + | HtmlBlockTbody + | HtmlBlockTd + | HtmlBlockTfoot + | HtmlBlockTh + | HtmlBlockThead + | HtmlBlockTr + | HtmlBlockScript + | HtmlBlockHead + +HtmlBlock = < ( HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing ) > + BlankLine+ + { if (extension(EXT_FILTER_HTML)) { + $$ = mk_list(LIST, NULL); + } else { + $$ = mk_str(yytext); + $$->key = HTMLBLOCK; + } + } + +HtmlBlockSelfClosing = '<' Spnl HtmlBlockType Spnl HtmlAttribute* '/' Spnl '>' + +HtmlBlockType = "address" | "blockquote" | "center" | "dir" | "div" | "dl" | "fieldset" | "form" | "h1" | "h2" | "h3" | + "h4" | "h5" | "h6" | "hr" | "isindex" | "menu" | "noframes" | "noscript" | "ol" | "p" | "pre" | "table" | + "ul" | "dd" | "dt" | "frameset" | "li" | "tbody" | "td" | "tfoot" | "th" | "thead" | "tr" | "script" | + "ADDRESS" | "BLOCKQUOTE" | "CENTER" | "DIR" | "DIV" | "DL" | "FIELDSET" | "FORM" | "H1" | "H2" | "H3" | + "H4" | "H5" | "H6" | "HR" | "ISINDEX" | "MENU" | "NOFRAMES" | "NOSCRIPT" | "OL" | "P" | "PRE" | "TABLE" | + "UL" | "DD" | "DT" | "FRAMESET" | "LI" | "TBODY" | "TD" | "TFOOT" | "TH" | "THEAD" | "TR" | "SCRIPT" + +StyleOpen = '<' Spnl ("style" | "STYLE") Spnl HtmlAttribute* '>' +StyleClose = '<' Spnl '/' ("style" | "STYLE") Spnl '>' +InStyleTags = StyleOpen (!StyleClose .)* StyleClose +StyleBlock = < InStyleTags > + BlankLine* + { if (extension(EXT_FILTER_STYLES)) { + $$ = mk_list(LIST, NULL); + } else { + $$ = mk_str(yytext); + $$->key = HTMLBLOCK; + } + } + +Inlines = a:StartList ( !Endline Inline { a = cons($$, a); } + | c:Endline &Inline { a = cons(c, a); } )+ Endline? + { $$ = mk_list(LIST, a); } + +Inline = Str + | Endline + | UlOrStarLine + | Space + | Strong + | Emph + | Strike + | Image + | Link + | NoteReference + | InlineNote + | Code + | RawHtml + | Entity + | EscapedChar + | Smart + | Symbol + +Space = Spacechar+ + { $$ = mk_str(" "); + $$->key = SPACE; } + +Str = a:StartList < NormalChar+ > { a = cons(mk_str(yytext), a); } + ( StrChunk { a = cons($$, a); } )* + { if (a->next == NULL) { $$ = a; } else { $$ = mk_list(LIST, a); } } + +StrChunk = < (NormalChar | '_'+ &Alphanumeric)+ > { $$ = mk_str(yytext); } | + AposChunk + +AposChunk = &{ extension(EXT_SMART) } '\'' &Alphanumeric + { $$ = mk_element(APOSTROPHE); } + +EscapedChar = '\\' !Newline < [-\\`|*_{}[\]()#+.!><] > + { $$ = mk_str(yytext); } + +Entity = ( HexEntity | DecEntity | CharEntity ) + { $$ = mk_str(yytext); $$->key = HTML; } + +Endline = LineBreak | TerminalEndline | NormalEndline + +NormalEndline = Sp Newline !BlankLine !'>' !AtxStart + !(Line ('='+ | '-'+) Newline) + { $$ = mk_str("\n"); + $$->key = SPACE; } + +TerminalEndline = Sp Newline Eof + { $$ = NULL; } + +LineBreak = " " NormalEndline + { $$ = mk_element(LINEBREAK); } + +Symbol = < SpecialChar > + { $$ = mk_str(yytext); } + +# This keeps the parser from getting bogged down on long strings of '*' or '_', +# or strings of '*' or '_' with space on each side: +UlOrStarLine = (UlLine | StarLine) { $$ = mk_str(yytext); } +StarLine = < "****" '*'* > | < Spacechar '*'+ &Spacechar > +UlLine = < "____" '_'* > | < Spacechar '_'+ &Spacechar > + +Emph = EmphStar | EmphUl + +Whitespace = Spacechar | Newline + +EmphStar = '*' !Whitespace + a:StartList + ( !'*' b:Inline { a = cons(b, a); } + | b:StrongStar { a = cons(b, a); } + )+ + '*' + { $$ = mk_list(EMPH, a); } + +EmphUl = '_' !Whitespace + a:StartList + ( !'_' b:Inline { a = cons(b, a); } + | b:StrongUl { a = cons(b, a); } + )+ + '_' + { $$ = mk_list(EMPH, a); } + +Strong = StrongStar | StrongUl + +StrongStar = "**" !Whitespace + a:StartList + ( !"**" b:Inline { a = cons(b, a); })+ + "**" + { $$ = mk_list(STRONG, a); } + +StrongUl = "__" !Whitespace + a:StartList + ( !"__" b:Inline { a = cons(b, a); })+ + "__" + { $$ = mk_list(STRONG, a); } + +Strike = &{ extension(EXT_STRIKE) } + "~~" !Whitespace + a:StartList + ( !"~~" b:Inline { a = cons(b, a); })+ + "~~" + { $$ = mk_list(STRIKE, a); } + +Image = '!' ( ExplicitLink | ReferenceLink ) + { if ($$->key == LINK) { + $$->key = IMAGE; + } else { + element *result; + result = $$; + $$->children = cons(mk_str("!"), result->children); + } } + +Link = ExplicitLink | ReferenceLink | AutoLink + +ReferenceLink = ReferenceLinkDouble | ReferenceLinkSingle + +ReferenceLinkDouble = a:Label < Spnl > !"[]" b:Label + { link match; + if (find_reference(&match, b->children)) { + $$ = mk_link(a->children, match.url, match.title); + free(a); + free_element_list(b); + } else { + element *result; + result = mk_element(LIST); + result->children = cons(mk_str("["), cons(a, cons(mk_str("]"), cons(mk_str(yytext), + cons(mk_str("["), cons(b, mk_str("]"))))))); + $$ = result; + } + } + +ReferenceLinkSingle = a:Label < (Spnl "[]")? > + { link match; + if (find_reference(&match, a->children)) { + $$ = mk_link(a->children, match.url, match.title); + free(a); + } + else { + element *result; + result = mk_element(LIST); + result->children = cons(mk_str("["), cons(a, cons(mk_str("]"), mk_str(yytext)))); + $$ = result; + } + } + +ExplicitLink = l:Label '(' Sp s:Source Spnl t:Title Sp ')' + { $$ = mk_link(l->children, s->contents.str, t->contents.str); + free_element(s); + free_element(t); + free(l); } + +Source = ( '<' < SourceContents > '>' | < SourceContents > ) + { $$ = mk_str(yytext); } + +SourceContents = ( ( !'(' !')' !'>' Nonspacechar )+ | '(' SourceContents ')')* + +Title = ( TitleSingle | TitleDouble | < "" > ) + { $$ = mk_str(yytext); } + +TitleSingle = '\'' < ( !( '\'' Sp ( ')' | Newline ) ) . )* > '\'' + +TitleDouble = '"' < ( !( '"' Sp ( ')' | Newline ) ) . )* > '"' + +AutoLink = AutoLinkUrl | AutoLinkEmail + +AutoLinkUrl = '<' < [A-Za-z]+ "://" ( !Newline !'>' . )+ > '>' + { $$ = mk_link(mk_str(yytext), yytext, ""); } + +AutoLinkEmail = '<' ( "mailto:" )? < [-A-Za-z0-9+_./!%~$]+ '@' ( !Newline !'>' . )+ > '>' + { char *mailto = malloc(strlen(yytext) + 8); + sprintf(mailto, "mailto:%s", yytext); + $$ = mk_link(mk_str(yytext), mailto, ""); + free(mailto); + } + +Reference = NonindentSpace !"[]" l:Label ':' Spnl s:RefSrc t:RefTitle BlankLine+ + { $$ = mk_link(l->children, s->contents.str, t->contents.str); + free_element(s); + free_element(t); + free(l); + $$->key = REFERENCE; } + +Label = '[' ( !'^' &{ extension(EXT_NOTES) } | &. &{ !extension(EXT_NOTES) } ) + a:StartList + ( !']' Inline { a = cons($$, a); } )* + ']' + { $$ = mk_list(LIST, a); } + +RefSrc = < Nonspacechar+ > + { $$ = mk_str(yytext); + $$->key = HTML; } + +RefTitle = ( RefTitleSingle | RefTitleDouble | RefTitleParens | EmptyTitle ) + { $$ = mk_str(yytext); } + +EmptyTitle = < "" > + +RefTitleSingle = Spnl '\'' < ( !( '\'' Sp Newline | Newline ) . )* > '\'' + +RefTitleDouble = Spnl '"' < ( !('"' Sp Newline | Newline) . )* > '"' + +RefTitleParens = Spnl '(' < ( !(')' Sp Newline | Newline) . )* > ')' + +References = a:StartList + ( b:Reference { a = cons(b, a); } | SkipBlock )* + { references = reverse(a); } + +Ticks1 = "`" !'`' +Ticks2 = "``" !'`' +Ticks3 = "```" !'`' +Ticks4 = "````" !'`' +Ticks5 = "`````" !'`' + +Code = ( Ticks1 Sp < ( ( !'`' Nonspacechar )+ | !Ticks1 '`'+ | !( Sp Ticks1 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks1 + | Ticks2 Sp < ( ( !'`' Nonspacechar )+ | !Ticks2 '`'+ | !( Sp Ticks2 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks2 + | Ticks3 Sp < ( ( !'`' Nonspacechar )+ | !Ticks3 '`'+ | !( Sp Ticks3 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks3 + | Ticks4 Sp < ( ( !'`' Nonspacechar )+ | !Ticks4 '`'+ | !( Sp Ticks4 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks4 + | Ticks5 Sp < ( ( !'`' Nonspacechar )+ | !Ticks5 '`'+ | !( Sp Ticks5 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks5 + ) + { $$ = mk_str(yytext); $$->key = CODE; } + +RawHtml = < (HtmlComment | HtmlBlockScript | HtmlTag) > + { if (extension(EXT_FILTER_HTML)) { + $$ = mk_list(LIST, NULL); + } else { + $$ = mk_str(yytext); + $$->key = HTML; + } + } + +BlankLine = Sp Newline + +Quoted = '"' (!'"' .)* '"' | '\'' (!'\'' .)* '\'' +HtmlAttribute = (AlphanumericAscii | '-')+ Spnl ('=' Spnl (Quoted | (!'>' Nonspacechar)+))? Spnl +HtmlComment = "<!--" (!"-->" .)* "-->" +HtmlTag = '<' Spnl '/'? AlphanumericAscii+ Spnl HtmlAttribute* '/'? Spnl '>' +Eof = !. +Spacechar = ' ' | '\t' +Nonspacechar = !Spacechar !Newline . +Newline = '\n' | '\r' '\n'? +Sp = Spacechar* +Spnl = Sp (Newline Sp)? +SpecialChar = '~' | '*' | '_' | '`' | '&' | '[' | ']' | '(' | ')' | '<' | '!' | '#' | '\\' | '\'' | '"' | ExtendedSpecialChar +NormalChar = !( SpecialChar | Spacechar | Newline ) . +Alphanumeric = [0-9A-Za-z] | '\200' | '\201' | '\202' | '\203' | '\204' | '\205' | '\206' | '\207' | '\210' | '\211' | '\212' | '\213' | '\214' | '\215' | '\216' | '\217' | '\220' | '\221' | '\222' | '\223' | '\224' | '\225' | '\226' | '\227' | '\230' | '\231' | '\232' | '\233' | '\234' | '\235' | '\236' | '\237' | '\240' | '\241' | '\242' | '\243' | '\244' | '\245' | '\246' | '\247' | '\250' | '\251' | '\252' | '\253' | '\254' | '\255' | '\256' | '\257' | '\260' | '\261' | '\262' | '\263' | '\264' | '\265' | '\266' | '\267' | '\270' | '\271' | '\272' | '\273' | '\274' | '\275' | '\276' | '\277' | '\300' | '\301' | '\302' | '\303' | '\304' | '\305' | '\306' | '\307' | '\310' | '\311' | '\312' | '\313' | '\314' | '\315' | '\316' | '\317' | '\320' | '\321' | '\322' | '\323' | '\324' | '\325' | '\326' | '\327' | '\330' | '\331' | '\332' | '\333' | '\334' | '\335' | '\336' | '\337' | '\340' | '\341' | '\342' | '\343' | '\344' | '\345' | '\346' | '\347' | '\350' | '\351' | '\352' | '\353' | '\354' | '\355' | '\356' | '\357' | '\360' | '\361' | '\362' | '\363' | '\364' | '\365' | '\366' | '\367' | '\370' | '\371' | '\372' | '\373' | '\374' | '\375' | '\376' | '\377' +AlphanumericAscii = [A-Za-z0-9] +Digit = [0-9] +BOM = "\357\273\277" + +HexEntity = < '&' '#' [Xx] [0-9a-fA-F]+ ';' > +DecEntity = < '&' '#' [0-9]+ > ';' > +CharEntity = < '&' [A-Za-z0-9]+ ';' > + +NonindentSpace = " " | " " | " " | "" +Indent = "\t" | " " +IndentedLine = Indent Line +OptionallyIndentedLine = Indent? Line + +# StartList starts a list data structure that can be added to with cons: +StartList = &. + { $$ = NULL; } + +Line = RawLine + { $$ = mk_str(yytext); } +RawLine = ( < (!'\r' !'\n' .)* Newline > | < .+ > Eof ) + +SkipBlock = HtmlBlock + | ( !'#' !SetextBottom1 !SetextBottom2 !BlankLine RawLine )+ BlankLine* + | BlankLine+ + | RawLine + +# Syntax extensions + +ExtendedSpecialChar = &{ extension(EXT_SMART) } ('.' | '-' | '\'' | '"') + | &{ extension(EXT_NOTES) } ( '^' ) + +Smart = &{ extension(EXT_SMART) } + ( Ellipsis | Dash | SingleQuoted | DoubleQuoted | Apostrophe ) + +Apostrophe = '\'' + { $$ = mk_element(APOSTROPHE); } + +Ellipsis = ("..." | ". . .") + { $$ = mk_element(ELLIPSIS); } + +Dash = EmDash | EnDash + +EnDash = '-' &Digit + { $$ = mk_element(ENDASH); } + +EmDash = ("---" | "--") + { $$ = mk_element(EMDASH); } + +SingleQuoteStart = '\'' !(Spacechar | Newline) + +SingleQuoteEnd = '\'' !Alphanumeric + +SingleQuoted = SingleQuoteStart + a:StartList + ( !SingleQuoteEnd b:Inline { a = cons(b, a); } )+ + SingleQuoteEnd + { $$ = mk_list(SINGLEQUOTED, a); } + +DoubleQuoteStart = '"' + +DoubleQuoteEnd = '"' + +DoubleQuoted = DoubleQuoteStart + a:StartList + ( !DoubleQuoteEnd b:Inline { a = cons(b, a); } )+ + DoubleQuoteEnd + { $$ = mk_list(DOUBLEQUOTED, a); } + +NoteReference = &{ extension(EXT_NOTES) } + ref:RawNoteReference + { element *match; + if (find_note(&match, ref->contents.str)) { + $$ = mk_element(NOTE); + assert(match->children != NULL); + $$->children = match->children; + $$->contents.str = 0; + } else { + char *s; + s = malloc(strlen(ref->contents.str) + 4); + sprintf(s, "[^%s]", ref->contents.str); + $$ = mk_str(s); + free(s); + } + } + +RawNoteReference = "[^" < ( !Newline !']' . )+ > ']' + { $$ = mk_str(yytext); } + +Note = &{ extension(EXT_NOTES) } + NonindentSpace ref:RawNoteReference ':' Sp + a:StartList + ( RawNoteBlock { a = cons($$, a); } ) + ( &Indent RawNoteBlock { a = cons($$, a); } )* + { $$ = mk_list(NOTE, a); + $$->contents.str = strdup(ref->contents.str); + } + +InlineNote = &{ extension(EXT_NOTES) } + "^[" + a:StartList + ( !']' Inline { a = cons($$, a); } )+ + ']' + { $$ = mk_list(NOTE, a); + $$->contents.str = 0; } + +Notes = a:StartList + ( b:Note { a = cons(b, a); } | SkipBlock )* + { notes = reverse(a); } + +RawNoteBlock = a:StartList + ( !BlankLine OptionallyIndentedLine { a = cons($$, a); } )+ + ( < BlankLine* > { a = cons(mk_str(yytext), a); } ) + { $$ = mk_str_from_list(a, true); + $$->key = RAW; + } + +%% + diff --git a/src/Model/DocumentationContent.kt b/src/Model/DocumentationContent.kt index cebb429b..77e8c764 100644 --- a/src/Model/DocumentationContent.kt +++ b/src/Model/DocumentationContent.kt @@ -9,40 +9,38 @@ public class DocumentationContentSection(public val label: String, public val te } } -// TODO: refactor sections to map -public class DocumentationContent(public val summary: RichString, - public val description: RichString, - public val sections: List<DocumentationContentSection>) { +public class DocumentationContent(public val sections: Map<String, DocumentationContentSection>) { + + public val summary: RichString get() = sections["\$summary"]?.text ?: RichString.empty + public val description: RichString get() = sections["\$description"]?.text ?: RichString.empty override fun equals(other: Any?): Boolean { if (other !is DocumentationContent) return false - if (summary != other.summary) - return false if (sections.size != other.sections.size) return false - for (index in sections.indices) - if (sections[index] != other.sections[index]) + for (keys in sections.keySet()) + if (sections[keys] != other.sections[keys]) return false return true } override fun hashCode(): Int { - return summary.hashCode() + sections.map { it.hashCode() }.sum() + return sections.map { it.hashCode() }.sum() } override fun toString(): String { if (sections.isEmpty()) - return summary.toString() - return "$summary | " + sections.joinToString() + return "<empty>" + return sections.values().joinToString() } val isEmpty: Boolean get() = description.isEmpty() && sections.none() class object { - val Empty = DocumentationContent(RichString.empty, RichString.empty, listOf()) + val Empty = DocumentationContent(mapOf()) } } @@ -50,31 +48,33 @@ public class DocumentationContent(public val summary: RichString, fun BindingContext.getDocumentation(descriptor: DeclarationDescriptor): DocumentationContent { val docText = getDocumentationElements(descriptor).map { it.extractText() }.join("\n") val sections = docText.parseSections() - val (summary, description) = sections.extractSummaryAndDescription() - return DocumentationContent(summary, description, sections.drop(1)) + sections.createSummaryAndDescription() + return DocumentationContent(sections) } -fun List<DocumentationContentSection>.extractSummaryAndDescription() : Pair<RichString, RichString> { - // TODO: rework to unify - // if no $summary and $description is present, parse unnamed section and create specific sections - // otherwise, create empty sections for missing +fun MutableMap<String, DocumentationContentSection>.createSummaryAndDescription() { - val summary = firstOrNull { it.label == "\$summary" } - if (summary != null) { - val description = firstOrNull { it.label == "\$description" } - return Pair(summary.text, description?.text ?: RichString.empty) + val summary = get("\$summary") + val description = get("\$description") + if (summary != null && description == null) { + return } - val description = firstOrNull { it.label == "\$description" } - if (description != null) { - return Pair(RichString.empty, description.text) + if (summary == null && description != null) { + return } - val default = firstOrNull { it.label == "" }?.text - if (default == null) - return Pair(RichString.empty, RichString.empty) + val unnamed = get("") + if (unnamed == null) { + return + } - return default.splitBy("\n") + val split = unnamed.text.splitBy("\n") + remove("") + if (!split.first.isEmpty()) + put("\$summary", DocumentationContentSection("\$summary", split.first)) + if (!split.second.isEmpty()) + put("\$description", DocumentationContentSection("\$description", split.second)) } fun String.parseLabel(index: Int): Pair<String, Int> { @@ -104,8 +104,8 @@ fun String.parseLabel(index: Int): Pair<String, Int> { return "" to -1 } -fun String.parseSections(): List<DocumentationContentSection> { - val sections = arrayListOf<DocumentationContentSection>() +fun String.parseSections(): MutableMap<String, DocumentationContentSection> { + val sections = hashMapOf<String, DocumentationContentSection>() var currentLabel = "" var currentSectionStart = 0 var currentIndex = 0 @@ -117,7 +117,7 @@ fun String.parseSections(): List<DocumentationContentSection> { // section starts, add previous section val currentContent = substring(currentSectionStart, currentIndex).trim() val section = DocumentationContentSection(currentLabel, currentContent.toRichString()) - sections.add(section) + sections.put(section.label, section) currentLabel = label currentIndex = index + 1 @@ -131,12 +131,20 @@ fun String.parseSections(): List<DocumentationContentSection> { val currentContent = substring(currentSectionStart, currentIndex).trim() val section = DocumentationContentSection(currentLabel, currentContent.toRichString()) - sections.add(section) + sections.put(section.label, section) return sections } fun String.toRichString() : RichString { val content = RichString() + for(index in indices) { + val ch = get(index) + when { + ch == '\\' -> continue + ch == '*' && index < length-1 && !get(index + 1).isWhitespace() -> ch + } + } + content.addSlice(this, NormalStyle) return content }
\ No newline at end of file diff --git a/src/Model/DocumentationNodeBuilder.kt b/src/Model/DocumentationNodeBuilder.kt index c8744172..f724c444 100644 --- a/src/Model/DocumentationNodeBuilder.kt +++ b/src/Model/DocumentationNodeBuilder.kt @@ -35,7 +35,7 @@ class DocumentationNodeBuilder(val context: BindingContext) : DeclarationDescrip val classifierDescriptor = typeConstructor.getDeclarationDescriptor() val name = when (classifierDescriptor) { is Named -> classifierDescriptor.getName().asString() - else -> "<BAD>" + else -> "<anonymous>" } val node = DocumentationNode(descriptor, name, DocumentationContent.Empty, DocumentationNode.Kind.Type) reference(data, node, DocumentationReference.Kind.Detail) diff --git a/src/Processing/CrossReferences.kt b/src/Processing/CrossReferences.kt new file mode 100644 index 00000000..9f21da6e --- /dev/null +++ b/src/Processing/CrossReferences.kt @@ -0,0 +1,12 @@ +package org.jetbrains.dokka + +public fun DocumentationNode.buildCrossReferences() { + for (member in members) { + member.buildCrossReferences() + member.details(DocumentationNode.Kind.Receiver).forEach { detail -> + + + } + } +} + diff --git a/src/RichContent/RichString.kt b/src/RichContent/RichString.kt index f09e4715..2110c47f 100644 --- a/src/RichContent/RichString.kt +++ b/src/RichContent/RichString.kt @@ -5,7 +5,8 @@ public class RichString { public val slices: List<RichStringSlice> get() = sliceList public fun addSlice(slice: RichStringSlice) { - sliceList.add(slice) + if (slice.text.length() > 0) + sliceList.add(slice) } public fun addSlice(text: String, style: RichStringStyle) { diff --git a/src/main.kt b/src/main.kt index 2f2ac93c..a541831d 100644 --- a/src/main.kt +++ b/src/main.kt @@ -59,10 +59,23 @@ public fun main(args: Array<String>) { val timeAnalyse = System.currentTimeMillis() - startAnalyse println("done in ${timeAnalyse / 1000} secs") + print("Processing cross references... ") + val startProcessing = System.currentTimeMillis() + documentation.buildCrossReferences() + val timeProcessing = System.currentTimeMillis() - startProcessing + println("done in ${timeProcessing / 1000} secs") + val startBuild = System.currentTimeMillis() val signatureGenerator = KotlinLanguageService() val locationService = FoldersLocationService(arguments.outputDir) - val formatter = JekyllFormatService(locationService, signatureGenerator) + val templateService = HtmlTemplateService.default("/dokka/styles/style.css") + val resolutionService = object : ResolutionService { + override fun resolve(text: String): DocumentationNode { + return documentation + } + } + + val formatter = HtmlFormatService(locationService, resolutionService, signatureGenerator, templateService) val generator = FileGenerator(signatureGenerator, locationService, formatter) print("Building pages... ") generator.buildPage(documentation) |