From c9ca0d86e810ac7240b249b145810f6862a58e93 Mon Sep 17 00:00:00 2001 From: Ilya Ryzhenkov Date: Mon, 15 Dec 2014 20:54:16 +0300 Subject: Migrate to non-PsiBuilder fully-featured markdown parser. --- .idea/libraries/idea_markdown.xml | 9 + .idea/libraries/kotlin_compiler.xml | 4 - README.md | 2 +- dokka.iml | 1 + .../jetbrains/markdown/MarkdownElementTypes.java | 49 - gen/org/jetbrains/markdown/MarkdownParser.java | 1724 -------------------- gen/org/jetbrains/markdown/lexer/JFlex.jar | Bin 193380 -> 0 bytes .../jetbrains/markdown/lexer/MarkdownLexer.java | 9 - .../jetbrains/markdown/lexer/_MarkdownLexer.flex | 44 - .../jetbrains/markdown/lexer/_MarkdownLexer.java | 508 ------ .../jetbrains/markdown/lexer/idea-flex.skeleton | 251 --- lib/idea-markdown.jar | Bin 0 -> 200046 bytes src/Kotlin/ContentBuilder.kt | 81 +- src/Kotlin/DocumentationBuilder.kt | 19 +- src/Markdown/GeneratedParserUtilBase.java | 1031 ------------ src/Markdown/MarkdownProcessor.kt | 156 +- src/Markdown/MarkdownTokenType.kt | 6 - src/Markdown/markdown.bnf | 98 -- src/Markdown/markdown.leg | 781 --------- src/main.kt | 3 +- test/src/markdown/ParserTest.kt | 23 +- 21 files changed, 148 insertions(+), 4651 deletions(-) create mode 100644 .idea/libraries/idea_markdown.xml delete mode 100644 gen/org/jetbrains/markdown/MarkdownElementTypes.java delete mode 100644 gen/org/jetbrains/markdown/MarkdownParser.java delete mode 100644 gen/org/jetbrains/markdown/lexer/JFlex.jar delete mode 100644 gen/org/jetbrains/markdown/lexer/MarkdownLexer.java delete mode 100644 gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex delete mode 100644 gen/org/jetbrains/markdown/lexer/_MarkdownLexer.java delete mode 100644 gen/org/jetbrains/markdown/lexer/idea-flex.skeleton create mode 100644 lib/idea-markdown.jar delete mode 100644 src/Markdown/GeneratedParserUtilBase.java delete mode 100644 src/Markdown/MarkdownTokenType.kt delete mode 100644 src/Markdown/markdown.bnf delete mode 100644 src/Markdown/markdown.leg diff --git a/.idea/libraries/idea_markdown.xml b/.idea/libraries/idea_markdown.xml new file mode 100644 index 00000000..d2fae338 --- /dev/null +++ b/.idea/libraries/idea_markdown.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/kotlin_compiler.xml b/.idea/libraries/kotlin_compiler.xml index 306c11be..d3199a12 100644 --- a/.idea/libraries/kotlin_compiler.xml +++ b/.idea/libraries/kotlin_compiler.xml @@ -64,8 +64,6 @@ - - @@ -128,8 +126,6 @@ - - diff --git a/README.md b/README.md index b4392f47..950f6493 100644 --- a/README.md +++ b/README.md @@ -84,4 +84,4 @@ KDoc is a flavour of markdown with symbol processing extensions. * \[name\] - link to `name` (markdown style) * $name - link to `name` (Kotlin string interpolation style), or ${java.lang.String} for longer references * $name: - named section, optionally bound to symbol `name`, e.g. param doc -* !{reference} -- include content of a file or symbol denoted by reference, e.g. code example +* ${code reference} -- include content of a symbol denoted by reference, e.g. code example diff --git a/dokka.iml b/dokka.iml index f48bdd8f..509dfa7e 100644 --- a/dokka.iml +++ b/dokka.iml @@ -13,5 +13,6 @@ + \ No newline at end of file diff --git a/gen/org/jetbrains/markdown/MarkdownElementTypes.java b/gen/org/jetbrains/markdown/MarkdownElementTypes.java deleted file mode 100644 index 5c4aad41..00000000 --- a/gen/org/jetbrains/markdown/MarkdownElementTypes.java +++ /dev/null @@ -1,49 +0,0 @@ -// This is a generated file. Not intended for manual editing. -package org.jetbrains.markdown; - -import com.intellij.psi.tree.IElementType; -import org.jetbrains.dokka.Markdown.MarkdownTokenType; - -public interface MarkdownElementTypes { - - IElementType ANONYMOUS_SECTION = new IElementType("ANONYMOUS_SECTION", null); - IElementType BLANK_LINE = new IElementType("BLANK_LINE", null); - IElementType BLOCK = new IElementType("BLOCK", null); - IElementType BULLET = new IElementType("BULLET", null); - IElementType BULLET_LIST = new IElementType("BULLET_LIST", null); - IElementType CODE = new IElementType("CODE", null); - IElementType DIRECTIVE = new IElementType("DIRECTIVE", null); - IElementType DIRECTIVE_NAME = new IElementType("DIRECTIVE_NAME", null); - IElementType DIRECTIVE_PARAMS = new IElementType("DIRECTIVE_PARAMS", null); - IElementType EMPH = new IElementType("EMPH", null); - IElementType END_LINE = new IElementType("END_LINE", null); - IElementType ENUMERATOR = new IElementType("ENUMERATOR", null); - IElementType HORIZONTAL_RULE = new IElementType("HORIZONTAL_RULE", null); - IElementType HREF = new IElementType("HREF", null); - IElementType INLINE = new IElementType("INLINE", null); - IElementType LINK = new IElementType("LINK", null); - IElementType LIST_BLOCK = new IElementType("LIST_BLOCK", null); - IElementType LIST_BLOCK_LINE = new IElementType("LIST_BLOCK_LINE", null); - IElementType LIST_CONTINUATION_BLOCK = new IElementType("LIST_CONTINUATION_BLOCK", null); - IElementType LIST_ITEM = new IElementType("LIST_ITEM", null); - IElementType NAMED_SECTION = new IElementType("NAMED_SECTION", null); - IElementType ORDERED_LIST = new IElementType("ORDERED_LIST", null); - IElementType PARA = new IElementType("PARA", null); - IElementType PLAIN_TEXT = new IElementType("PLAIN_TEXT", null); - IElementType SECTION_BODY = new IElementType("SECTION_BODY", null); - IElementType SECTION_NAME = new IElementType("SECTION_NAME", null); - IElementType STRONG = new IElementType("STRONG", null); - IElementType STRONG_STAR = new IElementType("STRONG_STAR", null); - IElementType STRONG_UNDERSCORE = new IElementType("STRONG_UNDERSCORE", null); - IElementType TARGET = new IElementType("TARGET", null); - IElementType UNUSED = new IElementType("UNUSED", null); - IElementType WHITESPACE = new IElementType("WHITESPACE", null); - - IElementType BOM = new MarkdownTokenType("BOM"); - IElementType EOL = new MarkdownTokenType("EOL"); - IElementType EOP = new MarkdownTokenType("EOP"); - IElementType NUMBER = new MarkdownTokenType("Number"); - IElementType SPACE = new MarkdownTokenType("Space"); - IElementType SPECIAL = new MarkdownTokenType("Special"); - IElementType WORD = new MarkdownTokenType("Word"); -} diff --git a/gen/org/jetbrains/markdown/MarkdownParser.java b/gen/org/jetbrains/markdown/MarkdownParser.java deleted file mode 100644 index 05a1ada8..00000000 --- a/gen/org/jetbrains/markdown/MarkdownParser.java +++ /dev/null @@ -1,1724 +0,0 @@ -// This is a generated file. Not intended for manual editing. -package org.jetbrains.markdown; - -import com.intellij.lang.PsiBuilder; -import com.intellij.lang.PsiBuilder.Marker; -import static org.jetbrains.markdown.MarkdownElementTypes.*; -import static org.jetbrains.dokka.Markdown.GeneratedParserUtilBase.*; -import com.intellij.psi.tree.IElementType; -import com.intellij.lang.ASTNode; -import com.intellij.psi.tree.TokenSet; -import com.intellij.lang.PsiParser; - -@SuppressWarnings({"SimplifiableIfStatement", "UnusedAssignment"}) -public class MarkdownParser implements PsiParser { - - public ASTNode parse(IElementType t, PsiBuilder b) { - parse_only_(t, b); - return b.getTreeBuilt(); - } - - public void parse_only_(IElementType t, PsiBuilder b) { - boolean r; - b = adapt_builder_(t, b, this, null); - Marker m = enter_section_(b, 0, _COLLAPSE_, null); - if (t == ANONYMOUS_SECTION) { - r = AnonymousSection(b, 0); - } - else if (t == BLANK_LINE) { - r = BlankLine(b, 0); - } - else if (t == BLOCK) { - r = Block(b, 0); - } - else if (t == BULLET) { - r = Bullet(b, 0); - } - else if (t == BULLET_LIST) { - r = BulletList(b, 0); - } - else if (t == CODE) { - r = Code(b, 0); - } - else if (t == DIRECTIVE) { - r = Directive(b, 0); - } - else if (t == DIRECTIVE_NAME) { - r = DirectiveName(b, 0); - } - else if (t == DIRECTIVE_PARAMS) { - r = DirectiveParams(b, 0); - } - else if (t == EMPH) { - r = Emph(b, 0); - } - else if (t == END_LINE) { - r = EndLine(b, 0); - } - else if (t == ENUMERATOR) { - r = Enumerator(b, 0); - } - else if (t == HORIZONTAL_RULE) { - r = HorizontalRule(b, 0); - } - else if (t == HREF) { - r = Href(b, 0); - } - else if (t == INLINE) { - r = Inline(b, 0); - } - else if (t == LINK) { - r = Link(b, 0); - } - else if (t == LIST_BLOCK) { - r = ListBlock(b, 0); - } - else if (t == LIST_BLOCK_LINE) { - r = ListBlockLine(b, 0); - } - else if (t == LIST_CONTINUATION_BLOCK) { - r = ListContinuationBlock(b, 0); - } - else if (t == LIST_ITEM) { - r = ListItem(b, 0); - } - else if (t == NAMED_SECTION) { - r = NamedSection(b, 0); - } - else if (t == ORDERED_LIST) { - r = OrderedList(b, 0); - } - else if (t == PARA) { - r = Para(b, 0); - } - else if (t == PLAIN_TEXT) { - r = PlainText(b, 0); - } - else if (t == SECTION_BODY) { - r = SectionBody(b, 0); - } - else if (t == SECTION_NAME) { - r = SectionName(b, 0); - } - else if (t == STRONG) { - r = Strong(b, 0); - } - else if (t == STRONG_STAR) { - r = StrongStar(b, 0); - } - else if (t == STRONG_UNDERSCORE) { - r = StrongUnderscore(b, 0); - } - else if (t == TARGET) { - r = Target(b, 0); - } - else if (t == UNUSED) { - r = Unused(b, 0); - } - else if (t == WHITESPACE) { - r = Whitespace(b, 0); - } - else { - r = parse_root_(t, b, 0); - } - exit_section_(b, 0, m, t, r, true, TRUE_CONDITION); - } - - protected boolean parse_root_(IElementType t, PsiBuilder b, int l) { - return Document(b, l + 1); - } - - /* ********************************************************** */ - // SectionBody - public static boolean AnonymousSection(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "AnonymousSection")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = SectionBody(b, l + 1); - exit_section_(b, l, m, ANONYMOUS_SECTION, r, false, null); - return r; - } - - /* ********************************************************** */ - // OptionalSpace EOL - public static boolean BlankLine(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "BlankLine")) return false; - if (!nextTokenIs(b, "", EOL, SPACE)) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = OptionalSpace(b, l + 1); - r = r && consumeToken(b, EOL); - exit_section_(b, l, m, BLANK_LINE, r, false, null); - return r; - } - - /* ********************************************************** */ - // BlankLine* ( - // OrderedList - // | BulletList - // | HorizontalRule - // | Directive - // | Para - // ) - public static boolean Block(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Block")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = Block_0(b, l + 1); - r = r && Block_1(b, l + 1); - exit_section_(b, l, m, BLOCK, r, false, null); - return r; - } - - // BlankLine* - private static boolean Block_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Block_0")) return false; - int c = current_position_(b); - while (true) { - if (!BlankLine(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "Block_0", c)) break; - c = current_position_(b); - } - return true; - } - - // OrderedList - // | BulletList - // | HorizontalRule - // | Directive - // | Para - private static boolean Block_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Block_1")) return false; - boolean r; - Marker m = enter_section_(b); - r = OrderedList(b, l + 1); - if (!r) r = BulletList(b, l + 1); - if (!r) r = HorizontalRule(b, l + 1); - if (!r) r = Directive(b, l + 1); - if (!r) r = Para(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // !HorizontalRule NonindentSpace ('+' | '*' | '-') Space+ - public static boolean Bullet(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Bullet")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = Bullet_0(b, l + 1); - r = r && NonindentSpace(b, l + 1); - r = r && Bullet_2(b, l + 1); - r = r && Bullet_3(b, l + 1); - exit_section_(b, l, m, BULLET, r, false, null); - return r; - } - - // !HorizontalRule - private static boolean Bullet_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Bullet_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !HorizontalRule(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // '+' | '*' | '-' - private static boolean Bullet_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Bullet_2")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "+"); - if (!r) r = consumeToken(b, "*"); - if (!r) r = consumeToken(b, "-"); - exit_section_(b, m, null, r); - return r; - } - - // Space+ - private static boolean Bullet_3(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Bullet_3")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, SPACE); - int c = current_position_(b); - while (r) { - if (!consumeToken(b, SPACE)) break; - if (!empty_element_parsed_guard_(b, "Bullet_3", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // &Bullet List - public static boolean BulletList(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "BulletList")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = BulletList_0(b, l + 1); - r = r && List(b, l + 1); - exit_section_(b, l, m, BULLET_LIST, r, false, null); - return r; - } - - // &Bullet - private static boolean BulletList_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "BulletList_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _AND_, null); - r = Bullet(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - /* ********************************************************** */ - // '`' !Whitespace (!'`' Inline)+ '`' - public static boolean Code(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Code")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = consumeToken(b, "`"); - r = r && Code_1(b, l + 1); - r = r && Code_2(b, l + 1); - r = r && consumeToken(b, "`"); - exit_section_(b, l, m, CODE, r, false, null); - return r; - } - - // !Whitespace - private static boolean Code_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Code_1")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !Whitespace(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // (!'`' Inline)+ - private static boolean Code_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Code_2")) return false; - boolean r; - Marker m = enter_section_(b); - r = Code_2_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!Code_2_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "Code_2", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - // !'`' Inline - private static boolean Code_2_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Code_2_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = Code_2_0_0(b, l + 1); - r = r && Inline(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !'`' - private static boolean Code_2_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Code_2_0_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !consumeToken(b, "`"); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - /* ********************************************************** */ - // '{' DirectiveName Space+ DirectiveParams '}' - public static boolean Directive(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Directive")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = consumeToken(b, "{"); - r = r && DirectiveName(b, l + 1); - r = r && Directive_2(b, l + 1); - r = r && DirectiveParams(b, l + 1); - r = r && consumeToken(b, "}"); - exit_section_(b, l, m, DIRECTIVE, r, false, null); - return r; - } - - // Space+ - private static boolean Directive_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Directive_2")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, SPACE); - int c = current_position_(b); - while (r) { - if (!consumeToken(b, SPACE)) break; - if (!empty_element_parsed_guard_(b, "Directive_2", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // Word - public static boolean DirectiveName(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "DirectiveName")) return false; - if (!nextTokenIs(b, WORD)) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, WORD); - exit_section_(b, m, DIRECTIVE_NAME, r); - return r; - } - - /* ********************************************************** */ - // PlainText - public static boolean DirectiveParams(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "DirectiveParams")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = PlainText(b, l + 1); - exit_section_(b, l, m, DIRECTIVE_PARAMS, r, false, null); - return r; - } - - /* ********************************************************** */ - // BOM? Whitespace* AnonymousSection? (Whitespace* NamedSection)* - static boolean Document(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Document")) return false; - boolean r; - Marker m = enter_section_(b); - r = Document_0(b, l + 1); - r = r && Document_1(b, l + 1); - r = r && Document_2(b, l + 1); - r = r && Document_3(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // BOM? - private static boolean Document_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Document_0")) return false; - consumeToken(b, BOM); - return true; - } - - // Whitespace* - private static boolean Document_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Document_1")) return false; - int c = current_position_(b); - while (true) { - if (!Whitespace(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "Document_1", c)) break; - c = current_position_(b); - } - return true; - } - - // AnonymousSection? - private static boolean Document_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Document_2")) return false; - AnonymousSection(b, l + 1); - return true; - } - - // (Whitespace* NamedSection)* - private static boolean Document_3(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Document_3")) return false; - int c = current_position_(b); - while (true) { - if (!Document_3_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "Document_3", c)) break; - c = current_position_(b); - } - return true; - } - - // Whitespace* NamedSection - private static boolean Document_3_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Document_3_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = Document_3_0_0(b, l + 1); - r = r && NamedSection(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // Whitespace* - private static boolean Document_3_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Document_3_0_0")) return false; - int c = current_position_(b); - while (true) { - if (!Whitespace(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "Document_3_0_0", c)) break; - c = current_position_(b); - } - return true; - } - - /* ********************************************************** */ - // EmphStar | EmphUnderscore - public static boolean Emph(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Emph")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = EmphStar(b, l + 1); - if (!r) r = EmphUnderscore(b, l + 1); - exit_section_(b, l, m, EMPH, r, false, null); - return r; - } - - /* ********************************************************** */ - // '*' !Whitespace (!'*' Inline)+ '*' - static boolean EmphStar(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphStar")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "*"); - r = r && EmphStar_1(b, l + 1); - r = r && EmphStar_2(b, l + 1); - r = r && consumeToken(b, "*"); - exit_section_(b, m, null, r); - return r; - } - - // !Whitespace - private static boolean EmphStar_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphStar_1")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !Whitespace(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // (!'*' Inline)+ - private static boolean EmphStar_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphStar_2")) return false; - boolean r; - Marker m = enter_section_(b); - r = EmphStar_2_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!EmphStar_2_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "EmphStar_2", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - // !'*' Inline - private static boolean EmphStar_2_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphStar_2_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = EmphStar_2_0_0(b, l + 1); - r = r && Inline(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !'*' - private static boolean EmphStar_2_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphStar_2_0_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !consumeToken(b, "*"); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - /* ********************************************************** */ - // '_' !Whitespace (!'_' Inline)+ '_' - static boolean EmphUnderscore(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphUnderscore")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "_"); - r = r && EmphUnderscore_1(b, l + 1); - r = r && EmphUnderscore_2(b, l + 1); - r = r && consumeToken(b, "_"); - exit_section_(b, m, null, r); - return r; - } - - // !Whitespace - private static boolean EmphUnderscore_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphUnderscore_1")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !Whitespace(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // (!'_' Inline)+ - private static boolean EmphUnderscore_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphUnderscore_2")) return false; - boolean r; - Marker m = enter_section_(b); - r = EmphUnderscore_2_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!EmphUnderscore_2_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "EmphUnderscore_2", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - // !'_' Inline - private static boolean EmphUnderscore_2_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphUnderscore_2_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = EmphUnderscore_2_0_0(b, l + 1); - r = r && Inline(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !'_' - private static boolean EmphUnderscore_2_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EmphUnderscore_2_0_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !consumeToken(b, "_"); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - /* ********************************************************** */ - // TerminalEndline | NormalEndline - public static boolean EndLine(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "EndLine")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = TerminalEndline(b, l + 1); - if (!r) r = NormalEndline(b, l + 1); - exit_section_(b, l, m, END_LINE, r, false, null); - return r; - } - - /* ********************************************************** */ - // NonindentSpace Number '.' Space+ - public static boolean Enumerator(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Enumerator")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = NonindentSpace(b, l + 1); - r = r && consumeToken(b, NUMBER); - r = r && consumeToken(b, "."); - r = r && Enumerator_3(b, l + 1); - exit_section_(b, l, m, ENUMERATOR, r, false, null); - return r; - } - - // Space+ - private static boolean Enumerator_3(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Enumerator_3")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, SPACE); - int c = current_position_(b); - while (r) { - if (!consumeToken(b, SPACE)) break; - if (!empty_element_parsed_guard_(b, "Enumerator_3", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // NonindentSpace - // ( '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')* - // | '-' OptionalSpace '-' OptionalSpace '-' (OptionalSpace '-')* - // | '_' OptionalSpace '_' OptionalSpace '_' (OptionalSpace '_')*) - // OptionalSpace EOL BlankLine+ - public static boolean HorizontalRule(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = NonindentSpace(b, l + 1); - r = r && HorizontalRule_1(b, l + 1); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, EOL); - r = r && HorizontalRule_4(b, l + 1); - exit_section_(b, l, m, HORIZONTAL_RULE, r, false, null); - return r; - } - - // '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')* - // | '-' OptionalSpace '-' OptionalSpace '-' (OptionalSpace '-')* - // | '_' OptionalSpace '_' OptionalSpace '_' (OptionalSpace '_')* - private static boolean HorizontalRule_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1")) return false; - boolean r; - Marker m = enter_section_(b); - r = HorizontalRule_1_0(b, l + 1); - if (!r) r = HorizontalRule_1_1(b, l + 1); - if (!r) r = HorizontalRule_1_2(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')* - private static boolean HorizontalRule_1_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "*"); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, "*"); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, "*"); - r = r && HorizontalRule_1_0_5(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // (OptionalSpace '*')* - private static boolean HorizontalRule_1_0_5(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_0_5")) return false; - int c = current_position_(b); - while (true) { - if (!HorizontalRule_1_0_5_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "HorizontalRule_1_0_5", c)) break; - c = current_position_(b); - } - return true; - } - - // OptionalSpace '*' - private static boolean HorizontalRule_1_0_5_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_0_5_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = OptionalSpace(b, l + 1); - r = r && consumeToken(b, "*"); - exit_section_(b, m, null, r); - return r; - } - - // '-' OptionalSpace '-' OptionalSpace '-' (OptionalSpace '-')* - private static boolean HorizontalRule_1_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_1")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "-"); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, "-"); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, "-"); - r = r && HorizontalRule_1_1_5(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // (OptionalSpace '-')* - private static boolean HorizontalRule_1_1_5(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_1_5")) return false; - int c = current_position_(b); - while (true) { - if (!HorizontalRule_1_1_5_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "HorizontalRule_1_1_5", c)) break; - c = current_position_(b); - } - return true; - } - - // OptionalSpace '-' - private static boolean HorizontalRule_1_1_5_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_1_5_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = OptionalSpace(b, l + 1); - r = r && consumeToken(b, "-"); - exit_section_(b, m, null, r); - return r; - } - - // '_' OptionalSpace '_' OptionalSpace '_' (OptionalSpace '_')* - private static boolean HorizontalRule_1_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_2")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "_"); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, "_"); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, "_"); - r = r && HorizontalRule_1_2_5(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // (OptionalSpace '_')* - private static boolean HorizontalRule_1_2_5(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_2_5")) return false; - int c = current_position_(b); - while (true) { - if (!HorizontalRule_1_2_5_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "HorizontalRule_1_2_5", c)) break; - c = current_position_(b); - } - return true; - } - - // OptionalSpace '_' - private static boolean HorizontalRule_1_2_5_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_1_2_5_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = OptionalSpace(b, l + 1); - r = r && consumeToken(b, "_"); - exit_section_(b, m, null, r); - return r; - } - - // BlankLine+ - private static boolean HorizontalRule_4(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "HorizontalRule_4")) return false; - boolean r; - Marker m = enter_section_(b); - r = BlankLine(b, l + 1); - int c = current_position_(b); - while (r) { - if (!BlankLine(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "HorizontalRule_4", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // (Word | Number | ':' | '/')+ - public static boolean Href(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Href")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = Href_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!Href_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "Href", c)) break; - c = current_position_(b); - } - exit_section_(b, l, m, HREF, r, false, null); - return r; - } - - // Word | Number | ':' | '/' - private static boolean Href_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Href_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, WORD); - if (!r) r = consumeToken(b, NUMBER); - if (!r) r = consumeToken(b, ":"); - if (!r) r = consumeToken(b, "/"); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // "\t" | " " - static boolean Indent(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Indent")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "\t"); - if (!r) r = consumeToken(b, " "); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // Strong | Emph | Code | Link | PlainText - public static boolean Inline(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inline")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = Strong(b, l + 1); - if (!r) r = Emph(b, l + 1); - if (!r) r = Code(b, l + 1); - if (!r) r = Link(b, l + 1); - if (!r) r = PlainText(b, l + 1); - exit_section_(b, l, m, INLINE, r, false, null); - return r; - } - - /* ********************************************************** */ - // (!EndLine Inline | EndLine &Inline )+ EndLine? - static boolean Inlines(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inlines")) return false; - boolean r; - Marker m = enter_section_(b); - r = Inlines_0(b, l + 1); - r = r && Inlines_1(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // (!EndLine Inline | EndLine &Inline )+ - private static boolean Inlines_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inlines_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = Inlines_0_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!Inlines_0_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "Inlines_0", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - // !EndLine Inline | EndLine &Inline - private static boolean Inlines_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inlines_0_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = Inlines_0_0_0(b, l + 1); - if (!r) r = Inlines_0_0_1(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !EndLine Inline - private static boolean Inlines_0_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inlines_0_0_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = Inlines_0_0_0_0(b, l + 1); - r = r && Inline(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !EndLine - private static boolean Inlines_0_0_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inlines_0_0_0_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !EndLine(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // EndLine &Inline - private static boolean Inlines_0_0_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inlines_0_0_1")) return false; - boolean r; - Marker m = enter_section_(b); - r = EndLine(b, l + 1); - r = r && Inlines_0_0_1_1(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // &Inline - private static boolean Inlines_0_0_1_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inlines_0_0_1_1")) return false; - boolean r; - Marker m = enter_section_(b, l, _AND_, null); - r = Inline(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // EndLine? - private static boolean Inlines_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Inlines_1")) return false; - EndLine(b, l + 1); - return true; - } - - /* ********************************************************** */ - // '[' Target ']' ('(' Href ')')? - public static boolean Link(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Link")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = consumeToken(b, "["); - r = r && Target(b, l + 1); - r = r && consumeToken(b, "]"); - r = r && Link_3(b, l + 1); - exit_section_(b, l, m, LINK, r, false, null); - return r; - } - - // ('(' Href ')')? - private static boolean Link_3(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Link_3")) return false; - Link_3_0(b, l + 1); - return true; - } - - // '(' Href ')' - private static boolean Link_3_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Link_3_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "("); - r = r && Href(b, l + 1); - r = r && consumeToken(b, ")"); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // (ListItem BlankLine*)+ - static boolean List(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "List")) return false; - boolean r; - Marker m = enter_section_(b); - r = List_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!List_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "List", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - // ListItem BlankLine* - private static boolean List_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "List_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = ListItem(b, l + 1); - r = r && List_0_1(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // BlankLine* - private static boolean List_0_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "List_0_1")) return false; - int c = current_position_(b); - while (true) { - if (!BlankLine(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "List_0_1", c)) break; - c = current_position_(b); - } - return true; - } - - /* ********************************************************** */ - // !BlankLine Inlines ( ListBlockLine )* - public static boolean ListBlock(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlock")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = ListBlock_0(b, l + 1); - r = r && Inlines(b, l + 1); - r = r && ListBlock_2(b, l + 1); - exit_section_(b, l, m, LIST_BLOCK, r, false, null); - return r; - } - - // !BlankLine - private static boolean ListBlock_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlock_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !BlankLine(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // ( ListBlockLine )* - private static boolean ListBlock_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlock_2")) return false; - int c = current_position_(b); - while (true) { - if (!ListBlock_2_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "ListBlock_2", c)) break; - c = current_position_(b); - } - return true; - } - - // ( ListBlockLine ) - private static boolean ListBlock_2_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlock_2_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = ListBlockLine(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Inlines - public static boolean ListBlockLine(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlockLine")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = ListBlockLine_0(b, l + 1); - r = r && ListBlockLine_1(b, l + 1); - r = r && ListBlockLine_2(b, l + 1); - r = r && ListBlockLine_3(b, l + 1); - r = r && Inlines(b, l + 1); - exit_section_(b, l, m, LIST_BLOCK_LINE, r, false, null); - return r; - } - - // !BlankLine - private static boolean ListBlockLine_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlockLine_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !BlankLine(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // !(Indent? (Bullet | Enumerator)) - private static boolean ListBlockLine_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlockLine_1")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !ListBlockLine_1_0(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // Indent? (Bullet | Enumerator) - private static boolean ListBlockLine_1_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlockLine_1_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = ListBlockLine_1_0_0(b, l + 1); - r = r && ListBlockLine_1_0_1(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // Indent? - private static boolean ListBlockLine_1_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlockLine_1_0_0")) return false; - Indent(b, l + 1); - return true; - } - - // Bullet | Enumerator - private static boolean ListBlockLine_1_0_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlockLine_1_0_1")) return false; - boolean r; - Marker m = enter_section_(b); - r = Bullet(b, l + 1); - if (!r) r = Enumerator(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !HorizontalRule - private static boolean ListBlockLine_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlockLine_2")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !HorizontalRule(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // Indent? - private static boolean ListBlockLine_3(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListBlockLine_3")) return false; - Indent(b, l + 1); - return true; - } - - /* ********************************************************** */ - // BlankLine* (Indent ListBlock)+ - public static boolean ListContinuationBlock(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListContinuationBlock")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = ListContinuationBlock_0(b, l + 1); - r = r && ListContinuationBlock_1(b, l + 1); - exit_section_(b, l, m, LIST_CONTINUATION_BLOCK, r, false, null); - return r; - } - - // BlankLine* - private static boolean ListContinuationBlock_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListContinuationBlock_0")) return false; - int c = current_position_(b); - while (true) { - if (!BlankLine(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "ListContinuationBlock_0", c)) break; - c = current_position_(b); - } - return true; - } - - // (Indent ListBlock)+ - private static boolean ListContinuationBlock_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListContinuationBlock_1")) return false; - boolean r; - Marker m = enter_section_(b); - r = ListContinuationBlock_1_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!ListContinuationBlock_1_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "ListContinuationBlock_1", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - // Indent ListBlock - private static boolean ListContinuationBlock_1_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListContinuationBlock_1_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = Indent(b, l + 1); - r = r && ListBlock(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // (Bullet | Enumerator) ListBlock ( ListContinuationBlock )* - public static boolean ListItem(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListItem")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = ListItem_0(b, l + 1); - r = r && ListBlock(b, l + 1); - r = r && ListItem_2(b, l + 1); - exit_section_(b, l, m, LIST_ITEM, r, false, null); - return r; - } - - // Bullet | Enumerator - private static boolean ListItem_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListItem_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = Bullet(b, l + 1); - if (!r) r = Enumerator(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // ( ListContinuationBlock )* - private static boolean ListItem_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListItem_2")) return false; - int c = current_position_(b); - while (true) { - if (!ListItem_2_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "ListItem_2", c)) break; - c = current_position_(b); - } - return true; - } - - // ( ListContinuationBlock ) - private static boolean ListItem_2_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "ListItem_2_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = ListContinuationBlock(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // SectionHeader SectionBody - public static boolean NamedSection(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "NamedSection")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = SectionHeader(b, l + 1); - r = r && SectionBody(b, l + 1); - exit_section_(b, l, m, NAMED_SECTION, r, false, null); - return r; - } - - /* ********************************************************** */ - // (" " | " " | " ")? - static boolean NonindentSpace(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "NonindentSpace")) return false; - NonindentSpace_0(b, l + 1); - return true; - } - - // " " | " " | " " - private static boolean NonindentSpace_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "NonindentSpace_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, " "); - if (!r) r = consumeToken(b, " "); - if (!r) r = consumeToken(b, " "); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // EOL !BlankLine - static boolean NormalEndline(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "NormalEndline")) return false; - if (!nextTokenIs(b, EOL)) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, EOL); - r = r && NormalEndline_1(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !BlankLine - private static boolean NormalEndline_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "NormalEndline_1")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !BlankLine(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - /* ********************************************************** */ - // Space* - static boolean OptionalSpace(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "OptionalSpace")) return false; - int c = current_position_(b); - while (true) { - if (!consumeToken(b, SPACE)) break; - if (!empty_element_parsed_guard_(b, "OptionalSpace", c)) break; - c = current_position_(b); - } - return true; - } - - /* ********************************************************** */ - // &Enumerator List - public static boolean OrderedList(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "OrderedList")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = OrderedList_0(b, l + 1); - r = r && List(b, l + 1); - exit_section_(b, l, m, ORDERED_LIST, r, false, null); - return r; - } - - // &Enumerator - private static boolean OrderedList_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "OrderedList_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _AND_, null); - r = Enumerator(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - /* ********************************************************** */ - // Inlines (EOP | Space* <>)? - public static boolean Para(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Para")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = Inlines(b, l + 1); - r = r && Para_1(b, l + 1); - exit_section_(b, l, m, PARA, r, false, null); - return r; - } - - // (EOP | Space* <>)? - private static boolean Para_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Para_1")) return false; - Para_1_0(b, l + 1); - return true; - } - - // EOP | Space* <> - private static boolean Para_1_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Para_1_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, EOP); - if (!r) r = Para_1_0_1(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // Space* <> - private static boolean Para_1_0_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Para_1_0_1")) return false; - boolean r; - Marker m = enter_section_(b); - r = Para_1_0_1_0(b, l + 1); - r = r && eof(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // Space* - private static boolean Para_1_0_1_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Para_1_0_1_0")) return false; - int c = current_position_(b); - while (true) { - if (!consumeToken(b, SPACE)) break; - if (!empty_element_parsed_guard_(b, "Para_1_0_1_0", c)) break; - c = current_position_(b); - } - return true; - } - - /* ********************************************************** */ - // (Word | Number | Space | ':')+ - public static boolean PlainText(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "PlainText")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = PlainText_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!PlainText_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "PlainText", c)) break; - c = current_position_(b); - } - exit_section_(b, l, m, PLAIN_TEXT, r, false, null); - return r; - } - - // Word | Number | Space | ':' - private static boolean PlainText_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "PlainText_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, WORD); - if (!r) r = consumeToken(b, NUMBER); - if (!r) r = consumeToken(b, SPACE); - if (!r) r = consumeToken(b, ":"); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // Block* - public static boolean SectionBody(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionBody")) return false; - Marker m = enter_section_(b, l, _NONE_, "
"); - int c = current_position_(b); - while (true) { - if (!Block(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "SectionBody", c)) break; - c = current_position_(b); - } - exit_section_(b, l, m, SECTION_BODY, true, false, null); - return true; - } - - /* ********************************************************** */ - // '$' SectionName OptionalSpace ':' OptionalSpace - static boolean SectionHeader(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionHeader")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "$"); - r = r && SectionName(b, l + 1); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, ":"); - r = r && OptionalSpace(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // SectionNameStart | '{' OptionalSpace SectionNameStart (Space+ Word)* OptionalSpace '}' - public static boolean SectionName(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionName")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, "
"); - r = SectionNameStart(b, l + 1); - if (!r) r = SectionName_1(b, l + 1); - exit_section_(b, l, m, SECTION_NAME, r, false, null); - return r; - } - - // '{' OptionalSpace SectionNameStart (Space+ Word)* OptionalSpace '}' - private static boolean SectionName_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionName_1")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, "{"); - r = r && OptionalSpace(b, l + 1); - r = r && SectionNameStart(b, l + 1); - r = r && SectionName_1_3(b, l + 1); - r = r && OptionalSpace(b, l + 1); - r = r && consumeToken(b, "}"); - exit_section_(b, m, null, r); - return r; - } - - // (Space+ Word)* - private static boolean SectionName_1_3(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionName_1_3")) return false; - int c = current_position_(b); - while (true) { - if (!SectionName_1_3_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "SectionName_1_3", c)) break; - c = current_position_(b); - } - return true; - } - - // Space+ Word - private static boolean SectionName_1_3_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionName_1_3_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = SectionName_1_3_0_0(b, l + 1); - r = r && consumeToken(b, WORD); - exit_section_(b, m, null, r); - return r; - } - - // Space+ - private static boolean SectionName_1_3_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionName_1_3_0_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, SPACE); - int c = current_position_(b); - while (r) { - if (!consumeToken(b, SPACE)) break; - if (!empty_element_parsed_guard_(b, "SectionName_1_3_0_0", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // '$'? Word - static boolean SectionNameStart(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionNameStart")) return false; - boolean r; - Marker m = enter_section_(b); - r = SectionNameStart_0(b, l + 1); - r = r && consumeToken(b, WORD); - exit_section_(b, m, null, r); - return r; - } - - // '$'? - private static boolean SectionNameStart_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "SectionNameStart_0")) return false; - consumeToken(b, "$"); - return true; - } - - /* ********************************************************** */ - // StrongStar | StrongUnderscore - public static boolean Strong(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Strong")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = StrongStar(b, l + 1); - if (!r) r = StrongUnderscore(b, l + 1); - exit_section_(b, l, m, STRONG, r, false, null); - return r; - } - - /* ********************************************************** */ - // '**' !Whitespace (!'**' Inline)+ '**' - public static boolean StrongStar(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongStar")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = consumeToken(b, "**"); - r = r && StrongStar_1(b, l + 1); - r = r && StrongStar_2(b, l + 1); - r = r && consumeToken(b, "**"); - exit_section_(b, l, m, STRONG_STAR, r, false, null); - return r; - } - - // !Whitespace - private static boolean StrongStar_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongStar_1")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !Whitespace(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // (!'**' Inline)+ - private static boolean StrongStar_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongStar_2")) return false; - boolean r; - Marker m = enter_section_(b); - r = StrongStar_2_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!StrongStar_2_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "StrongStar_2", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - // !'**' Inline - private static boolean StrongStar_2_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongStar_2_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = StrongStar_2_0_0(b, l + 1); - r = r && Inline(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !'**' - private static boolean StrongStar_2_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongStar_2_0_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !consumeToken(b, "**"); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - /* ********************************************************** */ - // '__' !Whitespace (!'__' Inline)+ '__' - public static boolean StrongUnderscore(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongUnderscore")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = consumeToken(b, "__"); - r = r && StrongUnderscore_1(b, l + 1); - r = r && StrongUnderscore_2(b, l + 1); - r = r && consumeToken(b, "__"); - exit_section_(b, l, m, STRONG_UNDERSCORE, r, false, null); - return r; - } - - // !Whitespace - private static boolean StrongUnderscore_1(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongUnderscore_1")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !Whitespace(b, l + 1); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - // (!'__' Inline)+ - private static boolean StrongUnderscore_2(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongUnderscore_2")) return false; - boolean r; - Marker m = enter_section_(b); - r = StrongUnderscore_2_0(b, l + 1); - int c = current_position_(b); - while (r) { - if (!StrongUnderscore_2_0(b, l + 1)) break; - if (!empty_element_parsed_guard_(b, "StrongUnderscore_2", c)) break; - c = current_position_(b); - } - exit_section_(b, m, null, r); - return r; - } - - // !'__' Inline - private static boolean StrongUnderscore_2_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongUnderscore_2_0")) return false; - boolean r; - Marker m = enter_section_(b); - r = StrongUnderscore_2_0_0(b, l + 1); - r = r && Inline(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - // !'__' - private static boolean StrongUnderscore_2_0_0(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "StrongUnderscore_2_0_0")) return false; - boolean r; - Marker m = enter_section_(b, l, _NOT_, null); - r = !consumeToken(b, "__"); - exit_section_(b, l, m, null, r, false, null); - return r; - } - - /* ********************************************************** */ - // Word+ - public static boolean Target(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Target")) return false; - if (!nextTokenIs(b, WORD)) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, WORD); - int c = current_position_(b); - while (r) { - if (!consumeToken(b, WORD)) break; - if (!empty_element_parsed_guard_(b, "Target", c)) break; - c = current_position_(b); - } - exit_section_(b, m, TARGET, r); - return r; - } - - /* ********************************************************** */ - // OptionalSpace <> - static boolean TerminalEndline(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "TerminalEndline")) return false; - boolean r; - Marker m = enter_section_(b); - r = OptionalSpace(b, l + 1); - r = r && eof(b, l + 1); - exit_section_(b, m, null, r); - return r; - } - - /* ********************************************************** */ - // Special - public static boolean Unused(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Unused")) return false; - if (!nextTokenIs(b, SPECIAL)) return false; - boolean r; - Marker m = enter_section_(b); - r = consumeToken(b, SPECIAL); - exit_section_(b, m, UNUSED, r); - return r; - } - - /* ********************************************************** */ - // Space | EOL | EOP - public static boolean Whitespace(PsiBuilder b, int l) { - if (!recursion_guard_(b, l, "Whitespace")) return false; - boolean r; - Marker m = enter_section_(b, l, _NONE_, ""); - r = consumeToken(b, SPACE); - if (!r) r = consumeToken(b, EOL); - if (!r) r = consumeToken(b, EOP); - exit_section_(b, l, m, WHITESPACE, r, false, null); - return r; - } - -} diff --git a/gen/org/jetbrains/markdown/lexer/JFlex.jar b/gen/org/jetbrains/markdown/lexer/JFlex.jar deleted file mode 100644 index 8bef2fdb..00000000 Binary files a/gen/org/jetbrains/markdown/lexer/JFlex.jar and /dev/null differ diff --git a/gen/org/jetbrains/markdown/lexer/MarkdownLexer.java b/gen/org/jetbrains/markdown/lexer/MarkdownLexer.java deleted file mode 100644 index 40728972..00000000 --- a/gen/org/jetbrains/markdown/lexer/MarkdownLexer.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.jetbrains.markdown.lexer; - -import com.intellij.lexer.FlexAdapter; - -public class MarkdownLexer extends FlexAdapter { - public MarkdownLexer() { - super(new _MarkdownLexer()); - } -} diff --git a/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex b/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex deleted file mode 100644 index b4c5c9a7..00000000 --- a/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex +++ /dev/null @@ -1,44 +0,0 @@ -package org.jetbrains.markdown.lexer; - -import com.intellij.lexer.*; -import com.intellij.psi.tree.IElementType; -import static org.jetbrains.markdown.MarkdownElementTypes.*; - -%% - -%{ - public _MarkdownLexer() { - this((java.io.Reader)null); - } -%} - -%public -%class _MarkdownLexer -%implements FlexLexer -%function advance -%type IElementType -%unicode - -LineTerminator = \r|\n|\r\n -Word = {Letter}+ -Number = [:digit:]+ -Space = [ \t\f] -Letter = [^~:{}$\*_`&\[\]() { - {Space} { return SPACE; } - {EOP} { return EOP; } - {EOL} { return EOL; } - "\\357\\273\\277" { return BOM; } - - {Number} { return NUMBER; } - - {Special} { return SPECIAL; } - {Word} { return WORD; } - - [^] { return com.intellij.psi.TokenType.BAD_CHARACTER; } -} diff --git a/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.java b/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.java deleted file mode 100644 index 086512a1..00000000 --- a/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.java +++ /dev/null @@ -1,508 +0,0 @@ -/* The following code was generated by JFlex 1.4.3 on 9/29/14 10:13 PM */ - -package org.jetbrains.markdown.lexer; - -import com.intellij.lexer.*; -import com.intellij.psi.tree.IElementType; -import static org.jetbrains.markdown.MarkdownElementTypes.*; - - -/** - * This class is a scanner generated by - * JFlex 1.4.3 - * on 9/29/14 10:13 PM from the specification file - * /Users/orangy/Projects/dokka/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex - */ -public class _MarkdownLexer implements FlexLexer { - /** initial size of the lookahead buffer */ - private static final int ZZ_BUFFERSIZE = 16384; - - /** lexical states */ - public static final int YYINITIAL = 0; - - /** - * ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l - * ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l - * at the beginning of a line - * l is of the form l = 2*k, k a non negative integer - */ - private static final int ZZ_LEXSTATE[] = { - 0, 0 - }; - - /** - * Translates characters to character classes - */ - private static final String ZZ_CMAP_PACKED = - "\11\0\1\6\1\2\1\0\1\4\1\1\22\0\1\6\1\5\1\0"+ - "\2\5\1\0\1\5\1\0\3\5\5\0\2\3\1\13\1\10\1\3"+ - "\1\11\1\3\1\12\2\3\1\5\1\0\1\5\36\0\1\5\1\7"+ - "\1\5\1\0\2\5\32\0\1\5\1\0\2\5\u05e1\0\12\3\206\0"+ - "\12\3\u026c\0\12\3\166\0\12\3\166\0\12\3\166\0\12\3\166\0"+ - "\12\3\167\0\11\3\166\0\12\3\166\0\12\3\166\0\12\3\340\0"+ - "\12\3\166\0\12\3\106\0\12\3\u0116\0\12\3\u031f\0\11\3\u046e\0"+ - "\12\3\46\0\12\3\u012c\0\12\3\ue5c0\0\12\3\346\0"; - - /** - * Translates characters to character classes - */ - private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED); - - /** - * Translates DFA states to action switch labels. - */ - private static final int [] ZZ_ACTION = zzUnpackAction(); - - private static final String ZZ_ACTION_PACKED_0 = - "\1\0\1\1\2\2\1\3\1\4\1\5\1\4\1\5"+ - "\2\6\1\0\1\6\1\1\13\0\1\7"; - - private static int [] zzUnpackAction() { - int [] result = new int[26]; - int offset = 0; - offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackAction(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - - /** - * Translates a state to a row index in the transition table - */ - private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); - - private static final String ZZ_ROWMAP_PACKED_0 = - "\0\0\0\14\0\30\0\44\0\60\0\74\0\110\0\124"+ - "\0\140\0\154\0\44\0\44\0\110\0\74\0\124\0\170"+ - "\0\204\0\220\0\234\0\250\0\264\0\300\0\314\0\330"+ - "\0\344\0\110"; - - private static int [] zzUnpackRowMap() { - int [] result = new int[26]; - int offset = 0; - offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackRowMap(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int high = packed.charAt(i++) << 16; - result[j++] = high | packed.charAt(i++); - } - return j; - } - - /** - * The transition table of the DFA - */ - private static final int [] ZZ_TRANS = zzUnpackTrans(); - - private static final String ZZ_TRANS_PACKED_0 = - "\1\2\1\3\1\4\1\5\1\6\1\7\1\10\1\11"+ - "\4\5\1\2\2\0\2\2\3\0\4\2\1\0\1\12"+ - "\1\13\1\0\1\14\1\0\1\14\6\0\1\12\1\15"+ - "\1\0\1\14\1\0\1\14\5\0\1\2\2\0\1\5"+ - "\1\2\3\0\4\5\1\2\1\3\1\4\1\2\1\16"+ - "\1\0\1\17\1\0\4\2\15\0\1\3\1\4\1\0"+ - "\1\17\1\0\1\17\15\0\1\20\5\0\1\15\22\0"+ - "\1\21\14\0\1\22\10\0\1\23\17\0\1\24\12\0"+ - "\1\25\11\0\1\26\12\0\1\27\17\0\1\30\12\0"+ - "\1\31\13\0\1\32\1\0"; - - private static int [] zzUnpackTrans() { - int [] result = new int[240]; - int offset = 0; - offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackTrans(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - value--; - do result[j++] = value; while (--count > 0); - } - return j; - } - - - /* error codes */ - private static final int ZZ_UNKNOWN_ERROR = 0; - private static final int ZZ_NO_MATCH = 1; - private static final int ZZ_PUSHBACK_2BIG = 2; - private static final char[] EMPTY_BUFFER = new char[0]; - private static final int YYEOF = -1; - private static java.io.Reader zzReader = null; // Fake - - /* error messages for the codes above */ - private static final String ZZ_ERROR_MSG[] = { - "Unkown internal scanner error", - "Error: could not match input", - "Error: pushback value was too large" - }; - - /** - * ZZ_ATTRIBUTE[aState] contains the attributes of state aState - */ - private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute(); - - private static final String ZZ_ATTRIBUTE_PACKED_0 = - "\1\0\5\1\1\11\4\1\1\0\1\11\1\1\13\0"+ - "\1\11"; - - private static int [] zzUnpackAttribute() { - int [] result = new int[26]; - int offset = 0; - offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); - return result; - } - - private static int zzUnpackAttribute(String packed, int offset, int [] result) { - int i = 0; /* index in packed string */ - int j = offset; /* index in unpacked array */ - int l = packed.length(); - while (i < l) { - int count = packed.charAt(i++); - int value = packed.charAt(i++); - do result[j++] = value; while (--count > 0); - } - return j; - } - - /** the current state of the DFA */ - private int zzState; - - /** the current lexical state */ - private int zzLexicalState = YYINITIAL; - - /** this buffer contains the current text to be matched and is - the source of the yytext() string */ - private CharSequence zzBuffer = ""; - - /** this buffer may contains the current text array to be matched when it is cheap to acquire it */ - private char[] zzBufferArray; - - /** the textposition at the last accepting state */ - private int zzMarkedPos; - - /** the textposition at the last state to be included in yytext */ - private int zzPushbackPos; - - /** the current text position in the buffer */ - private int zzCurrentPos; - - /** startRead marks the beginning of the yytext() string in the buffer */ - private int zzStartRead; - - /** endRead marks the last character in the buffer, that has been read - from input */ - private int zzEndRead; - - /** - * zzAtBOL == true <=> the scanner is currently at the beginning of a line - */ - private boolean zzAtBOL = true; - - /** zzAtEOF == true <=> the scanner is at the EOF */ - private boolean zzAtEOF; - - /* user code: */ - public _MarkdownLexer() { - this((java.io.Reader)null); - } - - - public _MarkdownLexer(java.io.Reader in) { - this.zzReader = in; - } - - /** - * Creates a new scanner. - * There is also java.io.Reader version of this constructor. - * - * @param in the java.io.Inputstream to read input from. - */ - public _MarkdownLexer(java.io.InputStream in) { - this(new java.io.InputStreamReader(in)); - } - - /** - * Unpacks the compressed character translation table. - * - * @param packed the packed character translation table - * @return the unpacked character translation table - */ - private static char [] zzUnpackCMap(String packed) { - char [] map = new char[0x10000]; - int i = 0; /* index in packed string */ - int j = 0; /* index in unpacked array */ - while (i < 156) { - int count = packed.charAt(i++); - char value = packed.charAt(i++); - do map[j++] = value; while (--count > 0); - } - return map; - } - - public final int getTokenStart(){ - return zzStartRead; - } - - public final int getTokenEnd(){ - return getTokenStart() + yylength(); - } - - public void reset(CharSequence buffer, int start, int end,int initialState){ - zzBuffer = buffer; - zzBufferArray = com.intellij.util.text.CharArrayUtil.fromSequenceWithoutCopying(buffer); - zzCurrentPos = zzMarkedPos = zzStartRead = start; - zzPushbackPos = 0; - zzAtEOF = false; - zzAtBOL = true; - zzEndRead = end; - yybegin(initialState); - } - - /** - * Refills the input buffer. - * - * @return false, iff there was new input. - * - * @exception java.io.IOException if any I/O-Error occurs - */ - private boolean zzRefill() throws java.io.IOException { - return true; - } - - - /** - * Returns the current lexical state. - */ - public final int yystate() { - return zzLexicalState; - } - - - /** - * Enters a new lexical state - * - * @param newState the new lexical state - */ - public final void yybegin(int newState) { - zzLexicalState = newState; - } - - - /** - * Returns the text matched by the current regular expression. - */ - public final CharSequence yytext() { - return zzBuffer.subSequence(zzStartRead, zzMarkedPos); - } - - - /** - * Returns the character at position pos from the - * matched text. - * - * It is equivalent to yytext().charAt(pos), but faster - * - * @param pos the position of the character to fetch. - * A value from 0 to yylength()-1. - * - * @return the character at position pos - */ - public final char yycharat(int pos) { - return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer.charAt(zzStartRead+pos); - } - - - /** - * Returns the length of the matched text region. - */ - public final int yylength() { - return zzMarkedPos-zzStartRead; - } - - - /** - * Reports an error that occured while scanning. - * - * In a wellformed scanner (no or only correct usage of - * yypushback(int) and a match-all fallback rule) this method - * will only be called with things that "Can't Possibly Happen". - * If this method is called, something is seriously wrong - * (e.g. a JFlex bug producing a faulty scanner etc.). - * - * Usual syntax/scanner level error handling should be done - * in error fallback rules. - * - * @param errorCode the code of the errormessage to display - */ - private void zzScanError(int errorCode) { - String message; - try { - message = ZZ_ERROR_MSG[errorCode]; - } - catch (ArrayIndexOutOfBoundsException e) { - message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; - } - - throw new Error(message); - } - - - /** - * Pushes the specified amount of characters back into the input stream. - * - * They will be read again by then next call of the scanning method - * - * @param number the number of characters to be read again. - * This number must not be greater than yylength()! - */ - public void yypushback(int number) { - if ( number > yylength() ) - zzScanError(ZZ_PUSHBACK_2BIG); - - zzMarkedPos -= number; - } - - - /** - * Resumes scanning until the next regular expression is matched, - * the end of input is encountered or an I/O-Error occurs. - * - * @return the next token - * @exception java.io.IOException if any I/O-Error occurs - */ - public IElementType advance() throws java.io.IOException { - int zzInput; - int zzAction; - - // cached fields: - int zzCurrentPosL; - int zzMarkedPosL; - int zzEndReadL = zzEndRead; - CharSequence zzBufferL = zzBuffer; - char[] zzBufferArrayL = zzBufferArray; - char [] zzCMapL = ZZ_CMAP; - - int [] zzTransL = ZZ_TRANS; - int [] zzRowMapL = ZZ_ROWMAP; - int [] zzAttrL = ZZ_ATTRIBUTE; - - while (true) { - zzMarkedPosL = zzMarkedPos; - - zzAction = -1; - - zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; - - zzState = ZZ_LEXSTATE[zzLexicalState]; - - - zzForAction: { - while (true) { - - if (zzCurrentPosL < zzEndReadL) - zzInput = (zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++] : zzBufferL.charAt(zzCurrentPosL++)); - else if (zzAtEOF) { - zzInput = YYEOF; - break zzForAction; - } - else { - // store back cached positions - zzCurrentPos = zzCurrentPosL; - zzMarkedPos = zzMarkedPosL; - boolean eof = zzRefill(); - // get translated positions and possibly new buffer - zzCurrentPosL = zzCurrentPos; - zzMarkedPosL = zzMarkedPos; - zzBufferL = zzBuffer; - zzEndReadL = zzEndRead; - if (eof) { - zzInput = YYEOF; - break zzForAction; - } - else { - zzInput = (zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++] : zzBufferL.charAt(zzCurrentPosL++)); - } - } - int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ]; - if (zzNext == -1) break zzForAction; - zzState = zzNext; - - int zzAttributes = zzAttrL[zzState]; - if ( (zzAttributes & 1) == 1 ) { - zzAction = zzState; - zzMarkedPosL = zzCurrentPosL; - if ( (zzAttributes & 8) == 8 ) break zzForAction; - } - - } - } - - // store back cached position - zzMarkedPos = zzMarkedPosL; - - switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) { - case 3: - { return NUMBER; - } - case 8: break; - case 1: - { return WORD; - } - case 9: break; - case 6: - { return EOP; - } - case 10: break; - case 7: - { return BOM; - } - case 11: break; - case 2: - { return EOL; - } - case 12: break; - case 5: - { return SPECIAL; - } - case 13: break; - case 4: - { return SPACE; - } - case 14: break; - default: - if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { - zzAtEOF = true; - return null; - } - else { - zzScanError(ZZ_NO_MATCH); - } - } - } - } - - -} diff --git a/gen/org/jetbrains/markdown/lexer/idea-flex.skeleton b/gen/org/jetbrains/markdown/lexer/idea-flex.skeleton deleted file mode 100644 index 234a62c4..00000000 --- a/gen/org/jetbrains/markdown/lexer/idea-flex.skeleton +++ /dev/null @@ -1,251 +0,0 @@ - /** initial size of the lookahead buffer */ ---- private static final int ZZ_BUFFERSIZE = ...; - - /** lexical states */ ---- lexical states, charmap - - /* error codes */ - private static final int ZZ_UNKNOWN_ERROR = 0; - private static final int ZZ_NO_MATCH = 1; - private static final int ZZ_PUSHBACK_2BIG = 2; - private static final char[] EMPTY_BUFFER = new char[0]; - private static final int YYEOF = -1; - private static java.io.Reader zzReader = null; // Fake - - /* error messages for the codes above */ - private static final String ZZ_ERROR_MSG[] = { - "Unkown internal scanner error", - "Error: could not match input", - "Error: pushback value was too large" - }; - ---- isFinal list - /** the current state of the DFA */ - private int zzState; - - /** the current lexical state */ - private int zzLexicalState = YYINITIAL; - - /** this buffer contains the current text to be matched and is - the source of the yytext() string */ - private CharSequence zzBuffer = ""; - - /** this buffer may contains the current text array to be matched when it is cheap to acquire it */ - private char[] zzBufferArray; - - /** the textposition at the last accepting state */ - private int zzMarkedPos; - - /** the textposition at the last state to be included in yytext */ - private int zzPushbackPos; - - /** the current text position in the buffer */ - private int zzCurrentPos; - - /** startRead marks the beginning of the yytext() string in the buffer */ - private int zzStartRead; - - /** endRead marks the last character in the buffer, that has been read - from input */ - private int zzEndRead; - - /** - * zzAtBOL == true <=> the scanner is currently at the beginning of a line - */ - private boolean zzAtBOL = true; - - /** zzAtEOF == true <=> the scanner is at the EOF */ - private boolean zzAtEOF; - ---- user class code - ---- constructor declaration - - public final int getTokenStart(){ - return zzStartRead; - } - - public final int getTokenEnd(){ - return getTokenStart() + yylength(); - } - - public void reset(CharSequence buffer, int start, int end,int initialState){ - zzBuffer = buffer; - zzBufferArray = com.intellij.util.text.CharArrayUtil.fromSequenceWithoutCopying(buffer); - zzCurrentPos = zzMarkedPos = zzStartRead = start; - zzPushbackPos = 0; - zzAtEOF = false; - zzAtBOL = true; - zzEndRead = end; - yybegin(initialState); - } - - /** - * Refills the input buffer. - * - * @return false, iff there was new input. - * - * @exception java.io.IOException if any I/O-Error occurs - */ - private boolean zzRefill() throws java.io.IOException { - return true; - } - - - /** - * Returns the current lexical state. - */ - public final int yystate() { - return zzLexicalState; - } - - - /** - * Enters a new lexical state - * - * @param newState the new lexical state - */ - public final void yybegin(int newState) { - zzLexicalState = newState; - } - - - /** - * Returns the text matched by the current regular expression. - */ - public final CharSequence yytext() { - return zzBuffer.subSequence(zzStartRead, zzMarkedPos); - } - - - /** - * Returns the character at position pos from the - * matched text. - * - * It is equivalent to yytext().charAt(pos), but faster - * - * @param pos the position of the character to fetch. - * A value from 0 to yylength()-1. - * - * @return the character at position pos - */ - public final char yycharat(int pos) { - return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer.charAt(zzStartRead+pos); - } - - - /** - * Returns the length of the matched text region. - */ - public final int yylength() { - return zzMarkedPos-zzStartRead; - } - - - /** - * Reports an error that occured while scanning. - * - * In a wellformed scanner (no or only correct usage of - * yypushback(int) and a match-all fallback rule) this method - * will only be called with things that "Can't Possibly Happen". - * If this method is called, something is seriously wrong - * (e.g. a JFlex bug producing a faulty scanner etc.). - * - * Usual syntax/scanner level error handling should be done - * in error fallback rules. - * - * @param errorCode the code of the errormessage to display - */ ---- zzScanError declaration - String message; - try { - message = ZZ_ERROR_MSG[errorCode]; - } - catch (ArrayIndexOutOfBoundsException e) { - message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; - } - ---- throws clause - } - - - /** - * Pushes the specified amount of characters back into the input stream. - * - * They will be read again by then next call of the scanning method - * - * @param number the number of characters to be read again. - * This number must not be greater than yylength()! - */ ---- yypushback decl (contains zzScanError exception) - if ( number > yylength() ) - zzScanError(ZZ_PUSHBACK_2BIG); - - zzMarkedPos -= number; - } - - ---- zzDoEOF - /** - * Resumes scanning until the next regular expression is matched, - * the end of input is encountered or an I/O-Error occurs. - * - * @return the next token - * @exception java.io.IOException if any I/O-Error occurs - */ ---- yylex declaration - int zzInput; - int zzAction; - - // cached fields: - int zzCurrentPosL; - int zzMarkedPosL; - int zzEndReadL = zzEndRead; - CharSequence zzBufferL = zzBuffer; - char[] zzBufferArrayL = zzBufferArray; - char [] zzCMapL = ZZ_CMAP; - ---- local declarations - - while (true) { - zzMarkedPosL = zzMarkedPos; - ---- start admin (line, char, col count) - zzAction = -1; - - zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL; - ---- start admin (lexstate etc) - - zzForAction: { - while (true) { - ---- next input, line, col, char count, next transition, isFinal action - zzAction = zzState; - zzMarkedPosL = zzCurrentPosL; ---- line count update - } - - } - } - - // store back cached position - zzMarkedPos = zzMarkedPosL; ---- char count update - ---- actions - default: - if (zzInput == YYEOF && zzStartRead == zzCurrentPos) { - zzAtEOF = true; ---- eofvalue - } - else { ---- no match - } - } - } - } - ---- main - -} diff --git a/lib/idea-markdown.jar b/lib/idea-markdown.jar new file mode 100644 index 00000000..ba830192 Binary files /dev/null and b/lib/idea-markdown.jar differ diff --git a/src/Kotlin/ContentBuilder.kt b/src/Kotlin/ContentBuilder.kt index 635fc74f..8079fb4c 100644 --- a/src/Kotlin/ContentBuilder.kt +++ b/src/Kotlin/ContentBuilder.kt @@ -1,23 +1,22 @@ package org.jetbrains.dokka -import org.jetbrains.markdown.MarkdownElementTypes import java.util.ArrayDeque -import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor -import org.jetbrains.jet.lang.resolve.name.Name -import org.jetbrains.jet.lang.resolve.DescriptorToSourceUtils -import org.jetbrains.jet.lang.resolve.scopes.JetScope -import org.jetbrains.jet.lang.resolve.name.FqName +import org.jetbrains.jet.lang.descriptors.* +import org.jetbrains.jet.lang.resolve.* +import org.jetbrains.jet.lang.resolve.scopes.* +import org.jetbrains.jet.lang.resolve.name.* +import net.nicoulaj.idea.markdown.lang.* -public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: DeclarationDescriptor): Content { +public fun DocumentationBuilder.buildContent(tree: MarkdownNode, descriptor: DeclarationDescriptor): Content { val nodeStack = ArrayDeque() nodeStack.push(Content()) - tree.visit {(node, text, processChildren) -> + tree.visit {(node, processChildren) -> val parent = nodeStack.peek()!! - val nodeType = node.getTokenType() - val nodeText = tree.getNodeText(node) + val nodeType = node.type + val nodeText = tree.text when (nodeType) { - MarkdownElementTypes.BULLET_LIST -> { + MarkdownElementTypes.UNORDERED_LIST -> { nodeStack.push(ContentList()) processChildren() parent.append(nodeStack.pop()) @@ -27,9 +26,7 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: Dec processChildren() parent.append(nodeStack.pop()) } - MarkdownElementTypes.HORIZONTAL_RULE -> { - } - MarkdownElementTypes.LIST_BLOCK -> { + MarkdownElementTypes.LIST_ITEM -> { nodeStack.push(ContentBlock()) processChildren() parent.append(nodeStack.pop()) @@ -44,50 +41,42 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: Dec processChildren() parent.append(nodeStack.pop()) } - MarkdownElementTypes.CODE -> { + MarkdownTokenTypes.CODE -> { nodeStack.push(ContentCode()) processChildren() parent.append(nodeStack.pop()) } - MarkdownElementTypes.ANONYMOUS_SECTION -> { - nodeStack.push(ContentSection("")) - processChildren() - parent.append(nodeStack.pop()) - } - MarkdownElementTypes.DIRECTIVE -> { - val name = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_NAME)?.let { tree.getNodeText(it) } ?: "" - val params = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_PARAMS)?.let { tree.getNodeText(it) } ?: "" - when (name) { - "code" -> parent.append(functionBody(descriptor, params)) - } - } - MarkdownElementTypes.NAMED_SECTION -> { - val label = tree.findChildByType(node, MarkdownElementTypes.SECTION_NAME)?.let { tree.getNodeText(it) } ?: "" - nodeStack.push(ContentSection(label)) - processChildren() - parent.append(nodeStack.pop()) - } - MarkdownElementTypes.LINK -> { - val target = tree.findChildByType(node, MarkdownElementTypes.TARGET)?.let { tree.getNodeText(it) } ?: "" - val href = tree.findChildByType(node, MarkdownElementTypes.HREF)?.let { tree.getNodeText(it) } + /* MarkdownElementTypes.ANONYMOUS_SECTION -> { + nodeStack.push(ContentSection("")) + processChildren() + parent.append(nodeStack.pop()) + } + MarkdownElementTypes.DIRECTIVE -> { + val name = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_NAME)?.let { tree.getNodeText(it) } ?: "" + val params = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_PARAMS)?.let { tree.getNodeText(it) } ?: "" + when (name) { + "code" -> parent.append(functionBody(descriptor, params)) + } + } + MarkdownElementTypes.NAMED_SECTION -> { + val label = tree.findChildByType(node, MarkdownElementTypes.SECTION_NAME)?.let { tree.getNodeText(it) } ?: "" + nodeStack.push(ContentSection(label)) + processChildren() + parent.append(nodeStack.pop()) + }*/ + MarkdownElementTypes.INLINE_LINK -> { + val target = node.child(MarkdownElementTypes.LINK_TITLE)?.let { it.text } ?: "" + val href = node.child(MarkdownElementTypes.LINK_DESTINATION)?.let { it.text } val link = if (href != null) ContentExternalLink(href) else ContentExternalLink(target) link.append(ContentText(target)) parent.append(link) } - MarkdownElementTypes.PLAIN_TEXT -> { + MarkdownTokenTypes.TEXT -> { nodeStack.push(ContentText(nodeText)) processChildren() parent.append(nodeStack.pop()) } - MarkdownElementTypes.END_LINE -> { - nodeStack.push(ContentText(nodeText)) - processChildren() - parent.append(nodeStack.pop()) - } - MarkdownElementTypes.BLANK_LINE -> { - processChildren() - } - MarkdownElementTypes.PARA -> { + MarkdownElementTypes.PARAGRAPH -> { nodeStack.push(ContentParagraph()) processChildren() parent.append(nodeStack.pop()) diff --git a/src/Kotlin/DocumentationBuilder.kt b/src/Kotlin/DocumentationBuilder.kt index 0acff1dc..d8be9d5a 100644 --- a/src/Kotlin/DocumentationBuilder.kt +++ b/src/Kotlin/DocumentationBuilder.kt @@ -1,17 +1,12 @@ package org.jetbrains.dokka import org.jetbrains.jet.lang.descriptors.* -import org.jetbrains.dokka.DocumentationNode.Kind -import org.jetbrains.jet.lang.types.TypeProjection -import org.jetbrains.jet.lang.types.Variance -import org.jetbrains.jet.lang.types.JetType -import org.jetbrains.jet.lang.types.lang.KotlinBuiltIns -import org.jetbrains.jet.lang.resolve.BindingContext -import org.jetbrains.jet.lang.resolve.name.Name -import org.jetbrains.jet.lang.resolve.scopes.JetScope -import org.jetbrains.jet.lang.psi.JetFile -import org.jetbrains.jet.lang.resolve.name.FqName -import org.jetbrains.jet.lang.resolve.lazy.ResolveSession +import org.jetbrains.dokka.DocumentationNode.* +import org.jetbrains.jet.lang.types.* +import org.jetbrains.jet.lang.types.lang.* +import org.jetbrains.jet.lang.resolve.scopes.* +import org.jetbrains.jet.lang.resolve.name.* +import org.jetbrains.jet.lang.resolve.lazy.* public data class DocumentationOptions(val includeNonPublic: Boolean = false) @@ -24,7 +19,7 @@ class DocumentationBuilder(val session: ResolveSession, val options: Documentati fun parseDocumentation(descriptor: DeclarationDescriptor): Content { val docText = descriptor.getDocumentationElements().map { it.extractText() }.join("\n") - val tree = MarkdownProcessor.parse(docText) + val tree = parseMarkdown(docText) //println(tree.toTestString()) val content = buildContent(tree, descriptor) return content diff --git a/src/Markdown/GeneratedParserUtilBase.java b/src/Markdown/GeneratedParserUtilBase.java deleted file mode 100644 index 9dd999b5..00000000 --- a/src/Markdown/GeneratedParserUtilBase.java +++ /dev/null @@ -1,1031 +0,0 @@ -/* - * Copyright 2011-2014 Gregory Shrago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.jetbrains.dokka.Markdown; - -import com.intellij.lang.*; -import com.intellij.lang.impl.PsiBuilderAdapter; -import com.intellij.lang.impl.PsiBuilderImpl; -import com.intellij.lexer.Lexer; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.Comparing; -import com.intellij.openapi.util.Key; -import com.intellij.openapi.util.Pair; -import com.intellij.openapi.util.text.StringHash; -import com.intellij.openapi.util.text.StringUtil; -import com.intellij.psi.PsiFile; -import com.intellij.psi.PsiReference; -import com.intellij.psi.TokenType; -import com.intellij.psi.impl.source.resolve.FileContextUtil; -import com.intellij.psi.impl.source.tree.CompositePsiElement; -import com.intellij.psi.tree.ICompositeElementType; -import com.intellij.psi.tree.IElementType; -import com.intellij.psi.tree.TokenSet; -import com.intellij.util.Function; -import com.intellij.util.PairProcessor; -import com.intellij.util.containers.ContainerUtil; -import com.intellij.util.containers.LimitedPool; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedList; - -/** - * @author gregsh - */ -public class GeneratedParserUtilBase { - - private static final Logger LOG = Logger.getInstance("org.intellij.grammar.parser.GeneratedParserUtilBase"); - - private static final int MAX_RECURSION_LEVEL = 1000; - private static final int MAX_VARIANTS_SIZE = 10000; - private static final int MAX_VARIANTS_TO_DISPLAY = 50; - - private static final int INITIAL_VARIANTS_SIZE = 1000; - private static final int VARIANTS_POOL_SIZE = 10000; - private static final int FRAMES_POOL_SIZE = 500; - - public static final IElementType DUMMY_BLOCK = new DummyBlockElementType(); - - public interface Parser { - boolean parse(PsiBuilder builder, int level); - } - - public static final Parser TOKEN_ADVANCER = new Parser() { - @Override - public boolean parse(PsiBuilder builder, int level) { - if (builder.eof()) return false; - builder.advanceLexer(); - return true; - } - }; - - public static final Parser TRUE_CONDITION = new Parser() { - @Override - public boolean parse(PsiBuilder builder, int level) { - return true; - } - }; - - public static boolean eof(PsiBuilder builder_, int level_) { - return builder_.eof(); - } - - public static int current_position_(PsiBuilder builder_) { - return builder_.rawTokenIndex(); - } - - public static boolean recursion_guard_(PsiBuilder builder_, int level_, String funcName_) { - if (level_ > MAX_RECURSION_LEVEL) { - builder_.error("Maximum recursion level (" + MAX_RECURSION_LEVEL + ") reached in '" + funcName_ + "'"); - return false; - } - return true; - } - - public static boolean empty_element_parsed_guard_(PsiBuilder builder_, String funcName_, int prev_position_) { - if (prev_position_ == current_position_(builder_)) { - builder_.error("Empty element parsed in '" + funcName_ + "' at offset " + builder_.getCurrentOffset()); - return false; - } - return true; - } - - public static boolean invalid_left_marker_guard_(PsiBuilder builder_, PsiBuilder.Marker marker_, String funcName_) { - //builder_.error("Invalid left marker encountered in " + funcName_ +" at offset " + builder_.getCurrentOffset()); - boolean goodMarker = marker_ != null; // && ((LighterASTNode)marker_).getTokenType() != TokenType.ERROR_ELEMENT; - if (!goodMarker) return false; - ErrorState state = ErrorState.get(builder_); - - return !state.frameStack.isEmpty(); - } - - public static TokenSet create_token_set_(IElementType... tokenTypes_) { - return TokenSet.create(tokenTypes_); - } - - private static boolean consumeTokens(PsiBuilder builder_, boolean smart, int pin, IElementType... tokens) { - ErrorState state = ErrorState.get(builder_); - if (state.completionState != null && state.predicateCount == 0) { - addCompletionVariant(builder_, state.completionState, tokens); - } - // suppress single token completion - CompletionState completionState = state.completionState; - state.completionState = null; - boolean result_ = true; - boolean pinned_ = false; - for (int i = 0, tokensLength = tokens.length; i < tokensLength; i++) { - if (pin > 0 && i == pin) pinned_ = result_; - if (result_ || pinned_) { - boolean fast = smart && i == 0; - if (!(fast ? consumeTokenFast(builder_, tokens[i]) : consumeToken(builder_, tokens[i]))) { - result_ = false; - if (pin < 0 || pinned_) report_error_(builder_, state, false); - } - } - } - state.completionState = completionState; - return pinned_ || result_; - } - - public static boolean consumeTokens(PsiBuilder builder_, int pin_, IElementType... token) { - return consumeTokens(builder_, false, pin_, token); - } - - public static boolean consumeTokensSmart(PsiBuilder builder_, int pin_, IElementType... token) { - return consumeTokens(builder_, true, pin_, token); - } - - public static boolean parseTokens(PsiBuilder builder_, int pin_, IElementType... tokens) { - return parseTokens(builder_, false, pin_, tokens); - } - - public static boolean parseTokensSmart(PsiBuilder builder_, int pin_, IElementType... tokens) { - return parseTokens(builder_, true, pin_, tokens); - } - - public static boolean parseTokens(PsiBuilder builder_, boolean smart, int pin_, IElementType... tokens) { - PsiBuilder.Marker marker_ = builder_.mark(); - boolean result_ = consumeTokens(builder_, smart, pin_, tokens); - if (!result_) { - marker_.rollbackTo(); - } - else { - marker_.drop(); - } - return result_; - } - - public static boolean consumeTokenSmart(PsiBuilder builder_, IElementType token) { - addCompletionVariantSmart(builder_, token); - return consumeTokenFast(builder_, token); - } - - public static boolean consumeTokenSmart(PsiBuilder builder_, String token) { - addCompletionVariantSmart(builder_, token); - return consumeTokenFast(builder_, token); - } - - public static boolean consumeToken(PsiBuilder builder_, IElementType token) { - addVariantSmart(builder_, token, true); - if (nextTokenIsFast(builder_, token)) { - builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeTokenFast(PsiBuilder builder_, IElementType token) { - if (nextTokenIsFast(builder_, token)) { - builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeToken(PsiBuilder builder_, String text) { - return consumeToken(builder_, text, ErrorState.get(builder_).caseSensitive); - } - - public static boolean consumeToken(PsiBuilder builder_, String text, boolean caseSensitive) { - addVariantSmart(builder_, text, true); - int count = nextTokenIsFast(builder_, text, caseSensitive); - if (count > 0) { - while (count-- > 0) builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean consumeTokenFast(PsiBuilder builder_, String text) { - int count = nextTokenIsFast(builder_, text, ErrorState.get(builder_).caseSensitive); - if (count > 0) { - while (count-- > 0) builder_.advanceLexer(); - return true; - } - return false; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType token) { - return builder_.getTokenType() == token; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType... tokens) { - IElementType tokenType = builder_.getTokenType(); - for (IElementType token : tokens) { - if (token == tokenType) return true; - } - return false; - } - - public static boolean nextTokenIs(PsiBuilder builder_, String frameName, IElementType... tokens) { - ErrorState state = ErrorState.get(builder_); - if (state.completionState != null) return true; - boolean track = !state.suppressErrors && state.predicateCount < 2 && state.predicateSign; - if (!track) return nextTokenIsFast(builder_, tokens); - IElementType tokenType = builder_.getTokenType(); - if (StringUtil.isNotEmpty(frameName)) { - addVariantInner(state, builder_.rawTokenIndex(), frameName); - } - else { - for (IElementType token : tokens) { - addVariant(builder_, state, token); - } - } - if (tokenType == null) return false; - for (IElementType token : tokens) { - if (tokenType == token) return true; - } - return false; - } - - public static boolean nextTokenIs(PsiBuilder builder_, IElementType token) { - if (!addVariantSmart(builder_, token, false)) return true; - return nextTokenIsFast(builder_, token); - } - - public static boolean nextTokenIs(PsiBuilder builder_, String tokenText) { - if (!addVariantSmart(builder_, tokenText, false)) return true; - return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0; - } - - public static boolean nextTokenIsFast(PsiBuilder builder_, String tokenText) { - return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0; - } - - public static int nextTokenIsFast(PsiBuilder builder_, String tokenText, boolean caseSensitive) { - CharSequence sequence = builder_.getOriginalText(); - int offset = builder_.getCurrentOffset(); - int endOffset = offset + tokenText.length(); - CharSequence subSequence = sequence.subSequence(offset, Math.min(endOffset, sequence.length())); - - if (!Comparing.equal(subSequence, tokenText, caseSensitive)) return 0; - - int count = 0; - while (true) { - int nextOffset = builder_.rawTokenTypeStart(++count); - if (nextOffset > endOffset) { - return -count; - } - else if (nextOffset == endOffset) { - break; - } - } - return count; - } - - private static void addCompletionVariantSmart(PsiBuilder builder_, Object token) { - ErrorState state = ErrorState.get(builder_); - CompletionState completionState = state.completionState; - if (completionState != null && state.predicateCount == 0) { - addCompletionVariant(builder_, completionState, token); - } - } - - private static boolean addVariantSmart(PsiBuilder builder_, Object token, boolean force) { - ErrorState state = ErrorState.get(builder_); - // skip FIRST check in completion mode - if (state.completionState != null && !force) return false; - builder_.eof(); - if (!state.suppressErrors && state.predicateCount < 2) { - addVariant(builder_, state, token); - } - return true; - } - - public static void addVariant(PsiBuilder builder_, String text) { - addVariant(builder_, ErrorState.get(builder_), text); - } - - private static void addVariant(PsiBuilder builder_, ErrorState state, Object o) { - builder_.eof(); // skip whitespaces - addVariantInner(state, builder_.rawTokenIndex(), o); - - CompletionState completionState = state.completionState; - if (completionState != null && state.predicateSign) { - addCompletionVariant(builder_, completionState, o); - } - } - - private static void addVariantInner(ErrorState state, int pos, Object o) { - Variant variant = state.VARIANTS.alloc().init(pos, o); - if (state.predicateSign) { - state.variants.add(variant); - if (state.lastExpectedVariantPos < variant.position) { - state.lastExpectedVariantPos = variant.position; - } - } - else { - state.unexpected.add(variant); - } - } - - private static void addCompletionVariant(@NotNull PsiBuilder builder_, @NotNull CompletionState completionState, Object o) { - int offset = builder_.getCurrentOffset(); - if (!builder_.eof() && offset == builder_.rawTokenTypeStart(1)) return; // suppress for zero-length tokens - - boolean add = false; - int diff = completionState.offset - offset; - String text = completionState.convertItem(o); - int length = text == null? 0 : text.length(); - if (length == 0) return; - if (diff == 0) { - add = true; - } - else if (diff > 0 && diff <= length) { - CharSequence fragment = builder_.getOriginalText().subSequence(offset, completionState.offset); - add = completionState.prefixMatches(fragment.toString(), text); - } - else if (diff < 0) { - for (int i=-1; ; i--) { - IElementType type = builder_.rawLookup(i); - int tokenStart = builder_.rawTokenTypeStart(i); - if (isWhitespaceOrComment(builder_, type)) { - diff = completionState.offset - tokenStart; - } - else if (type != null && tokenStart < completionState.offset) { - CharSequence fragment = builder_.getOriginalText().subSequence(tokenStart, completionState.offset); - if (completionState.prefixMatches(fragment.toString(), text)) { - diff = completionState.offset - tokenStart; - } - break; - } - else break; - } - add = diff >= 0 && diff < length; - } - add = add && length > 1 && !(text.charAt(0) == '<' && text.charAt(length - 1) == '>') && - !(text.charAt(0) == '\'' && text.charAt(length - 1) == '\'' && length < 5); - if (add) { - completionState.addItem(builder_, text); - } - } - - public static boolean isWhitespaceOrComment(@NotNull PsiBuilder builder_, @Nullable IElementType type) { - return ((PsiBuilderImpl)((Builder)builder_).getDelegate()).whitespaceOrComment(type); - } - - // here's the new section API for compact parsers & less IntelliJ platform API exposure - public static final int _NONE_ = 0x0; - public static final int _COLLAPSE_ = 0x1; - public static final int _LEFT_ = 0x2; - public static final int _LEFT_INNER_ = 0x4; - public static final int _AND_ = 0x8; - public static final int _NOT_ = 0x10; - - // simple enter/exit methods pair that doesn't require frame object - public static PsiBuilder.Marker enter_section_(PsiBuilder builder_) { - return builder_.mark(); - } - - public static void exit_section_(PsiBuilder builder_, - PsiBuilder.Marker marker, - @Nullable IElementType elementType, - boolean result) { - close_marker_impl_(ErrorState.get(builder_).frameStack.peekLast(), marker, elementType, result); - } - - // complex enter/exit methods pair with frame object - public static PsiBuilder.Marker enter_section_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) { - PsiBuilder.Marker marker = builder_.mark(); - enter_section_impl_(builder_, level, modifiers, frameName); - return marker; - } - - private static void enter_section_impl_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) { - ErrorState state = ErrorState.get(builder_); - Frame frame = state.FRAMES.alloc().init(builder_, state, level, modifiers, frameName); - Frame prevFrame = state.frameStack.peekLast(); - if (prevFrame != null && prevFrame.errorReportedAt > frame.position) { - // report error for previous unsuccessful frame - reportError(builder_, state, frame, true, false); - } - if (((frame.modifiers & _LEFT_) | (frame.modifiers & _LEFT_INNER_)) != 0) { - PsiBuilder.Marker left = (PsiBuilder.Marker)builder_.getLatestDoneMarker(); - if (invalid_left_marker_guard_(builder_, left, frameName)) { - frame.leftMarker = left; - } - } - state.frameStack.add(frame); - if ((modifiers & _AND_) != 0) { - if (state.predicateCount == 0 && !state.predicateSign) { - throw new AssertionError("Incorrect false predicate sign"); - } - state.predicateCount++; - } - else if ((modifiers & _NOT_) != 0) { - if (state.predicateCount == 0) { - state.predicateSign = false; - } - else { - state.predicateSign = !state.predicateSign; - } - state.predicateCount++; - } - } - - public static void exit_section_(PsiBuilder builder_, - int level, - PsiBuilder.Marker marker, - @Nullable IElementType elementType, - boolean result, - boolean pinned, - @Nullable Parser eatMore) { - ErrorState state = ErrorState.get(builder_); - - Frame frame = state.frameStack.pollLast(); - if (frame == null || level != frame.level) { - LOG.error("Unbalanced error section: got " + frame + ", expected level " + level); - if (frame != null) state.FRAMES.recycle(frame); - close_marker_impl_(frame, marker, elementType, result); - return; - } - - if (((frame.modifiers & _AND_) | (frame.modifiers & _NOT_)) != 0) { - close_marker_impl_(frame, marker, null, false); - state.predicateCount--; - if ((frame.modifiers & _NOT_) != 0) state.predicateSign = !state.predicateSign; - state.FRAMES.recycle(frame); - return; - } - exit_section_impl_(state, frame, builder_, marker, elementType, result, pinned); - - int initialPos = builder_.rawTokenIndex(); - boolean willFail = !result && !pinned; - if (willFail && initialPos == frame.position && state.lastExpectedVariantPos == frame.position && - frame.name != null && state.variants.size() - frame.variantCount > 1) { - state.clearVariants(true, frame.variantCount); - addVariantInner(state, initialPos, frame.name); - } - int lastErrorPos = getLastVariantPos(state, initialPos); - if (!state.suppressErrors && eatMore != null) { - state.suppressErrors = true; - final boolean eatMoreFlagOnce = !builder_.eof() && eatMore.parse(builder_, frame.level + 1); - boolean eatMoreFlag = eatMoreFlagOnce || !result && frame.position == initialPos && lastErrorPos > frame.position; - - PsiBuilderImpl.ProductionMarker latestDoneMarker = - (pinned || result) && (state.altMode || elementType != null) && - eatMoreFlagOnce ? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null; - PsiBuilder.Marker extensionMarker = null; - IElementType extensionTokenType = null; - // whitespace prefix makes the very first frame offset bigger than marker start offset which is always 0 - if (latestDoneMarker instanceof PsiBuilder.Marker && - frame.position >= latestDoneMarker.getStartIndex() && - frame.position <= latestDoneMarker.getEndIndex()) { - extensionMarker = ((PsiBuilder.Marker)latestDoneMarker).precede(); - extensionTokenType = latestDoneMarker.getTokenType(); - ((PsiBuilder.Marker)latestDoneMarker).drop(); - } - // advance to the last error pos - // skip tokens until lastErrorPos. parseAsTree might look better here... - int parenCount = 0; - while ((eatMoreFlag || parenCount > 0) && builder_.rawTokenIndex() < lastErrorPos) { - builder_.advanceLexer(); - eatMoreFlag = eatMore.parse(builder_, frame.level + 1); - } - boolean errorReported = frame.errorReportedAt == initialPos || !result && frame.errorReportedAt >= frame.position; - if (errorReported) { - if (eatMoreFlag) { - builder_.advanceLexer(); - parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); - } - } - else if (eatMoreFlag) { - errorReported = reportError(builder_, state, frame, true, true); - parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore); - } - else if (eatMoreFlagOnce || (!result && frame.position != builder_.rawTokenIndex()) || frame.errorReportedAt > initialPos) { - errorReported = reportError(builder_, state, frame, true, false); - } - if (extensionMarker != null) { - extensionMarker.done(extensionTokenType); - } - state.suppressErrors = false; - if (errorReported || result) { - state.clearVariants(true, 0); - state.clearVariants(false, 0); - state.lastExpectedVariantPos = -1; - } - } - else if (!result && pinned && frame.errorReportedAt < 0) { - // do not report if there are errors beyond current position - if (lastErrorPos == initialPos) { - // do not force, inner recoverRoot might have skipped some tokens - reportError(builder_, state, frame, false, false); - } - else if (lastErrorPos > initialPos) { - // set error pos here as if it is reported for future reference - frame.errorReportedAt = lastErrorPos; - } - } - // propagate errorReportedAt up the stack to avoid duplicate reporting - Frame prevFrame = willFail && eatMore == null ? null : state.frameStack.peekLast(); - if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) { - prevFrame.errorReportedAt = frame.errorReportedAt; - } - state.FRAMES.recycle(frame); - } - - private static void exit_section_impl_(ErrorState state, - Frame frame, - PsiBuilder builder_, - PsiBuilder.Marker marker, - IElementType elementType, - boolean result, - boolean pinned) { - if (elementType != null && marker != null) { - if ((frame.modifiers & _COLLAPSE_) != 0) { - PsiBuilderImpl.ProductionMarker last = result || pinned? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null; - if (last != null && last.getStartIndex() == frame.position && - state.typeExtends(last.getTokenType(), elementType)) { - IElementType resultType = last.getTokenType(); - ((PsiBuilder.Marker)last).drop(); - marker.done(resultType); - return; - } - } - if (result || pinned) { - if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) { - marker.done(elementType); - frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType()); - frame.leftMarker.drop(); - } - else if ((frame.modifiers & _LEFT_) != 0 && frame.leftMarker != null) { - marker.drop(); - frame.leftMarker.precede().done(elementType); - } - else { - if (frame.level == 0) builder_.eof(); // skip whitespaces - marker.done(elementType); - } - } - else { - close_marker_impl_(frame, marker, null, false); - } - } - else if (result || pinned) { - if (marker != null) marker.drop(); - if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) { - frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType()); - frame.leftMarker.drop(); - } - } - else { - close_marker_impl_(frame, marker, null, false); - } - } - - private static void close_marker_impl_(Frame frame, PsiBuilder.Marker marker, IElementType elementType, boolean result) { - if (marker == null) return; - if (result) { - if (elementType != null) { - marker.done(elementType); - } - else { - marker.drop(); - } - } - else { - if (frame != null) { - int position = ((PsiBuilderImpl.ProductionMarker)marker).getStartIndex(); - if (frame.errorReportedAt > position) { - frame.errorReportedAt = frame.errorReportedAtPrev; - } - } - marker.rollbackTo(); - } - } - - public static boolean report_error_(PsiBuilder builder_, boolean result_) { - if (!result_) report_error_(builder_, ErrorState.get(builder_), false); - return result_; - } - - public static void report_error_(PsiBuilder builder_, ErrorState state, boolean advance) { - Frame frame = state.frameStack.isEmpty()? null : state.frameStack.getLast(); - if (frame == null) { - LOG.error("unbalanced enter/exit section call: got null"); - return; - } - int position = builder_.rawTokenIndex(); - if (frame.errorReportedAt < position && getLastVariantPos(state, position + 1) <= position) { - reportError(builder_, state, frame, true, advance); - } - } - - private static int getLastVariantPos(ErrorState state, int defValue) { - return state.lastExpectedVariantPos < 0? defValue : state.lastExpectedVariantPos; - } - - private static boolean reportError(PsiBuilder builder_, - ErrorState state, - Frame frame, - boolean force, - boolean advance) { - String expectedText = state.getExpectedText(builder_); - boolean notEmpty = StringUtil.isNotEmpty(expectedText); - if (force || notEmpty || advance) { - String gotText = builder_.eof()? "unexpected end of file" : - notEmpty? "got '" + builder_.getTokenText() +"'" : - "'" + builder_.getTokenText() +"' unexpected"; - String message = expectedText + gotText; - if (advance) { - PsiBuilder.Marker mark = builder_.mark(); - builder_.advanceLexer(); - mark.error(message); - } - else { - builder_.error(message); - } - builder_.eof(); // skip whitespaces - frame.errorReportedAt = builder_.rawTokenIndex(); - return true; - } - return false; - } - - - public static final Key COMPLETION_STATE_KEY = Key.create("COMPLETION_STATE_KEY"); - - public static class CompletionState implements Function { - public final int offset; - public final Collection items = ContainerUtil.newTroveSet(); - - public CompletionState(int offset_) { - offset = offset_; - } - - @Nullable - public String convertItem(Object o) { - return o instanceof Object[] ? StringUtil.join((Object[]) o, this, " ") : o.toString(); - } - - @Override - public String fun(Object o) { - return o.toString(); - } - - public void addItem(@NotNull PsiBuilder builder, @NotNull String text) { - items.add(text); - } - - public boolean prefixMatches(@NotNull String prefix, @NotNull String variant) { - return StringUtil.startsWithIgnoreCase(variant, prefix); - } - } - - public static class Builder extends PsiBuilderAdapter { - public final ErrorState state; - public final PsiParser parser; - - public Builder(PsiBuilder builder_, ErrorState state_, PsiParser parser_) { - super(builder_); - state = state_; - parser = parser_; - } - - public Lexer getLexer() { - return ((PsiBuilderImpl)myDelegate).getLexer(); - } - } - - public static PsiBuilder adapt_builder_(IElementType root, PsiBuilder builder, PsiParser parser) { - return adapt_builder_(root, builder, parser, null); - } - - public static PsiBuilder adapt_builder_(IElementType root, PsiBuilder builder, PsiParser parser, TokenSet[] extendsSets) { - ErrorState state = new ErrorState(); - ErrorState.initState(state, builder, root, extendsSets); - return new Builder(builder, state, parser); - } - - public static class ErrorState { - TokenSet[] extendsSets; - public PairProcessor altExtendsChecker; - - int predicateCount; - boolean predicateSign = true; - boolean suppressErrors; - public final LinkedList frameStack = new LinkedList(); - public CompletionState completionState; - - private boolean caseSensitive; - public boolean altMode; - - int lastExpectedVariantPos = -1; - MyList variants = new MyList(INITIAL_VARIANTS_SIZE); - MyList unexpected = new MyList(INITIAL_VARIANTS_SIZE / 10); - - final LimitedPool VARIANTS = new LimitedPool(VARIANTS_POOL_SIZE, new LimitedPool.ObjectFactory() { - @Override - public Variant create() { - return new Variant(); - } - - @Override - public void cleanup(final Variant o) { - } - }); - final LimitedPool FRAMES = new LimitedPool(FRAMES_POOL_SIZE, new LimitedPool.ObjectFactory() { - @Override - public Frame create() { - return new Frame(); - } - - @Override - public void cleanup(final Frame o) { - } - }); - - public static ErrorState get(PsiBuilder builder) { - return ((Builder)builder).state; - } - - public static void initState(ErrorState state, PsiBuilder builder, IElementType root, TokenSet[] extendsSets) { - state.extendsSets = extendsSets; - PsiFile file = builder.getUserDataUnprotected(FileContextUtil.CONTAINING_FILE_KEY); - state.completionState = file == null? null: file.getUserData(COMPLETION_STATE_KEY); - Language language = file == null? root.getLanguage() : file.getLanguage(); - state.caseSensitive = language.isCaseSensitive(); - } - - public String getExpectedText(PsiBuilder builder_) { - int position = builder_.rawTokenIndex(); - StringBuilder sb = new StringBuilder(); - if (addExpected(sb, position, true)) { - sb.append(" expected, "); - } - else if (addExpected(sb, position, false)) sb.append(" unexpected, "); - return sb.toString(); - } - - private boolean addExpected(StringBuilder sb, int position, boolean expected) { - MyList list = expected ? variants : unexpected; - String[] strings = new String[list.size()]; - long[] hashes = new long[strings.length]; - Arrays.fill(strings, ""); - int count = 0; - loop: for (Variant variant : list) { - if (position == variant.position) { - String text = variant.object.toString(); - long hash = StringHash.calc(text); - for (int i=0; i 0) { - if (count > MAX_VARIANTS_TO_DISPLAY) { - sb.append(" and ..."); - break; - } - else { - sb.append(", "); - } - } - char c = s.charAt(0); - String displayText = c == '<' || StringUtil.isJavaIdentifierStart(c) ? s : '\'' + s + '\''; - sb.append(displayText); - } - if (count > 1 && count < MAX_VARIANTS_TO_DISPLAY) { - int idx = sb.lastIndexOf(", "); - sb.replace(idx, idx + 1, " or"); - } - return count > 0; - } - - public void clearVariants(boolean expected, int start) { - MyList list = expected? variants : unexpected; - if (start < 0 || start >= list.size()) return; - for (int i = start, len = list.size(); i < len; i ++) { - VARIANTS.recycle(list.get(i)); - } - list.setSize(start); - } - - boolean typeExtends(IElementType child_, IElementType parent_) { - if (child_ == parent_) return true; - if (extendsSets != null) { - for (TokenSet set : extendsSets) { - if (set.contains(child_) && set.contains(parent_)) return true; - } - } - return altExtendsChecker != null && altExtendsChecker.process(child_, parent_); - } - } - - public static class Frame { - public int offset; - public int position; - public int level; - public int modifiers; - public String name; - public int variantCount; - public int errorReportedAt; - public int errorReportedAtPrev; - public PsiBuilder.Marker leftMarker; - - public Frame() { - } - - public Frame init(PsiBuilder builder_, ErrorState state, int level_, int modifiers_, String name_) { - offset = builder_.getCurrentOffset(); - position = builder_.rawTokenIndex(); - level = level_; - modifiers = modifiers_; - name = name_; - variantCount = state.variants.size(); - errorReportedAt = -1; - - Frame prev = state.frameStack.peekLast(); - errorReportedAtPrev = prev == null? -1 : prev.errorReportedAt; - leftMarker = null; - return this; - } - - @Override - public String toString() { - String mod = modifiers == _NONE_ ? "_NONE_, " : - ((modifiers & _COLLAPSE_) != 0? "_CAN_COLLAPSE_, ": "") + - ((modifiers & _LEFT_) != 0? "_LEFT_, ": "") + - ((modifiers & _LEFT_INNER_) != 0? "_LEFT_INNER_, ": "") + - ((modifiers & _AND_) != 0? "_AND_, ": "") + - ((modifiers & _NOT_) != 0? "_NOT_, ": ""); - return String.format("{%s:%s:%d, %d, %s%s}", offset, position, level, errorReportedAt, mod, name); - } - } - - - private static class Variant { - int position; - Object object; - - public Variant init(int pos, Object o) { - position = pos; - object = o; - return this; - } - - @Override - public String toString() { - return "<" + position + ", " + object + ">"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Variant variant = (Variant)o; - - if (position != variant.position) return false; - if (!this.object.equals(variant.object)) return false; - - return true; - } - - @Override - public int hashCode() { - int result = position; - result = 31 * result + object.hashCode(); - return result; - } - } - - - private static final int MAX_CHILDREN_IN_TREE = 10; - public static boolean parseAsTree(ErrorState state, final PsiBuilder builder_, int level, final IElementType chunkType, - boolean checkBraces, final Parser parser, final Parser eatMoreCondition) { - final LinkedList> parenList = new LinkedList>(); - final LinkedList> siblingList = new LinkedList>(); - PsiBuilder.Marker marker = null; - - final Runnable checkSiblingsRunnable = new Runnable() { - @Override - public void run() { - main: - while (!siblingList.isEmpty()) { - final Pair parenPair = parenList.peek(); - final int rating = siblingList.getFirst().second; - int count = 0; - for (Pair pair : siblingList) { - if (pair.second != rating || parenPair != null && pair.first == parenPair.second) break main; - if (++count >= MAX_CHILDREN_IN_TREE) { - final PsiBuilder.Marker parentMarker = pair.first.precede(); - while (count-- > 0) { - siblingList.removeFirst(); - } - parentMarker.done(chunkType); - siblingList.addFirst(Pair.create(parentMarker, rating + 1)); - continue main; - } - } - break; - } - } - }; - int totalCount = 0; - int tokenCount = 0; - while (true) { - final IElementType tokenType = builder_.getTokenType(); - if (marker == null) { - marker = builder_.mark(); - } - final boolean result = (!parenList.isEmpty() || eatMoreCondition.parse(builder_, level + 1)) && parser.parse(builder_, level + 1); - if (result) { - tokenCount++; - totalCount++; - } - if (!result) { - break; - } - - if (tokenCount >= MAX_CHILDREN_IN_TREE && marker != null) { - marker.done(chunkType); - siblingList.addFirst(Pair.create(marker, 1)); - checkSiblingsRunnable.run(); - marker = null; - tokenCount = 0; - } - } - if (marker != null) { - marker.drop(); - } - for (Pair pair : parenList) { - pair.first.drop(); - } - return totalCount != 0; - } - - private static class DummyBlockElementType extends IElementType implements ICompositeElementType{ - DummyBlockElementType() { - super("DUMMY_BLOCK", Language.ANY); - } - - @NotNull - @Override - public ASTNode createCompositeNode() { - return new DummyBlock(); - } - } - - public static class DummyBlock extends CompositePsiElement { - DummyBlock() { - super(DUMMY_BLOCK); - } - - @NotNull - @Override - public PsiReference[] getReferences() { - return PsiReference.EMPTY_ARRAY; - } - - @NotNull - @Override - public Language getLanguage() { - return getParent().getLanguage(); - } - } - - private static class MyList extends ArrayList { - MyList(int initialCapacity) { - super(initialCapacity); - } - - protected void setSize(int fromIndex) { - removeRange(fromIndex, size()); - } - - @Override - public boolean add(E e) { - int size = size(); - if (size >= MAX_VARIANTS_SIZE) { - removeRange(MAX_VARIANTS_SIZE / 4, size - MAX_VARIANTS_SIZE / 4); - } - return super.add(e); - } - } -} diff --git a/src/Markdown/MarkdownProcessor.kt b/src/Markdown/MarkdownProcessor.kt index 040f2ad5..b5e18f92 100644 --- a/src/Markdown/MarkdownProcessor.kt +++ b/src/Markdown/MarkdownProcessor.kt @@ -1,80 +1,34 @@ package org.jetbrains.dokka -import org.jetbrains.markdown.* -import com.intellij.lang.impl.PsiBuilderImpl -import com.intellij.psi.tree.TokenSet -import com.intellij.lang.Language -import com.intellij.psi.tree.IFileElementType -import com.intellij.lang.LighterASTNode -import com.intellij.util.diff.FlyweightCapableTreeStructure -import com.intellij.openapi.util.Ref -import org.jetbrains.markdown.lexer.MarkdownLexer -import com.intellij.psi.tree.IElementType +import net.nicoulaj.idea.markdown.lang.ast.* +import net.nicoulaj.idea.markdown.lang.parser.dialects.commonmark.* +import net.nicoulaj.idea.markdown.lang.parser.* +import net.nicoulaj.idea.markdown.lang.* -public object MarkdownProcessor { - val EXPR_LANGUAGE = object : Language("MARKDOWN") {} - val DOCUMENT = IFileElementType("DOCUMENT", EXPR_LANGUAGE); - - public fun parse(markdown: String): MarkdownTree { - val parser = MarkdownParser() - val builder = PsiBuilderImpl(null, null, TokenSet.EMPTY, TokenSet.EMPTY, MarkdownLexer(), null, markdown, null, null) - parser.parse_only_(DOCUMENT, builder) - val light = builder.getLightTree()!! - return MarkdownTree(markdown, light) - } +class MarkdownNode(val node: ASTNode, val markdown: String) { + val children: List get() = node.children.map { MarkdownNode(it, markdown) } + val endOffset: Int get() = node.endOffset + val startOffset: Int get() = node.startOffset + val type: IElementType get() = node.type + val text: String get() = markdown.substring(startOffset, endOffset) + fun child(type: IElementType): MarkdownNode? = children.firstOrNull { it.type == type } } -public class MarkdownTree(private val text: String, private val structure: FlyweightCapableTreeStructure) { - fun visit(action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) { - visit(structure.getRoot(), action) - } - - fun findChildByType(node: LighterASTNode, findType: IElementType) : LighterASTNode? { - val ref: Ref?> = Ref.create?>() - val count = structure.getChildren(node, ref) - val children = ref.get() - if (children != null) { - for (index in 0..count - 1) { - val child = children[index] - val nodeType = child.getTokenType() - if (nodeType == findType) - return child - val nestedChild = findChildByType(child, findType) - if (nestedChild != null) - return nestedChild - } - } - return null - } - - fun getNodeText(node: LighterASTNode) : String { - return text.substring(node.getStartOffset(), node.getEndOffset()) - } - - fun visit(node: LighterASTNode, action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) { - action(node, text) { - val ref : Ref?> = Ref.create?>() - val count = structure.getChildren(node, ref) - val children = ref.get() - if (children != null) { - for (index in 0..count - 1) { - val child = children[index] - visit(child, action) - } - } +fun MarkdownNode.visit(action: (MarkdownNode, () -> Unit) -> Unit) { + action(this) { + for (child in children) { + child.visit(action) } } - } -public fun MarkdownTree.toTestString(): String { +public fun MarkdownNode.toTestString(): String { val sb = StringBuilder() var level = 0 - visit {(node, text, visitChildren) -> - val nodeText = text.substring(node.getStartOffset(), node.getEndOffset()) + visit {(node, visitChildren) -> sb.append(" ".repeat(level * 2)) - sb.append(node.getTokenType().toString()) - sb.append(":" + nodeText.replace("\n", "\u23CE")) + sb.append(node.type.toString()) + sb.append(":" + node.text.replace("\n", "\u23CE")) sb.appendln() level++ visitChildren() @@ -83,26 +37,23 @@ public fun MarkdownTree.toTestString(): String { return sb.toString() } -public fun MarkdownTree.toHtml(): String { +public fun MarkdownNode.toHtml(): String { val sb = StringBuilder() - visit {(node, text, processChildren) -> - val nodeType = node.getTokenType() - val nodeText = text.substring(node.getStartOffset(), node.getEndOffset()) + visit {(node, processChildren) -> + val nodeType = node.type + val nodeText = node.text when (nodeType) { - MarkdownElementTypes.BULLET_LIST -> { + MarkdownElementTypes.UNORDERED_LIST -> { sb.appendln("
    ") processChildren() sb.appendln("
") } - MarkdownElementTypes.HORIZONTAL_RULE -> { - sb.appendln("
") - } MarkdownElementTypes.ORDERED_LIST -> { sb.appendln("
    ") processChildren() sb.appendln("
") } - MarkdownElementTypes.LIST_BLOCK -> { + MarkdownElementTypes.LIST_ITEM -> { sb.append("
  • ") processChildren() sb.appendln("
    • ") @@ -117,20 +68,54 @@ public fun MarkdownTree.toHtml(): String { processChildren() sb.append("    ") } - MarkdownElementTypes.PLAIN_TEXT -> { - sb.append(nodeText) + MarkdownElementTypes.ATX_1 -> { + sb.append("

    ") + processChildren() + sb.append("

    ") } - MarkdownElementTypes.END_LINE -> { - sb.appendln() + MarkdownElementTypes.ATX_2 -> { + sb.append("

    ") + processChildren() + sb.append("

    ") } - MarkdownElementTypes.BLANK_LINE -> { - sb.appendln() + MarkdownElementTypes.ATX_3 -> { + sb.append("

    ") + processChildren() + sb.append("

    ") + } + MarkdownElementTypes.ATX_4 -> { + sb.append("

    ") + processChildren() + sb.append("

    ") + } + MarkdownElementTypes.ATX_5 -> { + sb.append("
    ") + processChildren() + sb.append("
    ") + } + MarkdownElementTypes.ATX_6 -> { + sb.append("
    ") + processChildren() + sb.append("
    ") } - MarkdownElementTypes.PARA -> { + MarkdownElementTypes.BLOCK_QUOTE -> { + sb.append("
    ") + processChildren() + sb.append("
    ") + } + MarkdownElementTypes.PARAGRAPH -> { sb.append("

    ") processChildren() sb.appendln("

    ") } + MarkdownTokenTypes.CODE -> { + sb.append("
    ")
+                sb.append(nodeText)
+                sb.append("
    ")
+            }
+            MarkdownTokenTypes.TEXT -> {
+                sb.append(nodeText)
+            }
             else -> {
                 processChildren()
             }
@@ -139,9 +124,16 @@ public fun MarkdownTree.toHtml(): String {
     return sb.toString()
 }
 
+fun parseMarkdown(markdown: String): MarkdownNode {
+    if (markdown.isEmpty())
+        return MarkdownNode(LeafASTNode(MarkdownElementTypes.MARKDOWN_FILE, 0, 0), markdown)
+    return MarkdownNode(MarkdownParser(CommonMarkMarkerProcessor()).buildMarkdownTreeFromString(markdown), markdown)
+}
 
 fun markdownToHtml(markdown: String): String {
-    val markdownTree = MarkdownProcessor.parse(markdown)
+
+    val tree = MarkdownParser(CommonMarkMarkerProcessor()).buildMarkdownTreeFromString(markdown)
+    val markdownTree = MarkdownNode(tree, markdown)
     val ast = markdownTree.toTestString()
     return markdownTree.toHtml()
 }
diff --git a/src/Markdown/MarkdownTokenType.kt b/src/Markdown/MarkdownTokenType.kt
deleted file mode 100644
index 293228c3..00000000
--- a/src/Markdown/MarkdownTokenType.kt
+++ /dev/null
@@ -1,6 +0,0 @@
-package org.jetbrains.dokka.Markdown
-
-import com.intellij.psi.tree.IElementType
-
-public class MarkdownTokenType(debugName: String) : IElementType(debugName, null) {
-}
\ No newline at end of file
diff --git a/src/Markdown/markdown.bnf b/src/Markdown/markdown.bnf
deleted file mode 100644
index 2e4979ec..00000000
--- a/src/Markdown/markdown.bnf
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  psiPackage = 'org.jetbrains.markdown'
-  psiImplPackage = 'org.jetbrains.markdown.impl'
-
-  parserClass="org.jetbrains.markdown.MarkdownParser"
-  parserUtilClass="org.jetbrains.dokka.Markdown.GeneratedParserUtilBase"
-  elementTypeHolderClass = 'org.jetbrains.markdown.MarkdownElementTypes'
-
-  tokenTypeClass = 'org.jetbrains.dokka.Markdown.MarkdownTokenType'
-
-  generatePsi = false
-
-  tokens=[
-    BOM
-    EOL
-    EOP
-    Number
-    SPECIAL
-    Word
-  ]
-
-}
-
-Document ::= BOM? Whitespace* AnonymousSection? (Whitespace* NamedSection)*
-
-AnonymousSection ::= SectionBody
-NamedSection ::= SectionHeader SectionBody
-private SectionHeader ::= '$' SectionName OptionalSpace ':' OptionalSpace
-SectionName ::= SectionNameStart | '{' OptionalSpace SectionNameStart (Space+ Word)* OptionalSpace '}'
-private SectionNameStart ::= '$'? Word
-SectionBody::= Block*
-
-Unused ::= Special
-BlankLine ::= OptionalSpace EOL
-
-Whitespace ::= Space | EOL | EOP
-private OptionalSpace ::= Space*
-private NonindentSpace ::= ("   " | "  " | " ")?
-
-EndLine ::= TerminalEndline | NormalEndline
-private NormalEndline ::=   EOL !BlankLine
-private TerminalEndline ::= OptionalSpace <>
-private Indent ::=            "\t" | "    "
-
-// ---- BLOCKS ----
-Block ::= BlankLine* (
-              OrderedList
-            | BulletList
-            | HorizontalRule
-            | Directive
-            | Para
-            )
-
-Para ::= Inlines (EOP | Space* <>)?
-
-HorizontalRule ::= NonindentSpace
-                 ( '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')*
-                 | '-' OptionalSpace '-' OptionalSpace '-' (OptionalSpace '-')*
-                 | '_' OptionalSpace '_' OptionalSpace '_' (OptionalSpace '_')*)
-                 OptionalSpace EOL BlankLine+
-
-Directive ::= '{' DirectiveName Space+ DirectiveParams '}'
-DirectiveName ::= Word
-DirectiveParams ::= PlainText
-
-Bullet ::= !HorizontalRule NonindentSpace ('+' | '*' | '-') Space+
-Enumerator ::= NonindentSpace Number '.' Space+
-
-BulletList ::= &Bullet List
-OrderedList ::= &Enumerator List
-
-private List ::=  (ListItem BlankLine*)+
-ListItem      ::= (Bullet | Enumerator) ListBlock ( ListContinuationBlock )*
-
-ListBlock ::= !BlankLine Inlines ( ListBlockLine )*
-ListBlockLine ::= !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Inlines
-
-ListContinuationBlock ::= BlankLine* (Indent ListBlock)+
-
-
-// ---- INLINES ----
-private Inlines ::= (!EndLine Inline | EndLine &Inline )+ EndLine?
-Inline  ::= Strong | Emph | Code | Link | PlainText
-PlainText ::= (Word | Number | Space | ':')+
-
-Emph ::= EmphStar | EmphUnderscore
-private EmphStar ::= '*' !Whitespace (!'*' Inline)+ '*'
-private EmphUnderscore ::= '_' !Whitespace (!'_' Inline)+ '_'
-
-Code ::= '`' !Whitespace (!'`' Inline)+ '`'
-
-Strong ::= StrongStar | StrongUnderscore
-StrongStar ::= '**' !Whitespace (!'**' Inline)+ '**'
-StrongUnderscore ::= '__' !Whitespace (!'__' Inline)+ '__'
-
-Link ::=  '[' Target ']' ('(' Href ')')?
-Target ::= Word+
-Href ::= (Word | Number | ':' | '/')+
\ No newline at end of file
diff --git a/src/Markdown/markdown.leg b/src/Markdown/markdown.leg
deleted file mode 100644
index ea8bc522..00000000
--- a/src/Markdown/markdown.leg
+++ /dev/null
@@ -1,781 +0,0 @@
-%{
-/**********************************************************************
-
-  markdown_parser.leg - markdown parser in C using a PEG grammar.
-  (c) 2008 John MacFarlane (jgm at berkeley dot edu).
-
-  This program is free software; you can redistribute it and/or modify
-  it under the terms of the GNU General Public License or the MIT
-  license.  See LICENSE for details.
-
-  This program is distributed in the hope that it will be useful,
-  but WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-  GNU General Public License for more details.
-
- ***********************************************************************/
-
-#include 
-#include 
-#include "markdown_peg.h"
-#include "utility_functions.h"
-
-
-
-/**********************************************************************
-
-  Definitions for leg parser generator.
-  YY_INPUT is the function the parser calls to get new input.
-  We take all new input from (static) charbuf.
-
- ***********************************************************************/
-
-
-
-# define YYSTYPE element *
-#ifdef __DEBUG__
-# define YY_DEBUG 1
-#endif
-
-#define YY_INPUT(buf, result, max_size)              \
-{                                                    \
-    int yyc;                                         \
-    if (charbuf && *charbuf != '\0') {               \
-        yyc= *charbuf++;                             \
-    } else {                                         \
-        yyc= EOF;                                    \
-    }                                                \
-    result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1);     \
-}
-
-#define YY_RULE(T)	T
-
-
-/**********************************************************************
-
-  PEG grammar and parser actions for markdown syntax.
-
- ***********************************************************************/
-
-%}
-
-Doc =       BOM? a:StartList ( Block { a = cons($$, a); } )*
-            { parse_result = reverse(a); }
-
-Block =     BlankLine*
-            ( BlockQuote
-            | Verbatim
-            | Note
-            | Reference
-            | HorizontalRule
-            | Heading
-            | OrderedList
-            | BulletList
-            | HtmlBlock
-            | StyleBlock
-            | Para
-            | Plain )
-
-Para =      NonindentSpace a:Inlines BlankLine+
-            { $$ = a; $$->key = PARA; }
-
-Plain =     a:Inlines
-            { $$ = a; $$->key = PLAIN; }
-
-AtxInline = !Newline !(Sp '#'* Sp Newline) Inline
-
-AtxStart =  < ( "######" | "#####" | "####" | "###" | "##" | "#" ) >
-            { $$ = mk_element(H1 + (strlen(yytext) - 1)); }
-
-AtxHeading = s:AtxStart Sp a:StartList ( AtxInline { a = cons($$, a); } )+ (Sp '#'* Sp)?  Newline
-            { $$ = mk_list(s->key, a);
-              free(s); }
-
-SetextHeading = SetextHeading1 | SetextHeading2
-
-SetextBottom1 = '='+ Newline
-
-SetextBottom2 = '-'+ Newline
-
-SetextHeading1 =  &(RawLine SetextBottom1)
-                  a:StartList ( !Endline Inline { a = cons($$, a); } )+ Sp Newline
-                  SetextBottom1 { $$ = mk_list(H1, a); }
-
-SetextHeading2 =  &(RawLine SetextBottom2)
-                  a:StartList ( !Endline Inline { a = cons($$, a); } )+ Sp Newline
-                  SetextBottom2 { $$ = mk_list(H2, a); }
-
-Heading = SetextHeading | AtxHeading
-
-BlockQuote = a:BlockQuoteRaw
-             {  $$ = mk_element(BLOCKQUOTE);
-                $$->children = a;
-             }
-
-BlockQuoteRaw =  a:StartList
-                 (( '>' ' '? Line { a = cons($$, a); } )
-                  ( !'>' !BlankLine Line { a = cons($$, a); } )*
-                  ( BlankLine { a = cons(mk_str("\n"), a); } )*
-                 )+
-                 {   $$ = mk_str_from_list(a, true);
-                     $$->key = RAW;
-                 }
-
-NonblankIndentedLine = !BlankLine IndentedLine
-
-VerbatimChunk = a:StartList
-                ( BlankLine { a = cons(mk_str("\n"), a); } )*
-                ( NonblankIndentedLine { a = cons($$, a); } )+
-                { $$ = mk_str_from_list(a, false); }
-
-Verbatim =     a:StartList ( VerbatimChunk { a = cons($$, a); } )+
-               { $$ = mk_str_from_list(a, false);
-                 $$->key = VERBATIM; }
-
-HorizontalRule = NonindentSpace
-                 ( '*' Sp '*' Sp '*' (Sp '*')*
-                 | '-' Sp '-' Sp '-' (Sp '-')*
-                 | '_' Sp '_' Sp '_' (Sp '_')*)
-                 Sp Newline BlankLine+
-                 { $$ = mk_element(HRULE); }
-
-Bullet = !HorizontalRule NonindentSpace ('+' | '*' | '-') Spacechar+
-
-BulletList = &Bullet (ListTight | ListLoose)
-             { $$->key = BULLETLIST; }
-
-ListTight = a:StartList
-            ( ListItemTight { a = cons($$, a); } )+
-            BlankLine* !(Bullet | Enumerator)
-            { $$ = mk_list(LIST, a); }
-
-ListLoose = a:StartList
-            ( b:ListItem BlankLine*
-              {   element *li;
-                  li = b->children;
-                  li->contents.str = realloc(li->contents.str, strlen(li->contents.str) + 3);
-                  strcat(li->contents.str, "\n\n");  /* In loose list, \n\n added to end of each element */
-                  a = cons(b, a);
-              } )+
-            { $$ = mk_list(LIST, a); }
-
-ListItem =  ( Bullet | Enumerator )
-            a:StartList
-            ListBlock { a = cons($$, a); }
-            ( ListContinuationBlock { a = cons($$, a); } )*
-            {  element *raw;
-               raw = mk_str_from_list(a, false);
-               raw->key = RAW;
-               $$ = mk_element(LISTITEM);
-               $$->children = raw;
-            }
-
-ListItemTight =
-            ( Bullet | Enumerator )
-            a:StartList
-            ListBlock { a = cons($$, a); }
-            ( !BlankLine
-              ListContinuationBlock { a = cons($$, a); } )*
-            !ListContinuationBlock
-            {  element *raw;
-               raw = mk_str_from_list(a, false);
-               raw->key = RAW;
-               $$ = mk_element(LISTITEM);
-               $$->children = raw;
-            }
-
-ListBlock = a:StartList
-            !BlankLine Line { a = cons($$, a); }
-            ( ListBlockLine { a = cons($$, a); } )*
-            { $$ = mk_str_from_list(a, false); }
-
-ListContinuationBlock = a:StartList
-                        ( < BlankLine* >
-                          {   if (strlen(yytext) == 0)
-                                   a = cons(mk_str("\001"), a); /* block separator */
-                              else
-                                   a = cons(mk_str(yytext), a); } )
-                        ( Indent ListBlock { a = cons($$, a); } )+
-                        {  $$ = mk_str_from_list(a, false); }
-
-Enumerator = NonindentSpace [0-9]+ '.' Spacechar+
-
-OrderedList = &Enumerator (ListTight | ListLoose)
-              { $$->key = ORDEREDLIST; }
-
-ListBlockLine = !BlankLine
-                !( Indent? (Bullet | Enumerator) )
-                !HorizontalRule
-                OptionallyIndentedLine
-
-# Parsers for different kinds of block-level HTML content.
-# This is repetitive due to constraints of PEG grammar.
-
-HtmlBlockOpenAddress = '<' Spnl ("address" | "ADDRESS") Spnl HtmlAttribute* '>'
-HtmlBlockCloseAddress = '<' Spnl '/' ("address" | "ADDRESS") Spnl '>'
-HtmlBlockAddress = HtmlBlockOpenAddress (HtmlBlockAddress | !HtmlBlockCloseAddress .)* HtmlBlockCloseAddress
-
-HtmlBlockOpenBlockquote = '<' Spnl ("blockquote" | "BLOCKQUOTE") Spnl HtmlAttribute* '>'
-HtmlBlockCloseBlockquote = '<' Spnl '/' ("blockquote" | "BLOCKQUOTE") Spnl '>'
-HtmlBlockBlockquote = HtmlBlockOpenBlockquote (HtmlBlockBlockquote | !HtmlBlockCloseBlockquote .)* HtmlBlockCloseBlockquote
-
-HtmlBlockOpenCenter = '<' Spnl ("center" | "CENTER") Spnl HtmlAttribute* '>'
-HtmlBlockCloseCenter = '<' Spnl '/' ("center" | "CENTER") Spnl '>'
-HtmlBlockCenter = HtmlBlockOpenCenter (HtmlBlockCenter | !HtmlBlockCloseCenter .)* HtmlBlockCloseCenter
-
-HtmlBlockOpenDir = '<' Spnl ("dir" | "DIR") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDir = '<' Spnl '/' ("dir" | "DIR") Spnl '>'
-HtmlBlockDir = HtmlBlockOpenDir (HtmlBlockDir | !HtmlBlockCloseDir .)* HtmlBlockCloseDir
-
-HtmlBlockOpenDiv = '<' Spnl ("div" | "DIV") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDiv = '<' Spnl '/' ("div" | "DIV") Spnl '>'
-HtmlBlockDiv = HtmlBlockOpenDiv (HtmlBlockDiv | !HtmlBlockCloseDiv .)* HtmlBlockCloseDiv
-
-HtmlBlockOpenDl = '<' Spnl ("dl" | "DL") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDl = '<' Spnl '/' ("dl" | "DL") Spnl '>'
-HtmlBlockDl = HtmlBlockOpenDl (HtmlBlockDl | !HtmlBlockCloseDl .)* HtmlBlockCloseDl
-
-HtmlBlockOpenFieldset = '<' Spnl ("fieldset" | "FIELDSET") Spnl HtmlAttribute* '>'
-HtmlBlockCloseFieldset = '<' Spnl '/' ("fieldset" | "FIELDSET") Spnl '>'
-HtmlBlockFieldset = HtmlBlockOpenFieldset (HtmlBlockFieldset | !HtmlBlockCloseFieldset .)* HtmlBlockCloseFieldset
-
-HtmlBlockOpenForm = '<' Spnl ("form" | "FORM") Spnl HtmlAttribute* '>'
-HtmlBlockCloseForm = '<' Spnl '/' ("form" | "FORM") Spnl '>'
-HtmlBlockForm = HtmlBlockOpenForm (HtmlBlockForm | !HtmlBlockCloseForm .)* HtmlBlockCloseForm
-
-HtmlBlockOpenH1 = '<' Spnl ("h1" | "H1") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH1 = '<' Spnl '/' ("h1" | "H1") Spnl '>'
-HtmlBlockH1 = HtmlBlockOpenH1 (HtmlBlockH1 | !HtmlBlockCloseH1 .)* HtmlBlockCloseH1
-
-HtmlBlockOpenH2 = '<' Spnl ("h2" | "H2") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH2 = '<' Spnl '/' ("h2" | "H2") Spnl '>'
-HtmlBlockH2 = HtmlBlockOpenH2 (HtmlBlockH2 | !HtmlBlockCloseH2 .)* HtmlBlockCloseH2
-
-HtmlBlockOpenH3 = '<' Spnl ("h3" | "H3") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH3 = '<' Spnl '/' ("h3" | "H3") Spnl '>'
-HtmlBlockH3 = HtmlBlockOpenH3 (HtmlBlockH3 | !HtmlBlockCloseH3 .)* HtmlBlockCloseH3
-
-HtmlBlockOpenH4 = '<' Spnl ("h4" | "H4") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH4 = '<' Spnl '/' ("h4" | "H4") Spnl '>'
-HtmlBlockH4 = HtmlBlockOpenH4 (HtmlBlockH4 | !HtmlBlockCloseH4 .)* HtmlBlockCloseH4
-
-HtmlBlockOpenH5 = '<' Spnl ("h5" | "H5") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH5 = '<' Spnl '/' ("h5" | "H5") Spnl '>'
-HtmlBlockH5 = HtmlBlockOpenH5 (HtmlBlockH5 | !HtmlBlockCloseH5 .)* HtmlBlockCloseH5
-
-HtmlBlockOpenH6 = '<' Spnl ("h6" | "H6") Spnl HtmlAttribute* '>'
-HtmlBlockCloseH6 = '<' Spnl '/' ("h6" | "H6") Spnl '>'
-HtmlBlockH6 = HtmlBlockOpenH6 (HtmlBlockH6 | !HtmlBlockCloseH6 .)* HtmlBlockCloseH6
-
-HtmlBlockOpenMenu = '<' Spnl ("menu" | "MENU") Spnl HtmlAttribute* '>'
-HtmlBlockCloseMenu = '<' Spnl '/' ("menu" | "MENU") Spnl '>'
-HtmlBlockMenu = HtmlBlockOpenMenu (HtmlBlockMenu | !HtmlBlockCloseMenu .)* HtmlBlockCloseMenu
-
-HtmlBlockOpenNoframes = '<' Spnl ("noframes" | "NOFRAMES") Spnl HtmlAttribute* '>'
-HtmlBlockCloseNoframes = '<' Spnl '/' ("noframes" | "NOFRAMES") Spnl '>'
-HtmlBlockNoframes = HtmlBlockOpenNoframes (HtmlBlockNoframes | !HtmlBlockCloseNoframes .)* HtmlBlockCloseNoframes
-
-HtmlBlockOpenNoscript = '<' Spnl ("noscript" | "NOSCRIPT") Spnl HtmlAttribute* '>'
-HtmlBlockCloseNoscript = '<' Spnl '/' ("noscript" | "NOSCRIPT") Spnl '>'
-HtmlBlockNoscript = HtmlBlockOpenNoscript (HtmlBlockNoscript | !HtmlBlockCloseNoscript .)* HtmlBlockCloseNoscript
-
-HtmlBlockOpenOl = '<' Spnl ("ol" | "OL") Spnl HtmlAttribute* '>'
-HtmlBlockCloseOl = '<' Spnl '/' ("ol" | "OL") Spnl '>'
-HtmlBlockOl = HtmlBlockOpenOl (HtmlBlockOl | !HtmlBlockCloseOl .)* HtmlBlockCloseOl
-
-HtmlBlockOpenP = '<' Spnl ("p" | "P") Spnl HtmlAttribute* '>'
-HtmlBlockCloseP = '<' Spnl '/' ("p" | "P") Spnl '>'
-HtmlBlockP = HtmlBlockOpenP (HtmlBlockP | !HtmlBlockCloseP .)* HtmlBlockCloseP
-
-HtmlBlockOpenPre = '<' Spnl ("pre" | "PRE") Spnl HtmlAttribute* '>'
-HtmlBlockClosePre = '<' Spnl '/' ("pre" | "PRE") Spnl '>'
-HtmlBlockPre = HtmlBlockOpenPre (HtmlBlockPre | !HtmlBlockClosePre .)* HtmlBlockClosePre
-
-HtmlBlockOpenTable = '<' Spnl ("table" | "TABLE") Spnl HtmlAttribute* '>'
-HtmlBlockCloseTable = '<' Spnl '/' ("table" | "TABLE") Spnl '>'
-HtmlBlockTable = HtmlBlockOpenTable (HtmlBlockTable | !HtmlBlockCloseTable .)* HtmlBlockCloseTable
-
-HtmlBlockOpenUl = '<' Spnl ("ul" | "UL") Spnl HtmlAttribute* '>'
-HtmlBlockCloseUl = '<' Spnl '/' ("ul" | "UL") Spnl '>'
-HtmlBlockUl = HtmlBlockOpenUl (HtmlBlockUl | !HtmlBlockCloseUl .)* HtmlBlockCloseUl
-
-HtmlBlockOpenDd = '<' Spnl ("dd" | "DD") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDd = '<' Spnl '/' ("dd" | "DD") Spnl '>'
-HtmlBlockDd = HtmlBlockOpenDd (HtmlBlockDd | !HtmlBlockCloseDd .)* HtmlBlockCloseDd
-
-HtmlBlockOpenDt = '<' Spnl ("dt" | "DT") Spnl HtmlAttribute* '>'
-HtmlBlockCloseDt = '<' Spnl '/' ("dt" | "DT") Spnl '>'
-HtmlBlockDt = HtmlBlockOpenDt (HtmlBlockDt | !HtmlBlockCloseDt .)* HtmlBlockCloseDt
-
-HtmlBlockOpenFrameset = '<' Spnl ("frameset" | "FRAMESET") Spnl HtmlAttribute* '>'
-HtmlBlockCloseFrameset = '<' Spnl '/' ("frameset" | "FRAMESET") Spnl '>'
-HtmlBlockFrameset = HtmlBlockOpenFrameset (HtmlBlockFrameset | !HtmlBlockCloseFrameset .)* HtmlBlockCloseFrameset
-
-HtmlBlockOpenLi = '<' Spnl ("li" | "LI") Spnl HtmlAttribute* '>'
-HtmlBlockCloseLi = '<' Spnl '/' ("li" | "LI") Spnl '>'
-HtmlBlockLi = HtmlBlockOpenLi (HtmlBlockLi | !HtmlBlockCloseLi .)* HtmlBlockCloseLi
-
-HtmlBlockOpenTbody = '<' Spnl ("tbody" | "TBODY") Spnl HtmlAttribute* '>'
-HtmlBlockCloseTbody = '<' Spnl '/' ("tbody" | "TBODY") Spnl '>'
-HtmlBlockTbody = HtmlBlockOpenTbody (HtmlBlockTbody | !HtmlBlockCloseTbody .)* HtmlBlockCloseTbody
-
-HtmlBlockOpenTd = '<' Spnl ("td" | "TD") Spnl HtmlAttribute* '>'
-HtmlBlockCloseTd = '<' Spnl '/' ("td" | "TD") Spnl '>'
-HtmlBlockTd = HtmlBlockOpenTd (HtmlBlockTd | !HtmlBlockCloseTd .)* HtmlBlockCloseTd
-
-HtmlBlockOpenTfoot = '<' Spnl ("tfoot" | "TFOOT") Spnl HtmlAttribute* '>'
-HtmlBlockCloseTfoot = '<' Spnl '/' ("tfoot" | "TFOOT") Spnl '>'
-HtmlBlockTfoot = HtmlBlockOpenTfoot (HtmlBlockTfoot | !HtmlBlockCloseTfoot .)* HtmlBlockCloseTfoot
-
-HtmlBlockOpenTh = '<' Spnl ("th" | "TH") Spnl HtmlAttribute* '>'
-HtmlBlockCloseTh = '<' Spnl '/' ("th" | "TH") Spnl '>'
-HtmlBlockTh = HtmlBlockOpenTh (HtmlBlockTh | !HtmlBlockCloseTh .)* HtmlBlockCloseTh
-
-HtmlBlockOpenThead = '<' Spnl ("thead" | "THEAD") Spnl HtmlAttribute* '>'
-HtmlBlockCloseThead = '<' Spnl '/' ("thead" | "THEAD") Spnl '>'
-HtmlBlockThead = HtmlBlockOpenThead (HtmlBlockThead | !HtmlBlockCloseThead .)* HtmlBlockCloseThead
-
-HtmlBlockOpenTr = '<' Spnl ("tr" | "TR") Spnl HtmlAttribute* '>'
-HtmlBlockCloseTr = '<' Spnl '/' ("tr" | "TR") Spnl '>'
-HtmlBlockTr = HtmlBlockOpenTr (HtmlBlockTr | !HtmlBlockCloseTr .)* HtmlBlockCloseTr
-
-HtmlBlockOpenScript = '<' Spnl ("script" | "SCRIPT") Spnl HtmlAttribute* '>'
-HtmlBlockCloseScript = '<' Spnl '/' ("script" | "SCRIPT") Spnl '>'
-HtmlBlockScript = HtmlBlockOpenScript (!HtmlBlockCloseScript .)* HtmlBlockCloseScript
-
-HtmlBlockOpenHead = '<' Spnl ("head" | "HEAD") Spnl HtmlAttribute* '>'
-HtmlBlockCloseHead = '<' Spnl '/' ("head" | "HEAD") Spnl '>'
-HtmlBlockHead = HtmlBlockOpenHead (!HtmlBlockCloseHead .)* HtmlBlockCloseHead
-
-HtmlBlockInTags = HtmlBlockAddress
-                | HtmlBlockBlockquote
-                | HtmlBlockCenter
-                | HtmlBlockDir
-                | HtmlBlockDiv
-                | HtmlBlockDl
-                | HtmlBlockFieldset
-                | HtmlBlockForm
-                | HtmlBlockH1
-                | HtmlBlockH2
-                | HtmlBlockH3
-                | HtmlBlockH4
-                | HtmlBlockH5
-                | HtmlBlockH6
-                | HtmlBlockMenu
-                | HtmlBlockNoframes
-                | HtmlBlockNoscript
-                | HtmlBlockOl
-                | HtmlBlockP
-                | HtmlBlockPre
-                | HtmlBlockTable
-                | HtmlBlockUl
-                | HtmlBlockDd
-                | HtmlBlockDt
-                | HtmlBlockFrameset
-                | HtmlBlockLi
-                | HtmlBlockTbody
-                | HtmlBlockTd
-                | HtmlBlockTfoot
-                | HtmlBlockTh
-                | HtmlBlockThead
-                | HtmlBlockTr
-                | HtmlBlockScript
-                | HtmlBlockHead
-
-HtmlBlock = < ( HtmlBlockInTags | HtmlComment | HtmlBlockSelfClosing ) >
-            BlankLine+
-            {   if (extension(EXT_FILTER_HTML)) {
-                    $$ = mk_list(LIST, NULL);
-                } else {
-                    $$ = mk_str(yytext);
-                    $$->key = HTMLBLOCK;
-                }
-            }
-
-HtmlBlockSelfClosing = '<' Spnl HtmlBlockType Spnl HtmlAttribute* '/' Spnl '>'
-
-HtmlBlockType = "address" | "blockquote" | "center" | "dir" | "div" | "dl" | "fieldset" | "form" | "h1" | "h2" | "h3" |
-                "h4" | "h5" | "h6" | "hr" | "isindex" | "menu" | "noframes" | "noscript" | "ol" | "p" | "pre" | "table" |
-                "ul" | "dd" | "dt" | "frameset" | "li" | "tbody" | "td" | "tfoot" | "th" | "thead" | "tr" | "script" |
-                "ADDRESS" | "BLOCKQUOTE" | "CENTER" | "DIR" | "DIV" | "DL" | "FIELDSET" | "FORM" | "H1" | "H2" | "H3" |
-                "H4" | "H5" | "H6" | "HR" | "ISINDEX" | "MENU" | "NOFRAMES" | "NOSCRIPT" | "OL" | "P" | "PRE" | "TABLE" |
-                "UL" | "DD" | "DT" | "FRAMESET" | "LI" | "TBODY" | "TD" | "TFOOT" | "TH" | "THEAD" | "TR" | "SCRIPT"
-
-StyleOpen =     '<' Spnl ("style" | "STYLE") Spnl HtmlAttribute* '>'
-StyleClose =    '<' Spnl '/' ("style" | "STYLE") Spnl '>'
-InStyleTags =   StyleOpen (!StyleClose .)* StyleClose
-StyleBlock =    < InStyleTags >
-                BlankLine*
-                {   if (extension(EXT_FILTER_STYLES)) {
-                        $$ = mk_list(LIST, NULL);
-                    } else {
-                        $$ = mk_str(yytext);
-                        $$->key = HTMLBLOCK;
-                    }
-                }
-
-Inlines  =  a:StartList ( !Endline Inline { a = cons($$, a); }
-                        | c:Endline &Inline { a = cons(c, a); } )+ Endline?
-            { $$ = mk_list(LIST, a); }
-
-Inline  = Str
-        | Endline
-        | UlOrStarLine
-        | Space
-        | Strong
-        | Emph
-        | Strike
-        | Image
-        | Link
-        | NoteReference
-        | InlineNote
-        | Code
-        | RawHtml
-        | Entity
-        | EscapedChar
-        | Smart
-        | Symbol
-
-Space = Spacechar+
-        { $$ = mk_str(" ");
-          $$->key = SPACE; }
-
-Str = a:StartList < NormalChar+ > { a = cons(mk_str(yytext), a); }
-      ( StrChunk { a = cons($$, a); } )*
-      { if (a->next == NULL) { $$ = a; } else { $$ = mk_list(LIST, a); } }
-
-StrChunk = < (NormalChar | '_'+ &Alphanumeric)+ > { $$ = mk_str(yytext); } |
-           AposChunk
-
-AposChunk = &{ extension(EXT_SMART) } '\'' &Alphanumeric
-      { $$ = mk_element(APOSTROPHE); }
-
-EscapedChar =   '\\' !Newline < [-\\`|*_{}[\]()#+.!><] >
-                { $$ = mk_str(yytext); }
-
-Entity =    ( HexEntity | DecEntity | CharEntity )
-            { $$ = mk_str(yytext); $$->key = HTML; }
-
-Endline =   LineBreak | TerminalEndline | NormalEndline
-
-NormalEndline =   Sp Newline !BlankLine !'>' !AtxStart
-                  !(Line ('='+ | '-'+) Newline)
-                  { $$ = mk_str("\n");
-                    $$->key = SPACE; }
-
-TerminalEndline = Sp Newline Eof
-                  { $$ = NULL; }
-
-LineBreak = "  " NormalEndline
-            { $$ = mk_element(LINEBREAK); }
-
-Symbol =    < SpecialChar >
-            { $$ = mk_str(yytext); }
-
-# This keeps the parser from getting bogged down on long strings of '*' or '_',
-# or strings of '*' or '_' with space on each side:
-UlOrStarLine =  (UlLine | StarLine) { $$ = mk_str(yytext); }
-StarLine =      < "****" '*'* > | < Spacechar '*'+ &Spacechar >
-UlLine   =      < "____" '_'* > | < Spacechar '_'+ &Spacechar >
-
-Emph =      EmphStar | EmphUl
-
-Whitespace = Spacechar | Newline
-
-EmphStar =  '*' !Whitespace
-            a:StartList
-            ( !'*' b:Inline { a = cons(b, a); }
-            | b:StrongStar  { a = cons(b, a); }
-            )+
-            '*'
-            { $$ = mk_list(EMPH, a); }
-
-EmphUl =    '_' !Whitespace
-            a:StartList
-            ( !'_' b:Inline { a = cons(b, a); }
-            | b:StrongUl  { a = cons(b, a); }
-            )+
-            '_'
-            { $$ = mk_list(EMPH, a); }
-
-Strong = StrongStar | StrongUl
-
-StrongStar =    "**" !Whitespace
-                a:StartList
-                ( !"**" b:Inline { a = cons(b, a); })+
-                "**"
-                { $$ = mk_list(STRONG, a); }
-
-StrongUl   =    "__" !Whitespace
-                a:StartList
-                ( !"__" b:Inline { a = cons(b, a); })+
-                "__"
-                { $$ = mk_list(STRONG, a); }
-
-Strike = &{ extension(EXT_STRIKE) }
-         "~~" !Whitespace
-         a:StartList
-         ( !"~~" b:Inline { a = cons(b, a); })+
-         "~~"
-         { $$ = mk_list(STRIKE, a); }
-
-Image = '!' ( ExplicitLink | ReferenceLink )
-        { if ($$->key == LINK) {
-              $$->key = IMAGE;
-          } else {
-              element *result;
-              result = $$;
-              $$->children = cons(mk_str("!"), result->children);
-          } }
-
-Link =  ExplicitLink | ReferenceLink | AutoLink
-
-ReferenceLink = ReferenceLinkDouble | ReferenceLinkSingle
-
-ReferenceLinkDouble =  a:Label < Spnl > !"[]" b:Label
-                       {   link match;
-                           if (find_reference(&match, b->children)) {
-                               $$ = mk_link(a->children, match.url, match.title);
-                               free(a);
-                               free_element_list(b);
-                           } else {
-                               element *result;
-                               result = mk_element(LIST);
-                               result->children = cons(mk_str("["), cons(a, cons(mk_str("]"), cons(mk_str(yytext),
-                                                   cons(mk_str("["), cons(b, mk_str("]")))))));
-                               $$ = result;
-                           }
-                       }
-
-ReferenceLinkSingle =  a:Label < (Spnl "[]")? >
-                       {   link match;
-                           if (find_reference(&match, a->children)) {
-                               $$ = mk_link(a->children, match.url, match.title);
-                               free(a);
-                           }
-                           else {
-                               element *result;
-                               result = mk_element(LIST);
-                               result->children = cons(mk_str("["), cons(a, cons(mk_str("]"), mk_str(yytext))));
-                               $$ = result;
-                           }
-                       }
-
-ExplicitLink =  l:Label '(' Sp s:Source Spnl t:Title Sp ')'
-                { $$ = mk_link(l->children, s->contents.str, t->contents.str);
-                  free_element(s);
-                  free_element(t);
-                  free(l); }
-
-Source  = ( '<' < SourceContents > '>' | < SourceContents > )
-          { $$ = mk_str(yytext); }
-
-SourceContents = ( ( !'(' !')' !'>' Nonspacechar )+ | '(' SourceContents ')')*
-
-Title = ( TitleSingle | TitleDouble | < "" > )
-        { $$ = mk_str(yytext); }
-
-TitleSingle = '\'' < ( !( '\'' Sp ( ')' | Newline ) ) . )* > '\''
-
-TitleDouble = '"' < ( !( '"' Sp ( ')' | Newline ) ) . )* > '"'
-
-AutoLink = AutoLinkUrl | AutoLinkEmail
-
-AutoLinkUrl =   '<' < [A-Za-z]+ "://" ( !Newline !'>' . )+ > '>'
-                {   $$ = mk_link(mk_str(yytext), yytext, ""); }
-
-AutoLinkEmail = '<' ( "mailto:" )? < [-A-Za-z0-9+_./!%~$]+ '@' ( !Newline !'>' . )+ > '>'
-                {   char *mailto = malloc(strlen(yytext) + 8);
-                    sprintf(mailto, "mailto:%s", yytext);
-                    $$ = mk_link(mk_str(yytext), mailto, "");
-                    free(mailto);
-                }
-
-Reference = NonindentSpace !"[]" l:Label ':' Spnl s:RefSrc t:RefTitle BlankLine+
-            { $$ = mk_link(l->children, s->contents.str, t->contents.str);
-              free_element(s);
-              free_element(t);
-              free(l);
-              $$->key = REFERENCE; }
-
-Label = '[' ( !'^' &{ extension(EXT_NOTES) } | &. &{ !extension(EXT_NOTES) } )
-        a:StartList
-        ( !']' Inline { a = cons($$, a); } )*
-        ']'
-        { $$ = mk_list(LIST, a); }
-
-RefSrc = < Nonspacechar+ >
-         { $$ = mk_str(yytext);
-           $$->key = HTML; }
-
-RefTitle =  ( RefTitleSingle | RefTitleDouble | RefTitleParens | EmptyTitle )
-            { $$ = mk_str(yytext); }
-
-EmptyTitle = < "" >
-
-RefTitleSingle = Spnl '\'' < ( !( '\'' Sp Newline | Newline ) . )* > '\''
-
-RefTitleDouble = Spnl '"' < ( !('"' Sp Newline | Newline) . )* > '"'
-
-RefTitleParens = Spnl '(' < ( !(')' Sp Newline | Newline) . )* > ')'
-
-References = a:StartList
-             ( b:Reference { a = cons(b, a); } | SkipBlock )*
-             { references = reverse(a); }
-
-Ticks1 = "`" !'`'
-Ticks2 = "``" !'`'
-Ticks3 = "```" !'`'
-Ticks4 = "````" !'`'
-Ticks5 = "`````" !'`'
-
-Code = ( Ticks1 Sp < ( ( !'`' Nonspacechar )+ | !Ticks1 '`'+ | !( Sp Ticks1 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks1
-       | Ticks2 Sp < ( ( !'`' Nonspacechar )+ | !Ticks2 '`'+ | !( Sp Ticks2 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks2
-       | Ticks3 Sp < ( ( !'`' Nonspacechar )+ | !Ticks3 '`'+ | !( Sp Ticks3 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks3
-       | Ticks4 Sp < ( ( !'`' Nonspacechar )+ | !Ticks4 '`'+ | !( Sp Ticks4 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks4
-       | Ticks5 Sp < ( ( !'`' Nonspacechar )+ | !Ticks5 '`'+ | !( Sp Ticks5 ) ( Spacechar | Newline !BlankLine ) )+ > Sp Ticks5
-       )
-       { $$ = mk_str(yytext); $$->key = CODE; }
-
-RawHtml =   < (HtmlComment | HtmlBlockScript | HtmlTag) >
-            {   if (extension(EXT_FILTER_HTML)) {
-                    $$ = mk_list(LIST, NULL);
-                } else {
-                    $$ = mk_str(yytext);
-                    $$->key = HTML;
-                }
-            }
-
-BlankLine =     Sp Newline
-
-Quoted =        '"' (!'"' .)* '"' | '\'' (!'\'' .)* '\''
-HtmlAttribute = (AlphanumericAscii | '-')+ Spnl ('=' Spnl (Quoted | (!'>' Nonspacechar)+))? Spnl
-HtmlComment =   "" .)* "-->"
-HtmlTag =       '<' Spnl '/'? AlphanumericAscii+ Spnl HtmlAttribute* '/'? Spnl '>'
-Eof =           !.
-Spacechar =     ' ' | '\t'
-Nonspacechar =  !Spacechar !Newline .
-Newline =       '\n' | '\r' '\n'?
-Sp =            Spacechar*
-Spnl =          Sp (Newline Sp)?
-SpecialChar =   '~' | '*' | '_' | '`' | '&' | '[' | ']' | '(' | ')' | '<' | '!' | '#' | '\\' | '\'' | '"' | ExtendedSpecialChar
-NormalChar =    !( SpecialChar | Spacechar | Newline ) .
-Alphanumeric = [0-9A-Za-z] | '\200' | '\201' | '\202' | '\203' | '\204' | '\205' | '\206' | '\207' | '\210' | '\211' | '\212' | '\213' | '\214' | '\215' | '\216' | '\217' | '\220' | '\221' | '\222' | '\223' | '\224' | '\225' | '\226' | '\227' | '\230' | '\231' | '\232' | '\233' | '\234' | '\235' | '\236' | '\237' | '\240' | '\241' | '\242' | '\243' | '\244' | '\245' | '\246' | '\247' | '\250' | '\251' | '\252' | '\253' | '\254' | '\255' | '\256' | '\257' | '\260' | '\261' | '\262' | '\263' | '\264' | '\265' | '\266' | '\267' | '\270' | '\271' | '\272' | '\273' | '\274' | '\275' | '\276' | '\277' | '\300' | '\301' | '\302' | '\303' | '\304' | '\305' | '\306' | '\307' | '\310' | '\311' | '\312' | '\313' | '\314' | '\315' | '\316' | '\317' | '\320' | '\321' | '\322' | '\323' | '\324' | '\325' | '\326' | '\327' | '\330' | '\331' | '\332' | '\333' | '\334' | '\335' | '\336' | '\337' | '\340' | '\341' | '\342' | '\343' | '\344' | '\345' | '\346' | '\347' | '\350' | '\351' | '\352' | '\353' | '\354' | '\355' | '\356' | '\357' | '\360' | '\361' | '\362' | '\363' | '\364' | '\365' | '\366' | '\367' | '\370' | '\371' | '\372' | '\373' | '\374' | '\375' | '\376' | '\377'
-AlphanumericAscii = [A-Za-z0-9]
-Digit = [0-9]
-BOM = "\357\273\277"
-
-HexEntity =     < '&' '#' [Xx] [0-9a-fA-F]+ ';' >
-DecEntity =     < '&' '#' [0-9]+ > ';' >
-CharEntity =    < '&' [A-Za-z0-9]+ ';' >
-
-NonindentSpace =    "   " | "  " | " " | ""
-Indent =            "\t" | "    "
-IndentedLine =      Indent Line
-OptionallyIndentedLine = Indent? Line
-
-# StartList starts a list data structure that can be added to with cons:
-StartList = &.
-            { $$ = NULL; }
-
-Line =  RawLine
-        { $$ = mk_str(yytext); }
-RawLine = ( < (!'\r' !'\n' .)* Newline > | < .+ > Eof )
-
-SkipBlock = HtmlBlock
-          | ( !'#' !SetextBottom1 !SetextBottom2 !BlankLine RawLine )+ BlankLine*
-          | BlankLine+
-          | RawLine
-
-# Syntax extensions
-
-ExtendedSpecialChar = &{ extension(EXT_SMART) } ('.' | '-' | '\'' | '"')
-                    | &{ extension(EXT_NOTES) } ( '^' )
-
-Smart = &{ extension(EXT_SMART) }
-        ( Ellipsis | Dash | SingleQuoted | DoubleQuoted | Apostrophe )
-
-Apostrophe = '\''
-             { $$ = mk_element(APOSTROPHE); }
-
-Ellipsis = ("..." | ". . .")
-           { $$ = mk_element(ELLIPSIS); }
-
-Dash = EmDash | EnDash
-
-EnDash = '-' &Digit
-         { $$ = mk_element(ENDASH); }
-
-EmDash = ("---" | "--")
-         { $$ = mk_element(EMDASH); }
-
-SingleQuoteStart = '\'' !(Spacechar | Newline)
-
-SingleQuoteEnd = '\'' !Alphanumeric
-
-SingleQuoted = SingleQuoteStart
-               a:StartList
-               ( !SingleQuoteEnd b:Inline { a = cons(b, a); } )+
-               SingleQuoteEnd
-               { $$ = mk_list(SINGLEQUOTED, a); }
-
-DoubleQuoteStart = '"'
-
-DoubleQuoteEnd = '"'
-
-DoubleQuoted =  DoubleQuoteStart
-                a:StartList
-                ( !DoubleQuoteEnd b:Inline { a = cons(b, a); } )+
-                DoubleQuoteEnd
-                { $$ = mk_list(DOUBLEQUOTED, a); }
-
-NoteReference = &{ extension(EXT_NOTES) }
-                ref:RawNoteReference
-                {   element *match;
-                    if (find_note(&match, ref->contents.str)) {
-                        $$ = mk_element(NOTE);
-                        assert(match->children != NULL);
-                        $$->children = match->children;
-                        $$->contents.str = 0;
-                    } else {
-                        char *s;
-                        s = malloc(strlen(ref->contents.str) + 4);
-                        sprintf(s, "[^%s]", ref->contents.str);
-                        $$ = mk_str(s);
-                        free(s);
-                    }
-                }
-
-RawNoteReference = "[^" < ( !Newline !']' . )+ > ']'
-                   { $$ = mk_str(yytext); }
-
-Note =          &{ extension(EXT_NOTES) }
-                NonindentSpace ref:RawNoteReference ':' Sp
-                a:StartList
-                ( RawNoteBlock { a = cons($$, a); } )
-                ( &Indent RawNoteBlock { a = cons($$, a); } )*
-                {   $$ = mk_list(NOTE, a);
-                    $$->contents.str = strdup(ref->contents.str);
-                }
-
-InlineNote =    &{ extension(EXT_NOTES) }
-                "^["
-                a:StartList
-                ( !']' Inline { a = cons($$, a); } )+
-                ']'
-                { $$ = mk_list(NOTE, a);
-                  $$->contents.str = 0; }
-
-Notes =         a:StartList
-                ( b:Note { a = cons(b, a); } | SkipBlock )*
-                { notes = reverse(a); }
-
-RawNoteBlock =  a:StartList
-                    ( !BlankLine OptionallyIndentedLine { a = cons($$, a); } )+
-                ( < BlankLine* > { a = cons(mk_str(yytext), a); } )
-                {   $$ = mk_str_from_list(a, true);
-                    $$->key = RAW;
-                }
-
-%%
-
diff --git a/src/main.kt b/src/main.kt
index 6136300f..4e627376 100644
--- a/src/main.kt
+++ b/src/main.kt
@@ -8,7 +8,6 @@ import org.jetbrains.jet.utils.PathUtil
 import java.io.File
 import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor
 import org.jetbrains.jet.lang.resolve.name.FqName
-import java.lang.reflect.Constructor
 
 class DokkaArguments {
     Argument(value = "src", description = "Source file or directory (allows many paths separated by the system path separator)")
@@ -86,7 +85,7 @@ public fun main(args: Array) {
                 val file = File(include)
                 if (file.exists()) {
                     val text = file.readText()
-                    val tree = MarkdownProcessor.parse(text)
+                    val tree = parseMarkdown(text)
                     val content = buildContent(tree, session.getPackageFragment(FqName.ROOT))
                     moduleContent.children.addAll(content.children)
                 } else {
diff --git a/test/src/markdown/ParserTest.kt b/test/src/markdown/ParserTest.kt
index 201708c6..16892f23 100644
--- a/test/src/markdown/ParserTest.kt
+++ b/test/src/markdown/ParserTest.kt
@@ -2,13 +2,13 @@ package org.jetbrains.dokka.tests
 
 import org.junit.Test
 import org.jetbrains.dokka
-import org.jetbrains.dokka.MarkdownProcessor
 import org.jetbrains.dokka.toTestString
 import org.jetbrains.dokka.toHtml
+import org.jetbrains.dokka.parseMarkdown
 
 public class ParserTest {
     fun runTestFor(text : String) {
-        val markdownTree = MarkdownProcessor.parse(text)
+        val markdownTree = parseMarkdown(text)
         println(markdownTree.toTestString())
         println(markdownTree.toHtml())
     }
@@ -85,7 +85,7 @@ number two
     }
 
     Test fun directive() {
-        runTestFor("A text {code with.another.value} with directive")
+        runTestFor("A text \${code with.another.value} with directive")
     }
 
     Test fun emphAndEmptySection() {
@@ -119,5 +119,22 @@ number two
                 "Plain text \$summary:")
     }
 
+    val b = "$"
+    Test fun pair() {
+        runTestFor(
+                """Represents a generic pair of two values.
+
+There is no meaning attached to values in this class, it can be used for any purpose.
+Pair exhibits value semantics, i.e. two pairs are equal if both components are equal.
+
+An example of decomposing it into values:
+${b}{code test.tuples.PairTest.pairMultiAssignment}
+
+${b}constructor: Creates new instance of [Pair]
+${b}first: First value
+${b}second: Second value""""
+                  )
+    }
+
 }
 
-- 
cgit