Migrate to non-PsiBuilder fully-featured markdown parser.

8 files changed, 117 insertions, 2058 deletions

diff --git a/src/Kotlin/ContentBuilder.kt b/src/Kotlin/ContentBuilder.kt
index 635fc74f..8079fb4c 100644
--- a/src/Kotlin/ContentBuilder.kt
+++ b/src/Kotlin/ContentBuilder.kt
@@ -1,23 +1,22 @@
 package org.jetbrains.dokka
 
-import org.jetbrains.markdown.MarkdownElementTypes
 import java.util.ArrayDeque
-import org.jetbrains.jet.lang.descriptors.DeclarationDescriptor
-import org.jetbrains.jet.lang.resolve.name.Name
-import org.jetbrains.jet.lang.resolve.DescriptorToSourceUtils
-import org.jetbrains.jet.lang.resolve.scopes.JetScope
-import org.jetbrains.jet.lang.resolve.name.FqName
+import org.jetbrains.jet.lang.descriptors.*
+import org.jetbrains.jet.lang.resolve.*
+import org.jetbrains.jet.lang.resolve.scopes.*
+import org.jetbrains.jet.lang.resolve.name.*
+import net.nicoulaj.idea.markdown.lang.*
 
-public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: DeclarationDescriptor): Content {
+public fun DocumentationBuilder.buildContent(tree: MarkdownNode, descriptor: DeclarationDescriptor): Content {
     val nodeStack = ArrayDeque<ContentNode>()
     nodeStack.push(Content())
 
-    tree.visit {(node, text, processChildren) ->
+    tree.visit {(node, processChildren) ->
         val parent = nodeStack.peek()!!
-        val nodeType = node.getTokenType()
-        val nodeText = tree.getNodeText(node)
+        val nodeType = node.type
+        val nodeText = tree.text
         when (nodeType) {
-            MarkdownElementTypes.BULLET_LIST -> {
+            MarkdownElementTypes.UNORDERED_LIST -> {
                 nodeStack.push(ContentList())
                 processChildren()
                 parent.append(nodeStack.pop())
@@ -27,9 +26,7 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: Dec
                 processChildren()
                 parent.append(nodeStack.pop())
             }
-            MarkdownElementTypes.HORIZONTAL_RULE -> {
-            }
-            MarkdownElementTypes.LIST_BLOCK -> {
+            MarkdownElementTypes.LIST_ITEM -> {
                 nodeStack.push(ContentBlock())
                 processChildren()
                 parent.append(nodeStack.pop())
@@ -44,50 +41,42 @@ public fun DocumentationBuilder.buildContent(tree: MarkdownTree, descriptor: Dec
                 processChildren()
                 parent.append(nodeStack.pop())
             }
-            MarkdownElementTypes.CODE -> {
+            MarkdownTokenTypes.CODE -> {
                 nodeStack.push(ContentCode())
                 processChildren()
                 parent.append(nodeStack.pop())
             }
-            MarkdownElementTypes.ANONYMOUS_SECTION -> {
-                nodeStack.push(ContentSection(""))
-                processChildren()
-                parent.append(nodeStack.pop())
-            }
-            MarkdownElementTypes.DIRECTIVE -> {
-                val name = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_NAME)?.let { tree.getNodeText(it) } ?: ""
-                val params = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_PARAMS)?.let { tree.getNodeText(it) } ?: ""
-                when (name) {
-                    "code" -> parent.append(functionBody(descriptor, params))
-                }
-            }
-            MarkdownElementTypes.NAMED_SECTION -> {
-                val label = tree.findChildByType(node, MarkdownElementTypes.SECTION_NAME)?.let { tree.getNodeText(it) } ?: ""
-                nodeStack.push(ContentSection(label))
-                processChildren()
-                parent.append(nodeStack.pop())
-            }
-            MarkdownElementTypes.LINK -> {
-                val target = tree.findChildByType(node, MarkdownElementTypes.TARGET)?.let { tree.getNodeText(it) } ?: ""
-                val href = tree.findChildByType(node, MarkdownElementTypes.HREF)?.let { tree.getNodeText(it) }
+        /*            MarkdownElementTypes.ANONYMOUS_SECTION -> {
+                        nodeStack.push(ContentSection(""))
+                        processChildren()
+                        parent.append(nodeStack.pop())
+                    }
+                    MarkdownElementTypes.DIRECTIVE -> {
+                        val name = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_NAME)?.let { tree.getNodeText(it) } ?: ""
+                        val params = tree.findChildByType(node, MarkdownElementTypes.DIRECTIVE_PARAMS)?.let { tree.getNodeText(it) } ?: ""
+                        when (name) {
+                            "code" -> parent.append(functionBody(descriptor, params))
+                        }
+                    }
+                    MarkdownElementTypes.NAMED_SECTION -> {
+                        val label = tree.findChildByType(node, MarkdownElementTypes.SECTION_NAME)?.let { tree.getNodeText(it) } ?: ""
+                        nodeStack.push(ContentSection(label))
+                        processChildren()
+                        parent.append(nodeStack.pop())
+                    }*/
+            MarkdownElementTypes.INLINE_LINK -> {
+                val target = node.child(MarkdownElementTypes.LINK_TITLE)?.let { it.text } ?: ""
+                val href = node.child(MarkdownElementTypes.LINK_DESTINATION)?.let { it.text }
                 val link = if (href != null) ContentExternalLink(href) else ContentExternalLink(target)
                 link.append(ContentText(target))
                 parent.append(link)
             }
-            MarkdownElementTypes.PLAIN_TEXT -> {
+            MarkdownTokenTypes.TEXT -> {
                 nodeStack.push(ContentText(nodeText))
                 processChildren()
                 parent.append(nodeStack.pop())
             }
-            MarkdownElementTypes.END_LINE -> {
-                nodeStack.push(ContentText(nodeText))
-                processChildren()
-                parent.append(nodeStack.pop())
-            }
-            MarkdownElementTypes.BLANK_LINE -> {
-                processChildren()
-            }
-            MarkdownElementTypes.PARA -> {
+            MarkdownElementTypes.PARAGRAPH -> {
                 nodeStack.push(ContentParagraph())
                 processChildren()
                 parent.append(nodeStack.pop())
diff --git a/src/Kotlin/DocumentationBuilder.kt b/src/Kotlin/DocumentationBuilder.kt
index 0acff1dc..d8be9d5a 100644
--- a/src/Kotlin/DocumentationBuilder.kt
+++ b/src/Kotlin/DocumentationBuilder.kt
@@ -1,17 +1,12 @@
 package org.jetbrains.dokka
 
 import org.jetbrains.jet.lang.descriptors.*
-import org.jetbrains.dokka.DocumentationNode.Kind
-import org.jetbrains.jet.lang.types.TypeProjection
-import org.jetbrains.jet.lang.types.Variance
-import org.jetbrains.jet.lang.types.JetType
-import org.jetbrains.jet.lang.types.lang.KotlinBuiltIns
-import org.jetbrains.jet.lang.resolve.BindingContext
-import org.jetbrains.jet.lang.resolve.name.Name
-import org.jetbrains.jet.lang.resolve.scopes.JetScope
-import org.jetbrains.jet.lang.psi.JetFile
-import org.jetbrains.jet.lang.resolve.name.FqName
-import org.jetbrains.jet.lang.resolve.lazy.ResolveSession
+import org.jetbrains.dokka.DocumentationNode.*
+import org.jetbrains.jet.lang.types.*
+import org.jetbrains.jet.lang.types.lang.*
+import org.jetbrains.jet.lang.resolve.scopes.*
+import org.jetbrains.jet.lang.resolve.name.*
+import org.jetbrains.jet.lang.resolve.lazy.*
 
 public data class DocumentationOptions(val includeNonPublic: Boolean = false)
 
@@ -24,7 +19,7 @@ class DocumentationBuilder(val session: ResolveSession, val options: Documentati
 
     fun parseDocumentation(descriptor: DeclarationDescriptor): Content {
         val docText = descriptor.getDocumentationElements().map { it.extractText() }.join("\n")
-        val tree = MarkdownProcessor.parse(docText)
+        val tree = parseMarkdown(docText)
         //println(tree.toTestString())
         val content = buildContent(tree, descriptor)
         return content
diff --git a/src/Markdown/GeneratedParserUtilBase.java b/src/Markdown/GeneratedParserUtilBase.java
deleted file mode 100644
index 9dd999b5..00000000
--- a/src/Markdown/GeneratedParserUtilBase.java
+++ /dev/null
@@ -1,1031 +0,0 @@
-/*
- * Copyright 2011-2014 Gregory Shrago
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.dokka.Markdown;
-
-import com.intellij.lang.*;
-import com.intellij.lang.impl.PsiBuilderAdapter;
-import com.intellij.lang.impl.PsiBuilderImpl;
-import com.intellij.lexer.Lexer;
-import com.intellij.openapi.diagnostic.Logger;
-import com.intellij.openapi.util.Comparing;
-import com.intellij.openapi.util.Key;
-import com.intellij.openapi.util.Pair;
-import com.intellij.openapi.util.text.StringHash;
-import com.intellij.openapi.util.text.StringUtil;
-import com.intellij.psi.PsiFile;
-import com.intellij.psi.PsiReference;
-import com.intellij.psi.TokenType;
-import com.intellij.psi.impl.source.resolve.FileContextUtil;
-import com.intellij.psi.impl.source.tree.CompositePsiElement;
-import com.intellij.psi.tree.ICompositeElementType;
-import com.intellij.psi.tree.IElementType;
-import com.intellij.psi.tree.TokenSet;
-import com.intellij.util.Function;
-import com.intellij.util.PairProcessor;
-import com.intellij.util.containers.ContainerUtil;
-import com.intellij.util.containers.LimitedPool;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.LinkedList;
-
-/**
- * @author gregsh
- */
-public class GeneratedParserUtilBase {
-
-  private static final Logger LOG = Logger.getInstance("org.intellij.grammar.parser.GeneratedParserUtilBase");
-
-  private static final int MAX_RECURSION_LEVEL = 1000;
-  private static final int MAX_VARIANTS_SIZE = 10000;
-  private static final int MAX_VARIANTS_TO_DISPLAY = 50;
-
-  private static final int INITIAL_VARIANTS_SIZE = 1000;
-  private static final int VARIANTS_POOL_SIZE = 10000;
-  private static final int FRAMES_POOL_SIZE = 500;
-
-  public static final IElementType DUMMY_BLOCK = new DummyBlockElementType();
-
-  public interface Parser {
-    boolean parse(PsiBuilder builder, int level);
-  }
-
-  public static final Parser TOKEN_ADVANCER = new Parser() {
-    @Override
-    public boolean parse(PsiBuilder builder, int level) {
-      if (builder.eof()) return false;
-      builder.advanceLexer();
-      return true;
-    }
-  };
-
-  public static final Parser TRUE_CONDITION = new Parser() {
-    @Override
-    public boolean parse(PsiBuilder builder, int level) {
-      return true;
-    }
-  };
-
-  public static boolean eof(PsiBuilder builder_, int level_) {
-    return builder_.eof();
-  }
-
-  public static int current_position_(PsiBuilder builder_) {
-    return builder_.rawTokenIndex();
-  }
-
-  public static boolean recursion_guard_(PsiBuilder builder_, int level_, String funcName_) {
-    if (level_ > MAX_RECURSION_LEVEL) {
-      builder_.error("Maximum recursion level (" + MAX_RECURSION_LEVEL + ") reached in '" + funcName_ + "'");
-      return false;
-    }
-    return true;
-  }
-
-  public static boolean empty_element_parsed_guard_(PsiBuilder builder_, String funcName_, int prev_position_) {
-    if (prev_position_ == current_position_(builder_)) {
-      builder_.error("Empty element parsed in '" + funcName_ + "' at offset " + builder_.getCurrentOffset());
-      return false;
-    }
-    return true;
-  }
-
-  public static boolean invalid_left_marker_guard_(PsiBuilder builder_, PsiBuilder.Marker marker_, String funcName_) {
-    //builder_.error("Invalid left marker encountered in " + funcName_ +" at offset " + builder_.getCurrentOffset());
-    boolean goodMarker = marker_ != null; // && ((LighterASTNode)marker_).getTokenType() != TokenType.ERROR_ELEMENT;
-    if (!goodMarker) return false;
-    ErrorState state = ErrorState.get(builder_);
-
-    return !state.frameStack.isEmpty();
-  }
-
-  public static TokenSet create_token_set_(IElementType... tokenTypes_) {
-    return TokenSet.create(tokenTypes_);
-  }
-
-  private static boolean consumeTokens(PsiBuilder builder_, boolean smart, int pin, IElementType... tokens) {
-    ErrorState state = ErrorState.get(builder_);
-    if (state.completionState != null && state.predicateCount == 0) {
-      addCompletionVariant(builder_, state.completionState, tokens);
-    }
-    // suppress single token completion
-    CompletionState completionState = state.completionState;
-    state.completionState = null;
-    boolean result_ = true;
-    boolean pinned_ = false;
-    for (int i = 0, tokensLength = tokens.length; i < tokensLength; i++) {
-      if (pin > 0 && i == pin) pinned_ = result_;
-      if (result_ || pinned_) {
-        boolean fast = smart && i == 0;
-        if (!(fast ? consumeTokenFast(builder_, tokens[i]) : consumeToken(builder_, tokens[i]))) {
-          result_ = false;
-          if (pin < 0 || pinned_) report_error_(builder_, state, false);
-        }
-      }
-    }
-    state.completionState = completionState;
-    return pinned_ || result_;
-  }
-
-  public static boolean consumeTokens(PsiBuilder builder_, int pin_, IElementType... token) {
-    return consumeTokens(builder_, false, pin_, token);
-  }
-
-  public static boolean consumeTokensSmart(PsiBuilder builder_, int pin_, IElementType... token) {
-    return consumeTokens(builder_, true, pin_, token);
-  }
-
-  public static boolean parseTokens(PsiBuilder builder_, int pin_, IElementType... tokens) {
-    return parseTokens(builder_, false, pin_, tokens);
-  }
-
-  public static boolean parseTokensSmart(PsiBuilder builder_, int pin_, IElementType... tokens) {
-    return parseTokens(builder_, true, pin_, tokens);
-  }
-
-  public static boolean parseTokens(PsiBuilder builder_, boolean smart, int pin_, IElementType... tokens) {
-    PsiBuilder.Marker marker_ = builder_.mark();
-    boolean result_ = consumeTokens(builder_, smart, pin_, tokens);
-    if (!result_) {
-      marker_.rollbackTo();
-    }
-    else {
-      marker_.drop();
-    }
-    return result_;
-  }
-
-  public static boolean consumeTokenSmart(PsiBuilder builder_, IElementType token) {
-    addCompletionVariantSmart(builder_, token);
-    return consumeTokenFast(builder_, token);
-  }
-
-  public static boolean consumeTokenSmart(PsiBuilder builder_, String token) {
-    addCompletionVariantSmart(builder_, token);
-    return consumeTokenFast(builder_, token);
-  }
-
-  public static boolean consumeToken(PsiBuilder builder_, IElementType token) {
-    addVariantSmart(builder_, token, true);
-    if (nextTokenIsFast(builder_, token)) {
-      builder_.advanceLexer();
-      return true;
-    }
-    return false;
-  }
-
-  public static boolean consumeTokenFast(PsiBuilder builder_, IElementType token) {
-    if (nextTokenIsFast(builder_, token)) {
-      builder_.advanceLexer();
-      return true;
-    }
-    return false;
-  }
-
-  public static boolean consumeToken(PsiBuilder builder_, String text) {
-    return consumeToken(builder_, text, ErrorState.get(builder_).caseSensitive);
-  }
-
-  public static boolean consumeToken(PsiBuilder builder_, String text, boolean caseSensitive) {
-    addVariantSmart(builder_, text, true);
-    int count = nextTokenIsFast(builder_, text, caseSensitive);
-    if (count > 0) {
-      while (count-- > 0) builder_.advanceLexer();
-      return true;
-    }
-    return false;
-  }
-
-  public static boolean consumeTokenFast(PsiBuilder builder_, String text) {
-    int count = nextTokenIsFast(builder_, text, ErrorState.get(builder_).caseSensitive);
-    if (count > 0) {
-      while (count-- > 0) builder_.advanceLexer();
-      return true;
-    }
-    return false;
-  }
-
-  public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType token) {
-    return builder_.getTokenType() == token;
-  }
-
-  public static boolean nextTokenIsFast(PsiBuilder builder_, IElementType... tokens) {
-    IElementType tokenType = builder_.getTokenType();
-    for (IElementType token : tokens) {
-      if (token == tokenType) return true;
-    }
-    return false;
-  }
-
-  public static boolean nextTokenIs(PsiBuilder builder_, String frameName, IElementType... tokens) {
-    ErrorState state = ErrorState.get(builder_);
-    if (state.completionState != null) return true;
-    boolean track = !state.suppressErrors && state.predicateCount < 2 && state.predicateSign;
-    if (!track) return nextTokenIsFast(builder_, tokens);
-    IElementType tokenType = builder_.getTokenType();
-    if (StringUtil.isNotEmpty(frameName)) {
-      addVariantInner(state, builder_.rawTokenIndex(), frameName);
-    }
-    else {
-      for (IElementType token : tokens) {
-        addVariant(builder_, state, token);
-      }
-    }
-    if (tokenType == null) return false;
-    for (IElementType token : tokens) {
-      if (tokenType == token) return true;
-    }
-    return false;
-  }
-
-  public static boolean nextTokenIs(PsiBuilder builder_, IElementType token) {
-    if (!addVariantSmart(builder_, token, false)) return true;
-    return nextTokenIsFast(builder_, token);
-  }
-
-  public static boolean nextTokenIs(PsiBuilder builder_, String tokenText) {
-    if (!addVariantSmart(builder_, tokenText, false)) return true;
-    return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0;
-  }
-
-  public static boolean nextTokenIsFast(PsiBuilder builder_, String tokenText) {
-    return nextTokenIsFast(builder_, tokenText, ErrorState.get(builder_).caseSensitive) > 0;
-  }
-
-  public static int nextTokenIsFast(PsiBuilder builder_, String tokenText, boolean caseSensitive) {
-    CharSequence sequence = builder_.getOriginalText();
-    int offset = builder_.getCurrentOffset();
-    int endOffset = offset + tokenText.length();
-    CharSequence subSequence = sequence.subSequence(offset, Math.min(endOffset, sequence.length()));
-
-    if (!Comparing.equal(subSequence, tokenText, caseSensitive)) return 0;
-
-    int count = 0;
-    while (true) {
-      int nextOffset = builder_.rawTokenTypeStart(++count);
-      if (nextOffset > endOffset) {
-        return -count;
-      }
-      else if (nextOffset == endOffset) {
-        break;
-      }
-    }
-    return count;
-  }
-
-  private static void addCompletionVariantSmart(PsiBuilder builder_, Object token) {
-    ErrorState state = ErrorState.get(builder_);
-    CompletionState completionState = state.completionState;
-    if (completionState != null && state.predicateCount == 0) {
-      addCompletionVariant(builder_, completionState, token);
-    }
-  }
-
-  private static boolean addVariantSmart(PsiBuilder builder_, Object token, boolean force) {
-    ErrorState state = ErrorState.get(builder_);
-    // skip FIRST check in completion mode
-    if (state.completionState != null && !force) return false;
-    builder_.eof();
-    if (!state.suppressErrors && state.predicateCount < 2) {
-      addVariant(builder_, state, token);
-    }
-    return true;
-  }
-
-  public static void addVariant(PsiBuilder builder_, String text) {
-    addVariant(builder_, ErrorState.get(builder_), text);
-  }
-
-  private static void addVariant(PsiBuilder builder_, ErrorState state, Object o) {
-    builder_.eof(); // skip whitespaces
-    addVariantInner(state, builder_.rawTokenIndex(), o);
-
-    CompletionState completionState = state.completionState;
-    if (completionState != null && state.predicateSign) {
-      addCompletionVariant(builder_, completionState, o);
-    }
-  }
-
-  private static void addVariantInner(ErrorState state, int pos, Object o) {
-    Variant variant = state.VARIANTS.alloc().init(pos, o);
-    if (state.predicateSign) {
-      state.variants.add(variant);
-      if (state.lastExpectedVariantPos < variant.position) {
-        state.lastExpectedVariantPos = variant.position;
-      }
-    }
-    else {
-      state.unexpected.add(variant);
-    }
-  }
-
-  private static void addCompletionVariant(@NotNull PsiBuilder builder_, @NotNull CompletionState completionState, Object o) {
-    int offset = builder_.getCurrentOffset();
-    if (!builder_.eof() && offset == builder_.rawTokenTypeStart(1)) return; // suppress for zero-length tokens
-
-    boolean add = false;
-    int diff = completionState.offset - offset;
-    String text = completionState.convertItem(o);
-    int length = text == null? 0 : text.length();
-    if (length == 0) return;
-    if (diff == 0) {
-      add = true;
-    }
-    else if (diff > 0 && diff <= length) {
-      CharSequence fragment = builder_.getOriginalText().subSequence(offset, completionState.offset);
-      add = completionState.prefixMatches(fragment.toString(), text);
-    }
-    else if (diff < 0) {
-      for (int i=-1; ; i--) {
-        IElementType type = builder_.rawLookup(i);
-        int tokenStart = builder_.rawTokenTypeStart(i);
-        if (isWhitespaceOrComment(builder_, type)) {
-          diff = completionState.offset - tokenStart;
-        }
-        else if (type != null && tokenStart < completionState.offset) {
-          CharSequence fragment = builder_.getOriginalText().subSequence(tokenStart, completionState.offset);
-          if (completionState.prefixMatches(fragment.toString(), text)) {
-            diff = completionState.offset - tokenStart;
-          }
-          break;
-        }
-        else break;
-      }
-      add = diff >= 0 && diff < length;
-    }
-    add = add && length > 1 && !(text.charAt(0) == '<' && text.charAt(length - 1) == '>') &&
-          !(text.charAt(0) == '\'' && text.charAt(length - 1) == '\'' && length < 5);
-    if (add) {
-      completionState.addItem(builder_, text);
-    }
-  }
-
-  public static boolean isWhitespaceOrComment(@NotNull PsiBuilder builder_, @Nullable IElementType type) {
-    return ((PsiBuilderImpl)((Builder)builder_).getDelegate()).whitespaceOrComment(type);
-  }
-
-  // here's the new section API for compact parsers & less IntelliJ platform API exposure
-  public static final int _NONE_       = 0x0;
-  public static final int _COLLAPSE_   = 0x1;
-  public static final int _LEFT_       = 0x2;
-  public static final int _LEFT_INNER_ = 0x4;
-  public static final int _AND_        = 0x8;
-  public static final int _NOT_        = 0x10;
-
-  // simple enter/exit methods pair that doesn't require frame object
-  public static PsiBuilder.Marker enter_section_(PsiBuilder builder_) {
-    return builder_.mark();
-  }
-
-  public static void exit_section_(PsiBuilder builder_,
-                                   PsiBuilder.Marker marker,
-                                   @Nullable IElementType elementType,
-                                   boolean result) {
-    close_marker_impl_(ErrorState.get(builder_).frameStack.peekLast(), marker, elementType, result);
-  }
-
-  // complex enter/exit methods pair with frame object
-  public static PsiBuilder.Marker enter_section_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) {
-    PsiBuilder.Marker marker = builder_.mark();
-    enter_section_impl_(builder_, level, modifiers, frameName);
-    return marker;
-  }
-
-  private static void enter_section_impl_(PsiBuilder builder_, int level, int modifiers, @Nullable String frameName) {
-    ErrorState state = ErrorState.get(builder_);
-    Frame frame = state.FRAMES.alloc().init(builder_, state, level, modifiers, frameName);
-    Frame prevFrame = state.frameStack.peekLast();
-    if (prevFrame != null && prevFrame.errorReportedAt > frame.position) {
-      // report error for previous unsuccessful frame
-      reportError(builder_, state, frame, true, false);
-    }
-    if (((frame.modifiers & _LEFT_) | (frame.modifiers & _LEFT_INNER_)) != 0) {
-      PsiBuilder.Marker left = (PsiBuilder.Marker)builder_.getLatestDoneMarker();
-      if (invalid_left_marker_guard_(builder_, left, frameName)) {
-        frame.leftMarker = left;
-      }
-    }
-    state.frameStack.add(frame);
-    if ((modifiers & _AND_) != 0) {
-      if (state.predicateCount == 0 && !state.predicateSign) {
-        throw new AssertionError("Incorrect false predicate sign");
-      }
-      state.predicateCount++;
-    }
-    else if ((modifiers & _NOT_) != 0) {
-      if (state.predicateCount == 0) {
-        state.predicateSign = false;
-      }
-      else {
-        state.predicateSign = !state.predicateSign;
-      }
-      state.predicateCount++;
-    }
-  }
-
-  public static void exit_section_(PsiBuilder builder_,
-                                   int level,
-                                   PsiBuilder.Marker marker,
-                                   @Nullable IElementType elementType,
-                                   boolean result,
-                                   boolean pinned,
-                                   @Nullable Parser eatMore) {
-    ErrorState state = ErrorState.get(builder_);
-
-    Frame frame = state.frameStack.pollLast();
-    if (frame == null || level != frame.level) {
-      LOG.error("Unbalanced error section: got " + frame + ", expected level " + level);
-      if (frame != null) state.FRAMES.recycle(frame);
-      close_marker_impl_(frame, marker, elementType, result);
-      return;
-    }
-
-    if (((frame.modifiers & _AND_) | (frame.modifiers & _NOT_)) != 0) {
-      close_marker_impl_(frame, marker, null, false);
-      state.predicateCount--;
-      if ((frame.modifiers & _NOT_) != 0) state.predicateSign = !state.predicateSign;
-      state.FRAMES.recycle(frame);
-      return;
-    }
-    exit_section_impl_(state, frame, builder_, marker, elementType, result, pinned);
-
-    int initialPos = builder_.rawTokenIndex();
-    boolean willFail = !result && !pinned;
-    if (willFail && initialPos == frame.position && state.lastExpectedVariantPos == frame.position &&
-        frame.name != null && state.variants.size() - frame.variantCount > 1) {
-      state.clearVariants(true, frame.variantCount);
-      addVariantInner(state, initialPos, frame.name);
-    }
-    int lastErrorPos = getLastVariantPos(state, initialPos);
-    if (!state.suppressErrors && eatMore != null) {
-      state.suppressErrors = true;
-      final boolean eatMoreFlagOnce = !builder_.eof() && eatMore.parse(builder_, frame.level + 1);
-      boolean eatMoreFlag = eatMoreFlagOnce || !result && frame.position == initialPos && lastErrorPos > frame.position;
-
-      PsiBuilderImpl.ProductionMarker latestDoneMarker =
-        (pinned || result) && (state.altMode || elementType != null) &&
-        eatMoreFlagOnce ? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null;
-      PsiBuilder.Marker extensionMarker = null;
-      IElementType extensionTokenType = null;
-      // whitespace prefix makes the very first frame offset bigger than marker start offset which is always 0
-      if (latestDoneMarker instanceof PsiBuilder.Marker &&
-          frame.position >= latestDoneMarker.getStartIndex() &&
-          frame.position <= latestDoneMarker.getEndIndex()) {
-        extensionMarker = ((PsiBuilder.Marker)latestDoneMarker).precede();
-        extensionTokenType = latestDoneMarker.getTokenType();
-        ((PsiBuilder.Marker)latestDoneMarker).drop();
-      }
-      // advance to the last error pos
-      // skip tokens until lastErrorPos. parseAsTree might look better here...
-      int parenCount = 0;
-      while ((eatMoreFlag || parenCount > 0) && builder_.rawTokenIndex() < lastErrorPos) {
-        builder_.advanceLexer();
-        eatMoreFlag = eatMore.parse(builder_, frame.level + 1);
-      }
-      boolean errorReported = frame.errorReportedAt == initialPos || !result && frame.errorReportedAt >= frame.position;
-      if (errorReported) {
-        if (eatMoreFlag) {
-          builder_.advanceLexer();
-          parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore);
-        }
-      }
-      else if (eatMoreFlag) {
-        errorReported = reportError(builder_, state, frame, true, true);
-        parseAsTree(state, builder_, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore);
-      }
-      else if (eatMoreFlagOnce || (!result && frame.position != builder_.rawTokenIndex()) || frame.errorReportedAt > initialPos) {
-        errorReported = reportError(builder_, state, frame, true, false);
-      }
-      if (extensionMarker != null) {
-        extensionMarker.done(extensionTokenType);
-      }
-      state.suppressErrors = false;
-      if (errorReported || result) {
-        state.clearVariants(true, 0);
-        state.clearVariants(false, 0);
-        state.lastExpectedVariantPos = -1;
-      }
-    }
-    else if (!result && pinned && frame.errorReportedAt < 0) {
-      // do not report if there are errors beyond current position
-      if (lastErrorPos == initialPos) {
-        // do not force, inner recoverRoot might have skipped some tokens
-        reportError(builder_, state, frame, false, false);
-      }
-      else if (lastErrorPos > initialPos) {
-        // set error pos here as if it is reported for future reference
-        frame.errorReportedAt = lastErrorPos;
-      }
-    }
-    // propagate errorReportedAt up the stack to avoid duplicate reporting
-    Frame prevFrame = willFail && eatMore == null ? null : state.frameStack.peekLast();
-    if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) {
-      prevFrame.errorReportedAt = frame.errorReportedAt;
-    }
-    state.FRAMES.recycle(frame);
-  }
-
-  private static void exit_section_impl_(ErrorState state,
-                                         Frame frame,
-                                         PsiBuilder builder_,
-                                         PsiBuilder.Marker marker,
-                                         IElementType elementType,
-                                         boolean result,
-                                         boolean pinned) {
-    if (elementType != null && marker != null) {
-      if ((frame.modifiers & _COLLAPSE_) != 0) {
-        PsiBuilderImpl.ProductionMarker last = result || pinned? (PsiBuilderImpl.ProductionMarker)builder_.getLatestDoneMarker() : null;
-        if (last != null && last.getStartIndex() == frame.position &&
-            state.typeExtends(last.getTokenType(), elementType)) {
-          IElementType resultType = last.getTokenType();
-          ((PsiBuilder.Marker)last).drop();
-          marker.done(resultType);
-          return;
-        }
-      }
-      if (result || pinned) {
-        if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
-          marker.done(elementType);
-          frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType());
-          frame.leftMarker.drop();
-        }
-        else if ((frame.modifiers & _LEFT_) != 0 && frame.leftMarker != null) {
-          marker.drop();
-          frame.leftMarker.precede().done(elementType);
-        }
-        else {
-          if (frame.level == 0) builder_.eof(); // skip whitespaces
-          marker.done(elementType);
-        }
-      }
-      else {
-        close_marker_impl_(frame, marker, null, false);
-      }
-    }
-    else if (result || pinned) {
-      if (marker != null) marker.drop();
-      if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
-        frame.leftMarker.precede().done(((LighterASTNode)frame.leftMarker).getTokenType());
-        frame.leftMarker.drop();
-      }
-    }
-    else {
-      close_marker_impl_(frame, marker, null, false);
-    }
-  }
-
-  private static void close_marker_impl_(Frame frame, PsiBuilder.Marker marker, IElementType elementType, boolean result) {
-    if (marker == null) return;
-    if (result) {
-      if (elementType != null) {
-        marker.done(elementType);
-      }
-      else {
-        marker.drop();
-      }
-    }
-    else {
-      if (frame != null) {
-        int position = ((PsiBuilderImpl.ProductionMarker)marker).getStartIndex();
-        if (frame.errorReportedAt > position) {
-          frame.errorReportedAt = frame.errorReportedAtPrev;
-        }
-      }
-      marker.rollbackTo();
-    }
-  }
-
-  public static boolean report_error_(PsiBuilder builder_, boolean result_) {
-    if (!result_) report_error_(builder_, ErrorState.get(builder_), false);
-    return result_;
-  }
-
-  public static void report_error_(PsiBuilder builder_, ErrorState state, boolean advance) {
-    Frame frame = state.frameStack.isEmpty()? null : state.frameStack.getLast();
-    if (frame == null) {
-      LOG.error("unbalanced enter/exit section call: got null");
-      return;
-    }
-    int position = builder_.rawTokenIndex();
-    if (frame.errorReportedAt < position && getLastVariantPos(state, position + 1) <= position) {
-      reportError(builder_, state, frame, true, advance);
-    }
-  }
-
-  private static int getLastVariantPos(ErrorState state, int defValue) {
-    return state.lastExpectedVariantPos < 0? defValue : state.lastExpectedVariantPos;
-  }
-
-  private static boolean reportError(PsiBuilder builder_,
-                                     ErrorState state,
-                                     Frame frame,
-                                     boolean force,
-                                     boolean advance) {
-    String expectedText = state.getExpectedText(builder_);
-    boolean notEmpty = StringUtil.isNotEmpty(expectedText);
-    if (force || notEmpty || advance) {
-      String gotText = builder_.eof()? "unexpected end of file" :
-                             notEmpty? "got '" + builder_.getTokenText() +"'" :
-                             "'" + builder_.getTokenText() +"' unexpected";
-      String message = expectedText + gotText;
-      if (advance) {
-        PsiBuilder.Marker mark = builder_.mark();
-        builder_.advanceLexer();
-        mark.error(message);
-      }
-      else {
-        builder_.error(message);
-      }
-      builder_.eof(); // skip whitespaces
-      frame.errorReportedAt = builder_.rawTokenIndex();
-      return true;
-    }
-    return false;
-  }
-
-
-  public static final Key<CompletionState> COMPLETION_STATE_KEY = Key.create("COMPLETION_STATE_KEY");
-
-  public static class CompletionState implements Function<Object, String> {
-    public final int offset;
-    public final Collection<String> items = ContainerUtil.newTroveSet();
-
-    public CompletionState(int offset_) {
-      offset = offset_;
-    }
-
-    @Nullable
-    public String convertItem(Object o) {
-      return o instanceof Object[] ? StringUtil.join((Object[]) o, this, " ") : o.toString();
-    }
-
-    @Override
-    public String fun(Object o) {
-      return o.toString();
-    }
-
-    public void addItem(@NotNull PsiBuilder builder, @NotNull String text) {
-      items.add(text);
-    }
-
-    public boolean prefixMatches(@NotNull String prefix, @NotNull String variant) {
-      return StringUtil.startsWithIgnoreCase(variant, prefix);
-    }
-  }
-
-  public static class Builder extends PsiBuilderAdapter {
-    public final ErrorState state;
-    public final PsiParser parser;
-
-    public Builder(PsiBuilder builder_, ErrorState state_, PsiParser parser_) {
-      super(builder_);
-      state = state_;
-      parser = parser_;
-    }
-</