Continue markdown processing

6 files changed, 303 insertions, 117 deletions

diff --git a/gen/org/jetbrains/markdown/MarkdownElementTypes.java b/gen/org/jetbrains/markdown/MarkdownElementTypes.java
index 48c423eb..757cc62f 100644
--- a/gen/org/jetbrains/markdown/MarkdownElementTypes.java
+++ b/gen/org/jetbrains/markdown/MarkdownElementTypes.java
@@ -7,7 +7,6 @@ import org.jetbrains.dokka.Markdown.MarkdownTokenType;
 public interface MarkdownElementTypes {
 
   IElementType BLANK_LINE = new IElementType("BLANK_LINE", null);
-  IElementType BLOCK = new IElementType("BLOCK", null);
   IElementType BULLET = new IElementType("BULLET", null);
   IElementType BULLET_LIST = new IElementType("BULLET_LIST", null);
   IElementType EMPH = new IElementType("EMPH", null);
@@ -15,18 +14,22 @@ public interface MarkdownElementTypes {
   IElementType ENUMERATOR = new IElementType("ENUMERATOR", null);
   IElementType HORIZONTAL_RULE = new IElementType("HORIZONTAL_RULE", null);
   IElementType HREF = new IElementType("HREF", null);
-  IElementType INLINE = new IElementType("INLINE", null);
+  IElementType INDENTED_LINE = new IElementType("INDENTED_LINE", null);
   IElementType LINK = new IElementType("LINK", null);
   IElementType LIST_BLOCK = new IElementType("LIST_BLOCK", null);
   IElementType LIST_BLOCK_LINE = new IElementType("LIST_BLOCK_LINE", null);
   IElementType LIST_CONTINUATION_BLOCK = new IElementType("LIST_CONTINUATION_BLOCK", null);
   IElementType LIST_ITEM = new IElementType("LIST_ITEM", null);
+  IElementType NONBLANK_INDENTED_LINE = new IElementType("NONBLANK_INDENTED_LINE", null);
   IElementType ORDERED_LIST = new IElementType("ORDERED_LIST", null);
   IElementType PARA = new IElementType("PARA", null);
+  IElementType PLAIN_TEXT = new IElementType("PLAIN_TEXT", null);
   IElementType STRONG = new IElementType("STRONG", null);
   IElementType STRONG_STAR = new IElementType("STRONG_STAR", null);
   IElementType STRONG_UNDERSCORE = new IElementType("STRONG_UNDERSCORE", null);
   IElementType TARGET = new IElementType("TARGET", null);
+  IElementType VERBATIM = new IElementType("VERBATIM", null);
+  IElementType VERBATIM_ITEM = new IElementType("VERBATIM_ITEM", null);
   IElementType WHITESPACE = new IElementType("WHITESPACE", null);
 
   IElementType ANYCHAR = new MarkdownTokenType("AnyChar");
diff --git a/gen/org/jetbrains/markdown/MarkdownParser.java b/gen/org/jetbrains/markdown/MarkdownParser.java
index ef1d8375..90c69985 100644
--- a/gen/org/jetbrains/markdown/MarkdownParser.java
+++ b/gen/org/jetbrains/markdown/MarkdownParser.java
@@ -25,9 +25,6 @@ public class MarkdownParser implements PsiParser {
     if (root_ == BLANK_LINE) {
       result_ = BlankLine(builder_, 0);
     }
-    else if (root_ == BLOCK) {
-      result_ = Block(builder_, 0);
-    }
     else if (root_ == BULLET) {
       result_ = Bullet(builder_, 0);
     }
@@ -49,8 +46,8 @@ public class MarkdownParser implements PsiParser {
     else if (root_ == HREF) {
       result_ = Href(builder_, 0);
     }
-    else if (root_ == INLINE) {
-      result_ = Inline(builder_, 0);
+    else if (root_ == INDENTED_LINE) {
+      result_ = IndentedLine(builder_, 0);
     }
     else if (root_ == LINK) {
       result_ = Link(builder_, 0);
@@ -67,12 +64,18 @@ public class MarkdownParser implements PsiParser {
     else if (root_ == LIST_ITEM) {
       result_ = ListItem(builder_, 0);
     }
+    else if (root_ == NONBLANK_INDENTED_LINE) {
+      result_ = NonblankIndentedLine(builder_, 0);
+    }
     else if (root_ == ORDERED_LIST) {
       result_ = OrderedList(builder_, 0);
     }
     else if (root_ == PARA) {
       result_ = Para(builder_, 0);
     }
+    else if (root_ == PLAIN_TEXT) {
+      result_ = PlainText(builder_, 0);
+    }
     else if (root_ == STRONG) {
       result_ = Strong(builder_, 0);
     }
@@ -85,6 +88,12 @@ public class MarkdownParser implements PsiParser {
     else if (root_ == TARGET) {
       result_ = Target(builder_, 0);
     }
+    else if (root_ == VERBATIM) {
+      result_ = Verbatim(builder_, 0);
+    }
+    else if (root_ == VERBATIM_ITEM) {
+      result_ = VerbatimItem(builder_, 0);
+    }
     else if (root_ == WHITESPACE) {
       result_ = Whitespace(builder_, 0);
     }
@@ -113,18 +122,19 @@ public class MarkdownParser implements PsiParser {
 
   /* ********************************************************** */
   // BlankLine* (
-  //         Para
-  //         | Plain
+  //           Para
+  //         | Verbatim
   //         | OrderedList
   //         | BulletList
+  //         | Inlines
   //         )
-  public static boolean Block(PsiBuilder builder_, int level_) {
+  static boolean Block(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "Block")) return false;
     boolean result_;
-    Marker marker_ = enter_section_(builder_, level_, _NONE_, "<block>");
+    Marker marker_ = enter_section_(builder_);
     result_ = Block_0(builder_, level_ + 1);
     result_ = result_ && Block_1(builder_, level_ + 1);
-    exit_section_(builder_, level_, marker_, BLOCK, result_, false, null);
+    exit_section_(builder_, marker_, null, result_);
     return result_;
   }
 
@@ -141,17 +151,19 @@ public class MarkdownParser implements PsiParser {
   }
 
   // Para
-  //         | Plain
+  //         | Verbatim
   //         | OrderedList
   //         | BulletList
+  //         | Inlines
   private static boolean Block_1(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "Block_1")) return false;
     boolean result_;
     Marker marker_ = enter_section_(builder_);
     result_ = Para(builder_, level_ + 1);
-    if (!result_) result_ = Plain(builder_, level_ + 1);
+    if (!result_) result_ = Verbatim(builder_, level_ + 1);
     if (!result_) result_ = OrderedList(builder_, level_ + 1);
     if (!result_) result_ = BulletList(builder_, level_ + 1);
+    if (!result_) result_ = Inlines(builder_, level_ + 1);
     exit_section_(builder_, marker_, null, result_);
     return result_;
   }
@@ -409,7 +421,6 @@ public class MarkdownParser implements PsiParser {
   // TerminalEndline | NormalEndline
   public static boolean EndLine(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "EndLine")) return false;
-    if (!nextTokenIs(builder_, "<end line>", NEWLINE, SPACECHAR)) return false;
     boolean result_;
     Marker marker_ = enter_section_(builder_, level_, _NONE_, "<end line>");
     result_ = TerminalEndline(builder_, level_ + 1);
@@ -652,34 +663,27 @@ public class MarkdownParser implements PsiParser {
   }
 
   /* ********************************************************** */
-  // String | Number | EndLine | Spacechar+ | Strong | Emph | Link
-  public static boolean Inline(PsiBuilder builder_, int level_) {
-    if (!recursion_guard_(builder_, level_, "Inline")) return false;
+  // Indent PlainText
+  public static boolean IndentedLine(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "IndentedLine")) return false;
     boolean result_;
-    Marker marker_ = enter_section_(builder_, level_, _NONE_, "<inline>");
-    result_ = consumeToken(builder_, STRING);
-    if (!result_) result_ = consumeToken(builder_, NUMBER);
-    if (!result_) result_ = EndLine(builder_, level_ + 1);
-    if (!result_) result_ = Inline_3(builder_, level_ + 1);
-    if (!result_) result_ = Strong(builder_, level_ + 1);
-    if (!result_) result_ = Emph(builder_, level_ + 1);
-    if (!result_) result_ = Link(builder_, level_ + 1);
-    exit_section_(builder_, level_, marker_, INLINE, result_, false, null);
+    Marker marker_ = enter_section_(builder_, level_, _NONE_, "<indented line>");
+    result_ = Indent(builder_, level_ + 1);
+    result_ = result_ && PlainText(builder_, level_ + 1);
+    exit_section_(builder_, level_, marker_, INDENTED_LINE, result_, false, null);
     return result_;
   }
 
-  // Spacechar+
-  private static boolean Inline_3(PsiBuilder builder_, int level_) {
-    if (!recursion_guard_(builder_, level_, "Inline_3")) return false;
+  /* ********************************************************** */
+  // Strong | Emph | Link | PlainText
+  static boolean Inline(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "Inline")) return false;
     boolean result_;
     Marker marker_ = enter_section_(builder_);
-    result_ = consumeToken(builder_, SPACECHAR);
-    int pos_ = current_position_(builder_);
-    while (result_) {
-      if (!consumeToken(builder_, SPACECHAR)) break;
-      if (!empty_element_parsed_guard_(builder_, "Inline_3", pos_)) break;
-      pos_ = current_position_(builder_);
-    }
+    result_ = Strong(builder_, level_ + 1);
+    if (!result_) result_ = Emph(builder_, level_ + 1);
+    if (!result_) result_ = Link(builder_, level_ + 1);
+    if (!result_) result_ = PlainText(builder_, level_ + 1);
     exit_section_(builder_, marker_, null, result_);
     return result_;
   }
@@ -825,13 +829,13 @@ public class MarkdownParser implements PsiParser {
   }
 
   /* ********************************************************** */
-  // !BlankLine Plain ( ListBlockLine )*
+  // !BlankLine Inlines ( ListBlockLine )*
   public static boolean ListBlock(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "ListBlock")) return false;
     boolean result_;
     Marker marker_ = enter_section_(builder_, level_, _NONE_, "<list block>");
     result_ = ListBlock_0(builder_, level_ + 1);
-    result_ = result_ && Plain(builder_, level_ + 1);
+    result_ = result_ && Inlines(builder_, level_ + 1);
     result_ = result_ && ListBlock_2(builder_, level_ + 1);
     exit_section_(builder_, level_, marker_, LIST_BLOCK, result_, false, null);
     return result_;
@@ -870,7 +874,7 @@ public class MarkdownParser implements PsiParser {
   }
 
   /* ********************************************************** */
-  // !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Plain
+  // !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Inlines
   public static boolean ListBlockLine(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "ListBlockLine")) return false;
     boolean result_;
@@ -879,7 +883,7 @@ public class MarkdownParser implements PsiParser {
     result_ = result_ && ListBlockLine_1(builder_, level_ + 1);
     result_ = result_ && ListBlockLine_2(builder_, level_ + 1);
     result_ = result_ && ListBlockLine_3(builder_, level_ + 1);
-    result_ = result_ && Plain(builder_, level_ + 1);
+    result_ = result_ && Inlines(builder_, level_ + 1);
     exit_section_(builder_, level_, marker_, LIST_BLOCK_LINE, result_, false, null);
     return result_;
   }
@@ -1048,6 +1052,28 @@ public class MarkdownParser implements PsiParser {
   }
 
   /* ********************************************************** */
+  // !BlankLine IndentedLine
+  public static boolean NonblankIndentedLine(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "NonblankIndentedLine")) return false;
+    boolean result_;
+    Marker marker_ = enter_section_(builder_, level_, _NONE_, "<nonblank indented line>");
+    result_ = NonblankIndentedLine_0(builder_, level_ + 1);
+    result_ = result_ && IndentedLine(builder_, level_ + 1);
+    exit_section_(builder_, level_, marker_, NONBLANK_INDENTED_LINE, result_, false, null);
+    return result_;
+  }
+
+  // !BlankLine
+  private static boolean NonblankIndentedLine_0(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "NonblankIndentedLine_0")) return false;
+    boolean result_;
+    Marker marker_ = enter_section_(builder_, level_, _NOT_, null);
+    result_ = !BlankLine(builder_, level_ + 1);
+    exit_section_(builder_, level_, marker_, null, result_, false, null);
+    return result_;
+  }
+
+  /* ********************************************************** */
   // ("   " | "  " | " ")?
   static boolean NonindentSpace(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "NonindentSpace")) return false;
@@ -1127,7 +1153,7 @@ public class MarkdownParser implements PsiParser {
   }
 
   /* ********************************************************** */
-  // NonindentSpace Inlines (BlankLine+ | TerminalEndline)
+  // NonindentSpace Inlines (BlankLine | TerminalEndline)
   public static boolean Para(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "Para")) return false;
     boolean result_;
@@ -1139,37 +1165,44 @@ public class MarkdownParser implements PsiParser {
     return result_;
   }
 
-  // BlankLine+ | TerminalEndline
+  // BlankLine | TerminalEndline
   private static boolean Para_2(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "Para_2")) return false;
     boolean result_;
     Marker marker_ = enter_section_(builder_);
-    result_ = Para_2_0(builder_, level_ + 1);
+    result_ = BlankLine(builder_, level_ + 1);
     if (!result_) result_ = TerminalEndline(builder_, level_ + 1);
     exit_section_(builder_, marker_, null, result_);
     return result_;
   }
 
-  // BlankLine+
-  private static boolean Para_2_0(PsiBuilder builder_, int level_) {
-    if (!recursion_guard_(builder_, level_, "Para_2_0")) return false;
+  /* ********************************************************** */
+  // (String | Number | Spacechar)+
+  public static boolean PlainText(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "PlainText")) return false;
     boolean result_;
-    Marker marker_ = enter_section_(builder_);
-    result_ = BlankLine(builder_, level_ + 1);
+    Marker marker_ = enter_section_(builder_, level_, _NONE_, "<plain text>");
+    result_ = PlainText_0(builder_, level_ + 1);
     int pos_ = current_position_(builder_);
     while (result_) {
-      if (!BlankLine(builder_, level_ + 1)) break;
-      if (!empty_element_parsed_guard_(builder_, "Para_2_0", pos_)) break;
+      if (!PlainText_0(builder_, level_ + 1)) break;
+      if (!empty_element_parsed_guard_(builder_, "PlainText", pos_)) break;
       pos_ = current_position_(builder_);
     }
-    exit_section_(builder_, marker_, null, result_);
+    exit_section_(builder_, level_, marker_, PLAIN_TEXT, result_, false, null);
     return result_;
   }
 
-  /* ********************************************************** */
-  // Inlines
-  static boolean Plain(PsiBuilder builder_, int level_) {
-    return Inlines(builder_, level_ + 1);
+  // String | Number | Spacechar
+  private static boolean PlainText_0(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "PlainText_0")) return false;
+    boolean result_;
+    Marker marker_ = enter_section_(builder_);
+    result_ = consumeToken(builder_, STRING);
+    if (!result_) result_ = consumeToken(builder_, NUMBER);
+    if (!result_) result_ = consumeToken(builder_, SPACECHAR);
+    exit_section_(builder_, marker_, null, result_);
+    return result_;
   }
 
   /* ********************************************************** */
@@ -1350,10 +1383,20 @@ public class MarkdownParser implements PsiParser {
   }
 
   /* ********************************************************** */
-  // OptionalSpace Newline <<eof>>
+  // (OptionalSpace Newline <<eof>>) | (OptionalSpace <<eof>>)
   static boolean TerminalEndline(PsiBuilder builder_, int level_) {
     if (!recursion_guard_(builder_, level_, "TerminalEndline")) return false;
-    if (!nextTokenIs(builder_, "", NEWLINE, SPACECHAR)) return false;
+    boolean result_;
+    Marker marker_ = enter_section_(builder_);
+    result_ = TerminalEndline_0(builder_, level_ + 1);
+    if (!result_) result_ = TerminalEndline_1(builder_, level_ + 1);
+    exit_section_(builder_, marker_, null, result_);
+    return result_;
+  }
+
+  // OptionalSpace Newline <<eof>>
+  private static boolean TerminalEndline_0(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "TerminalEndline_0")) return false;
     boolean result_;
     Marker marker_ = enter_section_(builder_);
     result_ = OptionalSpace(builder_, level_ + 1);
@@ -1363,6 +1406,58 @@ public class MarkdownParser implements PsiParser {
     return result_;
   }
 
+  // OptionalSpace <<eof>>
+  private static boolean TerminalEndline_1(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "TerminalEndline_1")) return false;
+    boolean result_;
+    Marker marker_ = enter_section_(builder_);
+    result_ = OptionalSpace(builder_, level_ + 1);
+    result_ = result_ && eof(builder_, level_ + 1);
+    exit_section_(builder_, marker_, null, result_);
+    return result_;
+  }
+
+  /* ********************************************************** */
+  // VerbatimItem+
+  public static boolean Verbatim(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "Verbatim")) return false;
+    boolean result_;
+    Marker marker_ = enter_section_(builder_, level_, _NONE_, "<verbatim>");
+    result_ = VerbatimItem(builder_, level_ + 1);
+    int pos_ = current_position_(builder_);
+    while (result_) {
+      if (!VerbatimItem(builder_, level_ + 1)) break;
+      if (!empty_element_parsed_guard_(builder_, "Verbatim", pos_)) break;
+      pos_ = current_position_(builder_);
+    }
+    exit_section_(builder_, level_, marker_, VERBATIM, result_, false, null);
+    return result_;
+  }
+
+  /* ********************************************************** */
+  // BlankLine* NonblankIndentedLine
+  public static boolean VerbatimItem(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "VerbatimItem")) return false;
+    boolean result_;
+    Marker marker_ = enter_section_(builder_, level_, _NONE_, "<verbatim item>");
+    result_ = VerbatimItem_0(builder_, level_ + 1);
+    result_ = result_ && NonblankIndentedLine(builder_, level_ + 1);
+    exit_section_(builder_, level_, marker_, VERBATIM_ITEM, result_, false, null);
+    return result_;
+  }
+
+  // BlankLine*
+  private static boolean VerbatimItem_0(PsiBuilder builder_, int level_) {
+    if (!recursion_guard_(builder_, level_, "VerbatimItem_0")) return false;
+    int pos_ = current_position_(builder_);
+    while (true) {
+      if (!BlankLine(builder_, level_ + 1)) break;
+      if (!empty_element_parsed_guard_(builder_, "VerbatimItem_0", pos_)) break;
+      pos_ = current_position_(builder_);
+    }
+    return true;
+  }
+
   /* ********************************************************** */
   // Spacechar | Newline
   public static boolean Whitespace(PsiBuilder builder_, int level_) {
diff --git a/src/Markdown/MarkdownProcessor.kt b/src/Markdown/MarkdownProcessor.kt
index 4453e750..fe6e8436 100644
--- a/src/Markdown/MarkdownProcessor.kt
+++ b/src/Markdown/MarkdownProcessor.kt
@@ -26,31 +26,111 @@ public class MarkdownProcessor {
 }
 
 public class MarkdownTree(private val text: String, private val structure: FlyweightCapableTreeStructure<LighterASTNode>) {
-    public fun dump(): String {
-        val sb = StringBuilder()
-        visit(sb, "", structure.getRoot(), structure, text)
-        return sb.toString()
+    fun visit(action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) {
+        visit(structure.getRoot(), action)
+    }
+
+    fun visit(node: LighterASTNode, action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) {
+        action(node, text) {
+            val ref = Ref.create<Array<LighterASTNode>?>()
+            val count = structure.getChildren(node, ref)
+            val children = ref.get()
+            if (children != null) {
+                for (index in 0..count - 1) {
+                    val child = children[index]
+                    visit(child, action)
+                }
+            }
+        }
     }
-}
 
-fun markdownToHtml(markdown : String) : String {
-    return MarkdownProcessor().parse(markdown).dump()
 }
 
+public fun MarkdownTree.dump(): String {
+    val sb = StringBuilder()
+    var level = 0
+    visit {(node, text, visitChildren) ->
+        val nodeText = text.substring(node.getStartOffset(), node.getEndOffset())
+        sb.append(" ".repeat(level * 2))
+        sb.append(node.getTokenType().toString())
+        sb.append(":" + nodeText.replace("\n", "\u23CE"))
+        sb.appendln()
+        level++
+        visitChildren()
+        level--
+    }
+    return sb.toString()
+}
 
-fun visit(sb: StringBuilder, indent: String, node: LighterASTNode, structure: FlyweightCapableTreeStructure<LighterASTNode>, markdown: String) {
-    sb.append(indent)
-    sb.append(node.getTokenType().toString())
-    val nodeText = markdown.substring(node.getStartOffset(), node.getEndOffset())
-    sb.append(":" + nodeText.replace("\n","\u23CE"))
-    sb.appendln()
-    val ref = Ref.create<Array<LighterASTNode>?>()
-    val count = structure.getChildren(node, ref)
-    val children = ref.get()
-    if (children == null)
-        return
-    for (index in 0..count - 1) {
-        val child = children[index]
-        visit(sb, indent + "  ", child, structure, markdown)
+public fun MarkdownTree.toHtml(): String {
+    val sb = StringBuilder()
+    var level = 0
+    visit {(node, text, processChildren) ->
+        val nodeType = node.getTokenType()
+        val nodeText = text.substring(node.getStartOffset(), node.getEndOffset())
+        val indent = " ".repeat(level * 2)
+        when (nodeType) {
+            MarkdownElementTypes.BULLET_LIST -> {
+                sb.appendln("$indent<ul>")
+                level++
+                processChildren()
+                level--
+                sb.appendln("$indent</ul>")
+            }
+            MarkdownElementTypes.HORIZONTAL_RULE -> {
+                sb.appendln("$indent<hr/>")
+            }
+            MarkdownElementTypes.ORDERED_LIST -> {
+                sb.appendln("$indent<ol>")
+                processChildren()
+                sb.appendln("$indent</ol>")
+            }
+            MarkdownElementTypes.LIST_BLOCK -> {
+                sb.append("$indent<li>")
+                processChildren()
+                sb.appendln("</li>")
+            }
+            MarkdownElementTypes.EMPH -> {
+                sb.append("<em>")
+                processChildren()
+                sb.append("</em>")
+            }
+            MarkdownElementTypes.STRONG -> {
+                sb.append("<strong>")
+                processChildren()
+                sb.append("</strong>")
+            }
+            MarkdownElementTypes.PLAIN_TEXT -> {
+                sb.append(nodeText)
+            }
+            MarkdownElementTypes.END_LINE -> {
+                sb.appendln()
+            }
+            MarkdownElementTypes.BLANK_LINE -> {
+                sb.appendln()
+            }
+            MarkdownElementTypes.PARA -> {
+                sb.appendln("$indent<p>")
+                processChildren()
+                sb.appendln("$indent</p>")
+            }
+            MarkdownElementTypes.VERBATIM -> {
+                sb.appendln("$indent<pre><code>")
+                processChildren()
+                sb.appendln("$indent</code></pre>")
+            }
+            else -> {
+                processChildren()
+            }
+        }
     }
-}
\ No newline at end of file
+    return sb.toString()
+}
+
+
+fun markdownToHtml(markdown: String): String {
+    val markdownTree = MarkdownProcessor().parse(markdown)
+    val ast = markdownTree.dump()
+    return markdownTree.toHtml()
+}
+
diff --git a/src/Markdown/markdown.bnf b/src/Markdown/markdown.bnf
index 71100a6f..ca254295 100644
--- a/src/Markdown/markdown.bnf
+++ b/src/Markdown/markdown.bnf
@@ -32,19 +32,24 @@ BlankLine ::= OptionalSpace Newline
 Whitespace ::= Spacechar | Newline
 EndLine ::= TerminalEndline | NormalEndline
 private NormalEndline ::=   OptionalSpace Newline !BlankLine
-private TerminalEndline ::= OptionalSpace Newline <<eof>>
+private TerminalEndline ::= (OptionalSpace Newline <<eof>>) | (OptionalSpace <<eof>>)
 private Indent ::=            "\t" | "    "
+NonblankIndentedLine ::= !BlankLine IndentedLine
+IndentedLine ::= Indent PlainText
 
 // ---- BLOCKS ----
-Block ::= BlankLine* (
-        Para
-        | Plain
+private Block ::= BlankLine* (
+          Para
+        | Verbatim
         | OrderedList
         | BulletList
+        | Inlines
         )
 
-Para ::= NonindentSpace Inlines (BlankLine+ | TerminalEndline)
-private Plain ::= Inlines
+Para ::= NonindentSpace Inlines (BlankLine | TerminalEndline)
+
+Verbatim ::= VerbatimItem+
+VerbatimItem ::= BlankLine* NonblankIndentedLine
 
 HorizontalRule ::= NonindentSpace
                  ( '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')*
@@ -61,15 +66,16 @@ OrderedList ::= &Enumerator List
 private List ::=  (ListItem BlankLine*)+
 ListItem      ::= (Bullet | Enumerator) ListBlock ( ListContinuationBlock )*
 
-ListBlock ::= !BlankLine Plain ( ListBlockLine )*
-ListBlockLine ::= !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Plain
+ListBlock ::= !BlankLine Inlines ( ListBlockLine )*
+ListBlockLine ::= !BlankLine !(Indent? (Bullet | Enumerator)) !HorizontalRule Indent? Inlines
 
 ListContinuationBlock ::= BlankLine* (Indent ListBlock)+
 
 
 // ---- INLINES ----
 private Inlines ::= (!EndLine Inline | EndLine &Inline )+ EndLine?
-Inline  ::= String | Number | EndLine | Spacechar+ | Strong | Emph | Link
+private Inline  ::= Strong | Emph | Link | PlainText
+PlainText ::= (String | Number | Spacechar)+
 
 Emph ::= EmphStar | EmphUnderscore
 private EmphStar ::= '*' !Whitespace (!'*' Inline)+ '*'
diff --git a/test/src/markdown/MarkdownTestRunner.kt b/test/src/markdown/MarkdownTestRunner.kt
index bf1d9516..4ba9675e 100644
--- a/test/src/markdown/MarkdownTestRunner.kt
+++ b/test/src/markdown/MarkdownTestRunner.kt
@@ -84,7 +84,10 @@ public class MarkdownTestRunner(specificationClass: Class<MarkdownSpecification>
     private fun createTests(parent: MarkdownTestSection, lines: List<String>): Int {
         val testMark = lines.takeWhile { it.trim() != "." }
         val testHtml = lines.drop(testMark.size).drop(1).takeWhile { it.trim() != "." }
-        parent.children.add(MarkdownTestCase(spec, testMark.join("\n", postfix = "\n"), testHtml.join("\n", postfix = "\n")))
+        val markdown = testMark.join("\n", postfix = "\n", prefix = "\n")
+        val html = testHtml.join("\n", postfix = "\n")
+        val markdownTestCase = MarkdownTestCase(spec, markdown, html)
+        parent.children.add(markdownTestCase)
         return testMark.size + testHtml.size + 3
     }
 
diff --git a/test/src/markdown/ParserTest.kt b/test/src/markdown/ParserTest.kt
index 7b1522bc..726505cf 100644
--- a/test/src/markdown/ParserTest.kt
+++ b/test/src/markdown/ParserTest.kt
@@ -1,84 +1,83 @@
 package org.jetbrains.dokka.tests
 
 import org.junit.Test
-import org.jetbrains.dokka.*
+import org.jetbrains.dokka
+import org.jetbrains.dokka.MarkdownProcessor
+import org.jetbrains.dokka.dump
+import org.jetbrains.dokka.toHtml
 
 public class ParserTest {
+    fun runTestFor(text : String) {
+        val markdownTree = MarkdownProcessor().parse(text)
+        println(markdownTree.dump())
+        println(markdownTree.toHtml())
+    }
+
     Test fun text() {
-        val markdown = MarkdownProcessor().parse("text")
-        println(markdown.dump())
+        runTestFor("text")
     }
 
     Test fun textWithSpaces() {
-        val markdown = MarkdownProcessor().parse("text and string")
-        println(markdown.dump())
+        runTestFor("text and string")
     }
 
     Test fun link() {
-        val markdown = MarkdownProcessor().parse("text [links]")
-        println(markdown.dump())
+        runTestFor("text [links]")
     }
 
     Test fun linkWithHref() {
-        val markdown = MarkdownProcessor().parse("text [links](http://destination)")
-        println(markdown.dump())
+        runTestFor("text [links](http://destination)")
     }
 
     Test fun multiline() {
-        val markdown = MarkdownProcessor().parse(
-"""
+        runTestFor(
+                """
 text
 and
 string
 """)
-        println(markdown.dump())
     }
 
     Test fun para() {
-        val markdown = MarkdownProcessor().parse(
-"""paragraph number
+        runTestFor(
+                """
+paragraph number
 one
 
 paragraph
 number two
 """)
-        println(markdown.dump())
     }
 
     Test fun bulletList() {
-        val markdown = MarkdownProcessor().parse(
-"""
-* list item 1
+        runTestFor(
+                """* list item 1
 * list item 2
 """)
-        println(markdown.dump())
     }
 
     Test fun bulletListWithLines() {
-        val markdown = MarkdownProcessor().parse(
-"""
+        runTestFor(
+                """
 * list item 1
   continue 1
 * list item 2
   continue 2
  """)
-        println(markdown.dump())
     }
 
     Test fun bulletListStrong() {
-        val markdown = MarkdownProcessor().parse(
-"""
+        runTestFor(
+                """
 * list *item* 1
   continue 1
 * list *item* 2
   continue 2
  """)
-        println(markdown.dump())
     }
 
     Test fun emph() {
-        val markdown = MarkdownProcessor().parse("*text*")
-        println(markdown.dump())
+        runTestFor("*text*")
     }
 }