Extract content model, make doc model independent from descriptors, parse doccomments with custom parser, some tests failing due to hanging new lines.

3 files changed, 58 insertions, 86 deletions

diff --git a/src/Markdown/MarkdownProcessor.kt b/src/Markdown/MarkdownProcessor.kt
index fe6e8436..19d5a8fb 100644
--- a/src/Markdown/MarkdownProcessor.kt
+++ b/src/Markdown/MarkdownProcessor.kt
@@ -9,12 +9,11 @@ import com.intellij.lang.LighterASTNode
 import com.intellij.util.diff.FlyweightCapableTreeStructure
 import com.intellij.openapi.util.Ref
 import org.jetbrains.markdown.lexer.MarkdownLexer
+import com.intellij.psi.tree.IElementType
 
-public class MarkdownProcessor {
-    class object {
-        val EXPR_LANGUAGE = object : Language("MARKDOWN") {}
-        val DOCUMENT = IFileElementType("DOCUMENT", EXPR_LANGUAGE);
-    }
+public object MarkdownProcessor {
+    val EXPR_LANGUAGE = object : Language("MARKDOWN") {}
+    val DOCUMENT = IFileElementType("DOCUMENT", EXPR_LANGUAGE);
 
     public fun parse(markdown: String): MarkdownTree {
         val parser = MarkdownParser()
@@ -30,6 +29,28 @@ public class MarkdownTree(private val text: String, private val structure: Flywe
         visit(structure.getRoot(), action)
     }
 
+    fun findChildByType(node: LighterASTNode, findType: IElementType) : LighterASTNode? {
+        val ref = Ref.create<Array<LighterASTNode>?>()
+        val count = structure.getChildren(node, ref)
+        val children = ref.get()
+        if (children != null) {
+            for (index in 0..count - 1) {
+                val child = children[index]
+                val nodeType = child.getTokenType()
+                if (nodeType == findType)
+                    return child
+                val nestedChild = findChildByType(child, findType)
+                if (nestedChild != null)
+                    return nestedChild
+            }
+        }
+        return null
+    }
+
+    fun getNodeText(node: LighterASTNode) : String {
+        return text.substring(node.getStartOffset(), node.getEndOffset())
+    }
+
     fun visit(node: LighterASTNode, action: (LighterASTNode, String, visitChildren: () -> Unit) -> Unit) {
         action(node, text) {
             val ref = Ref.create<Array<LighterASTNode>?>()
@@ -46,7 +67,7 @@ public class MarkdownTree(private val text: String, private val structure: Flywe
 
 }
 
-public fun MarkdownTree.dump(): String {
+public fun MarkdownTree.toTestString(): String {
     val sb = StringBuilder()
     var level = 0
     visit {(node, text, visitChildren) ->
@@ -64,29 +85,25 @@ public fun MarkdownTree.dump(): String {
 
 public fun MarkdownTree.toHtml(): String {
     val sb = StringBuilder()
-    var level = 0
     visit {(node, text, processChildren) ->
         val nodeType = node.getTokenType()
         val nodeText = text.substring(node.getStartOffset(), node.getEndOffset())
-        val indent = " ".repeat(level * 2)
         when (nodeType) {
             MarkdownElementTypes.BULLET_LIST -> {
-                sb.appendln("$indent<ul>")
-                level++
+                sb.appendln("<ul>")
                 processChildren()
-                level--
-                sb.appendln("$indent</ul>")
+                sb.appendln("</ul>")
             }
             MarkdownElementTypes.HORIZONTAL_RULE -> {
-                sb.appendln("$indent<hr/>")
+                sb.appendln("<hr/>")
             }
             MarkdownElementTypes.ORDERED_LIST -> {
-                sb.appendln("$indent<ol>")
+                sb.appendln("<ol>")
                 processChildren()
-                sb.appendln("$indent</ol>")
+                sb.appendln("</ol>")
             }
             MarkdownElementTypes.LIST_BLOCK -> {
-                sb.append("$indent<li>")
+                sb.append("<li>")
                 processChildren()
                 sb.appendln("</li>")
             }
@@ -110,14 +127,9 @@ public fun MarkdownTree.toHtml(): String {
                 sb.appendln()
             }
             MarkdownElementTypes.PARA -> {
-                sb.appendln("$indent<p>")
-                processChildren()
-                sb.appendln("$indent</p>")
-            }
-            MarkdownElementTypes.VERBATIM -> {
-                sb.appendln("$indent<pre><code>")
+                sb.append("<p>")
                 processChildren()
-                sb.appendln("$indent</code></pre>")
+                sb.appendln("</p>")
             }
             else -> {
                 processChildren()
@@ -129,8 +141,8 @@ public fun MarkdownTree.toHtml(): String {
 
 
 fun markdownToHtml(markdown: String): String {
-    val markdownTree = MarkdownProcessor().parse(markdown)
-    val ast = markdownTree.dump()
+    val markdownTree = MarkdownProcessor.parse(markdown)
+    val ast = markdownTree.toTestString()
     return markdownTree.toHtml()
 }
 
diff --git a/src/Markdown/_MarkdownLexer.flex b/src/Markdown/_MarkdownLexer.flex
deleted file mode 100644
index 0a15e41a..00000000
--- a/src/Markdown/_MarkdownLexer.flex
+++ /dev/null
@@ -1,40 +0,0 @@
-package org.jetbrains.markdown.lexer;
-
-import com.intellij.lexer.*;
-import com.intellij.psi.tree.IElementType;
-import static org.jetbrains.markdown.MarkdownElementTypes.*;
-
-%%
-
-%{
-  public _MarkdownLexer() {
-    this((java.io.Reader)null);
-  }
-%}
-
-%public
-%class _MarkdownLexer
-%implements FlexLexer
-%function advance
-%type IElementType
-%unicode
-
-Newline="\r"|"\n"|"\r\n"
-Spacechar=[\ \t\f]
-Number=[0-9]+(\.[0-9]*)?
-String=[^~\*_`&\[\]()<!#\\ \t\n\r]+
-AnyChar=.
-Line=!'\r' !'\n' .* {Newline}
-
-%%
-<YYINITIAL> {
-  {Spacechar}                         { return SPACECHAR; }
-  {Newline}                             { return NEWLINE; }
-  "\\357\\273\\277"                 { return BOM; }
-
-  {Number}                          { return NUMBER; }
-  {String}                          { return STRING; }
-  {AnyChar}                         { return ANYCHAR; }
-
-  [^] { return com.intellij.psi.TokenType.BAD_CHARACTER; }
-}
diff --git a/src/Markdown/markdown.bnf b/src/Markdown/markdown.bnf
index ca254295..79e9e0b4 100644
--- a/src/Markdown/markdown.bnf
+++ b/src/Markdown/markdown.bnf
@@ -21,35 +21,35 @@
 
 }
 
-Document ::= BOM? ( Block )*
+Document ::= BOM? Whitespace* AnonymousSection? (Whitespace* NamedSection)*
 
-private OptionalSpace ::= Spacechar*
-private RequiredSpace ::= Spacechar+
-private NonindentSpace ::= ("   " | "  " | " ")?
+AnonymousSection ::= SectionBody
+NamedSection ::= SectionHeader SectionBody
+private SectionHeader ::= '$' SectionName OptionalSpace ':' OptionalSpace
+SectionName ::= SectionNameStart | '{' OptionalSpace SectionNameStart (Spacechar+ String)* OptionalSpace '}'
+private SectionNameStart ::= '$'? String
+SectionBody::= Block*
 
 BlankLine ::= OptionalSpace Newline
 
 Whitespace ::= Spacechar | Newline
+private OptionalSpace ::= Spacechar*
+private NonindentSpace ::= ("   " | "  " | " ")?
+
 EndLine ::= TerminalEndline | NormalEndline
-private NormalEndline ::=   OptionalSpace Newline !BlankLine
-private TerminalEndline ::= (OptionalSpace Newline <<eof>>) | (OptionalSpace <<eof>>)
+private NormalEndline ::=   Newline !BlankLine
+private TerminalEndline ::= OptionalSpace <<eof>>
 private Indent ::=            "\t" | "    "
-NonblankIndentedLine ::= !BlankLine IndentedLine
-IndentedLine ::= Indent PlainText
 
 // ---- BLOCKS ----
-private Block ::= BlankLine* (
-          Para
-        | Verbatim
-        | OrderedList
-        | BulletList
-        | Inlines
-        )
-
-Para ::= NonindentSpace Inlines (BlankLine | TerminalEndline)
+Block ::= BlankLine* (
+              OrderedList
+            | BulletList
+            | HorizontalRule
+            | Para
+            )
 
-Verbatim ::= VerbatimItem+
-VerbatimItem ::= BlankLine* NonblankIndentedLine
+Para ::= Inlines (BlankLine | TerminalEndline)?
 
 HorizontalRule ::= NonindentSpace
                  ( '*' OptionalSpace '*' OptionalSpace '*' (OptionalSpace '*')*
@@ -74,8 +74,8 @@ ListContinuationBlock ::= BlankLine* (Indent ListBlock)+
 
 // ---- INLINES ----
 private Inlines ::= (!EndLine Inline | EndLine &Inline )+ EndLine?
-private Inline  ::= Strong | Emph | Link | PlainText
-PlainText ::= (String | Number | Spacechar)+
+Inline  ::= Strong | Emph | Link | PlainText
+PlainText ::= (String | Number | Spacechar+)+
 
 Emph ::= EmphStar | EmphUnderscore
 private EmphStar ::= '*' !Whitespace (!'*' Inline)+ '*'