Extract content model, make doc model independent from descriptors, parse doccomments with custom parser, some tests failing due to hanging new lines.

2 files changed, 49 insertions, 7 deletions

diff --git a/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex b/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex
new file mode 100644
index 00000000..f5aab899
--- /dev/null
+++ b/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex
@@ -0,0 +1,42 @@
+package org.jetbrains.markdown.lexer;
+
+import com.intellij.lexer.*;
+import com.intellij.psi.tree.IElementType;
+import static org.jetbrains.markdown.MarkdownElementTypes.*;
+
+%%
+
+%{
+  public _MarkdownLexer() {
+    this((java.io.Reader)null);
+  }
+%}
+
+%public
+%class _MarkdownLexer
+%implements FlexLexer
+%function advance
+%type IElementType
+%unicode
+
+Newline="\r"|"\n"|"\r\n"
+Spacechar=[\ \t\f]
+Number=[0-9]+(\.[0-9]*)?
+String=[^~:{}$\*_`&\[\]()<!#\\ \t\n\r]+
+AnyChar=.
+Line=!'\r' !'\n' .* {Newline}
+
+%%
+<YYINITIAL> {
+  {Spacechar}                         { return SPACECHAR; }
+  {Newline}                             { return NEWLINE; }
+  "\\357\\273\\277"                 { return BOM; }
+
+
+  {Number}                          { return NUMBER; }
+  {String}                          { return STRING; }
+
+  {AnyChar}                         { return ANYCHAR; }
+
+  [^] { return com.intellij.psi.TokenType.BAD_CHARACTER; }
+}
diff --git a/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.java b/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.java
index 0052e3ea..c84a83d8 100644
--- a/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.java
+++ b/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.java
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.4.3 on 9/25/14 10:26 PM */
+/* The following code was generated by JFlex 1.4.3 on 9/29/14 3:24 PM */
 
 package org.jetbrains.markdown.lexer;
 
@@ -10,8 +10,8 @@ import static org.jetbrains.markdown.MarkdownElementTypes.*;
 /**
  * This class is a scanner generated by 
  * <a href="http://www.jflex.de/">JFlex</a> 1.4.3
- * on 9/25/14 10:26 PM from the specification file
- * <tt>/Users/orangy/Projects/dokka/src/Markdown/_MarkdownLexer.flex</tt>
+ * on 9/29/14 3:24 PM from the specification file
+ * <tt>/Users/orangy/Projects/dokka/gen/org/jetbrains/markdown/lexer/_MarkdownLexer.flex</tt>
  */
 public class _MarkdownLexer implements FlexLexer {
   /** initial size of the lookahead buffer */
@@ -35,9 +35,9 @@ public class _MarkdownLexer implements FlexLexer {
    */
   private static final String ZZ_CMAP_PACKED = 
     "\11\0\1\7\1\2\1\0\1\3\1\1\22\0\1\7\1\6\1\0"+
-    "\1\6\2\0\1\6\1\0\3\6\3\0\1\5\1\0\2\4\1\14"+
-    "\1\11\1\4\1\12\1\4\1\13\2\4\2\0\1\6\36\0\1\6"+
-    "\1\10\1\6\1\0\2\6\35\0\1\6\uff81\0";
+    "\2\6\1\0\1\6\1\0\3\6\3\0\1\5\1\0\2\4\1\14"+
+    "\1\11\1\4\1\12\1\4\1\13\2\4\1\6\1\0\1\6\36\0"+
+    "\1\6\1\10\1\6\1\0\2\6\32\0\1\6\1\0\2\6\uff81\0";
 
   /** 
    * Translates characters to character classes
@@ -245,7 +245,7 @@ public class _MarkdownLexer implements FlexLexer {
     char [] map = new char[0x10000];
     int i = 0;  /* index in packed string  */
     int j = 0;  /* index in unpacked array */
-    while (i < 74) {
+    while (i < 80) {
       int  count = packed.charAt(i++);
       char value = packed.charAt(i++);
       do map[j++] = value; while (--count > 0);