refactored ast, started lexer

include/lexer/Cursor.h

@@ -0,0 +1,2 @@
+#pragma once
+

include/lexer/Lexer.h

@@ -0,0 +1,15 @@
+#pragma once
+#include <string>
+
+namespace lexer {
+    struct State {
+        std::string sourceText;
+        char* currentPosition;
+        int line;
+        int column;
+    };
+
+    struct Lexer {
+
+    };
+}

include/lexer/Token.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string>
+#include "lexer/TokenKind.h"
+
+namespace lexer {
+    struct Token {
+        TokenKind kind;
+        std::string lexeme;
+
+        int line;
+        int column;
+    };
+}

include/lexer/TokenKind.h

@@ -0,0 +1,13 @@
+#pragma once
+
+namespace lexer {
+    enum class TokenKind {
+        Text,        // text
+        NewLine,     // new line
+        EOFile,      // end of file
+
+        HeaderMark,  // header mark ("#")
+        ListMark,    // list mark 
+        CodeFence    // code fence
+    };
+}