add Lua code tokenizer

2024-12-04 16:26:55 +03:00
parent 3e6e897ce8
commit 28f49ac948
5 changed files with 256 additions and 0 deletions
@@ -0,0 +1,35 @@
+#pragma once
+
+#include <string>
+#include <vector>
+
+namespace lua {
+    struct Location {
+        int pos;
+        int lineStart;
+        int line;
+    };
+
+    enum class TokenTag {
+        KEYWORD, NAME, INTEGER, NUMBER, OPEN_BRACKET, CLOSE_BRACKET, STRING,
+        OPERATOR, COMMA, SEMICOLON, UNEXPECTED
+    };
+
+    struct Token {
+        TokenTag tag;
+        std::string text;
+        Location start;
+        Location end;
+
+        Token(TokenTag tag, std::string text, Location start, Location end)
+            : tag(tag),
+              text(std::move(text)),
+              start(std::move(start)),
+              end(std::move(end)) {
+        }
+    };
+
+    bool is_lua_keyword(std::string_view view);
+
+    std::vector<Token> tokenize(std::string_view file, std::string_view source);
+}