Fix syntax highlighting unicode support (#475)

* convert BasicParser to a template * fix syntax hightlighting with unicode characters
2025-02-22 01:01:20 +03:00
parent b61f594495
commit 8a8c1525fd
19 changed files with 644 additions and 509 deletions
@@ -13,9 +13,11 @@ TEST(lua_parsing, Tokenizer) {
    auto filename = "res:scripts/stdlib.lua";
    auto source = io::read_string(filename);
    try {
-        auto tokens = lua::tokenize(filename, source);
+        auto tokens = lua::tokenize(filename, util::str2wstr_utf8(source));
        for (const auto& token : tokens) {
-            std::cout << (int)token.tag << " " << util::quote(token.text) << std::endl;
+            std::cout << (int)token.tag << " "
+                      << util::quote(util::wstr2str_utf8(token.text))
+                      << std::endl;
        }
    } catch (const parsing_error& err) {
        std::cerr << err.errorLog() << std::endl;