From 4260742d6e8a13da916f59f68a58c4c80097b595 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sun, 6 Apr 2025 22:22:35 +0300 Subject: [PATCH 01/10] add new Bytearray class --- res/modules/internal/bytearray.lua | 141 +++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 res/modules/internal/bytearray.lua diff --git a/res/modules/internal/bytearray.lua b/res/modules/internal/bytearray.lua new file mode 100644 index 00000000..95d3dc20 --- /dev/null +++ b/res/modules/internal/bytearray.lua @@ -0,0 +1,141 @@ +local MIN_CAPACITY = 8 +local _type = type +local FFI = ffi + +FFI.cdef[[ + void* malloc(size_t); + void free(void*); + typedef struct { + unsigned char* bytes; + int size; + int capacity; + } bytearray_t; +]] + +local malloc = FFI.C.malloc +local free = FFI.C.free + +local function grow_buffer(self, elems) + local new_capacity = math.ceil(self.capacity / 0.75 + elems) + local prev = self.bytes + self.bytes = malloc(new_capacity) + FFI.copy(self.bytes, prev, self.size) + self.capacity = new_capacity + free(prev) +end + +local function count_elements(b) + local elems = 1 + if _type(b) ~= "number" then + elems = #b + end + return elems +end + +local bytearray_methods = { + append=function(self, b) + local elems = count_elements(b) + if self.size + elems > self.capacity then + grow_buffer(self, elems) + end + if _type(b) == "number" then + self.bytes[self.size] = b + else + for i=1, #b do + self.bytes[self.size + i - 1] = b[i] + end + end + self.size = self.size + elems + end, + insert=function(self, index, b) + local elems = count_elements(b) + if self.size + elems >= self.capacity then + grow_buffer(self, elems) + end + if _type(b) == "number" then + self.bytes[index] = b + else + for i=1, #b do + self.bytes[index + i - 1] = b[i] + end + end + self.size = self.size + elems + end, + remove=function(self, index, elems) + if index <= 0 or index > self.size then + return + end + if elems == nil then + elems = 1 + end + if index + elems > self.size then + elems = self.size - index + 1 + end + for i=index, self.size - elems do + self.bytes[i] = self.bytes[i + elems] + end + self.size = self.size - elems + end, + clear=function(self) + self.size = 0 + end, + reserve=function(self, new_capacity) + if new_capacity <= self.capacity then + return + end + local prev = self.bytes + self.bytes = malloc(new_capacity) + FFI.copy(self.bytes, prev, self.size) + self.capacity = new_capacity + free(prev) + end, +} + +local bytearray_mt = { + __index = function(self, key) + if _type(key) == "string" then + return bytearray_methods[key] + end + if key <= 0 or key > self.size then + return + end + return self.bytes[key - 1] + end, + __newindex = function(self, key, value) + if key <= 0 or key > self.size then + return + end + self.bytes[key - 1] = value + end, + __tostring = function(self) + return string.format("FFIBytearray[%s]{...}", tonumber(self.size)) + end, + __len = function(self) + return tonumber(self.size) + end, + __gc = function(self) + free(self.bytes) + end +} + +local utf8tostring = utf8.tostring + +function utf8.tostring(bytes) + local type = _type(bytes) + if type == "cdata" then + return FFI.string(bytes.bytes, bytes.size) + else + return utf8tostring(bytes) + end +end + +local bytearray_type = FFI.metatype("bytearray_t", bytearray_mt) + +return function (n) + n = n or 0 + if n < MIN_CAPACITY then + return bytearray_type(malloc(MIN_CAPACITY), n, MIN_CAPACITY) + else + return bytearray_type(malloc(n), n, n) + end +end From 36c2971fcfe7d98ee298cf364c4b226cd249f4c0 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sun, 6 Apr 2025 22:23:39 +0300 Subject: [PATCH 02/10] add FFIBytearray --- res/scripts/stdlib.lua | 2 ++ res/scripts/stdmin.lua | 15 ++++++--------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/res/scripts/stdlib.lua b/res/scripts/stdlib.lua index d881f3fa..29b74e64 100644 --- a/res/scripts/stdlib.lua +++ b/res/scripts/stdlib.lua @@ -263,6 +263,8 @@ entities.get_all = function(uids) return stdcomp.get_all(uids) end end +FFIBytearray = require "core:internal/bytearray" +ffi = nil math.randomseed(time.uptime() * 1536227939) diff --git a/res/scripts/stdmin.lua b/res/scripts/stdmin.lua index 8a2af237..06c86832 100644 --- a/res/scripts/stdmin.lua +++ b/res/scripts/stdmin.lua @@ -1,6 +1,3 @@ -local _ffi = ffi -ffi = nil - -- Lua has no parallelizm, also _set_data does not call any lua functions so -- may be reused one global ffi buffer per lua_State local canvas_ffi_buffer @@ -8,14 +5,14 @@ local canvas_ffi_buffer_size = 0 function __vc_Canvas_set_data(self, data) if type(data) == "cdata" then - self:_set_data(tostring(_ffi.cast("uintptr_t", data))) + self:_set_data(tostring(ffi.cast("uintptr_t", data))) end local width = self.width local height = self.height local size = width * height * 4 if size > canvas_ffi_buffer_size then - canvas_ffi_buffer = _ffi.new( + canvas_ffi_buffer = ffi.new( string.format("unsigned char[%s]", size) ) canvas_ffi_buffer_size = size @@ -23,7 +20,7 @@ function __vc_Canvas_set_data(self, data) for i=0, size - 1 do canvas_ffi_buffer[i] = data[i + 1] end - self:_set_data(tostring(_ffi.cast("uintptr_t", canvas_ffi_buffer))) + self:_set_data(tostring(ffi.cast("uintptr_t", canvas_ffi_buffer))) end function crc32(bytes, chksum) @@ -31,14 +28,14 @@ function crc32(bytes, chksum) local length = #bytes if type(bytes) == "table" then - local buffer_len = _ffi.new('int[1]', length) - local buffer = _ffi.new( + local buffer_len = ffi.new('int[1]', length) + local buffer = ffi.new( string.format("char[%s]", length) ) for i=1, length do buffer[i - 1] = bytes[i] end - bytes = _ffi.string(buffer, buffer_len[0]) + bytes = ffi.string(buffer, buffer_len[0]) end return _crc32(bytes, chksum) end From 794aa5b9cd941c23da97b8db7516a4065884e794 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 7 Apr 2025 20:14:46 +0300 Subject: [PATCH 03/10] add simple data_buffer test --- dev/tests/data_buffer.lua | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 dev/tests/data_buffer.lua diff --git a/dev/tests/data_buffer.lua b/dev/tests/data_buffer.lua new file mode 100644 index 00000000..5c6f4cf1 --- /dev/null +++ b/dev/tests/data_buffer.lua @@ -0,0 +1,4 @@ +local data_buffer = require "core:data_buffer" +local buffer = data_buffer({}, 'LE', true) +buffer:put_int16(2025) +debug.print(buffer) From 303e861fbb4bfb9ec48954d4bf5bae3c6e7f25ed Mon Sep 17 00:00:00 2001 From: MihailRis Date: Mon, 7 Apr 2025 20:15:48 +0300 Subject: [PATCH 04/10] replace Bytearray with FFI implementation --- res/modules/internal/bytearray.lua | 46 +++- res/scripts/stdlib.lua | 4 +- src/logic/scripting/lua/libs/libbase64.cpp | 21 +- src/logic/scripting/lua/libs/libbjson.cpp | 19 +- src/logic/scripting/lua/libs/libbyteutil.cpp | 6 +- src/logic/scripting/lua/libs/libcore.cpp | 8 +- src/logic/scripting/lua/libs/libfile.cpp | 18 +- src/logic/scripting/lua/libs/libnetwork.cpp | 15 +- src/logic/scripting/lua/libs/libutf8.cpp | 19 +- src/logic/scripting/lua/libs/libworld.cpp | 19 +- src/logic/scripting/lua/lua_custom_types.hpp | 20 -- src/logic/scripting/lua/lua_engine.cpp | 1 - src/logic/scripting/lua/lua_overrides.cpp | 10 +- src/logic/scripting/lua/lua_util.cpp | 20 +- src/logic/scripting/lua/lua_util.hpp | 26 ++- .../lua/usertypes/lua_type_bytearray.cpp | 206 ------------------ 16 files changed, 125 insertions(+), 333 deletions(-) delete mode 100644 src/logic/scripting/lua/usertypes/lua_type_bytearray.cpp diff --git a/res/modules/internal/bytearray.lua b/res/modules/internal/bytearray.lua index 95d3dc20..04eddc94 100644 --- a/res/modules/internal/bytearray.lua +++ b/res/modules/internal/bytearray.lua @@ -118,20 +118,22 @@ local bytearray_mt = { end } -local utf8tostring = utf8.tostring - -function utf8.tostring(bytes) - local type = _type(bytes) - if type == "cdata" then - return FFI.string(bytes.bytes, bytes.size) - else - return utf8tostring(bytes) - end -end - local bytearray_type = FFI.metatype("bytearray_t", bytearray_mt) -return function (n) +local function FFIBytearray (n) + local t = type(n) + if t == "string" then + local buffer = malloc(#n) + FFI.copy(buffer, n, #n) + return bytearray_type(buffer, #n, #n) + elseif t == "table" then + local capacity = math.max(#n, MIN_CAPACITY) + local buffer = malloc(capacity) + for i=1,#n do + buffer[i - 1] = n[i] + end + return bytearray_type(buffer, #n, capacity) + end n = n or 0 if n < MIN_CAPACITY then return bytearray_type(malloc(MIN_CAPACITY), n, MIN_CAPACITY) @@ -139,3 +141,23 @@ return function (n) return bytearray_type(malloc(n), n, n) end end + +local function FFIBytearray_as_string(bytes) + local t = type(bytes) + if t == "cdata" then + return FFI.string(bytes.bytes, bytes.size) + elseif t == "table" then + local buffer = FFI.new("unsigned char[?]", #bytes) + for i=1, #bytes do + buffer[i - 1] = bytes[i] + end + return FFI.string(buffer, #bytes) + else + error("Bytearray expected, got "..type(bytes)) + end +end + +return { + FFIBytearray = FFIBytearray, + FFIBytearray_as_string = FFIBytearray_as_string +} diff --git a/res/scripts/stdlib.lua b/res/scripts/stdlib.lua index 29b74e64..558ba0f2 100644 --- a/res/scripts/stdlib.lua +++ b/res/scripts/stdlib.lua @@ -263,7 +263,9 @@ entities.get_all = function(uids) return stdcomp.get_all(uids) end end -FFIBytearray = require "core:internal/bytearray" +local bytearray = require "core:internal/bytearray" +Bytearray = bytearray.FFIBytearray +Bytearray_as_string = bytearray.FFIBytearray_as_string ffi = nil math.randomseed(time.uptime() * 1536227939) diff --git a/src/logic/scripting/lua/libs/libbase64.cpp b/src/logic/scripting/lua/libs/libbase64.cpp index 26f77eb6..5d50f1e8 100644 --- a/src/logic/scripting/lua/libs/libbase64.cpp +++ b/src/logic/scripting/lua/libs/libbase64.cpp @@ -16,14 +16,14 @@ static int l_encode(lua::State* L) { return lua::pushstring(L, util::base64_encode( reinterpret_cast(buffer.data()), buffer.size() )); - } else if (auto bytes = lua::touserdata(L, 1)) { - return lua::pushstring( - L, - util::base64_encode( - bytes->data().data(), - bytes->data().size() - ) + } else { + auto string = lua::bytearray_as_string(L, 1); + auto out = util::base64_encode( + reinterpret_cast(string.data()), + string.size() ); + lua::pop(L); + return lua::pushstring(L, std::move(out)); } throw std::runtime_error("array or ByteArray expected"); } @@ -36,13 +36,10 @@ static int l_decode(lua::State* L) { lua::pushinteger(L, buffer[i] & 0xFF); lua::rawseti(L, i+1); } + return 1; } else { - lua::newuserdata(L, buffer.size()); - auto bytearray = lua::touserdata(L, -1); - bytearray->data().reserve(buffer.size()); - std::memcpy(bytearray->data().data(), buffer.data(), buffer.size()); + return lua::create_bytearray(L, buffer.data(), buffer.size()); } - return 1; } const luaL_Reg base64lib[] = { diff --git a/src/logic/scripting/lua/libs/libbjson.cpp b/src/logic/scripting/lua/libs/libbjson.cpp index fa3dcfd7..f9acd077 100644 --- a/src/logic/scripting/lua/libs/libbjson.cpp +++ b/src/logic/scripting/lua/libs/libbjson.cpp @@ -9,9 +9,7 @@ static int l_tobytes(lua::State* L) { if (lua::gettop(L) >= 2) { compress = lua::toboolean(L, 2); } - return lua::newuserdata( - L, json::to_binary(value, compress) - ); + return lua::create_bytearray(L, json::to_binary(value, compress)); } static int l_frombytes(lua::State* L) { @@ -24,17 +22,18 @@ static int l_frombytes(lua::State* L) { lua::pop(L); } return lua::pushvalue(L, json::from_binary(buffer.data(), len)); - } else if (auto bytes = lua::touserdata(L, -1)) { - const auto& buffer = bytes->data(); - return lua::pushvalue( - L, json::from_binary(buffer.data(), buffer.size()) - ); } else { - throw std::runtime_error("table or Bytearray expected"); + auto string = lua::bytearray_as_string(L, 1); + auto out = json::from_binary( + reinterpret_cast(string.data()), string.size() + ); + lua::pop(L); + return lua::pushvalue(L, std::move(out)); } } const luaL_Reg bjsonlib[] = { {"tobytes", lua::wrap}, {"frombytes", lua::wrap}, - {NULL, NULL}}; + {NULL, NULL} +}; diff --git a/src/logic/scripting/lua/libs/libbyteutil.cpp b/src/logic/scripting/lua/libs/libbyteutil.cpp index d2c13175..76bd9d37 100644 --- a/src/logic/scripting/lua/libs/libbyteutil.cpp +++ b/src/logic/scripting/lua/libs/libbyteutil.cpp @@ -99,7 +99,7 @@ static int pack(lua::State* L, const char* format, bool usetable) { } return 1; } else { - return lua::newuserdata(L, builder.build()); + return lua::create_bytearray(L, builder.build()); } } @@ -130,8 +130,8 @@ static int count_elements(const char* format) { static int l_unpack(lua::State* L) { const char* format = lua::require_string(L, 1); int count = count_elements(format); - auto bytes = lua::require_bytearray(L, 2); - ByteReader reader(bytes); + auto bytes = lua::bytearray_as_string(L, 2); + ByteReader reader(reinterpret_cast(bytes.data()), bytes.size()); bool bigEndian = false; for (size_t i = 0; format[i]; i++) { diff --git a/src/logic/scripting/lua/libs/libcore.cpp b/src/logic/scripting/lua/libs/libcore.cpp index 568802b3..7ebdbc38 100644 --- a/src/logic/scripting/lua/libs/libcore.cpp +++ b/src/logic/scripting/lua/libs/libcore.cpp @@ -248,12 +248,14 @@ static int l_load_texture(lua::State* L) { } lua::pop(L); load_texture(buffer.data(), buffer.size(), lua::require_string(L, 2)); - } else if (auto bytes = lua::touserdata(L, 1)) { + } else { + auto string = lua::bytearray_as_string(L, 1); load_texture( - bytes->data().data(), - bytes->data().size(), + reinterpret_cast(string.data()), + string.size(), lua::require_string(L, 2) ); + lua::pop(L); } return 0; } diff --git a/src/logic/scripting/lua/libs/libfile.cpp b/src/logic/scripting/lua/libs/libfile.cpp index 51c63e82..fa2b2954 100644 --- a/src/logic/scripting/lua/libs/libfile.cpp +++ b/src/logic/scripting/lua/libs/libfile.cpp @@ -128,7 +128,7 @@ static int l_read_bytes(lua::State* L) { auto bytes = io::read_bytes(path); if (lua::gettop(L) < 2 || !lua::toboolean(L, 2)) { - lua::newuserdata(L, std::move(bytes)); + lua::create_bytearray(L, std::move(bytes)); } else { lua::createtable(L, length, 0); int newTable = lua::gettop(L); @@ -148,18 +148,12 @@ static int l_read_bytes(lua::State* L) { static int l_write_bytes(lua::State* L) { io::path path = get_writeable_path(L); - if (auto bytearray = lua::touserdata(L, 2)) { - auto& bytes = bytearray->data(); - return lua::pushboolean( - L, io::write_bytes(path, bytes.data(), bytes.size()) - ); - } - - std::vector bytes; - lua::read_bytes_from_table(L, 2, bytes); - return lua::pushboolean( - L, io::write_bytes(path, bytes.data(), bytes.size()) + auto string = lua::bytearray_as_string(L, 2); + bool res = io::write_bytes( + path, reinterpret_cast(string.data()), string.size() ); + lua::pop(L); + return lua::pushboolean(L, res); } static int l_list_all_res(lua::State* L, const std::string& path) { diff --git a/src/logic/scripting/lua/libs/libnetwork.cpp b/src/logic/scripting/lua/libs/libnetwork.cpp index ffad16ee..9a77b843 100644 --- a/src/logic/scripting/lua/libs/libnetwork.cpp +++ b/src/logic/scripting/lua/libs/libnetwork.cpp @@ -109,13 +109,13 @@ static int l_send(lua::State* L, network::Network& network) { } lua::pop(L); connection->send(buffer.data(), size); - } else if (auto bytes = lua::touserdata(L, 2)) { - connection->send( - reinterpret_cast(bytes->data().data()), bytes->data().size() - ); } else if (lua::isstring(L, 2)) { auto string = lua::tolstring(L, 2); connection->send(string.data(), string.length()); + } else { + auto string = lua::bytearray_as_string(L, 2); + connection->send(string.data(), string.length()); + lua::pop(L); } return 0; } @@ -140,13 +140,10 @@ static int l_recv(lua::State* L, network::Network& network) { lua::pushinteger(L, buffer[i] & 0xFF); lua::rawseti(L, i+1); } + return 1; } else { - lua::newuserdata(L, size); - auto bytearray = lua::touserdata(L, -1); - bytearray->data().reserve(size); - std::memcpy(bytearray->data().data(), buffer.data(), size); + return lua::create_bytearray(L, buffer.data(), size); } - return 1; } static int l_available(lua::State* L, network::Network& network) { diff --git a/src/logic/scripting/lua/libs/libutf8.cpp b/src/logic/scripting/lua/libs/libutf8.cpp index 4c1f484f..c0801b4d 100644 --- a/src/logic/scripting/lua/libs/libutf8.cpp +++ b/src/logic/scripting/lua/libs/libutf8.cpp @@ -14,13 +14,10 @@ static int l_tobytes(lua::State* L) { lua::pushinteger(L, string[i] & 0xFF); lua::rawseti(L, i+1); } + return 1; } else { - lua::newuserdata(L, string.length()); - auto bytearray = lua::touserdata(L, -1); - bytearray->data().reserve(string.length()); - std::memcpy(bytearray->data().data(), string.data(), string.length()); + return lua::create_bytearray(L, string.data(), string.length()); } - return 1; } static int l_tostring(lua::State* L) { @@ -35,16 +32,10 @@ static int l_tostring(lua::State* L) { } lua::pop(L); return lua::pushlstring(L, buffer.data(), size); - } else if (auto bytes = lua::touserdata(L, 1)) { - return lua::pushstring( - L, - std::string( - reinterpret_cast(bytes->data().data()), - bytes->data().size() - ) - ); + } else { + lua::bytearray_as_string(L, 1); + return 1; } - return 1; } static int l_length(lua::State* L) { diff --git a/src/logic/scripting/lua/libs/libworld.cpp b/src/logic/scripting/lua/libs/libworld.cpp index acf602c3..3e107f0e 100644 --- a/src/logic/scripting/lua/libs/libworld.cpp +++ b/src/logic/scripting/lua/libs/libworld.cpp @@ -144,7 +144,7 @@ static int l_get_chunk_data(lua::State* L) { } else { chunkData = compressed_chunks::encode(*chunk); } - return lua::newuserdata(L, std::move(chunkData)); + return lua::create_bytearray(L, std::move(chunkData)); } static void integrate_chunk_client(Chunk& chunk) { @@ -175,14 +175,17 @@ static int l_set_chunk_data(lua::State* L) { int x = static_cast(lua::tointeger(L, 1)); int z = static_cast(lua::tointeger(L, 2)); - auto buffer = lua::require_bytearray(L, 3); + auto buffer = lua::bytearray_as_string(L, 3); auto chunk = level->chunks->getChunk(x, z); if (chunk == nullptr) { return lua::pushboolean(L, false); } compressed_chunks::decode( - *chunk, buffer.data(), buffer.size(), *content->getIndices() + *chunk, + reinterpret_cast(buffer.data()), + buffer.size(), + *content->getIndices() ); if (controller->getChunksController()->lighting == nullptr) { return lua::pushboolean(L, true); @@ -198,10 +201,16 @@ static int l_save_chunk_data(lua::State* L) { int x = static_cast(lua::tointeger(L, 1)); int z = static_cast(lua::tointeger(L, 2)); - auto buffer = lua::require_bytearray(L, 3); + auto buffer = lua::bytearray_as_string(L, 3); compressed_chunks::save( - x, z, std::move(buffer), level->getWorld()->wfile->getRegions() + x, + z, + std::vector( + reinterpret_cast(buffer.data()), + reinterpret_cast(buffer.data()) + buffer.size() + ), + level->getWorld()->wfile->getRegions() ); return 0; } diff --git a/src/logic/scripting/lua/lua_custom_types.hpp b/src/logic/scripting/lua/lua_custom_types.hpp index e126ee72..1185ecbe 100644 --- a/src/logic/scripting/lua/lua_custom_types.hpp +++ b/src/logic/scripting/lua/lua_custom_types.hpp @@ -18,26 +18,6 @@ namespace lua { virtual const std::string& getTypeName() const = 0; }; - class LuaBytearray : public Userdata { - std::vector buffer; - public: - LuaBytearray(size_t capacity); - LuaBytearray(std::vector buffer); - LuaBytearray(const ubyte* data, size_t size); - virtual ~LuaBytearray(); - - const std::string& getTypeName() const override { - return TYPENAME; - } - inline std::vector& data() { - return buffer; - } - - static int createMetatable(lua::State*); - inline static std::string TYPENAME = "Bytearray"; - }; - static_assert(!std::is_abstract()); - class LuaHeightmap : public Userdata { std::shared_ptr map; std::unique_ptr noise; diff --git a/src/logic/scripting/lua/lua_engine.cpp b/src/logic/scripting/lua/lua_engine.cpp index 1ae35d39..ca335c49 100644 --- a/src/logic/scripting/lua/lua_engine.cpp +++ b/src/logic/scripting/lua/lua_engine.cpp @@ -117,7 +117,6 @@ void lua::init_state(State* L, StateType stateType) { initialize_libs_extends(L); - newusertype(L); newusertype(L); newusertype(L); newusertype(L); diff --git a/src/logic/scripting/lua/lua_overrides.cpp b/src/logic/scripting/lua/lua_overrides.cpp index 2040392c..3af2fa65 100644 --- a/src/logic/scripting/lua/lua_overrides.cpp +++ b/src/logic/scripting/lua/lua_overrides.cpp @@ -39,14 +39,16 @@ int l_crc32(lua::State* L) { string.length() ) ); - } else if (auto bytearray = lua::touserdata(L, 1)) { - auto& bytes = bytearray->data(); - return lua::pushinteger(L, crc32(value, bytes.data(), bytes.size())); } else if (lua::istable(L, 1)) { std::vector bytes; lua::read_bytes_from_table(L, 1, bytes); return lua::pushinteger(L, crc32(value, bytes.data(), bytes.size())); } else { - throw std::runtime_error("invalid argument 1"); + auto string = lua::bytearray_as_string(L, 1); + auto res = crc32( + value, reinterpret_cast(string.data()), string.size() + ); + lua::pop(L); + return lua::pushinteger(L, res); } } diff --git a/src/logic/scripting/lua/lua_util.cpp b/src/logic/scripting/lua/lua_util.cpp index 8cd77fb1..77937aef 100644 --- a/src/logic/scripting/lua/lua_util.cpp +++ b/src/logic/scripting/lua/lua_util.cpp @@ -61,7 +61,7 @@ int lua::pushvalue(State* L, const dv::value& value) { break; case value_type::bytes: { const auto& bytes = value.asBytes(); - newuserdata(L, bytes.data(), bytes.size()); + create_bytearray(L, bytes.data(), bytes.size()); break; } } @@ -128,18 +128,14 @@ dv::value lua::tovalue(State* L, int idx) { return map; } } - case LUA_TUSERDATA: { - if (auto bytes = touserdata(L, idx)) { - const auto& data = bytes->data(); - return std::make_shared(data.data(), data.size()); - } - [[fallthrough]]; - } - default: - throw std::runtime_error( - "lua type " + std::string(lua_typename(L, type)) + - " is not supported" + default: { + auto data = bytearray_as_string(L, idx); + auto bytes = std::make_shared( + reinterpret_cast(data.data()), data.size() ); + pop(L); + return bytes; + } } } diff --git a/src/logic/scripting/lua/lua_util.hpp b/src/logic/scripting/lua/lua_util.hpp index ab45edb1..ba0cd3d7 100644 --- a/src/logic/scripting/lua/lua_util.hpp +++ b/src/logic/scripting/lua/lua_util.hpp @@ -746,14 +746,22 @@ namespace lua { } } - inline std::vector require_bytearray(lua::State* L, int idx) { - if (auto* bytearray = lua::touserdata(L, idx)) { - return bytearray->data(); - } else if (lua::istable(L, idx)) { - std::vector bytes; - read_bytes_from_table(L, idx, bytes); - return bytes; - } - throw std::runtime_error("bytearray expected"); + inline int create_bytearray(lua::State* L, const void* bytes, size_t size) { + lua::requireglobal(L, "Bytearray"); + lua::pushlstring( + L, std::string_view(reinterpret_cast(bytes), size) + ); + return lua::call(L, 1, 1); + } + + inline int create_bytearray(lua::State* L, const std::vector& bytes) { + return create_bytearray(L, bytes.data(), bytes.size()); + } + + inline std::string_view bytearray_as_string(lua::State* L, int idx) { + lua::requireglobal(L, "Bytearray_as_string"); + lua::pushvalue(L, idx); + lua::call(L, 1, 1); + return lua::tolstring(L, -1); } } diff --git a/src/logic/scripting/lua/usertypes/lua_type_bytearray.cpp b/src/logic/scripting/lua/usertypes/lua_type_bytearray.cpp deleted file mode 100644 index 41b4ee59..00000000 --- a/src/logic/scripting/lua/usertypes/lua_type_bytearray.cpp +++ /dev/null @@ -1,206 +0,0 @@ -#include "../lua_custom_types.hpp" - -#include - -#include "util/listutil.hpp" -#include "../lua_util.hpp" - -using namespace lua; - -LuaBytearray::LuaBytearray(size_t capacity) : buffer(capacity) { - buffer.resize(capacity); -} - -LuaBytearray::LuaBytearray(std::vector buffer) : buffer(std::move(buffer)) { -} - -LuaBytearray::LuaBytearray(const ubyte* data, size_t size) : buffer(data, data + size) { -} - -LuaBytearray::~LuaBytearray() { -} - -static int l_append(lua::State* L) { - if (auto buffer = touserdata(L, 1)) { - if (lua::isnumber(L, 2)) { - auto value = tointeger(L, 2); - buffer->data().push_back(static_cast(value)); - } else if (lua::istable(L, 2)) { - lua::read_bytes_from_table(L, 2, buffer->data()); - } else if (auto extension = lua::touserdata(L, 2)) { - util::concat(buffer->data(), extension->data()); - } else { - throw std::runtime_error("integer/table/Bytearray expected"); - } - } - return 0; -} - -static int l_insert(lua::State* L) { - auto buffer = touserdata(L, 1); - if (buffer == nullptr) { - return 0; - } - auto& data = buffer->data(); - auto index = tointeger(L, 2) - 1; - if (static_cast(index) > data.size()) { - return 0; - } - if (lua::isnumber(L, 3)) { - auto value = tointeger(L, 3); - data.insert(data.begin() + index, static_cast(value)); - } else if (lua::istable(L, 3)) { - std::vector temp; - lua::read_bytes_from_table(L, 3, temp); - data.insert(data.begin() + index, temp.begin(), temp.end()); - } else if (auto extension = lua::touserdata(L, 3)) { - const std::vector& src = extension->data(); - data.insert(data.begin() + index, src.begin(), src.end()); - } else { - throw std::runtime_error("integer/table/Bytearray expected"); - } - return 0; -} - -static int l_remove(lua::State* L) { - auto buffer = touserdata(L, 1); - if (buffer == nullptr) { - return 0; - } - auto& data = buffer->data(); - auto index = tointeger(L, 2) - 1; - if (static_cast(index) > data.size()) { - return 0; - } - data.erase(data.begin() + index); - return 0; -} - -static std::unordered_map methods { - {"append", lua::wrap}, - {"insert", lua::wrap}, - {"remove", lua::wrap}, -}; - -static int l_meta_meta_call(lua::State* L) { - if (lua_istable(L, 2)) { - size_t len = objlen(L, 2); - std::vector buffer(len); - buffer.resize(len); - for (size_t i = 0; i < len; i++) { - rawgeti(L, i + 1); - buffer[i] = static_cast(tointeger(L, -1)); - pop(L); - } - return newuserdata(L, std::move(buffer)); - } - auto size = tointeger(L, 2); - if (size < 0) { - throw std::runtime_error("size can not be less than 0"); - } - return newuserdata(L, static_cast(size)); -} - -static int l_meta_index(lua::State* L) { - auto buffer = touserdata(L, 1); - if (buffer == nullptr) { - return 0; - } - auto& data = buffer->data(); - if (isstring(L, 2)) { - auto found = methods.find(tostring(L, 2)); - if (found != methods.end()) { - return pushcfunction(L, found->second); - } - } - auto index = tointeger(L, 2) - 1; - if (static_cast(index) > data.size()) { - return 0; - } - return pushinteger(L, data.at(index)); -} - -static int l_meta_newindex(lua::State* L) { - auto buffer = touserdata(L, 1); - if (buffer == nullptr) { - return 0; - } - auto& data = buffer->data(); - auto index = static_cast(tointeger(L, 2) - 1); - auto value = tointeger(L, 3); - if (index >= data.size()) { - if (index == data.size()) { - data.push_back(static_cast(value)); - } - return 0; - } - data[index] = static_cast(value); - return 0; -} - -static int l_meta_len(lua::State* L) { - if (auto buffer = touserdata(L, 1)) { - return pushinteger(L, buffer->data().size()); - } - return 0; -} - -static int l_meta_tostring(lua::State* L) { - auto buffer = touserdata(L, 1); - if (buffer == nullptr) { - return 0; - } - auto& data = buffer->data(); - if (data.size() > 512) { - return pushstring( - L, "bytearray[" + std::to_string(data.size()) + "]{...}" - ); - } else { - std::stringstream ss; - ss << "bytearray[" << std::to_string(data.size()) << "]{"; - for (size_t i = 0; i < data.size(); i++) { - if (i > 0) { - ss << " "; - } - ss << static_cast(data[i]); - } - ss << "}"; - return pushstring(L, ss.str()); - } -} - -static int l_meta_add(lua::State* L) { - auto bufferA = touserdata(L, 1); - auto bufferB = touserdata(L, 2); - if (bufferA == nullptr || bufferB == nullptr) { - return 0; - } - auto& dataA = bufferA->data(); - auto& dataB = bufferB->data(); - - std::vector ab; - ab.reserve(dataA.size() + dataB.size()); - ab.insert(ab.end(), dataA.begin(), dataA.end()); - ab.insert(ab.end(), dataB.begin(), dataB.end()); - return newuserdata(L, std::move(ab)); -} - -int LuaBytearray::createMetatable(lua::State* L) { - createtable(L, 0, 6); - pushcfunction(L, lua::wrap); - setfield(L, "__index"); - pushcfunction(L, lua::wrap); - setfield(L, "__newindex"); - pushcfunction(L, lua::wrap); - setfield(L, "__len"); - pushcfunction(L, lua::wrap); - setfield(L, "__tostring"); - pushcfunction(L, lua::wrap); - setfield(L, "__add"); - - createtable(L, 0, 1); - pushcfunction(L, lua::wrap); - setfield(L, "__call"); - setmetatable(L); - return 1; -} From d8feff56c0ed5598f80fc7d556251b2d41a599d8 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 12 Apr 2025 12:32:31 +0300 Subject: [PATCH 05/10] feat: Lua-style array append support in Bytearray --- dev/tests/bytearray.lua | 8 +++++++ res/modules/internal/bytearray.lua | 34 +++++++++++++++++------------- 2 files changed, 27 insertions(+), 15 deletions(-) create mode 100644 dev/tests/bytearray.lua diff --git a/dev/tests/bytearray.lua b/dev/tests/bytearray.lua new file mode 100644 index 00000000..9860b9fe --- /dev/null +++ b/dev/tests/bytearray.lua @@ -0,0 +1,8 @@ +local arr = Bytearray() +assert(#arr == 0) + +for i=1,10 do + arr[i] = 10 - i + assert(#arr == i) + assert(arr[i] == 10 - i) +end diff --git a/res/modules/internal/bytearray.lua b/res/modules/internal/bytearray.lua index 04eddc94..7178985e 100644 --- a/res/modules/internal/bytearray.lua +++ b/res/modules/internal/bytearray.lua @@ -32,21 +32,23 @@ local function count_elements(b) return elems end +local function append(self, b) + local elems = count_elements(b) + if self.size + elems > self.capacity then + grow_buffer(self, elems) + end + if _type(b) == "number" then + self.bytes[self.size] = b + else + for i=1, #b do + self.bytes[self.size + i - 1] = b[i] + end + end + self.size = self.size + elems +end + local bytearray_methods = { - append=function(self, b) - local elems = count_elements(b) - if self.size + elems > self.capacity then - grow_buffer(self, elems) - end - if _type(b) == "number" then - self.bytes[self.size] = b - else - for i=1, #b do - self.bytes[self.size + i - 1] = b[i] - end - end - self.size = self.size + elems - end, + append=append, insert=function(self, index, b) local elems = count_elements(b) if self.size + elems >= self.capacity then @@ -102,7 +104,9 @@ local bytearray_mt = { return self.bytes[key - 1] end, __newindex = function(self, key, value) - if key <= 0 or key > self.size then + if key == self.size + 1 then + return append(self, value) + elseif key <= 0 or key > self.size then return end self.bytes[key - 1] = value From 9169049530d2399d0bf05fd7e759c78463592ce1 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 12 Apr 2025 12:55:29 +0300 Subject: [PATCH 06/10] feat: ipairs(...) support in Bytearray --- dev/tests/bytearray.lua | 4 ++++ res/modules/internal/bytearray.lua | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/dev/tests/bytearray.lua b/dev/tests/bytearray.lua index 9860b9fe..e20d4318 100644 --- a/dev/tests/bytearray.lua +++ b/dev/tests/bytearray.lua @@ -6,3 +6,7 @@ for i=1,10 do assert(#arr == i) assert(arr[i] == 10 - i) end + +for i, v in ipairs(arr) do + assert(v == 10 - i) +end diff --git a/res/modules/internal/bytearray.lua b/res/modules/internal/bytearray.lua index 7178985e..03a2578d 100644 --- a/res/modules/internal/bytearray.lua +++ b/res/modules/internal/bytearray.lua @@ -119,6 +119,15 @@ local bytearray_mt = { end, __gc = function(self) free(self.bytes) + end, + __ipairs = function(self) + local i = 0 + return function() + i = i + 1 + if i <= self.size then + return i, self.bytes[i - 1] + end + end end } From fb07c86ea690d07475c509106c6083fbbba6db5f Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 12 Apr 2025 13:24:55 +0300 Subject: [PATCH 07/10] add static Lua-style functions: Bytearray.insert, Bytearray.remove, Bytearray.append --- dev/tests/bytearray.lua | 3 + res/modules/internal/bytearray.lua | 118 +++++++++++++++------------ res/scripts/stdlib.lua | 1 + src/logic/scripting/lua/lua_util.hpp | 2 +- 4 files changed, 73 insertions(+), 51 deletions(-) diff --git a/dev/tests/bytearray.lua b/dev/tests/bytearray.lua index e20d4318..79d8a89b 100644 --- a/dev/tests/bytearray.lua +++ b/dev/tests/bytearray.lua @@ -10,3 +10,6 @@ end for i, v in ipairs(arr) do assert(v == 10 - i) end + +Bytearray.remove(arr, 2) +assert(#arr == 9) diff --git a/res/modules/internal/bytearray.lua b/res/modules/internal/bytearray.lua index 03a2578d..fabfaee7 100644 --- a/res/modules/internal/bytearray.lua +++ b/res/modules/internal/bytearray.lua @@ -47,37 +47,50 @@ local function append(self, b) self.size = self.size + elems end +local function insert(self, index, b) + if index == nil then + index = self.size + 1 + end + if index <= 0 or index > self.size + 1 then + return + end + local elems = count_elements(b) + if self.size + elems > self.capacity then + grow_buffer(self, elems) + end + for i=self.size, index - 1, -1 do + self.bytes[i + elems] = self.bytes[i] + end + if _type(b) == "number" then + self.bytes[index - 1] = b + else + for i=1, #b do + self.bytes[index + i - 2] = b[i] + end + end + self.size = self.size + elems +end + +local function remove(self, index, elems) + if index <= 0 or index > self.size then + return + end + if elems == nil then + elems = 1 + end + if index + elems > self.size then + elems = self.size - index + 1 + end + for i=index, self.size - elems do + self.bytes[i] = self.bytes[i + elems] + end + self.size = self.size - elems +end + local bytearray_methods = { append=append, - insert=function(self, index, b) - local elems = count_elements(b) - if self.size + elems >= self.capacity then - grow_buffer(self, elems) - end - if _type(b) == "number" then - self.bytes[index] = b - else - for i=1, #b do - self.bytes[index + i - 1] = b[i] - end - end - self.size = self.size + elems - end, - remove=function(self, index, elems) - if index <= 0 or index > self.size then - return - end - if elems == nil then - elems = 1 - end - if index + elems > self.size then - elems = self.size - index + 1 - end - for i=index, self.size - elems do - self.bytes[i] = self.bytes[i + elems] - end - self.size = self.size - elems - end, + insert=insert, + remove=remove, clear=function(self) self.size = 0 end, @@ -133,27 +146,32 @@ local bytearray_mt = { local bytearray_type = FFI.metatype("bytearray_t", bytearray_mt) -local function FFIBytearray (n) - local t = type(n) - if t == "string" then - local buffer = malloc(#n) - FFI.copy(buffer, n, #n) - return bytearray_type(buffer, #n, #n) - elseif t == "table" then - local capacity = math.max(#n, MIN_CAPACITY) - local buffer = malloc(capacity) - for i=1,#n do - buffer[i - 1] = n[i] +local FFIBytearray = { + __call = function (n) + local t = type(n) + if t == "string" then + local buffer = malloc(#n) + FFI.copy(buffer, n, #n) + return bytearray_type(buffer, #n, #n) + elseif t == "table" then + local capacity = math.max(#n, MIN_CAPACITY) + local buffer = malloc(capacity) + for i=1,#n do + buffer[i - 1] = n[i] + end + return bytearray_type(buffer, #n, capacity) end - return bytearray_type(buffer, #n, capacity) - end - n = n or 0 - if n < MIN_CAPACITY then - return bytearray_type(malloc(MIN_CAPACITY), n, MIN_CAPACITY) - else - return bytearray_type(malloc(n), n, n) - end -end + n = n or 0 + if n < MIN_CAPACITY then + return bytearray_type(malloc(MIN_CAPACITY), n, MIN_CAPACITY) + else + return bytearray_type(malloc(n), n, n) + end + end, + append = append, + insert = insert, + remove = remove, +} local function FFIBytearray_as_string(bytes) local t = type(bytes) @@ -171,6 +189,6 @@ local function FFIBytearray_as_string(bytes) end return { - FFIBytearray = FFIBytearray, + FFIBytearray = setmetatable(FFIBytearray, FFIBytearray), FFIBytearray_as_string = FFIBytearray_as_string } diff --git a/res/scripts/stdlib.lua b/res/scripts/stdlib.lua index 558ba0f2..5fa9ca29 100644 --- a/res/scripts/stdlib.lua +++ b/res/scripts/stdlib.lua @@ -266,6 +266,7 @@ end local bytearray = require "core:internal/bytearray" Bytearray = bytearray.FFIBytearray Bytearray_as_string = bytearray.FFIBytearray_as_string +Bytearray_construct = Bytearray.__call ffi = nil math.randomseed(time.uptime() * 1536227939) diff --git a/src/logic/scripting/lua/lua_util.hpp b/src/logic/scripting/lua/lua_util.hpp index ba0cd3d7..7d545d49 100644 --- a/src/logic/scripting/lua/lua_util.hpp +++ b/src/logic/scripting/lua/lua_util.hpp @@ -747,7 +747,7 @@ namespace lua { } inline int create_bytearray(lua::State* L, const void* bytes, size_t size) { - lua::requireglobal(L, "Bytearray"); + lua::requireglobal(L, "Bytearray_construct"); lua::pushlstring( L, std::string_view(reinterpret_cast(bytes), size) ); From 1ade910fa1bcd374db0d7df2850c7e9d6eae73e7 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 12 Apr 2025 13:49:31 +0300 Subject: [PATCH 08/10] fix Bytearray.insert, Bytearray.remove --- dev/tests/bytearray.lua | 9 +++++++++ res/modules/internal/bytearray.lua | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dev/tests/bytearray.lua b/dev/tests/bytearray.lua index 79d8a89b..7b407cba 100644 --- a/dev/tests/bytearray.lua +++ b/dev/tests/bytearray.lua @@ -13,3 +13,12 @@ end Bytearray.remove(arr, 2) assert(#arr == 9) + +Bytearray.insert(arr, {5, 3, 6}) + +assert(#arr == 12) +Bytearray.insert(arr, 2, 8) +assert(#arr == 13) +for i=1,10 do + assert(arr[i] == 10 - i) +end diff --git a/res/modules/internal/bytearray.lua b/res/modules/internal/bytearray.lua index fabfaee7..e319751b 100644 --- a/res/modules/internal/bytearray.lua +++ b/res/modules/internal/bytearray.lua @@ -48,7 +48,8 @@ local function append(self, b) end local function insert(self, index, b) - if index == nil then + if b == nil then + b = index index = self.size + 1 end if index <= 0 or index > self.size + 1 then @@ -81,7 +82,7 @@ local function remove(self, index, elems) if index + elems > self.size then elems = self.size - index + 1 end - for i=index, self.size - elems do + for i=index - 1, self.size - elems - 1 do self.bytes[i] = self.bytes[i + elems] end self.size = self.size - elems From 0a49e4fa92880bc1682337517ab30a132a4e1a0a Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 12 Apr 2025 15:27:29 +0300 Subject: [PATCH 09/10] add Bytearray.trim, Bytearray.get_capacity --- dev/tests/bytearray.lua | 3 ++ res/modules/internal/bytearray.lua | 52 ++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/dev/tests/bytearray.lua b/dev/tests/bytearray.lua index 7b407cba..1a12bdeb 100644 --- a/dev/tests/bytearray.lua +++ b/dev/tests/bytearray.lua @@ -22,3 +22,6 @@ assert(#arr == 13) for i=1,10 do assert(arr[i] == 10 - i) end +print(#arr, arr:get_capacity()) +arr:trim() +assert(#arr == arr:get_capacity()) diff --git a/res/modules/internal/bytearray.lua b/res/modules/internal/bytearray.lua index e319751b..2481c4dd 100644 --- a/res/modules/internal/bytearray.lua +++ b/res/modules/internal/bytearray.lua @@ -24,6 +24,18 @@ local function grow_buffer(self, elems) free(prev) end +local function trim_buffer(self) + if self.size == self.capacity then + return + end + local size = self.size + local prev = self.bytes + self.bytes = malloc(size) + FFI.copy(self.bytes, prev, self.size) + self.capacity = size + free(prev) +end + local function count_elements(b) local elems = 1 if _type(b) ~= "number" then @@ -88,23 +100,33 @@ local function remove(self, index, elems) self.size = self.size - elems end +local function clear(self) + self.size = 0 +end + +local function reserve(self, new_capacity) + if new_capacity <= self.capacity then + return + end + local prev = self.bytes + self.bytes = malloc(new_capacity) + FFI.copy(self.bytes, prev, self.size) + self.capacity = new_capacity + free(prev) +end + +local function get_capacity(self) + return self.capacity +end + local bytearray_methods = { append=append, insert=insert, remove=remove, - clear=function(self) - self.size = 0 - end, - reserve=function(self, new_capacity) - if new_capacity <= self.capacity then - return - end - local prev = self.bytes - self.bytes = malloc(new_capacity) - FFI.copy(self.bytes, prev, self.size) - self.capacity = new_capacity - free(prev) - end, + trim=trim_buffer, + clear=clear, + reserve=reserve, + get_capacity=get_capacity, } local bytearray_mt = { @@ -172,6 +194,10 @@ local FFIBytearray = { append = append, insert = insert, remove = remove, + trim = trim_buffer, + clear = clear, + reserve = reserve, + get_capacity = get_capacity, } local function FFIBytearray_as_string(bytes) From da3f3d546fc84757791df6de801740fb72d941e8 Mon Sep 17 00:00:00 2001 From: MihailRis Date: Sat, 12 Apr 2025 15:32:26 +0300 Subject: [PATCH 10/10] cleanup --- res/modules/internal/bytearray.lua | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/res/modules/internal/bytearray.lua b/res/modules/internal/bytearray.lua index 2481c4dd..35bc6cc6 100644 --- a/res/modules/internal/bytearray.lua +++ b/res/modules/internal/bytearray.lua @@ -191,14 +191,8 @@ local FFIBytearray = { return bytearray_type(malloc(n), n, n) end end, - append = append, - insert = insert, - remove = remove, - trim = trim_buffer, - clear = clear, - reserve = reserve, - get_capacity = get_capacity, } +table.merge(FFIBytearray, bytearray_methods) local function FFIBytearray_as_string(bytes) local t = type(bytes)