add /coders/compression module

src/coders/compression.cpp

@@ -0,0 +1,87 @@
+#include "compression.hpp"
+
+#include <cstring>
+#include <stdexcept>
+
+#include "rle.hpp"
+#include "gzip.hpp"
+#include "util/BufferPool.hpp"
+
+using namespace compression;
+
+static util::BufferPool<ubyte> buffer_pools[] {
+    {255},
+    {UINT16_MAX},
+    {UINT16_MAX * 8},
+};
+
+static std::shared_ptr<ubyte[]> get_buffer(size_t minSize) {
+    for (auto& pool : buffer_pools) {
+        if (minSize <= pool.getBufferSize()) {
+            return pool.get();
+        }
+    }
+    return nullptr;
+}
+
+std::unique_ptr<ubyte[]> compression::compress(
+    const ubyte* src, size_t srclen, size_t& len, Method method
+) {
+    switch (method) {
+        case Method::NONE:
+            throw std::invalid_argument("compression method is NONE");
+        case Method::EXTRLE8: {
+            // max extrle out size is srcLen * 2
+            auto buffer = get_buffer(srclen * 2);
+            auto bytes = buffer.get();
+            std::unique_ptr<ubyte[]> uptr;
+            if (bytes == nullptr) {
+                uptr = std::make_unique<ubyte[]>(srclen * 2);
+                bytes = uptr.get();
+            }
+            len = extrle::encode(src, srclen, bytes);
+            if (uptr) {
+                return uptr;
+            }
+            auto data = std::make_unique<ubyte[]>(len);
+            std::memcpy(data.get(), bytes, len);
+            return data;
+        }
+        case Method::GZIP: {
+            auto buffer = gzip::compress(src, srclen);
+            auto data = std::make_unique<ubyte[]>(buffer.size());
+            std::memcpy(data.get(), buffer.data(), buffer.size());
+            len = buffer.size();
+            return data;
+        }
+        default:
+            throw std::runtime_error("not implemented");
+    }
+}
+
+std::unique_ptr<ubyte[]> compression::decompress(
+    const ubyte* src, size_t srclen, size_t dstlen, Method method
+) {
+    switch (method) {
+        case Method::NONE:
+            throw std::invalid_argument("compression method is NONE");
+        case Method::EXTRLE8: {
+            auto decompressed = std::make_unique<ubyte[]>(dstlen);
+            extrle::decode(src, srclen, decompressed.get());
+            return decompressed;
+        }
+        case Method::GZIP: {
+            auto buffer = gzip::decompress(src, srclen);
+            if (buffer.size() != dstlen) {
+                throw std::runtime_error(
+                    "expected decompressed size " + std::to_string(dstlen) +
+                    " got " + std::to_string(buffer.size()));
+            }
+            auto decompressed = std::make_unique<ubyte[]>(buffer.size());
+            std::memcpy(decompressed.get(), buffer.data(), buffer.size());
+            return decompressed;
+        }
+        default:
+            throw std::runtime_error("not implemented");
+    }
+}

src/coders/compression.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <memory>
+
+#include "typedefs.hpp"
+
+namespace compression {
+    enum class Method {
+        NONE, EXTRLE8, GZIP
+    };
+
+    /// @brief Compress buffer
+    /// @param src source buffer
+    /// @param srclen length of the source buffer
+    /// @param len (out argument) length of result buffer
+    /// @param method compression method
+    /// @return compressed bytes array
+    /// @throws std::invalid_argument if compression method is NONE
+    std::unique_ptr<ubyte[]> compress(
+        const ubyte* src, size_t srclen, size_t& len, Method method
+    );
+
+    /// @brief Decompress buffer
+    /// @param src compressed buffer
+    /// @param srclen length of compressed buffer
+    /// @param dstlen max expected length of source buffer
+    /// @return decompressed bytes array
+    std::unique_ptr<ubyte[]> decompress(
+        const ubyte* src, size_t srclen, size_t dstlen, Method method);
+}

src/files/RegionsLayer.cpp

@@ -11,7 +11,7 @@ static fs::path get_region_filename(int x, int z) {
 }
 
 /// @brief Read missing chunks data (null pointers) from region file
-static void fetchChunks(WorldRegion* region, int x, int z, regfile* file) {
+static void fetch_chunks(WorldRegion* region, int x, int z, regfile* file) {
     auto* chunks = region->getChunks();
     uint32_t* sizes = region->getSizes();
 
@@ -174,7 +174,7 @@ void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) {
 
     glm::ivec2 regcoord(x, z);
     if (auto regfile = getRegFile(regcoord, false)) {
-        fetchChunks(entry, x, z, regfile.get());
+        fetch_chunks(entry, x, z, regfile.get());
 
         std::lock_guard lock(regFilesMutex);
         regfile.reset();

src/files/WorldRegions.cpp

@@ -5,7 +5,6 @@
 #include <vector>
 
 #include "coders/byte_utils.hpp"
-#include "coders/rle.hpp"
 #include "data/dynamic.hpp"
 #include "items/Inventory.hpp"
 #include "maths/voxmaths.hpp"
@@ -55,8 +54,14 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) {
     for (size_t i = 0; i < sizeof(layers) / sizeof(RegionsLayer); i++) {
         layers[i].layer = i;
     }
-    layers[REGION_LAYER_VOXELS].folder = directory / fs::path("regions");
+    auto& voxels = layers[REGION_LAYER_VOXELS];
-    layers[REGION_LAYER_LIGHTS].folder = directory / fs::path("lights");
+    voxels.folder = directory / fs::path("regions");
+    voxels.compression = compression::Method::EXTRLE8;
+
+    auto& lights = layers[REGION_LAYER_LIGHTS];
+    lights.folder = directory / fs::path("lights");
+    lights.compression = compression::Method::EXTRLE8;
+
     layers[REGION_LAYER_INVENTORIES].folder =
         directory / fs::path("inventories");
     layers[REGION_LAYER_ENTITIES].folder = directory / fs::path("entities");
@@ -64,28 +69,6 @@ WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) {
 
 WorldRegions::~WorldRegions() = default;
 
-std::unique_ptr<ubyte[]> WorldRegions::compress(
-    const ubyte* src, size_t srclen, size_t& len
-) {
-    auto buffer = bufferPool.get();
-    auto bytes = buffer.get();
-
-    len = extrle::encode(src, srclen, bytes);
-    auto data = std::make_unique<ubyte[]>(len);
-    for (size_t i = 0; i < len; i++) {
-        data[i] = bytes[i];
-    }
-    return data;
-}
-
-std::unique_ptr<ubyte[]> WorldRegions::decompress(
-    const ubyte* src, size_t srclen, size_t dstlen
-) {
-    auto decompressed = std::make_unique<ubyte[]>(dstlen);
-    extrle::decode(src, srclen, decompressed.get());
-    return decompressed;
-}
-
 void RegionsLayer::writeAll() {
     for (auto& it : regions) {
         WorldRegion* region = it.second.get();
@@ -100,21 +83,19 @@ void RegionsLayer::writeAll() {
 void WorldRegions::put(
     int x,
     int z,
-    int layer,
+    int layerid,
     std::unique_ptr<ubyte[]> data,
-    size_t size,
+    size_t size
-    bool rle
 ) {
-    if (rle) {
+    auto& layer = layers[layerid];
-        size_t compressedSize;
+    if (layer.compression != compression::Method::NONE) {
-        auto compressed = compress(data.get(), size, compressedSize);
+        data = compression::compress(
-        put(x, z, layer, std::move(compressed), compressedSize, false);
+            data.get(), size, size, layer.compression);
-        return;
     }
     int regionX, regionZ, localX, localZ;
     calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
 
-    WorldRegion* region = layers[layer].getOrCreateRegion(regionX, regionZ);
+    WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ);
     region->setUnsaved(true);
     region->put(localX, localZ, std::move(data), size);
 }
@@ -139,7 +120,6 @@ static std::unique_ptr<ubyte[]> write_inventories(
     return data;
 }
 
-/// @brief Store chunk data (voxels and lights) in region (existing or new)
 void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
     assert(chunk != nullptr);
     if (!chunk->flags.lighted) {
@@ -157,8 +137,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
         chunk->z,
         REGION_LAYER_VOXELS,
         chunk->encode(),
-        CHUNK_DATA_LEN,
+        CHUNK_DATA_LEN);
-        true);
 
     // Writing lights cache
     if (doWriteLights && chunk->flags.lighted) {
@@ -166,8 +145,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
             chunk->z,
             REGION_LAYER_LIGHTS,
             chunk->lightmap.encode(),
-            LIGHTMAP_DATA_LEN,
+            LIGHTMAP_DATA_LEN);
-            true);
     }
     // Writing block inventories
     if (!chunk->inventories.empty()) {
@@ -177,8 +155,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
             chunk->z,
             REGION_LAYER_INVENTORIES,
             std::move(data),
-            datasize,
+            datasize);
-            false);
     }
     // Writing entities
     if (!entitiesData.empty()) {
@@ -190,29 +167,30 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
             chunk->z,
             REGION_LAYER_ENTITIES,
             std::move(data),
-            entitiesData.size(),
+            entitiesData.size());
-            false);
     }
 }
 
 std::unique_ptr<ubyte[]> WorldRegions::getVoxels(int x, int z) {
     uint32_t size;
-    auto* data = layers[REGION_LAYER_VOXELS].getData(x, z, size);
+    auto& layer = layers[REGION_LAYER_VOXELS];
+    auto* data = layer.getData(x, z, size);
     if (data == nullptr) {
         return nullptr;
     }
-    return decompress(data, size, CHUNK_DATA_LEN);
+    return compression::decompress(data, size, CHUNK_DATA_LEN, layer.compression);
 }
 
-/// @brief Get cached lights for chunk at x,z
-/// @return lights data or nullptr
 std::unique_ptr<light_t[]> WorldRegions::getLights(int x, int z) {
     uint32_t size;
-    auto* bytes = layers[REGION_LAYER_LIGHTS].getData(x, z, size);
+    auto& layer = layers[REGION_LAYER_LIGHTS];
+    auto* bytes = layer.getData(x, z, size);
     if (bytes == nullptr) {
         return nullptr;
     }
-    auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN);
+    auto data = compression::decompress(
+        bytes, size, LIGHTMAP_DATA_LEN, layer.compression
+    );
     return Lightmap::decode(data.get());
 }
 
@@ -254,10 +232,11 @@ dynamic::Map_sptr WorldRegions::fetchEntities(int x, int z) {
 }
 
 void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
-    if (layers[REGION_LAYER_VOXELS].getRegion(x, z)) {
+    auto& layer = layers[REGION_LAYER_VOXELS];
+    if (layer.getRegion(x, z)) {
         throw std::runtime_error("not implemented for in-memory regions");
     }
-    auto regfile = layers[REGION_LAYER_VOXELS].getRegFile({x, z});
+    auto regfile = layer.getRegFile({x, z});
     if (regfile == nullptr) {
         throw std::runtime_error("could not open region file");
     }
@@ -271,14 +250,15 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
             if (data == nullptr) {
                 continue;
             }
-            data = decompress(data.get(), length, CHUNK_DATA_LEN);
+            data = compression::decompress(
+                data.get(), length, CHUNK_DATA_LEN, layer.compression
+            );
             if (func(data.get())) {
                 put(gx,
                     gz,
                     REGION_LAYER_VOXELS,
                     std::move(data),
-                    CHUNK_DATA_LEN,
+                    CHUNK_DATA_LEN);
-                    true);
             }
         }
     }

src/files/WorldRegions.hpp

@@ -8,11 +8,12 @@
 #include <mutex>
 #include <unordered_map>
 
-#include "data/dynamic_fwd.hpp"
 #include "typedefs.hpp"
+#include "data/dynamic_fwd.hpp"
 #include "util/BufferPool.hpp"
 #include "voxels/Chunk.hpp"
 #include "maths/voxmaths.hpp"
+#include "coders/compression.hpp"
 #include "files.hpp"
 
 #define GLM_ENABLE_EXPERIMENTAL
@@ -126,6 +127,8 @@ struct RegionsLayer {
     /// @brief Regions layer folder
     fs::path folder;
 
+    compression::Method compression = compression::Method::NONE;
+
     /// @brief In-memory regions data
     regionsmap regions;
 
@@ -139,7 +142,7 @@ struct RegionsLayer {
     std::mutex regFilesMutex;
     std::condition_variable regFilesCv;
 
-    regfile_ptr getRegFile(glm::ivec2 coord, bool create = true);
+    [[nodiscard]] regfile_ptr getRegFile(glm::ivec2 coord, bool create = true);
     [[nodiscard]] regfile_ptr useRegFile(glm::ivec2 coord);
     regfile_ptr createRegFile(glm::ivec2 coord);
     void closeRegFile(glm::ivec2 coord);
@@ -178,26 +181,6 @@ class WorldRegions {
     fs::path directory;
 
     RegionsLayer layers[4] {};
-    util::BufferPool<ubyte> bufferPool {
-        std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2};
-
-    /// @brief Compress buffer with extrle
-    /// @param src source buffer
-    /// @param srclen length of the source buffer
-    /// @param len (out argument) length of result buffer
-    /// @return compressed bytes array
-    std::unique_ptr<ubyte[]> compress(
-        const ubyte* src, size_t srclen, size_t& len
-    );
-
-    /// @brief Decompress buffer with extrle
-    /// @param src compressed buffer
-    /// @param srclen length of compressed buffer
-    /// @param dstlen max expected length of source buffer
-    /// @return decompressed bytes array
-    std::unique_ptr<ubyte[]> decompress(
-        const ubyte* src, size_t srclen, size_t dstlen
-    );
 public:
     bool generatorTestMode = false;
     bool doWriteLights = true;
@@ -215,19 +198,22 @@ public:
     /// @param layer regions layer
     /// @param data target data
     /// @param size data size
-    /// @param rle compress with ext-RLE
     void put(
         int x,
         int z,
         int layer,
         std::unique_ptr<ubyte[]> data,
-        size_t size,
+        size_t size
-        bool rle
     );
 
     std::unique_ptr<ubyte[]> getVoxels(int x, int z);
+
+    /// @brief Get cached lights for chunk at x,z
+    /// @return lights data or nullptr
     std::unique_ptr<light_t[]> getLights(int x, int z);
+    
     chunk_inventories_map fetchInventories(int x, int z);
+    
     dynamic::Map_sptr fetchEntities(int x, int z);
 
     void processRegionVoxels(int x, int z, const regionproc& func);

src/util/BufferPool.hpp

@@ -35,5 +35,9 @@ namespace util {
                 freeBuffers.push(ptr);
             });
         }
+
+        size_t getBufferSize() const {
+            return bufferSize;
+        }
     };
 }

src/voxels/ChunksStorage.cpp

@@ -59,8 +59,7 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
 
     auto chunk = std::make_shared<Chunk>(x, z);
     store(chunk);
-    auto data = regions.getVoxels(chunk->x, chunk->z);
+    if (auto data = regions.getVoxels(chunk->x, chunk->z)) {
-    if (data) {
         chunk->decode(data.get());
 
         auto invs = regions.fetchInventories(chunk->x, chunk->z);
@@ -77,9 +76,7 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
         }
         verifyLoadedChunk(level->content->getIndices(), chunk.get());
     }
-
+    if (auto lights = regions.getLights(chunk->x, chunk->z)) {
-    auto lights = regions.getLights(chunk->x, chunk->z);
-    if (lights) {
         chunk->lightmap.set(lights.get());
         chunk->flags.loadedLights = true;
     }