update region file format 2 to 3 (WIP)

src/files/RegionsLayer.cpp

@@ -135,6 +135,10 @@ WorldRegion* RegionsLayer::getRegion(int x, int z) {
     return found->second.get();
 }
 
+fs::path RegionsLayer::getRegionFilePath(int x, int z) const {
+    return folder / get_region_filename(x, z);
+}
+
 WorldRegion* RegionsLayer::getOrCreateRegion(int x, int z) {
     if (auto region = getRegion(x, z)) {
         return region;

src/files/WorldConverter.cpp

@@ -6,6 +6,7 @@
 #include <utility>
 
 #include "content/ContentReport.hpp"
+#include "files/compatibility.hpp"
 #include "data/dynamic.hpp"
 #include "debug/Logger.hpp"
 #include "files/files.hpp"
@@ -48,7 +49,7 @@ void WorldConverter::addRegionsTasks(
             logger.error() << "could not parse region name " << name;
             continue;
         }
-        tasks.push(ConvertTask {taskType, file.path(), x, z});
+        tasks.push(ConvertTask {taskType, file.path(), x, z, layerid});
     }
 }
 
@@ -58,11 +59,7 @@ void WorldConverter::createUpgradeTasks() {
         if (issue.issueType != ContentIssueType::REGION_FORMAT_UPDATE) {
             continue;
         }
-        if (issue.regionLayer == REGION_LAYER_VOXELS) {
-            addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_VOXELS);
-        } else {
-            addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_REGION);
-        }
+        addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_REGION);
     }
 }
 
@@ -159,12 +156,13 @@ std::shared_ptr<Task> WorldConverter::startTask(
     return pool;
 }
 
-void WorldConverter::upgradeRegion(const fs::path& file, int x, int z) const {
-    throw std::runtime_error("unsupported region format");
-}
-
-void WorldConverter::upgradeVoxels(const fs::path& file, int x, int z) const {
-    throw std::runtime_error("unsupported region format");
+void WorldConverter::upgradeRegion(
+    const fs::path& file, int x, int z, RegionLayerIndex layer
+) const {
+    auto path = wfile->getRegions().getRegionFilePath(layer, x, z);
+    auto bytes = files::read_bytes_buffer(path);
+    auto buffer = compatibility::convertRegion2to3(bytes, layer);
+    files::write_bytes(path, buffer.data(), buffer.size());
 }
 
 void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const {
@@ -195,11 +193,7 @@ void WorldConverter::convert(const ConvertTask& task) const {
 
     switch (task.type) {
         case ConvertTaskType::UPGRADE_REGION:
-            upgradeRegion(task.file, task.x, task.z);
-            break;
-        case ConvertTaskType::UPGRADE_VOXELS:
-            upgradeRegion(task.file, task.x, task.z);
-            upgradeVoxels(task.file, task.x, task.z);
+            upgradeRegion(task.file, task.x, task.z, task.layer);
             break;
         case ConvertTaskType::VOXELS:
             convertVoxels(task.file, task.x, task.z);

src/files/WorldConverter.hpp

@@ -24,8 +24,6 @@ enum class ConvertTaskType {
     PLAYER,
     /// @brief refresh region file version
     UPGRADE_REGION,
-    /// @brief rewrite voxels region file to new format
-    UPGRADE_VOXELS,
 };
 
 struct ConvertTask {
@@ -34,6 +32,7 @@ struct ConvertTask {
 
     /// @brief region coords
     int x, z;
+    RegionLayerIndex layer;
 };
 
 class WorldConverter : public Task {
@@ -45,8 +44,8 @@ class WorldConverter : public Task {
     uint tasksDone = 0;
     bool upgradeMode;
 
-    void upgradeRegion(const fs::path& file, int x, int z) const;
-    void upgradeVoxels(const fs::path& file, int x, int z) const;
+    void upgradeRegion(
+        const fs::path& file, int x, int z, RegionLayerIndex layer) const;
     void convertPlayer(const fs::path& file) const;
     void convertVoxels(const fs::path& file, int x, int z) const;
     void convertInventories(const fs::path& file, int x, int z) const;

src/files/WorldRegions.cpp

@@ -290,6 +290,10 @@ const fs::path& WorldRegions::getRegionsFolder(RegionLayerIndex layerid) const {
     return layers[layerid].folder;
 }
 
+fs::path WorldRegions::getRegionFilePath(RegionLayerIndex layerid, int x, int z) const {
+    return layers[layerid].getRegionFilePath(x, z);
+}
+
 void WorldRegions::writeAll() {
     for (auto& layer : layers) {
         fs::create_directories(layer.folder);

src/files/WorldRegions.hpp

@@ -147,6 +147,8 @@ struct RegionsLayer {
     WorldRegion* getRegion(int x, int z);
     WorldRegion* getOrCreateRegion(int x, int z);
 
+    fs::path getRegionFilePath(int x, int z) const;
+
     /// @brief Get chunk data. Read from file if not loaded yet.
     /// @param x chunk x coord
     /// @param z chunk z coord
@@ -237,6 +239,8 @@ public:
     /// @return directory path
     const fs::path& getRegionsFolder(RegionLayerIndex layerid) const;
 
+    fs::path getRegionFilePath(RegionLayerIndex layerid, int x, int z) const;
+
     /// @brief Write all region layers
     void writeAll();
 

src/files/compatibility.cpp

@@ -0,0 +1,109 @@
+#include "compatibility.hpp"
+
+#include <stdexcept>
+
+#include "constants.hpp"
+#include "voxels/voxel.hpp"
+#include "coders/compression.hpp"
+#include "coders/byte_utils.hpp"
+#include "lighting/Lightmap.hpp"
+#include "util/data_io.hpp"
+
+static inline size_t VOXELS_DATA_SIZE_V1 = CHUNK_VOL * 4;
+static inline size_t VOXELS_DATA_SIZE_V2 = CHUNK_VOL * 4;
+
+static util::Buffer<ubyte> convert_voxels_1to2(const ubyte* buffer, uint32_t size) {
+    auto data = compression::decompress(
+        buffer, size, VOXELS_DATA_SIZE_V1, compression::Method::EXTRLE8);
+
+    util::Buffer<ubyte> dstBuffer(VOXELS_DATA_SIZE_V2);
+    auto dst = reinterpret_cast<uint16_t*>(dstBuffer.data());
+
+    for (size_t i = 0; i < CHUNK_VOL; i++) {
+        ubyte bid1 = data[i];
+        ubyte bid2 = data[CHUNK_VOL + i];
+
+        ubyte bst1 = data[CHUNK_VOL * 2 + i];
+        ubyte bst2 = data[CHUNK_VOL * 3 + i];
+
+        dst[i] =
+            (static_cast<blockid_t>(bid1) << 8) | static_cast<blockid_t>(bid2);
+        dst[CHUNK_VOL + i] = (
+            (static_cast<blockstate_t>(bst1) << 8) |
+            static_cast<blockstate_t>(bst2)
+        );
+    }
+    size_t outLen;
+    auto compressed = compression::compress(
+        data.get(), VOXELS_DATA_SIZE_V2, outLen, compression::Method::EXTRLE16);
+    return util::Buffer<ubyte>(std::move(compressed), outLen);
+}
+
+util::Buffer<ubyte> compatibility::convertRegion2to3(
+    const util::Buffer<ubyte>& src, RegionLayerIndex layer
+) {
+    const size_t REGION_CHUNKS = 1024;
+    const size_t HEADER_SIZE = 10;
+    const size_t OFFSET_TABLE_SIZE = REGION_CHUNKS * sizeof(uint32_t);
+    const ubyte COMPRESS_NONE = 0;
+    const ubyte COMPRESS_EXTRLE8 = 1;
+    const ubyte COMPRESS_EXTRLE16 = 2;
+
+    const ubyte* const ptr = src.data();
+
+    ByteBuilder builder;
+    builder.putCStr(".VOXREG");
+    builder.put(3);
+    switch (layer) {
+        case REGION_LAYER_VOXELS: builder.put(COMPRESS_EXTRLE16); break;
+        case REGION_LAYER_LIGHTS: builder.put(COMPRESS_EXTRLE8); break;
+        default: builder.put(COMPRESS_NONE); break;
+    }
+
+    uint32_t offsets[REGION_CHUNKS] {};
+    size_t chunkIndex = 0;
+
+    auto tablePtr = reinterpret_cast<const uint32_t*>(
+        ptr + src.size() - OFFSET_TABLE_SIZE
+    );
+
+    for (size_t i = 0; i < REGION_CHUNKS; i++) {
+        uint32_t srcOffset = dataio::be2h(tablePtr[i]);
+        if (srcOffset == 0) {
+            continue;
+        }
+        uint32_t size = *reinterpret_cast<const uint32_t*>(ptr + srcOffset);
+        size = dataio::be2h(size);
+
+        const ubyte* data = ptr + srcOffset + sizeof(uint32_t);
+        offsets[i] = builder.size();
+
+        switch (layer) {
+            case REGION_LAYER_VOXELS: {
+                auto dstdata = convert_voxels_1to2(data, size);
+                builder.putInt32(dstdata.size());
+                builder.putInt32(VOXELS_DATA_SIZE_V2);
+                builder.put(dstdata.data(), dstdata.size());
+                break;
+            }
+            case REGION_LAYER_LIGHTS:
+                builder.putInt32(size);
+                builder.putInt32(LIGHTMAP_DATA_LEN);
+                builder.put(data, size);
+                break;
+            case REGION_LAYER_ENTITIES:
+            case REGION_LAYER_INVENTORIES: {
+                builder.putInt32(size);
+                builder.putInt32(size);
+                builder.put(data, size);
+                break;
+            case REGION_LAYERS_COUNT: 
+                throw std::invalid_argument("invalid enum");
+            }
+        }
+    }
+    for (size_t i = 0; i < REGION_CHUNKS; i++) {
+        builder.putInt32(offsets[i]);
+    }
+    return util::Buffer<ubyte>(builder.build().data(), builder.size());
+}

src/files/compatibility.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "typedefs.hpp"
+#include "util/Buffer.hpp"
+#include "files/world_regions_fwd.hpp"
+
+namespace compatibility {
+    /// @brief Convert region file from version 2 to 3
+    /// @see /doc/specs/region_file_spec.md
+    /// @param src region file source content
+    /// @return new region file content
+    util::Buffer<ubyte> convertRegion2to3(
+        const util::Buffer<ubyte>& src, RegionLayerIndex layer);
+}

src/files/files.cpp

@@ -66,6 +66,12 @@ bool files::read(const fs::path& filename, char* data, size_t size) {
     return true;
 }
 
+util::Buffer<ubyte> files::read_bytes_buffer(const fs::path& path) {
+    size_t size;
+    auto bytes = files::read_bytes(path, size);
+    return util::Buffer<ubyte>(std::move(bytes), size);
+}
+
 std::unique_ptr<ubyte[]> files::read_bytes(
     const fs::path& filename, size_t& length
 ) {

src/files/files.hpp

@@ -7,6 +7,7 @@
 #include <vector>
 
 #include "typedefs.hpp"
+#include "util/Buffer.hpp"
 
 namespace fs = std::filesystem;
 
@@ -59,6 +60,7 @@ namespace files {
     );
 
     bool read(const fs::path&, char* data, size_t size);
+    util::Buffer<ubyte> read_bytes_buffer(const fs::path&);
     std::unique_ptr<ubyte[]> read_bytes(const fs::path&, size_t& length);
     std::vector<ubyte> read_bytes(const fs::path&);
     std::string read_string(const fs::path& filename);