PulseStudio/VoxelEngine
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Merge branch 'main' into heightmaps

Browse Source
This commit is contained in:
MihailRis
2024-10-03 19:12:37 +03:00
parent e73da1687a 414f18f7c7
commit ea9ad08bfd
64 changed files with 3472 additions and 732 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/files/RegionsLayer.cpp
+241
View File
@@ -0,0 +1,241 @@
#include "WorldRegions.hpp"
#include <cstring>
#include "util/data_io.hpp"
#define REGION_FORMAT_MAGIC ".VOXREG"
static fs::path get_region_filename(int x, int z) {
return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin");
}
/// @brief Read missing chunks data (null pointers) from region file
static void fetch_chunks(WorldRegion* region, int x, int z, regfile* file) {
auto* chunks = region->getChunks();
auto sizes = region->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE;
int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE;
if (chunks[i] == nullptr) {
chunks[i] = RegionsLayer::readChunkData(
chunk_x, chunk_z, sizes[i][0], sizes[i][1], file);
}
}
}
regfile::regfile(fs::path filename) : file(std::move(filename)) {
if (file.length() < REGION_HEADER_SIZE)
throw std::runtime_error("incomplete region file header");
char header[REGION_HEADER_SIZE];
file.read(header, REGION_HEADER_SIZE);
// avoid of use strcmp_s
if (std::string(header, std::strlen(REGION_FORMAT_MAGIC)) !=
REGION_FORMAT_MAGIC) {
throw std::runtime_error("invalid region file magic number");
}
version = header[8];
if (static_cast<uint>(version) > REGION_FORMAT_VERSION) {
throw illegal_region_format(
"region format " + std::to_string(version) + " is not supported"
);
}
}
std::unique_ptr<ubyte[]> regfile::read(int index, uint32_t& size, uint32_t& srcSize) {
size_t file_size = file.length();
size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
uint32_t buff32;
file.seekg(table_offset + index * 4);
file.read(reinterpret_cast<char*>(&buff32), 4);
uint32_t offset = dataio::le2h(buff32);
if (offset == 0) {
return nullptr;
}
file.seekg(offset);
file.read(reinterpret_cast<char*>(&buff32), 4);
size = dataio::le2h(buff32);
file.read(reinterpret_cast<char*>(&buff32), 4);
srcSize = dataio::le2h(buff32);
auto data = std::make_unique<ubyte[]>(size);
file.read(reinterpret_cast<char*>(data.get()), size);
return data;
}
void RegionsLayer::closeRegFile(glm::ivec2 coord) {
openRegFiles.erase(coord);
regFilesCv.notify_one();
}
regfile_ptr RegionsLayer::useRegFile(glm::ivec2 coord) {
auto* file = openRegFiles[coord].get();
file->inUse = true;
return regfile_ptr(file, &regFilesCv);
}
// Marks regfile as used and unmarks when shared_ptr dies
regfile_ptr RegionsLayer::getRegFile(glm::ivec2 coord, bool create) {
{
std::lock_guard lock(regFilesMutex);
const auto found = openRegFiles.find(coord);
if (found != openRegFiles.end()) {
if (found->second->inUse) {
throw std::runtime_error("regfile is currently in use");
}
return useRegFile(found->first);
}
}
if (create) {
return createRegFile(coord);
}
return nullptr;
}
regfile_ptr RegionsLayer::createRegFile(glm::ivec2 coord) {
auto file = folder / get_region_filename(coord[0], coord[1]);
if (!fs::exists(file)) {
return nullptr;
}
if (openRegFiles.size() == MAX_OPEN_REGION_FILES) {
std::unique_lock lock(regFilesMutex);
while (true) {
bool closed = false;
// FIXME: bad choosing algorithm
for (auto& entry : openRegFiles) {
if (!entry.second->inUse) {
closeRegFile(entry.first);
closed = true;
break;
}
}
if (closed) {
break;
}
// notified when any regfile gets out of use or closed
regFilesCv.wait(lock);
}
openRegFiles[coord] = std::make_unique<regfile>(file);
return useRegFile(coord);
} else {
std::lock_guard lock(regFilesMutex);
openRegFiles[coord] = std::make_unique<regfile>(file);
return useRegFile(coord);
}
}
WorldRegion* RegionsLayer::getRegion(int x, int z) {
std::lock_guard lock(mapMutex);
auto found = regions.find({x, z});
if (found == regions.end()) {
return nullptr;
}
return found->second.get();
}
fs::path RegionsLayer::getRegionFilePath(int x, int z) const {
return folder / get_region_filename(x, z);
}
WorldRegion* RegionsLayer::getOrCreateRegion(int x, int z) {
if (auto region = getRegion(x, z)) {
return region;
}
std::lock_guard lock(mapMutex);
auto region_ptr = std::make_unique<WorldRegion>();
auto region = region_ptr.get();
regions[{x, z}] = std::move(region_ptr);
return region;
}
ubyte* RegionsLayer::getData(int x, int z, uint32_t& size, uint32_t& srcSize) {
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = getOrCreateRegion(regionX, regionZ);
ubyte* data = region->getChunkData(localX, localZ);
if (data == nullptr) {
auto regfile = getRegFile({regionX, regionZ});
if (regfile != nullptr) {
auto dataptr = readChunkData(x, z, size, srcSize, regfile.get());
if (dataptr) {
data = dataptr.get();
region->put(localX, localZ, std::move(dataptr), size, srcSize);
}
}
}
if (data != nullptr) {
auto sizevec = region->getChunkDataSize(localX, localZ);
size = sizevec[0];
srcSize = sizevec[1];
return data;
}
return nullptr;
}
void RegionsLayer::writeRegion(int x, int z, WorldRegion* entry) {
fs::path filename = folder / get_region_filename(x, z);
glm::ivec2 regcoord(x, z);
if (auto regfile = getRegFile(regcoord, false)) {
fetch_chunks(entry, x, z, regfile.get());
std::lock_guard lock(regFilesMutex);
regfile.reset();
closeRegFile(regcoord);
}
char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC;
header[8] = REGION_FORMAT_VERSION;
header[9] = static_cast<ubyte>(compression); // FIXME
std::ofstream file(filename, std::ios::out | std::ios::binary);
file.write(header, REGION_HEADER_SIZE);
size_t offset = REGION_HEADER_SIZE;
uint32_t intbuf;
uint offsets[REGION_CHUNKS_COUNT] {};
auto region = entry->getChunks();
auto sizes = entry->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
ubyte* chunk = region[i].get();
if (chunk == nullptr) {
continue;
}
offsets[i] = offset;
auto sizevec = sizes[i];
uint32_t compressedSize = sizevec[0];
uint32_t srcSize = sizevec[1];
intbuf = dataio::h2le(compressedSize);
file.write(reinterpret_cast<const char*>(&intbuf), 4);
offset += 4;
intbuf = dataio::h2le(srcSize);
file.write(reinterpret_cast<const char*>(&intbuf), 4);
offset += 4;
file.write(reinterpret_cast<const char*>(chunk), compressedSize);
offset += compressedSize;
}
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
intbuf = dataio::h2le(offsets[i]);
file.write(reinterpret_cast<const char*>(&intbuf), 4);
}
}
std::unique_ptr<ubyte[]> RegionsLayer::readChunkData(
int x, int z, uint32_t& size, uint32_t& srcSize, regfile* rfile
) {
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
int chunkIndex = localZ * REGION_SIZE + localX;
return rfile->read(chunkIndex, size, srcSize);
}
src/files/WorldConverter.cpp
+195 -42
View File
@@ -5,49 +5,140 @@
#include <stdexcept>
#include <utility>
#include "content/ContentLUT.hpp"
#include "content/ContentReport.hpp"
#include "files/compatibility.hpp"
#include "debug/Logger.hpp"
#include "files/files.hpp"
#include "objects/Player.hpp"
#include "util/ThreadPool.hpp"
#include "voxels/Chunk.hpp"
#include "items/Inventory.hpp"
#include "voxels/Block.hpp"
#include "WorldFiles.hpp"
namespace fs = std::filesystem;
static debug::Logger logger("world-converter");
class ConverterWorker : public util::Worker<convert_task, int> {
class ConverterWorker : public util::Worker<ConvertTask, int> {
std::shared_ptr<WorldConverter> converter;
public:
ConverterWorker(std::shared_ptr<WorldConverter> converter)
: converter(std::move(converter)) {
}
int operator()(const std::shared_ptr<convert_task>& task) override {
int operator()(const std::shared_ptr<ConvertTask>& task) override {
converter->convert(*task);
return 0;
}
};
void WorldConverter::addRegionsTasks(
RegionLayerIndex layerid,
ConvertTaskType taskType
) {
const auto& regions = wfile->getRegions();
auto regionsFolder = regions.getRegionsFolder(layerid);
if (!fs::is_directory(regionsFolder)) {
return;
}
for (const auto& file : fs::directory_iterator(regionsFolder)) {
int x, z;
std::string name = file.path().stem().string();
if (!WorldRegions::parseRegionFilename(name, x, z)) {
logger.error() << "could not parse region name " << name;
continue;
}
tasks.push(ConvertTask {taskType, file.path(), x, z, layerid});
}
}
void WorldConverter::createUpgradeTasks() {
const auto& regions = wfile->getRegions();
for (auto& issue : report->getIssues()) {
if (issue.issueType != ContentIssueType::REGION_FORMAT_UPDATE) {
continue;
}
addRegionsTasks(issue.regionLayer, ConvertTaskType::UPGRADE_REGION);
}
}
void WorldConverter::createConvertTasks() {
auto handleReorder = [=](ContentType contentType) {
switch (contentType) {
case ContentType::BLOCK:
addRegionsTasks(
REGION_LAYER_VOXELS,
ConvertTaskType::VOXELS
);
break;
case ContentType::ITEM:
addRegionsTasks(
REGION_LAYER_INVENTORIES,
ConvertTaskType::INVENTORIES
);
break;
default:
break;
}
};
const auto& regions = wfile->getRegions();
for (auto& issue : report->getIssues()) {
switch (issue.issueType) {
case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE:
case ContentIssueType::REGION_FORMAT_UPDATE:
break;
case ContentIssueType::MISSING:
throw std::runtime_error("issue can't be resolved");
case ContentIssueType::REORDER:
handleReorder(issue.contentType);
break;
}
}
tasks.push(ConvertTask {ConvertTaskType::PLAYER, wfile->getPlayerFile()});
}
void WorldConverter::createBlockFieldsConvertTasks() {
// blocks data conversion requires correct block indices
// so it must be done AFTER voxels conversion
const auto& regions = wfile->getRegions();
for (auto& issue : report->getIssues()) {
switch (issue.issueType) {
case ContentIssueType::BLOCK_DATA_LAYOUTS_UPDATE:
addRegionsTasks(
REGION_LAYER_BLOCKS_DATA,
ConvertTaskType::CONVERT_BLOCKS_DATA
);
break;
default:
break;
}
}
}
WorldConverter::WorldConverter(
const std::shared_ptr<WorldFiles>& worldFiles,
const Content* content,
std::shared_ptr<ContentLUT> lut
std::shared_ptr<ContentReport> reportPtr,
ConvertMode mode
)
: wfile(worldFiles),
lut(std::move(lut)),
content(content) {
fs::path regionsFolder =
wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS);
if (!fs::is_directory(regionsFolder)) {
logger.error() << "nothing to convert";
return;
}
tasks.push(convert_task {convert_task_type::player, wfile->getPlayerFile()}
);
for (const auto& file : fs::directory_iterator(regionsFolder)) {
tasks.push(convert_task {convert_task_type::region, file.path()});
report(std::move(reportPtr)),
content(content),
mode(mode)
{
switch (mode) {
case ConvertMode::UPGRADE:
createUpgradeTasks();
break;
case ConvertMode::REINDEX:
createConvertTasks();
break;
case ConvertMode::BLOCK_FIELDS:
createBlockFieldsConvertTasks();
break;
}
}
@@ -57,11 +148,13 @@ WorldConverter::~WorldConverter() {
std::shared_ptr<Task> WorldConverter::startTask(
const std::shared_ptr<WorldFiles>& worldFiles,
const Content* content,
const std::shared_ptr<ContentLUT>& lut,
const std::shared_ptr<ContentReport>& report,
const runnable& onDone,
ConvertMode mode,
bool multithreading
) {
auto converter = std::make_shared<WorldConverter>(worldFiles, content, lut);
auto converter = std::make_shared<WorldConverter>(
worldFiles, content, report, mode);
if (!multithreading) {
converter->setOnComplete([=]() {
converter->write();
@@ -69,15 +162,15 @@ std::shared_ptr<Task> WorldConverter::startTask(
});
return converter;
}
auto pool = std::make_shared<util::ThreadPool<convert_task, int>>(
auto pool = std::make_shared<util::ThreadPool<ConvertTask, int>>(
"converter-pool",
[=]() { return std::make_shared<ConverterWorker>(converter); },
[=](int&) {}
);
auto& converterTasks = converter->tasks;
while (!converterTasks.empty()) {
const convert_task& task = converterTasks.front();
auto ptr = std::make_shared<convert_task>(task);
const ConvertTask& task = converterTasks.front();
auto ptr = std::make_shared<ConvertTask>(task);
pool->enqueueJob(ptr);
converterTasks.pop();
}
@@ -88,39 +181,85 @@ std::shared_ptr<Task> WorldConverter::startTask(
return pool;
}
void WorldConverter::convertRegion(const fs::path& file) const {
int x, z;
std::string name = file.stem().string();
if (!WorldRegions::parseRegionFilename(name, x, z)) {
logger.error() << "could not parse name " << name;
return;
}
logger.info() << "converting region " << name;
wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) {
if (lut) {
Chunk::convert(data, lut.get());
}
return true;
void WorldConverter::upgradeRegion(
const fs::path& file, int x, int z, RegionLayerIndex layer
) const {
auto path = wfile->getRegions().getRegionFilePath(layer, x, z);
auto bytes = files::read_bytes_buffer(path);
auto buffer = compatibility::convert_region_2to3(bytes, layer);
files::write_bytes(path, buffer.data(), buffer.size());
}
void WorldConverter::convertVoxels(const fs::path& file, int x, int z) const {
logger.info() << "converting voxels region " << x << "_" << z;
wfile->getRegions().processRegion(x, z, REGION_LAYER_VOXELS,
[=](std::unique_ptr<ubyte[]> data, uint32_t*) {
Chunk::convert(data.get(), report.get());
return data;
});
}
void WorldConverter::convertInventories(const fs::path& file, int x, int z) const {
logger.info() << "converting inventories region " << x << "_" << z;
wfile->getRegions().processInventories(x, z, [=](Inventory* inventory) {
inventory->convert(report.get());
});
}
void WorldConverter::convertPlayer(const fs::path& file) const {
logger.info() << "converting player " << file.u8string();
auto map = files::read_json(file);
Player::convert(map, lut.get());
Player::convert(map, report.get());
files::write_json(file, map);
}
void WorldConverter::convert(const convert_task& task) const {
void WorldConverter::convertBlocksData(int x, int z, const ContentReport& report) const {
logger.info() << "converting blocks data";
wfile->getRegions().processBlocksData(x, z,
[=](BlocksMetadata* heap, std::unique_ptr<ubyte[]> voxelsData) {
Chunk chunk(0, 0);
chunk.decode(voxelsData.get());
const auto& indices = content->getIndices()->blocks;
BlocksMetadata newHeap;
for (const auto& entry : *heap) {
size_t index = entry.index;
const auto& def = indices.require(chunk.voxels[index].id);
const auto& newStruct = *def.dataStruct;
const auto& found = report.blocksDataLayouts.find(def.name);
if (found == report.blocksDataLayouts.end()) {
logger.error() << "no previous fields layout found for block"
<< def.name << " - discard";
continue;
}
const auto& prevStruct = found->second;
uint8_t* dst = newHeap.allocate(index, newStruct.size());
newStruct.convert(prevStruct, entry.data(), dst, true);
}
*heap = std::move(newHeap);
});
}
void WorldConverter::convert(const ConvertTask& task) const {
if (!fs::is_regular_file(task.file)) return;
switch (task.type) {
case convert_task_type::region:
convertRegion(task.file);
case ConvertTaskType::UPGRADE_REGION:
upgradeRegion(task.file, task.x, task.z, task.layer);
break;
case convert_task_type::player:
case ConvertTaskType::VOXELS:
convertVoxels(task.file, task.x, task.z);
break;
case ConvertTaskType::INVENTORIES:
convertInventories(task.file, task.x, task.z);
break;
case ConvertTaskType::PLAYER:
convertPlayer(task.file);
break;
case ConvertTaskType::CONVERT_BLOCKS_DATA:
convertBlocksData(task.x, task.z, *report);
break;
}
}
@@ -128,7 +267,7 @@ void WorldConverter::convertNext() {
if (tasks.empty()) {
throw std::runtime_error("no more regions to convert");
}
convert_task task = tasks.front();
ConvertTask task = tasks.front();
tasks.pop();
tasksDone++;
@@ -155,8 +294,22 @@ bool WorldConverter::isActive() const {
}
void WorldConverter::write() {
logger.info() << "writing world";
wfile->write(nullptr, content);
logger.info() << "applying changes";
auto patch = dv::object();
switch (mode) {
case ConvertMode::UPGRADE:
patch["region-version"] = REGION_FORMAT_VERSION;
break;
case ConvertMode::REINDEX:
WorldFiles::createContentIndicesCache(content->getIndices(), patch);
break;
case ConvertMode::BLOCK_FIELDS:
WorldFiles::createBlockFieldsIndices(content->getIndices(), patch);
break;
}
wfile->patchIndicesFile(patch);
wfile->write(nullptr, nullptr);
}
void WorldConverter::waitForEnd() {
src/files/WorldConverter.hpp
+48 -10
View File
@@ -6,40 +6,77 @@
#include "delegates.hpp"
#include "interfaces/Task.hpp"
#include "files/world_regions_fwd.hpp"
#include "typedefs.hpp"
namespace fs = std::filesystem;
class Content;
class ContentLUT;
class ContentReport;
class WorldFiles;
enum class convert_task_type { region, player };
enum class ConvertTaskType {
/// @brief rewrite voxels region indices
VOXELS,
/// @brief rewrite inventories region indices
INVENTORIES,
/// @brief rewrite player
PLAYER,
/// @brief refresh region file version
UPGRADE_REGION,
/// @brief convert blocks data to updated layouts
CONVERT_BLOCKS_DATA,
};
struct convert_task {
convert_task_type type;
struct ConvertTask {
ConvertTaskType type;
fs::path file;
/// @brief region coords
int x, z;
RegionLayerIndex layer;
};
enum class ConvertMode {
UPGRADE,
REINDEX,
BLOCK_FIELDS,
};
class WorldConverter : public Task {
std::shared_ptr<WorldFiles> wfile;
std::shared_ptr<ContentLUT> const lut;
std::shared_ptr<ContentReport> const report;
const Content* const content;
std::queue<convert_task> tasks;
std::queue<ConvertTask> tasks;
runnable onComplete;
uint tasksDone = 0;
ConvertMode mode;
void upgradeRegion(
const fs::path& file, int x, int z, RegionLayerIndex layer) const;
void convertPlayer(const fs::path& file) const;
void convertRegion(const fs::path& file) const;
void convertVoxels(const fs::path& file, int x, int z) const;
void convertInventories(const fs::path& file, int x, int z) const;
void convertBlocksData(int x, int z, const ContentReport& report) const;
void addRegionsTasks(
RegionLayerIndex layerid,
ConvertTaskType taskType
);
void createUpgradeTasks();
void createConvertTasks();
void createBlockFieldsConvertTasks();
public:
WorldConverter(
const std::shared_ptr<WorldFiles>& worldFiles,
const Content* content,
std::shared_ptr<ContentLUT> lut
std::shared_ptr<ContentReport> report,
ConvertMode mode
);
~WorldConverter();
void convert(const convert_task& task) const;
void convert(const ConvertTask& task) const;
void convertNext();
void setOnComplete(runnable callback);
void write();
@@ -54,8 +91,9 @@ public:
static std::shared_ptr<Task> startTask(
const std::shared_ptr<WorldFiles>& worldFiles,
const Content* content,
const std::shared_ptr<ContentLUT>& lut,
const std::shared_ptr<ContentReport>& report,
const runnable& onDone,
ConvertMode mode,
bool multithreading
);
};
src/files/WorldFiles.cpp
+47 -6
View File
@@ -21,9 +21,11 @@
#include "objects/EntityDef.hpp"
#include "objects/Player.hpp"
#include "physics/Hitbox.hpp"
#include "data/StructLayout.hpp"
#include "settings.hpp"
#include "typedefs.hpp"
#include "util/data_io.hpp"
#include "util/stringutil.hpp"
#include "voxels/Block.hpp"
#include "voxels/Chunk.hpp"
#include "voxels/voxel.hpp"
@@ -73,7 +75,9 @@ fs::path WorldFiles::getPacksFile() const {
return directory / fs::path("packs.list");
}
void WorldFiles::write(const World* world, const Content* content) {
void WorldFiles::write(
const World* world, const Content* content
) {
if (world) {
writeWorldInfo(world->getInfo());
if (!fs::exists(getPacksFile())) {
@@ -83,9 +87,10 @@ void WorldFiles::write(const World* world, const Content* content) {
if (generatorTestMode) {
return;
}
writeIndices(content->getIndices());
regions.write();
if (content) {
writeIndices(content->getIndices());
}
regions.writeAll();
}
void WorldFiles::writePacks(const std::vector<ContentPack>& packs) {
@@ -107,11 +112,33 @@ static void write_indices(
}
}
void WorldFiles::writeIndices(const ContentIndices* indices) {
dv::value root = dv::object();
void WorldFiles::createContentIndicesCache(
const ContentIndices* indices, dv::value& root
) {
write_indices(indices->blocks, root.list("blocks"));
write_indices(indices->items, root.list("items"));
write_indices(indices->entities, root.list("entities"));
}
void WorldFiles::createBlockFieldsIndices(
const ContentIndices* indices, dv::value& root
) {
auto& structsMap = root.object("blocks-data");
for (const auto* def : indices->blocks.getIterable()) {
if (def->dataStruct == nullptr) {
continue;
}
structsMap[def->name] = def->dataStruct->serialize();
}
}
void WorldFiles::writeIndices(const ContentIndices* indices) {
dv::value root = dv::object();
root["region-version"] = REGION_FORMAT_VERSION;
createContentIndicesCache(indices, root);
createBlockFieldsIndices(indices, root);
files::write_json(getIndicesFile(), root);
}
@@ -164,6 +191,20 @@ bool WorldFiles::readResourcesData(const Content* content) {
return true;
}
void WorldFiles::patchIndicesFile(const dv::value& map) {
fs::path file = getIndicesFile();
if (!fs::is_regular_file(file)) {
logger.error() << file.filename().u8string() << " does not exists";
return;
}
auto root = files::read_json(file);
for (const auto& [key, value] : map.asObject()) {
logger.info() << "patching indices.json: update " << util::quote(key);
root[key] = value;
}
files::write_json(file, root, true);
}
static void erase_pack_indices(dv::value& root, const std::string& id) {
auto prefix = id + ":";
auto& blocks = root["blocks"];
src/files/WorldFiles.hpp
+11 -2
View File
@@ -51,6 +51,15 @@ public:
std::optional<WorldInfo> readWorldInfo();
bool readResourcesData(const Content* content);
static void createContentIndicesCache(
const ContentIndices* indices, dv::value& root
);
static void createBlockFieldsIndices(
const ContentIndices* indices, dv::value& root
);
void patchIndicesFile(const dv::value& map);
/// @brief Write all unsaved data to world files
/// @param world target world
/// @param content world content
@@ -63,8 +72,6 @@ public:
/// @return world folder
fs::path getFolder() const;
static const inline std::string WORLD_FILE = "world.json";
WorldRegions& getRegions() {
return regions;
}
@@ -72,4 +79,6 @@ public:
bool doesWriteLights() const {
return doWriteLights;
}
static const inline std::string WORLD_FILE = "world.json";
};
src/files/WorldRegions.cpp
+220 -336
View File
@@ -4,60 +4,24 @@
#include <utility>
#include <vector>
#include "debug/Logger.hpp"
#include "coders/json.hpp"
#include "coders/byte_utils.hpp"
#include "coders/rle.hpp"
#include "coders/binary_json.hpp"
#include "items/Inventory.hpp"
#include "maths/voxmaths.hpp"
#include "util/data_io.hpp"
#include "coders/binary_json.hpp"
#define REGION_FORMAT_MAGIC ".VOXREG"
regfile::regfile(fs::path filename) : file(std::move(filename)) {
if (file.length() < REGION_HEADER_SIZE)
throw std::runtime_error("incomplete region file header");
char header[REGION_HEADER_SIZE];
file.read(header, REGION_HEADER_SIZE);
// avoid of use strcmp_s
if (std::string(header, std::strlen(REGION_FORMAT_MAGIC)) !=
REGION_FORMAT_MAGIC) {
throw std::runtime_error("invalid region file magic number");
}
version = header[8];
if (static_cast<uint>(version) > REGION_FORMAT_VERSION) {
throw illegal_region_format(
"region format " + std::to_string(version) + " is not supported"
);
}
}
std::unique_ptr<ubyte[]> regfile::read(int index, uint32_t& length) {
size_t file_size = file.length();
size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
uint32_t offset;
file.seekg(table_offset + index * 4);
file.read(reinterpret_cast<char*>(&offset), 4);
offset = dataio::read_int32_big(reinterpret_cast<const ubyte*>(&offset), 0);
if (offset == 0) {
return nullptr;
}
file.seekg(offset);
file.read(reinterpret_cast<char*>(&offset), 4);
length = dataio::read_int32_big(reinterpret_cast<const ubyte*>(&offset), 0);
auto data = std::make_unique<ubyte[]>(length);
file.read(reinterpret_cast<char*>(data.get()), length);
return data;
}
static debug::Logger logger("world-regions");
WorldRegion::WorldRegion()
: chunksData(
std::make_unique<std::unique_ptr<ubyte[]>[]>(REGION_CHUNKS_COUNT)
),
sizes(std::make_unique<uint32_t[]>(REGION_CHUNKS_COUNT)) {
sizes(std::make_unique<glm::u32vec2[]>(REGION_CHUNKS_COUNT)) {
}
WorldRegion::~WorldRegion() = default;
@@ -73,293 +37,89 @@ std::unique_ptr<ubyte[]>* WorldRegion::getChunks() const {
return chunksData.get();
}
uint32_t* WorldRegion::getSizes() const {
glm::u32vec2* WorldRegion::getSizes() const {
return sizes.get();
}
void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) {
void WorldRegion::put(
uint x, uint z, std::unique_ptr<ubyte[]> data, uint32_t size, uint32_t srcSize
) {
size_t chunk_index = z * REGION_SIZE + x;
chunksData[chunk_index].reset(data);
sizes[chunk_index] = size;
chunksData[chunk_index] = std::move(data);
sizes[chunk_index] = glm::u32vec2(size, srcSize);
}
ubyte* WorldRegion::getChunkData(uint x, uint z) {
return chunksData[z * REGION_SIZE + x].get();
}
uint WorldRegion::getChunkDataSize(uint x, uint z) {
glm::u32vec2 WorldRegion::getChunkDataSize(uint x, uint z) {
return sizes[z * REGION_SIZE + x];
}
WorldRegions::WorldRegions(const fs::path& directory) : directory(directory) {
for (size_t i = 0; i < sizeof(layers) / sizeof(RegionsLayer); i++) {
layers[i].layer = i;
for (size_t i = 0; i < REGION_LAYERS_COUNT; i++) {
layers[i].layer = static_cast<RegionLayerIndex>(i);
}
layers[REGION_LAYER_VOXELS].folder = directory / fs::path("regions");
layers[REGION_LAYER_LIGHTS].folder = directory / fs::path("lights");
auto& voxels = layers[REGION_LAYER_VOXELS];
voxels.folder = directory / fs::path("regions");
voxels.compression = compression::Method::EXTRLE16;
auto& lights = layers[REGION_LAYER_LIGHTS];
lights.folder = directory / fs::path("lights");
lights.compression = compression::Method::EXTRLE8;
layers[REGION_LAYER_INVENTORIES].folder =
directory / fs::path("inventories");
layers[REGION_LAYER_ENTITIES].folder = directory / fs::path("entities");
auto& blocksData = layers[REGION_LAYER_BLOCKS_DATA];
blocksData.folder = directory / fs::path("blocksdata");
}
WorldRegions::~WorldRegions() = default;
WorldRegion* WorldRegions::getRegion(int x, int z, int layer) {
RegionsLayer& regions = layers[layer];
std::lock_guard lock(regions.mutex);
auto found = regions.regions.find(glm::ivec2(x, z));
if (found == regions.regions.end()) {
return nullptr;
}
return found->second.get();
}
WorldRegion* WorldRegions::getOrCreateRegion(int x, int z, int layer) {
if (auto region = getRegion(x, z, layer)) {
return region;
}
RegionsLayer& regions = layers[layer];
std::lock_guard lock(regions.mutex);
auto region_ptr = std::make_unique<WorldRegion>();
auto region = region_ptr.get();
regions.regions[{x, z}] = std::move(region_ptr);
return region;
}
std::unique_ptr<ubyte[]> WorldRegions::compress(
const ubyte* src, size_t srclen, size_t& len
) {
auto buffer = bufferPool.get();
auto bytes = buffer.get();
len = extrle::encode(src, srclen, bytes);
auto data = std::make_unique<ubyte[]>(len);
for (size_t i = 0; i < len; i++) {
data[i] = bytes[i];
}
return data;
}
std::unique_ptr<ubyte[]> WorldRegions::decompress(
const ubyte* src, size_t srclen, size_t dstlen
) {
auto decompressed = std::make_unique<ubyte[]>(dstlen);
extrle::decode(src, srclen, decompressed.get());
return decompressed;
}
inline void calc_reg_coords(
int x, int z, int& regionX, int& regionZ, int& localX, int& localZ
) {
regionX = floordiv(x, REGION_SIZE);
regionZ = floordiv(z, REGION_SIZE);
localX = x - (regionX * REGION_SIZE);
localZ = z - (regionZ * REGION_SIZE);
}
std::unique_ptr<ubyte[]> WorldRegions::readChunkData(
int x, int z, uint32_t& length, regfile* rfile
) {
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
int chunkIndex = localZ * REGION_SIZE + localX;
return rfile->read(chunkIndex, length);
}
/// @brief Read missing chunks data (null pointers) from region file
void WorldRegions::fetchChunks(
WorldRegion* region, int x, int z, regfile* file
) {
auto* chunks = region->getChunks();
uint32_t* sizes = region->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE;
int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE;
if (chunks[i] == nullptr) {
chunks[i] = readChunkData(chunk_x, chunk_z, sizes[i], file);
}
}
}
ubyte* WorldRegions::getData(int x, int z, int layer, uint32_t& size) {
if (generatorTestMode) {
return nullptr;
}
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer);
ubyte* data = region->getChunkData(localX, localZ);
if (data == nullptr) {
auto regfile = getRegFile(glm::ivec3(regionX, regionZ, layer));
if (regfile != nullptr) {
data = readChunkData(x, z, size, regfile.get()).release();
}
if (data != nullptr) {
region->put(localX, localZ, data, size);
}
}
if (data != nullptr) {
size = region->getChunkDataSize(localX, localZ);
return data;
}
return nullptr;
}
regfile_ptr WorldRegions::useRegFile(glm::ivec3 coord) {
auto* file = openRegFiles[coord].get();
file->inUse = true;
return regfile_ptr(file, &regFilesCv);
}
void WorldRegions::closeRegFile(glm::ivec3 coord) {
openRegFiles.erase(coord);
regFilesCv.notify_one();
}
// Marks regfile as used and unmarks when shared_ptr dies
regfile_ptr WorldRegions::getRegFile(glm::ivec3 coord, bool create) {
{
std::lock_guard lock(regFilesMutex);
const auto found = openRegFiles.find(coord);
if (found != openRegFiles.end()) {
if (found->second->inUse) {
throw std::runtime_error("regfile is currently in use");
}
return useRegFile(found->first);
}
}
if (create) {
return createRegFile(coord);
}
return nullptr;
}
regfile_ptr WorldRegions::createRegFile(glm::ivec3 coord) {
fs::path file =
layers[coord[2]].folder / getRegionFilename(coord[0], coord[1]);
if (!fs::exists(file)) {
return nullptr;
}
if (openRegFiles.size() == MAX_OPEN_REGION_FILES) {
std::unique_lock lock(regFilesMutex);
while (true) {
bool closed = false;
// FIXME: bad choosing algorithm
for (auto& entry : openRegFiles) {
if (!entry.second->inUse) {
closeRegFile(entry.first);
closed = true;
break;
}
}
if (closed) {
break;
}
// notified when any regfile gets out of use or closed
regFilesCv.wait(lock);
}
openRegFiles[coord] = std::make_unique<regfile>(file);
return useRegFile(coord);
} else {
std::lock_guard lock(regFilesMutex);
openRegFiles[coord] = std::make_unique<regfile>(file);
return useRegFile(coord);
}
}
fs::path WorldRegions::getRegionFilename(int x, int z) const {
return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin");
}
void WorldRegions::writeRegion(int x, int z, int layer, WorldRegion* entry) {
fs::path filename = layers[layer].folder / getRegionFilename(x, z);
glm::ivec3 regcoord(x, z, layer);
if (auto regfile = getRegFile(regcoord, false)) {
fetchChunks(entry, x, z, regfile.get());
std::lock_guard lock(regFilesMutex);
regfile.reset();
closeRegFile(regcoord);
}
char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC;
header[8] = REGION_FORMAT_VERSION;
header[9] = 0; // flags
std::ofstream file(filename, std::ios::out | std::ios::binary);
file.write(header, REGION_HEADER_SIZE);
size_t offset = REGION_HEADER_SIZE;
char intbuf[4] {};
uint offsets[REGION_CHUNKS_COUNT] {};
auto* region = entry->getChunks();
uint32_t* sizes = entry->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
ubyte* chunk = region[i].get();
if (chunk == nullptr) {
offsets[i] = 0;
} else {
offsets[i] = offset;
size_t compressedSize = sizes[i];
dataio::write_int32_big(
compressedSize, reinterpret_cast<ubyte*>(intbuf), 0
);
offset += 4 + compressedSize;
file.write(intbuf, 4);
file.write(reinterpret_cast<const char*>(chunk), compressedSize);
}
}
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
dataio::write_int32_big(
offsets[i], reinterpret_cast<ubyte*>(intbuf), 0
);
file.write(intbuf, 4);
}
}
void WorldRegions::writeRegions(int layer) {
for (auto& it : layers[layer].regions) {
void RegionsLayer::writeAll() {
for (auto& it : regions) {
WorldRegion* region = it.second.get();
if (region->getChunks() == nullptr || !region->isUnsaved()) {
continue;
}
glm::ivec2 key = it.first;
writeRegion(key[0], key[1], layer, region);
const auto& key = it.first;
writeRegion(key[0], key[1], region);
}
}
void WorldRegions::put(
int x,
int z,
int layer,
RegionLayerIndex layerid,
std::unique_ptr<ubyte[]> data,
size_t size,
bool rle
size_t srcSize
) {
if (rle) {
size_t compressedSize;
auto compressed = compress(data.get(), size, compressedSize);
put(x, z, layer, std::move(compressed), compressedSize, false);
return;
}
size_t size = srcSize;
auto& layer = layers[layerid];
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer);
WorldRegion* region = layer.getOrCreateRegion(regionX, regionZ);
region->setUnsaved(true);
region->put(localX, localZ, data.release(), size);
if (data == nullptr) {
region->put(localX, localZ, nullptr, 0, 0);
return;
}
if (layer.compression != compression::Method::NONE) {
data = compression::compress(
data.get(), size, size, layer.compression);
}
region->put(localX, localZ, std::move(data), size, srcSize);
}
static std::unique_ptr<ubyte[]> write_inventories(
Chunk* chunk, uint& datasize
const ChunkInventoriesMap& inventories, uint32_t& datasize
) {
auto& inventories = chunk->inventories;
ByteBuilder builder;
builder.putInt32(inventories.size());
for (auto& entry : inventories) {
@@ -376,7 +136,22 @@ static std::unique_ptr<ubyte[]> write_inventories(
return data;
}
/// @brief Store chunk data (voxels and lights) in region (existing or new)
static ChunkInventoriesMap load_inventories(const ubyte* src, uint32_t size) {
ChunkInventoriesMap inventories;
ByteReader reader(src, size);
auto count = reader.getInt32();
for (int i = 0; i < count; i++) {
uint index = reader.getInt32();
uint size = reader.getInt32();
auto map = json::from_binary(reader.pointer(), size);
reader.skip(size);
auto inv = std::make_shared<Inventory>(0, 0);
inv->deserialize(map);
inventories[index] = inv;
}
return inventories;
}
void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
assert(chunk != nullptr);
if (!chunk->flags.lighted) {
@@ -394,8 +169,7 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
chunk->z,
REGION_LAYER_VOXELS,
chunk->encode(),
CHUNK_DATA_LEN,
true);
CHUNK_DATA_LEN);
// Writing lights cache
if (doWriteLights && chunk->flags.lighted) {
@@ -403,19 +177,17 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
chunk->z,
REGION_LAYER_LIGHTS,
chunk->lightmap.encode(),
LIGHTMAP_DATA_LEN,
true);
LIGHTMAP_DATA_LEN);
}
// Writing block inventories
if (!chunk->inventories.empty()) {
uint datasize;
auto data = write_inventories(chunk, datasize);
auto data = write_inventories(chunk->inventories, datasize);
put(chunk->x,
chunk->z,
REGION_LAYER_INVENTORIES,
std::move(data),
datasize,
false);
datasize);
}
// Writing entities
if (!entitiesData.empty()) {
@@ -427,71 +199,164 @@ void WorldRegions::put(Chunk* chunk, std::vector<ubyte> entitiesData) {
chunk->z,
REGION_LAYER_ENTITIES,
std::move(data),
entitiesData.size(),
false);
entitiesData.size());
}
// Writing blocks data
if (chunk->flags.blocksData) {
auto bytes = chunk->blocksMetadata.serialize();
put(chunk->x,
chunk->z,
REGION_LAYER_BLOCKS_DATA,
bytes.release(),
bytes.size());
}
}
std::unique_ptr<ubyte[]> WorldRegions::getChunk(int x, int z) {
std::unique_ptr<ubyte[]> WorldRegions::getVoxels(int x, int z) {
uint32_t size;
auto* data = getData(x, z, REGION_LAYER_VOXELS, size);
uint32_t srcSize;
auto& layer = layers[REGION_LAYER_VOXELS];
auto* data = layer.getData(x, z, size, srcSize);
if (data == nullptr) {
return nullptr;
}
return decompress(data, size, CHUNK_DATA_LEN);
assert(srcSize == CHUNK_DATA_LEN);
return compression::decompress(data, size, srcSize, layer.compression);
}
/// @brief Get cached lights for chunk at x,z
/// @return lights data or nullptr
std::unique_ptr<light_t[]> WorldRegions::getLights(int x, int z) {
uint32_t size;
auto* bytes = getData(x, z, REGION_LAYER_LIGHTS, size);
uint32_t srcSize;
auto& layer = layers[REGION_LAYER_LIGHTS];
auto* bytes = layer.getData(x, z, size, srcSize);
if (bytes == nullptr) {
return nullptr;
}
auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN);
auto data = compression::decompress(
bytes, size, srcSize, layer.compression
);
assert(srcSize == LIGHTMAP_DATA_LEN);
return Lightmap::decode(data.get());
}
chunk_inventories_map WorldRegions::fetchInventories(int x, int z) {
chunk_inventories_map meta;
ChunkInventoriesMap WorldRegions::fetchInventories(int x, int z) {
uint32_t bytesSize;
const ubyte* data = getData(x, z, REGION_LAYER_INVENTORIES, bytesSize);
if (data == nullptr) {
return meta;
uint32_t srcSize;
auto bytes = layers[REGION_LAYER_INVENTORIES].getData(x, z, bytesSize, srcSize);
if (bytes == nullptr) {
return {};
}
ByteReader reader(data, bytesSize);
auto count = reader.getInt32();
for (int i = 0; i < count; i++) {
uint index = reader.getInt32();
uint size = reader.getInt32();
auto map = json::from_binary(reader.pointer(), size);
reader.skip(size);
auto inv = std::make_shared<Inventory>(0, 0);
inv->deserialize(map);
meta[index] = inv;
return load_inventories(bytes, bytesSize);
}
BlocksMetadata WorldRegions::getBlocksData(int x, int z) {
uint32_t bytesSize;
uint32_t srcSize;
auto bytes = layers[REGION_LAYER_BLOCKS_DATA].getData(x, z, bytesSize, srcSize);
if (bytes == nullptr) {
return {};
}
BlocksMetadata heap;
heap.deserialize(bytes, bytesSize);
return heap;
}
void WorldRegions::processInventories(int x, int z, const InventoryProc& func) {
processRegion(x, z, REGION_LAYER_INVENTORIES,
[=](std::unique_ptr<ubyte[]> data, uint32_t* size) {
auto inventories = load_inventories(data.get(), *size);
for (const auto& [_, inventory] : inventories) {
func(inventory.get());
}
return write_inventories(inventories, *size);
});
}
void WorldRegions::processBlocksData(int x, int z, const BlockDataProc& func) {
auto& voxLayer = layers[REGION_LAYER_VOXELS];
auto& datLayer = layers[REGION_LAYER_BLOCKS_DATA];
if (voxLayer.getRegion(x, z) || datLayer.getRegion(x, z)) {
throw std::runtime_error("not implemented for in-memory regions");
}
auto datRegfile = datLayer.getRegFile({x, z});
if (datRegfile == nullptr) {
throw std::runtime_error("could not open region file");
}
auto voxRegfile = voxLayer.getRegFile({x, z});
if (voxRegfile == nullptr) {
logger.warning() << "missing voxels region - discard blocks data for "
<< x << "_" << z;
deleteRegion(REGION_LAYER_BLOCKS_DATA, x, z);
return;
}
for (uint cz = 0; cz < REGION_SIZE; cz++) {
for (uint cx = 0; cx < REGION_SIZE; cx++) {
int gx = cx + x * REGION_SIZE;
int gz = cz + z * REGION_SIZE;
uint32_t datLength;
uint32_t datSrcSize;
auto datData = RegionsLayer::readChunkData(
gx, gz, datLength, datSrcSize, datRegfile.get()
);
if (datData == nullptr) {
continue;
}
uint32_t voxLength;
uint32_t voxSrcSize;
auto voxData = RegionsLayer::readChunkData(
gx, gz, voxLength, voxSrcSize, voxRegfile.get()
);
if (voxData == nullptr) {
logger.warning()
<< "missing voxels for chunk (" << gx << ", " << gz << ")";
put(gx, gz, REGION_LAYER_BLOCKS_DATA, nullptr, 0);
continue;
}
voxData = compression::decompress(
voxData.get(), voxLength, voxSrcSize, voxLayer.compression
);
BlocksMetadata blocksData;
blocksData.deserialize(datData.get(), datLength);
try {
func(&blocksData, std::move(voxData));
} catch (const std::exception& err) {
logger.error() << "an error ocurred while processing blocks "
"data in chunk (" << gx << ", " << gz << "): " << err.what();
blocksData = {};
}
auto bytes = blocksData.serialize();
put(gx, gz, REGION_LAYER_BLOCKS_DATA, bytes.release(), bytes.size());
}
}
return meta;
}
dv::value WorldRegions::fetchEntities(int x, int z) {
if (generatorTestMode) {
return nullptr;
}
uint32_t bytesSize;
const ubyte* data = getData(x, z, REGION_LAYER_ENTITIES, bytesSize);
uint32_t srcSize;
const ubyte* data = layers[REGION_LAYER_ENTITIES].getData(x, z, bytesSize, srcSize);
if (data == nullptr) {
return nullptr;
}
auto map = json::from_binary(data, bytesSize);
if (map.size() == 0) {
if (map.empty()) {
return nullptr;
}
return map;
}
void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
if (getRegion(x, z, REGION_LAYER_VOXELS)) {
void WorldRegions::processRegion(
int x, int z, RegionLayerIndex layerid, const RegionProc& func
) {
auto& layer = layers[layerid];
if (layer.getRegion(x, z)) {
throw std::runtime_error("not implemented for in-memory regions");
}
auto regfile = getRegFile(glm::ivec3(x, z, REGION_LAYER_VOXELS));
auto regfile = layer.getRegFile({x, z});
if (regfile == nullptr) {
throw std::runtime_error("could not open region file");
}
@@ -500,31 +365,50 @@ void WorldRegions::processRegionVoxels(int x, int z, const regionproc& func) {
int gx = cx + x * REGION_SIZE;
int gz = cz + z * REGION_SIZE;
uint32_t length;
auto data = readChunkData(gx, gz, length, regfile.get());
uint32_t srcSize;
auto data =
RegionsLayer::readChunkData(gx, gz, length, srcSize, regfile.get());
if (data == nullptr) {
continue;
}
data = decompress(data.get(), length, CHUNK_DATA_LEN);
if (func(data.get())) {
put(gx,
gz,
REGION_LAYER_VOXELS,
std::move(data),
CHUNK_DATA_LEN,
true);
if (layer.compression != compression::Method::NONE) {
data = compression::decompress(
data.get(), length, srcSize, layer.compression
);
} else {
srcSize = length;
}
if (auto writeData = func(std::move(data), &srcSize)) {
put(gx, gz, layerid, std::move(writeData), srcSize);
}
}
}
}
fs::path WorldRegions::getRegionsFolder(int layer) const {
return layers[layer].folder;
const fs::path& WorldRegions::getRegionsFolder(RegionLayerIndex layerid) const {
return layers[layerid].folder;
}
void WorldRegions::write() {
fs::path WorldRegions::getRegionFilePath(RegionLayerIndex layerid, int x, int z) const {
return layers[layerid].getRegionFilePath(x, z);
}
void WorldRegions::writeAll() {
for (auto& layer : layers) {
fs::create_directories(layer.folder);
writeRegions(layer.layer);
layer.writeAll();
}
}
void WorldRegions::deleteRegion(RegionLayerIndex layerid, int x, int z) {
auto& layer = layers[layerid];
if (layer.getRegFile({x, z}, false)) {
throw std::runtime_error("region file is currently in use");
}
auto file = layer.getRegionFilePath(x, z);
if (fs::exists(file)) {
logger.info() << "remove region file " << file.u8string();
fs::remove(file);
}
}
src/files/WorldRegions.hpp
+117 -72
View File
@@ -11,7 +11,10 @@
#include "typedefs.hpp"
#include "util/BufferPool.hpp"
#include "voxels/Chunk.hpp"
#include "maths/voxmaths.hpp"
#include "coders/compression.hpp"
#include "files.hpp"
#include "world_regions_fwd.hpp"
#define GLM_ENABLE_EXPERIMENTAL
#include <glm/gtx/hash.hpp>
@@ -20,16 +23,9 @@ namespace fs = std::filesystem;
inline constexpr uint REGION_HEADER_SIZE = 10;
inline constexpr uint REGION_LAYER_VOXELS = 0;
inline constexpr uint REGION_LAYER_LIGHTS = 1;
inline constexpr uint REGION_LAYER_INVENTORIES = 2;
inline constexpr uint REGION_LAYER_ENTITIES = 3;
inline constexpr uint REGION_SIZE_BIT = 5;
inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT));
inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE));
inline constexpr uint REGION_FORMAT_VERSION = 2;
inline constexpr uint MAX_OPEN_REGION_FILES = 16;
class illegal_region_format : public std::runtime_error {
public:
@@ -40,21 +36,21 @@ public:
class WorldRegion {
std::unique_ptr<std::unique_ptr<ubyte[]>[]> chunksData;
std::unique_ptr<uint32_t[]> sizes;
std::unique_ptr<glm::u32vec2[]> sizes;
bool unsaved = false;
public:
WorldRegion();
~WorldRegion();
void put(uint x, uint z, ubyte* data, uint32_t size);
void put(uint x, uint z, std::unique_ptr<ubyte[]> data, uint32_t size, uint32_t srcSize);
ubyte* getChunkData(uint x, uint z);
uint getChunkDataSize(uint x, uint z);
glm::u32vec2 getChunkDataSize(uint x, uint z);
void setUnsaved(bool unsaved);
bool isUnsaved() const;
std::unique_ptr<ubyte[]>* getChunks() const;
uint32_t* getSizes() const;
glm::u32vec2* getSizes() const;
};
struct regfile {
@@ -65,19 +61,15 @@ struct regfile {
regfile(fs::path filename);
regfile(const regfile&) = delete;
std::unique_ptr<ubyte[]> read(int index, uint32_t& length);
std::unique_ptr<ubyte[]> read(int index, uint32_t& size, uint32_t& srcSize);
};
using regionsmap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
using regionproc = std::function<bool(ubyte*)>;
struct RegionsLayer {
int layer;
fs::path folder;
regionsmap regions;
std::mutex mutex;
};
using RegionsMap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
using RegionProc = std::function<std::unique_ptr<ubyte[]>(std::unique_ptr<ubyte[]>,uint32_t*)>;
using InventoryProc = std::function<void(Inventory*)>;
using BlockDataProc = std::function<void(BlocksMetadata*, std::unique_ptr<ubyte[]>)>;
/// @brief Region file pointer keeping inUse flag on until destroyed
class regfile_ptr {
regfile* file;
std::condition_variable* cv;
@@ -115,58 +107,80 @@ public:
}
};
class WorldRegions {
fs::path directory;
std::unordered_map<glm::ivec3, std::unique_ptr<regfile>> openRegFiles;
inline void calc_reg_coords(
int x, int z, int& regionX, int& regionZ, int& localX, int& localZ
) {
regionX = floordiv(x, REGION_SIZE);
regionZ = floordiv(z, REGION_SIZE);
localX = x - (regionX * REGION_SIZE);
localZ = z - (regionZ * REGION_SIZE);
}
struct RegionsLayer {
/// @brief Layer index
RegionLayerIndex layer;
/// @brief Regions layer folder
fs::path folder;
compression::Method compression = compression::Method::NONE;
/// @brief In-memory regions data
RegionsMap regions;
/// @brief In-memory regions map mutex
std::mutex mapMutex;
/// @brief Open region files map
std::unordered_map<glm::ivec2, std::unique_ptr<regfile>> openRegFiles;
/// @brief Open region files map mutex
std::mutex regFilesMutex;
std::condition_variable regFilesCv;
RegionsLayer layers[4] {};
util::BufferPool<ubyte> bufferPool {
std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2};
WorldRegion* getRegion(int x, int z, int layer);
WorldRegion* getOrCreateRegion(int x, int z, int layer);
[[nodiscard]] regfile_ptr getRegFile(glm::ivec2 coord, bool create = true);
[[nodiscard]] regfile_ptr useRegFile(glm::ivec2 coord);
regfile_ptr createRegFile(glm::ivec2 coord);
void closeRegFile(glm::ivec2 coord);
/// @brief Compress buffer with extrle
/// @param src source buffer
/// @param srclen length of the source buffer
/// @param len (out argument) length of result buffer
/// @return compressed bytes array
std::unique_ptr<ubyte[]> compress(
const ubyte* src, size_t srclen, size_t& len
);
WorldRegion* getRegion(int x, int z);
WorldRegion* getOrCreateRegion(int x, int z);
/// @brief Decompress buffer with extrle
/// @param src compressed buffer
/// @param srclen length of compressed buffer
/// @param dstlen max expected length of source buffer
/// @return decompressed bytes array
std::unique_ptr<ubyte[]> decompress(
const ubyte* src, size_t srclen, size_t dstlen
);
fs::path getRegionFilePath(int x, int z) const;
std::unique_ptr<ubyte[]> readChunkData(
int x, int y, uint32_t& length, regfile* file
);
void fetchChunks(WorldRegion* region, int x, int y, regfile* file);
ubyte* getData(int x, int z, int layer, uint32_t& size);
regfile_ptr getRegFile(glm::ivec3 coord, bool create = true);
void closeRegFile(glm::ivec3 coord);
regfile_ptr useRegFile(glm::ivec3 coord);
regfile_ptr createRegFile(glm::ivec3 coord);
fs::path getRegionFilename(int x, int y) const;
void writeRegions(int layer);
/// @brief Get chunk data. Read from file if not loaded yet.
/// @param x chunk x coord
/// @param z chunk z coord
/// @param size [out] compressed chunk data length
/// @param size [out] source chunk data length
/// @return nullptr if no saved chunk data found
[[nodiscard]] ubyte* getData(int x, int z, uint32_t& size, uint32_t& srcSize);
/// @brief Write or rewrite region file
/// @param x region X
/// @param z region Z
/// @param layer regions layer
void writeRegion(int x, int y, int layer, WorldRegion* entry);
void writeRegion(int x, int y, WorldRegion* entry);
/// @brief Write all unsaved regions to files
void writeAll();
/// @brief Read chunk data from region file
/// @param x chunk x coord
/// @param z chunk z coord
/// @param size [out] compressed chunk data length
/// @param srcSize [out] source chunk data length
/// @param rfile region file
/// @return nullptr if chunk is not present in region file
[[nodiscard]] static std::unique_ptr<ubyte[]> readChunkData(
int x, int z, uint32_t& size, uint32_t& srcSize, regfile* rfile
);
};
class WorldRegions {
/// @brief World directory
fs::path directory;
RegionsLayer layers[REGION_LAYERS_COUNT] {};
public:
bool generatorTestMode = false;
bool doWriteLights = true;
@@ -184,26 +198,57 @@ public:
/// @param layer regions layer
/// @param data target data
/// @param size data size
/// @param rle compress with ext-RLE
void put(
int x,
int z,
int layer,
RegionLayerIndex layer,
std::unique_ptr<ubyte[]> data,
size_t size,
bool rle
size_t size
);
std::unique_ptr<ubyte[]> getChunk(int x, int z);
/// @brief Get chunk voxels data
/// @param x chunk.x
/// @param z chunk.z
/// @return voxels data buffer or nullptr
std::unique_ptr<ubyte[]> getVoxels(int x, int z);
/// @brief Get cached lights for chunk at x,z
/// @return lights data or nullptr
std::unique_ptr<light_t[]> getLights(int x, int z);
chunk_inventories_map fetchInventories(int x, int z);
ChunkInventoriesMap fetchInventories(int x, int z);
BlocksMetadata getBlocksData(int x, int z);
/// @brief Load saved entities data for chunk
/// @param x chunk.x
/// @param z chunk.z
/// @return map with entities list as "data"
dv::value fetchEntities(int x, int z);
void processRegionVoxels(int x, int z, const regionproc& func);
/// @brief Load, process and save processed region chunks data
/// @param x region X
/// @param z region Z
/// @param layerid regions layer index
/// @param func processing callback
void processRegion(
int x, int z, RegionLayerIndex layerid, const RegionProc& func);
fs::path getRegionsFolder(int layer) const;
void processInventories(int x, int z, const InventoryProc& func);
void write();
void processBlocksData(int x, int z, const BlockDataProc& func);
/// @brief Get regions directory by layer index
/// @param layerid layer index
/// @return directory path
const fs::path& getRegionsFolder(RegionLayerIndex layerid) const;
fs::path getRegionFilePath(RegionLayerIndex layerid, int x, int z) const;
/// @brief Write all region layers
void writeAll();
void deleteRegion(RegionLayerIndex layerid, int x, int z);
/// @brief Extract X and Z from 'X_Z.bin' region file name.
/// @param name source region file name
src/files/compatibility.cpp
+110
View File
@@ -0,0 +1,110 @@
#include "compatibility.hpp"
#include <stdexcept>
#include "constants.hpp"
#include "voxels/voxel.hpp"
#include "coders/compression.hpp"
#include "coders/byte_utils.hpp"
#include "lighting/Lightmap.hpp"
#include "util/data_io.hpp"
static inline size_t VOXELS_DATA_SIZE_V1 = CHUNK_VOL * 4;
static inline size_t VOXELS_DATA_SIZE_V2 = CHUNK_VOL * 4;
static util::Buffer<ubyte> convert_voxels_1to2(const ubyte* buffer, uint32_t size) {
auto data = compression::decompress(
buffer, size, VOXELS_DATA_SIZE_V1, compression::Method::EXTRLE8);
util::Buffer<ubyte> dstBuffer(VOXELS_DATA_SIZE_V2);
auto dst = reinterpret_cast<uint16_t*>(dstBuffer.data());
for (size_t i = 0; i < CHUNK_VOL; i++) {
ubyte bid1 = data[i];
ubyte bid2 = data[CHUNK_VOL + i];
ubyte bst1 = data[CHUNK_VOL * 2 + i];
ubyte bst2 = data[CHUNK_VOL * 3 + i];
dst[i] =
(static_cast<blockid_t>(bid1) << 8) | static_cast<blockid_t>(bid2);
dst[CHUNK_VOL + i] = (
(static_cast<blockstate_t>(bst1) << 8) |
static_cast<blockstate_t>(bst2)
);
}
size_t outLen;
auto compressed = compression::compress(
dstBuffer.data(), VOXELS_DATA_SIZE_V2, outLen, compression::Method::EXTRLE16);
return util::Buffer<ubyte>(std::move(compressed), outLen);
}
util::Buffer<ubyte> compatibility::convert_region_2to3(
const util::Buffer<ubyte>& src, RegionLayerIndex layer
) {
const size_t REGION_CHUNKS = 1024;
const size_t HEADER_SIZE = 10;
const size_t OFFSET_TABLE_SIZE = REGION_CHUNKS * sizeof(uint32_t);
const ubyte COMPRESS_NONE = 0;
const ubyte COMPRESS_EXTRLE8 = 1;
const ubyte COMPRESS_EXTRLE16 = 2;
const ubyte* const ptr = src.data();
ByteBuilder builder;
builder.putCStr(".VOXREG");
builder.put(3);
switch (layer) {
case REGION_LAYER_VOXELS: builder.put(COMPRESS_EXTRLE16); break;
case REGION_LAYER_LIGHTS: builder.put(COMPRESS_EXTRLE8); break;
default: builder.put(COMPRESS_NONE); break;
}
uint32_t offsets[REGION_CHUNKS] {};
size_t chunkIndex = 0;
auto tablePtr = reinterpret_cast<const uint32_t*>(
ptr + src.size() - OFFSET_TABLE_SIZE
);
for (size_t i = 0; i < REGION_CHUNKS; i++) {
uint32_t srcOffset = dataio::be2h(tablePtr[i]);
if (srcOffset == 0) {
continue;
}
uint32_t size = *reinterpret_cast<const uint32_t*>(ptr + srcOffset);
size = dataio::be2h(size);
const ubyte* data = ptr + srcOffset + sizeof(uint32_t);
offsets[i] = builder.size();
switch (layer) {
case REGION_LAYER_VOXELS: {
auto dstdata = convert_voxels_1to2(data, size);
builder.putInt32(dstdata.size());
builder.putInt32(VOXELS_DATA_SIZE_V2);
builder.put(dstdata.data(), dstdata.size());
break;
}
case REGION_LAYER_LIGHTS:
builder.putInt32(size);
builder.putInt32(LIGHTMAP_DATA_LEN);
builder.put(data, size);
break;
case REGION_LAYER_ENTITIES:
case REGION_LAYER_INVENTORIES:
case REGION_LAYER_BLOCKS_DATA: {
builder.putInt32(size);
builder.putInt32(size);
builder.put(data, size);
break;
case REGION_LAYERS_COUNT:
throw std::invalid_argument("invalid enum");
}
}
}
for (size_t i = 0; i < REGION_CHUNKS; i++) {
builder.putInt32(offsets[i]);
}
return util::Buffer<ubyte>(builder.build().data(), builder.size());
}
src/files/compatibility.hpp
+14
View File
@@ -0,0 +1,14 @@
#pragma once
#include "typedefs.hpp"
#include "util/Buffer.hpp"
#include "files/world_regions_fwd.hpp"
namespace compatibility {
/// @brief Convert region file from version 2 to 3
/// @see /doc/specs/region_file_spec.md
/// @param src region file source content
/// @return new region file content
util::Buffer<ubyte> convert_region_2to3(
const util::Buffer<ubyte>& src, RegionLayerIndex layer);
}
src/files/files.cpp
+6
View File
@@ -65,6 +65,12 @@ bool files::read(const fs::path& filename, char* data, size_t size) {
return true;
}
util::Buffer<ubyte> files::read_bytes_buffer(const fs::path& path) {
size_t size;
auto bytes = files::read_bytes(path, size);
return util::Buffer<ubyte>(std::move(bytes), size);
}
std::unique_ptr<ubyte[]> files::read_bytes(
const fs::path& filename, size_t& length
) {
src/files/files.hpp
+2
View File
@@ -8,6 +8,7 @@
#include "typedefs.hpp"
#include "data/dv.hpp"
#include "util/Buffer.hpp"
namespace fs = std::filesystem;
@@ -56,6 +57,7 @@ namespace files {
);
bool read(const fs::path&, char* data, size_t size);
util::Buffer<ubyte> read_bytes_buffer(const fs::path&);
std::unique_ptr<ubyte[]> read_bytes(const fs::path&, size_t& length);
std::vector<ubyte> read_bytes(const fs::path&);
std::string read_string(const fs::path& filename);
src/files/world_regions_fwd.hpp
+13
View File
@@ -0,0 +1,13 @@
#pragma once
#include "typedefs.hpp"
enum RegionLayerIndex : uint {
REGION_LAYER_VOXELS = 0,
REGION_LAYER_LIGHTS,
REGION_LAYER_INVENTORIES,
REGION_LAYER_ENTITIES,
REGION_LAYER_BLOCKS_DATA,
REGION_LAYERS_COUNT
};