contentpack remove feature WIP

This commit is contained in:
MihailRis
2024-02-29 23:47:07 +03:00
parent a1f313bedc
commit 3862dbda66
18 changed files with 541 additions and 433 deletions
+253 -231
View File
@@ -49,91 +49,91 @@ regfile::regfile(fs::path filename) : file(filename) {
}
WorldRegion::WorldRegion() {
chunksData = new ubyte*[REGION_CHUNKS_COUNT]{};
sizes = new uint32_t[REGION_CHUNKS_COUNT]{};
chunksData = new ubyte*[REGION_CHUNKS_COUNT]{};
sizes = new uint32_t[REGION_CHUNKS_COUNT]{};
}
WorldRegion::~WorldRegion() {
for (uint i = 0; i < REGION_CHUNKS_COUNT; i++) {
delete[] chunksData[i];
}
delete[] sizes;
delete[] chunksData;
for (uint i = 0; i < REGION_CHUNKS_COUNT; i++) {
delete[] chunksData[i];
}
delete[] sizes;
delete[] chunksData;
}
void WorldRegion::setUnsaved(bool unsaved) {
this->unsaved = unsaved;
this->unsaved = unsaved;
}
bool WorldRegion::isUnsaved() const {
return unsaved;
return unsaved;
}
ubyte** WorldRegion::getChunks() const {
return chunksData;
return chunksData;
}
uint32_t* WorldRegion::getSizes() const {
return sizes;
return sizes;
}
void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) {
size_t chunk_index = z * REGION_SIZE + x;
delete[] chunksData[chunk_index];
chunksData[chunk_index] = data;
sizes[chunk_index] = size;
size_t chunk_index = z * REGION_SIZE + x;
delete[] chunksData[chunk_index];
chunksData[chunk_index] = data;
sizes[chunk_index] = size;
}
ubyte* WorldRegion::getChunkData(uint x, uint z) {
return chunksData[z * REGION_SIZE + x];
return chunksData[z * REGION_SIZE + x];
}
uint WorldRegion::getChunkDataSize(uint x, uint z) {
return sizes[z * REGION_SIZE + x];
return sizes[z * REGION_SIZE + x];
}
const char* WorldFiles::WORLD_FILE = "world.json";
WorldFiles::WorldFiles(fs::path directory, const DebugSettings& settings)
: directory(directory),
generatorTestMode(settings.generatorTestMode),
doWriteLights(settings.doWriteLights) {
compressionBuffer.reset(new ubyte[CHUNK_DATA_LEN * 2]);
: directory(directory),
generatorTestMode(settings.generatorTestMode),
doWriteLights(settings.doWriteLights) {
compressionBuffer.reset(new ubyte[CHUNK_DATA_LEN * 2]);
}
WorldFiles::~WorldFiles(){
}
WorldRegion* WorldFiles::getRegion(regionsmap& regions, int x, int z) {
auto found = regions.find(glm::ivec2(x, z));
if (found == regions.end())
return nullptr;
return found->second.get();
auto found = regions.find(glm::ivec2(x, z));
if (found == regions.end())
return nullptr;
return found->second.get();
}
WorldRegion* WorldFiles::getOrCreateRegion(regionsmap& regions, int x, int z) {
WorldRegion* region = getRegion(regions, x, z);
if (region == nullptr) {
region = new WorldRegion();
regions[glm::ivec2(x, z)].reset(region);
}
return region;
WorldRegion* region = getRegion(regions, x, z);
if (region == nullptr) {
region = new WorldRegion();
regions[glm::ivec2(x, z)].reset(region);
}
return region;
}
ubyte* WorldFiles::compress(const ubyte* src, size_t srclen, size_t& len) {
ubyte* buffer = this->compressionBuffer.get();
len = extrle::encode(src, srclen, buffer);
ubyte* data = new ubyte[len];
for (size_t i = 0; i < len; i++) {
data[i] = buffer[i];
}
return data;
len = extrle::encode(src, srclen, buffer);
ubyte* data = new ubyte[len];
for (size_t i = 0; i < len; i++) {
data[i] = buffer[i];
}
return data;
}
ubyte* WorldFiles::decompress(const ubyte* src, size_t srclen, size_t dstlen) {
ubyte* decompressed = new ubyte[dstlen];
extrle::decode(src, srclen, decompressed);
return decompressed;
ubyte* decompressed = new ubyte[dstlen];
extrle::decode(src, srclen, decompressed);
return decompressed;
}
int WorldFiles::getVoxelRegionVersion(int x, int z) {
@@ -168,49 +168,49 @@ int WorldFiles::getVoxelRegionsVersion() {
*/
void WorldFiles::put(int x, int z, const ubyte* voxelData) {
int regionX = floordiv(x, REGION_SIZE);
int regionZ = floordiv(z, REGION_SIZE);
int localX = x - (regionX * REGION_SIZE);
int localZ = z - (regionZ * REGION_SIZE);
int regionZ = floordiv(z, REGION_SIZE);
int localX = x - (regionX * REGION_SIZE);
int localZ = z - (regionZ * REGION_SIZE);
/* Writing Voxels */ {
WorldRegion* region = getOrCreateRegion(regions, regionX, regionZ);
region->setUnsaved(true);
size_t compressedSize;
ubyte* data = compress(voxelData, CHUNK_DATA_LEN, compressedSize);
region->put(localX, localZ, data, compressedSize);
}
/* Writing Voxels */ {
WorldRegion* region = getOrCreateRegion(regions, regionX, regionZ);
region->setUnsaved(true);
size_t compressedSize;
ubyte* data = compress(voxelData, CHUNK_DATA_LEN, compressedSize);
region->put(localX, localZ, data, compressedSize);
}
}
/*
* Store chunk (voxels and lights) in region (existing or new)
*/
void WorldFiles::put(Chunk* chunk){
assert(chunk != nullptr);
assert(chunk != nullptr);
int regionX = floordiv(chunk->x, REGION_SIZE);
int regionZ = floordiv(chunk->z, REGION_SIZE);
int localX = chunk->x - (regionX * REGION_SIZE);
int localZ = chunk->z - (regionZ * REGION_SIZE);
int regionX = floordiv(chunk->x, REGION_SIZE);
int regionZ = floordiv(chunk->z, REGION_SIZE);
int localX = chunk->x - (regionX * REGION_SIZE);
int localZ = chunk->z - (regionZ * REGION_SIZE);
/* Writing voxels */ {
/* Writing voxels */ {
size_t compressedSize;
std::unique_ptr<ubyte[]> chunk_data (chunk->encode());
ubyte* data = compress(chunk_data.get(), CHUNK_DATA_LEN, compressedSize);
ubyte* data = compress(chunk_data.get(), CHUNK_DATA_LEN, compressedSize);
WorldRegion* region = getOrCreateRegion(regions, regionX, regionZ);
region->setUnsaved(true);
region->put(localX, localZ, data, compressedSize);
}
WorldRegion* region = getOrCreateRegion(regions, regionX, regionZ);
region->setUnsaved(true);
region->put(localX, localZ, data, compressedSize);
}
/* Writing lights cache */
if (doWriteLights && chunk->isLighted()) {
if (doWriteLights && chunk->isLighted()) {
size_t compressedSize;
std::unique_ptr<ubyte[]> light_data (chunk->lightmap.encode());
ubyte* data = compress(light_data.get(), LIGHTMAP_DATA_LEN, compressedSize);
ubyte* data = compress(light_data.get(), LIGHTMAP_DATA_LEN, compressedSize);
WorldRegion* region = getOrCreateRegion(lights, regionX, regionZ);
region->setUnsaved(true);
region->put(localX, localZ, data, compressedSize);
}
WorldRegion* region = getOrCreateRegion(lights, regionX, regionZ);
region->setUnsaved(true);
region->put(localX, localZ, data, compressedSize);
}
/* Writing block inventories */
if (!chunk->inventories.empty()){
auto& inventories = chunk->inventories;
@@ -237,19 +237,19 @@ void WorldFiles::put(Chunk* chunk){
}
fs::path WorldFiles::getRegionsFolder() const {
return directory/fs::path("regions");
return directory/fs::path("regions");
}
fs::path WorldFiles::getLightsFolder() const {
return directory/fs::path("lights");
return directory/fs::path("lights");
}
fs::path WorldFiles::getInventoriesFolder() const {
return directory/fs::path("inventories");
return directory/fs::path("inventories");
}
fs::path WorldFiles::getRegionFilename(int x, int z) const {
return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin");
return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin");
}
/*
@@ -275,78 +275,78 @@ bool WorldFiles::parseRegionFilename(const std::string& name, int& x, int& z) {
}
fs::path WorldFiles::getPlayerFile() const {
return directory/fs::path("player.json");
return directory/fs::path("player.json");
}
fs::path WorldFiles::getWorldFile() const {
return directory/fs::path(WORLD_FILE);
return directory/fs::path(WORLD_FILE);
}
fs::path WorldFiles::getIndicesFile() const {
return directory/fs::path("indices.json");
return directory/fs::path("indices.json");
}
fs::path WorldFiles::getPacksFile() const {
return directory/fs::path("packs.list");
return directory/fs::path("packs.list");
}
ubyte* WorldFiles::getChunk(int x, int z){
return getData(regions, getRegionsFolder(), x, z, REGION_LAYER_VOXELS, true);
return getData(regions, getRegionsFolder(), x, z, REGION_LAYER_VOXELS, true);
}
/* Get cached lights for chunk at x,z
* @return lights data or nullptr */
light_t* WorldFiles::getLights(int x, int z) {
std::unique_ptr<ubyte[]> data (getData(lights, getLightsFolder(), x, z, REGION_LAYER_LIGHTS, true));
if (data == nullptr)
return nullptr;
return Lightmap::decode(data.get());
std::unique_ptr<ubyte[]> data (getData(lights, getLightsFolder(), x, z, REGION_LAYER_LIGHTS, true));
if (data == nullptr)
return nullptr;
return Lightmap::decode(data.get());
}
chunk_inventories_map WorldFiles::fetchInventories(int x, int z) {
chunk_inventories_map inventories;
const ubyte* data = getData(storages, getInventoriesFolder(), x, z, REGION_LAYER_INVENTORIES, false);
if (data == nullptr)
return inventories;
ByteReader reader(data, BUFFER_SIZE_UNKNOWN);
int count = reader.getInt32();
for (int i = 0; i < count; i++) {
uint index = reader.getInt32();
uint size = reader.getInt32();
auto map = json::from_binary(reader.pointer(), size);
reader.skip(size);
auto inv = std::make_shared<Inventory>(0, 0);
inv->deserialize(map.get());
inventories[index] = inv;
}
return inventories;
chunk_inventories_map inventories;
const ubyte* data = getData(storages, getInventoriesFolder(), x, z, REGION_LAYER_INVENTORIES, false);
if (data == nullptr)
return inventories;
ByteReader reader(data, BUFFER_SIZE_UNKNOWN);
int count = reader.getInt32();
for (int i = 0; i < count; i++) {
uint index = reader.getInt32();
uint size = reader.getInt32();
auto map = json::from_binary(reader.pointer(), size);
reader.skip(size);
auto inv = std::make_shared<Inventory>(0, 0);
inv->deserialize(map.get());
inventories[index] = inv;
}
return inventories;
}
ubyte* WorldFiles::getData(regionsmap& regions, const fs::path& folder,
int x, int z, int layer, bool compression) {
int regionX = floordiv(x, REGION_SIZE);
int regionZ = floordiv(z, REGION_SIZE);
int regionX = floordiv(x, REGION_SIZE);
int regionZ = floordiv(z, REGION_SIZE);
int localX = x - (regionX * REGION_SIZE);
int localZ = z - (regionZ * REGION_SIZE);
int localX = x - (regionX * REGION_SIZE);
int localZ = z - (regionZ * REGION_SIZE);
WorldRegion* region = getOrCreateRegion(regions, regionX, regionZ);
ubyte* data = region->getChunkData(localX, localZ);
if (data == nullptr) {
uint32_t size;
data = readChunkData(x, z, size, folder, layer);
if (data != nullptr) {
region->put(localX, localZ, data, size);
}
}
if (data != nullptr) {
WorldRegion* region = getOrCreateRegion(regions, regionX, regionZ);
ubyte* data = region->getChunkData(localX, localZ);
if (data == nullptr) {
uint32_t size;
data = readChunkData(x, z, size, folder, layer);
if (data != nullptr) {
region->put(localX, localZ, data, size);
}
}
if (data != nullptr) {
size_t size = region->getChunkDataSize(localX, localZ);
if (compression) {
return decompress(data, size, CHUNK_DATA_LEN);
}
return data;
}
return nullptr;
if (compression) {
return decompress(data, size, CHUNK_DATA_LEN);
}
return data;
}
return nullptr;
}
@@ -360,7 +360,7 @@ regfile* WorldFiles::getRegFile(glm::ivec3 coord, const fs::path& folder) {
auto iter = std::next(openRegFiles.begin(), rand() % openRegFiles.size());
openRegFiles.erase(iter);
}
fs::path filename = folder / getRegionFilename(coord[0], coord[1]);
fs::path filename = folder / getRegionFilename(coord[0], coord[1]);
if (!fs::is_regular_file(filename)) {
return nullptr;
}
@@ -373,14 +373,14 @@ ubyte* WorldFiles::readChunkData(int x,
uint32_t& length,
fs::path folder,
int layer){
if (generatorTestMode)
return nullptr;
int regionX = floordiv(x, REGION_SIZE);
int regionZ = floordiv(z, REGION_SIZE);
int localX = x - (regionX * REGION_SIZE);
int localZ = z - (regionZ * REGION_SIZE);
int chunkIndex = localZ * REGION_SIZE + localX;
if (generatorTestMode)
return nullptr;
int regionX = floordiv(x, REGION_SIZE);
int regionZ = floordiv(z, REGION_SIZE);
int localX = x - (regionX * REGION_SIZE);
int localZ = z - (regionZ * REGION_SIZE);
int chunkIndex = localZ * REGION_SIZE + localX;
glm::ivec3 coord(regionX, regionZ, layer);
regfile* rfile = WorldFiles::getRegFile(coord, folder);
@@ -389,24 +389,24 @@ ubyte* WorldFiles::readChunkData(int x,
}
files::rafile& file = rfile->file;
size_t file_size = file.length();
size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
size_t file_size = file.length();
size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
uint32_t offset;
file.seekg(table_offset + chunkIndex * 4);
file.read((char*)(&offset), 4);
offset = dataio::read_int32_big((const ubyte*)(&offset), 0);
uint32_t offset;
file.seekg(table_offset + chunkIndex * 4);
file.read((char*)(&offset), 4);
offset = dataio::read_int32_big((const ubyte*)(&offset), 0);
if (offset == 0){
return nullptr;
}
if (offset == 0){
return nullptr;
}
file.seekg(offset);
file.read((char*)(&offset), 4);
length = dataio::read_int32_big((const ubyte*)(&offset), 0);
ubyte* data = new ubyte[length]{};
file.read((char*)data, length);
return data;
file.seekg(offset);
file.read((char*)(&offset), 4);
length = dataio::read_int32_big((const ubyte*)(&offset), 0);
ubyte* data = new ubyte[length]{};
file.read((char*)data, length);
return data;
}
/* Read missing chunks data (null pointers) from region file
@@ -415,7 +415,7 @@ ubyte* WorldFiles::readChunkData(int x,
*/
void WorldFiles::fetchChunks(WorldRegion* region, int x, int z, fs::path folder, int layer) {
ubyte** chunks = region->getChunks();
uint32_t* sizes = region->getSizes();
uint32_t* sizes = region->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE;
@@ -441,70 +441,70 @@ void WorldFiles::writeRegion(int x, int z, WorldRegion* entry, fs::path folder,
openRegFiles.erase(regcoord);
}
char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC;
header[8] = REGION_FORMAT_VERSION;
header[9] = 0; // flags
std::ofstream file(filename, std::ios::out | std::ios::binary);
file.write(header, REGION_HEADER_SIZE);
char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC;
header[8] = REGION_FORMAT_VERSION;
header[9] = 0; // flags
std::ofstream file(filename, std::ios::out | std::ios::binary);
file.write(header, REGION_HEADER_SIZE);
size_t offset = REGION_HEADER_SIZE;
char intbuf[4]{};
uint offsets[REGION_CHUNKS_COUNT]{};
ubyte** region = entry->getChunks();
uint32_t* sizes = entry->getSizes();
size_t offset = REGION_HEADER_SIZE;
char intbuf[4]{};
uint offsets[REGION_CHUNKS_COUNT]{};
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
ubyte* chunk = region[i];
if (chunk == nullptr){
offsets[i] = 0;
} else {
offsets[i] = offset;
ubyte** region = entry->getChunks();
uint32_t* sizes = entry->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
ubyte* chunk = region[i];
if (chunk == nullptr){
offsets[i] = 0;
} else {
offsets[i] = offset;
size_t compressedSize = sizes[i];
dataio::write_int32_big(compressedSize, (ubyte*)intbuf, 0);
offset += 4 + compressedSize;
size_t compressedSize = sizes[i];
dataio::write_int32_big(compressedSize, (ubyte*)intbuf, 0);
offset += 4 + compressedSize;
file.write(intbuf, 4);
file.write((const char*)chunk, compressedSize);
}
}
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
dataio::write_int32_big(offsets[i], (ubyte*)intbuf, 0);
file.write(intbuf, 4);
}
file.write(intbuf, 4);
file.write((const char*)chunk, compressedSize);
}
}
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
dataio::write_int32_big(offsets[i], (ubyte*)intbuf, 0);
file.write(intbuf, 4);
}
}
void WorldFiles::writeRegions(regionsmap& regions, const fs::path& folder, int layer) {
for (auto& it : regions){
WorldRegion* region = it.second.get();
if (region->getChunks() == nullptr || !region->isUnsaved())
continue;
glm::ivec2 key = it.first;
writeRegion(key[0], key[1], region, folder, layer);
}
for (auto& it : regions){
WorldRegion* region = it.second.get();
if (region->getChunks() == nullptr || !region->isUnsaved())
continue;
glm::ivec2 key = it.first;
writeRegion(key[0], key[1], region, folder, layer);
}
}
void WorldFiles::write(const World* world, const Content* content) {
fs::path regionsFolder = getRegionsFolder();
fs::path lightsFolder = getLightsFolder();
fs::path regionsFolder = getRegionsFolder();
fs::path lightsFolder = getLightsFolder();
fs::path inventoriesFolder = getInventoriesFolder();
fs::create_directories(regionsFolder);
fs::create_directories(regionsFolder);
fs::create_directories(inventoriesFolder);
fs::create_directories(lightsFolder);
fs::create_directories(lightsFolder);
if (world) {
writeWorldInfo(world);
writePacks(world);
}
if (generatorTestMode) {
return;
writeWorldInfo(world);
writePacks(world);
}
writeIndices(content->getIndices());
writeRegions(regions, regionsFolder, REGION_LAYER_VOXELS);
writeRegions(lights, lightsFolder, REGION_LAYER_LIGHTS);
if (generatorTestMode) {
return;
}
writeIndices(content->getIndices());
writeRegions(regions, regionsFolder, REGION_LAYER_VOXELS);
writeRegions(lights, lightsFolder, REGION_LAYER_LIGHTS);
writeRegions(storages, inventoriesFolder, REGION_LAYER_INVENTORIES);
}
@@ -514,64 +514,64 @@ void WorldFiles::writePacks(const World* world) {
return;
}
const auto& packs = world->getPacks();
std::stringstream ss;
ss << "# autogenerated; do not modify\n";
for (const auto& pack : packs) {
ss << pack.id << "\n";
}
files::write_string(packsFile, ss.str());
const auto& packs = world->getPacks();
std::stringstream ss;
ss << "# autogenerated; do not modify\n";
for (const auto& pack : packs) {
ss << pack.id << "\n";
}
files::write_string(packsFile, ss.str());
}
void WorldFiles::writeIndices(const ContentIndices* indices) {
dynamic::Map root;
dynamic::Map root;
uint count;
auto& blocks = root.putList("blocks");
count = indices->countBlockDefs();
for (uint i = 0; i < count; i++) {
const Block* def = indices->getBlockDef(i);
blocks.put(def->name);
}
auto& blocks = root.putList("blocks");
count = indices->countBlockDefs();
for (uint i = 0; i < count; i++) {
const Block* def = indices->getBlockDef(i);
blocks.put(def->name);
}
auto& items = root.putList("items");
count = indices->countItemDefs();
for (uint i = 0; i < count; i++) {
const ItemDef* def = indices->getItemDef(i);
items.put(def->name);
}
count = indices->countItemDefs();
for (uint i = 0; i < count; i++) {
const ItemDef* def = indices->getItemDef(i);
items.put(def->name);
}
files::write_json(getIndicesFile(), &root);
files::write_json(getIndicesFile(), &root);
}
void WorldFiles::writeWorldInfo(const World* world) {
files::write_json(getWorldFile(), world->serialize().get());
files::write_json(getWorldFile(), world->serialize().get());
}
bool WorldFiles::readWorldInfo(World* world) {
fs::path file = getWorldFile();
if (!fs::is_regular_file(file)) {
std::cerr << "warning: world.json does not exists" << std::endl;
return false;
}
fs::path file = getWorldFile();
if (!fs::is_regular_file(file)) {
std::cerr << "warning: world.json does not exists" << std::endl;
return false;
}
auto root = files::read_json(file);
auto root = files::read_json(file);
world->deserialize(root.get());
return true;
return true;
}
void WorldFiles::writePlayer(std::shared_ptr<Player> player) {
files::write_json(getPlayerFile(), player->serialize().release());
files::write_json(getPlayerFile(), player->serialize().release());
}
bool WorldFiles::readPlayer(std::shared_ptr<Player> player) {
fs::path file = getPlayerFile();
if (!fs::is_regular_file(file)) {
std::cerr << "warning: player.json does not exists" << std::endl;
return false;
}
fs::path file = getPlayerFile();
if (!fs::is_regular_file(file)) {
std::cerr << "warning: player.json does not exists" << std::endl;
return false;
}
player->deserialize(files::read_json(file).get());
return true;
player->deserialize(files::read_json(file).get());
return true;
}
void WorldFiles::addPack(const World* world, const std::string& id) {
@@ -585,10 +585,32 @@ void WorldFiles::addPack(const World* world, const std::string& id) {
auto packs = files::read_list(file);
packs.push_back(id);
std::stringstream ss;
ss << "# autogenerated; do not modify\n";
for (const auto& pack : packs) {
ss << pack << "\n";
}
files::write_string(file, ss.str());
std::stringstream ss;
ss << "# autogenerated; do not modify\n";
for (const auto& pack : packs) {
ss << pack << "\n";
}
files::write_string(file, ss.str());
}
void WorldFiles::removePack(const World* world, const std::string& id) {
fs::path file = getPacksFile();
if (!fs::is_regular_file(file)) {
if (!fs::is_directory(directory)) {
fs::create_directories(directory);
}
writePacks(world);
}
auto packs = files::read_list(file);
auto found = std::find(packs.begin(), packs.end(), id);
if (found != packs.end()) {
packs.erase(found);
}
std::stringstream ss;
ss << "# autogenerated; do not modify\n";
for (const auto& pack : packs) {
ss << pack << "\n";
}
files::write_string(file, ss.str());
}