binary json format introduced + specification

src/coders/binary_json_spec.md

@@ -0,0 +1,56 @@
+# Binary JSON Format Specification
+
+Format version: 1.0
+
+This binary data format is developed for use as binary version of JSON in [VoxelEngine-Cpp](https://github.com/MihailRis/VoxelEngine-Cpp) and not compatible with [BSON](https://bsonspec.org/spec.html) due to elements/entries syntax and type codes differences
+
+## Basic types
+
+byteorder: little-endian
+| Name    | Size    | Definition              |
+| ------- | ------- | ----------------------- |
+| byte    | 1 byte  | 8 bit unsigned integer  |
+| int16   | 2 bytes | 16 bit signed integer   |
+| int32   | 4 bytes | 32 bit signed integer   |
+| int64   | 8 bytes | 64 bit unsigned integer |
+| uint32  | 4 bytes | 32 bit unsigned integer |
+| float64 | 8 bytes | 64 bit floating point   |
+
+## Syntax (RFC 5234)
+
+```bnf
+file      = %x01 document / cdocument   file content
+document  = uint32 (*entry) %x00        uint32 stores bytes number 
+                                        of the encoded document 
+                                        including the uint32 size
+entry     = cstring value
+value     = %x01 document
+          / %x02 (*value) %x00          list of values
+          / %x03 byte                   8 bit integer
+          / %x04 int16                  16 bit integer
+          / %x05 int32                  32 bit integer
+          / %x06 int64                  64 bit integer
+          / %x07 float64                number
+          / %x08 string                 utf-8 encoded string
+          / %x09 uint32 (*byte)         bytes array
+          / %x0A                        boolean 'false'
+          / %x0B                        boolean 'true'
+          / %x0C                        null value
+cdocument = %x1F %x8B (16*byte)         gzip-compressed document
+cstring   = (*%x01-FF) %x00
+string    = uint32 (*byte)              uint32 stores number of the 
+                                        encoded string bytes
+
+float64   = 8byte
+int64     = 8byte
+uint32    = 4byte
+int32     = 4byte
+int16     = 2byte
+byte      = %x00-FF
+```
+
+## VoxelEngine format support
+
+Current implementation does not support types: bytes array, null, compressed document. 
+
+All unsupported types will be implemented in future.

src/coders/byte_utils.cpp

@@ -0,0 +1,212 @@
+#include "byte_utils.h"
+
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+
+void ByteBuilder::put(ubyte b) {
+    buffer.push_back(b);
+}
+
+void ByteBuilder::putCStr(const char* str) {
+    size_t size = strlen(str)+1;
+    buffer.reserve(buffer.size() + size);
+    for (size_t i = 0; i < size; i++) {
+        buffer.push_back(str[i]);
+    }
+}
+
+void ByteBuilder::put(const std::string& s) {
+    size_t len = s.length();
+    putInt32(len);
+    put((const ubyte*)s.data(), len);
+}
+
+void ByteBuilder::put(const ubyte* arr, size_t size) {
+    buffer.reserve(buffer.size() + size);
+    for (size_t i = 0; i < size; i++) {
+        buffer.push_back(arr[i]);
+    }
+}
+
+void ByteBuilder::putInt16(int16_t val) {
+    buffer.push_back((char) (val >> 0 & 255));
+    buffer.push_back((char) (val >> 8 & 255));
+}
+
+void ByteBuilder::putInt32(int32_t val) {
+    buffer.reserve(buffer.size() + 4);
+    buffer.push_back((char) (val >> 0 & 255));
+    buffer.push_back((char) (val >> 8 & 255));
+    buffer.push_back((char) (val >> 16 & 255));
+    buffer.push_back((char) (val >> 24 & 255));
+}
+
+void ByteBuilder::putInt64(int64_t val) {
+    buffer.reserve(buffer.size() + 8);
+    buffer.push_back((char) (val >> 0 & 255));
+    buffer.push_back((char) (val >> 8 & 255));
+    buffer.push_back((char) (val >> 16 & 255));
+    buffer.push_back((char) (val >> 24 & 255));
+    buffer.push_back((char) (val >> 32 & 255));
+    buffer.push_back((char) (val >> 40 & 255));
+    buffer.push_back((char) (val >> 48 & 255));
+    buffer.push_back((char) (val >> 56 & 255));
+}
+
+void ByteBuilder::putFloat32(float val) {
+    union {
+        int32_t vali32;
+        float valfloat;
+    } value;
+    value.valfloat = val;
+    putInt32(value.vali32);
+}
+
+void ByteBuilder::putFloat64(double val) {
+    union {
+        int64_t vali64;
+        double valfloat;
+    } value;
+    value.valfloat = val;
+    putInt64(value.vali64);
+}
+
+void ByteBuilder::set(size_t position, ubyte val) {
+    buffer[position] = val;
+}
+
+void ByteBuilder::setInt16(size_t position, int16_t val) {
+    buffer[position++] = val >> 0 & 255;
+    buffer[position] = val >> 8 & 255;
+}
+
+void ByteBuilder::setInt32(size_t position, int32_t val) {
+    buffer[position++] = val >> 0 & 255;
+    buffer[position++] = val >> 8 & 255;
+    buffer[position++] = val >> 16 & 255;
+    buffer[position] = val >> 24 & 255;
+}
+
+void ByteBuilder::setInt64(size_t position, int64_t val) {
+    buffer[position++] = val >> 0 & 255;
+    buffer[position++] = val >> 8 & 255;
+    buffer[position++] = val >> 16 & 255;
+    buffer[position++] = val >> 24 & 255;
+
+    buffer[position++] = val >> 32 & 255;
+    buffer[position++] = val >> 40 & 255;
+    buffer[position++] = val >> 48 & 255;
+    buffer[position] = val >> 56 & 255;
+}
+
+std::vector<ubyte> ByteBuilder::build() {
+    return buffer;
+}
+
+ByteReader::ByteReader(const ubyte* data, size_t size)
+    : data(data), size(size), pos(0) {
+}
+
+ByteReader::ByteReader(const ubyte* data)
+    : data(data), size(4), pos(0) {
+    size = getInt32();
+}
+
+void ByteReader::checkMagic(const char* data, size_t size) {
+    if (pos + size >= this->size) {
+        throw std::runtime_error("invalid magic number");
+    }
+    for (size_t i = 0; i < size; i++) {
+        if (this->data[pos + i] != (ubyte)data[i]){
+            throw std::runtime_error("invalid magic number");
+        }
+    }
+    pos += size;
+}
+
+ubyte ByteReader::get() {
+    if (pos == size) {
+        throw std::underflow_error("buffer underflow");
+    }
+    return data[pos++];
+}
+
+ubyte ByteReader::peek() {
+    if (pos == size) {
+        throw std::underflow_error("buffer underflow");
+    }
+    return data[pos]; 
+}
+
+int16_t ByteReader::getInt16() {
+    if (pos+2 > size) {
+        throw std::underflow_error("unexpected end");
+    }
+    pos += 2;
+    return (data[pos - 1] << 8) | 
+           (data[pos - 2]);
+}
+
+int32_t ByteReader::getInt32() {
+    if (pos+4 > size) {
+        throw std::underflow_error("unexpected end");
+    }
+    pos += 4;
+    return (data[pos - 1] << 24) | 
+           (data[pos - 2] << 16) | 
+           (data[pos - 3] << 8) | 
+           (data[pos - 4]);
+}
+
+int64_t ByteReader::getInt64() {
+    if (pos+8 > size) {
+        throw std::underflow_error("unexpected end");
+    }
+    pos += 8;
+    return ((int64_t)data[pos - 1] << 56) | 
+           ((int64_t)data[pos - 2] << 48) | 
+           ((int64_t)data[pos - 3] << 40) | 
+           ((int64_t)data[pos - 4] << 32) | 
+           ((int64_t)data[pos - 5] << 24) | 
+           ((int64_t)data[pos - 6] << 16) | 
+           ((int64_t)data[pos - 7] << 8) | 
+           ((int64_t)data[pos - 8]);
+}
+
+float ByteReader::getFloat32() {
+    union {
+        int32_t vali32;
+        float valfloat;
+    } value;
+    value.vali32 = getInt32();
+    return value.valfloat;
+}
+
+double ByteReader::getFloat64() {
+    union {
+        int64_t vali64;
+        double valfloat;
+    } value;
+    value.vali64 = getInt64();
+    return value.valfloat;
+}
+
+const char* ByteReader::getCString() {
+    const char* cstr = (const char*)(data+pos);
+    pos += strlen(cstr) + 1;
+    return cstr;
+}
+
+std::string ByteReader::getString() {
+    uint32_t length = (uint32_t)getInt32();
+    if (pos+length > size) {
+        throw std::underflow_error("unexpected end");
+    }
+    pos += length;
+    return std::string((const char*)(data+pos-length), length);
+}
+
+bool ByteReader::hasNext() const {
+    return pos < size;
+}

src/coders/byte_utils.h

@@ -0,0 +1,76 @@
+#ifndef CODERS_BYTE_UTILS_H_
+#define CODERS_BYTE_UTILS_H_
+
+#include <string>
+#include <vector>
+#include "../typedefs.h"
+
+/* byteorder: little-endian */
+class ByteBuilder {
+    std::vector<ubyte> buffer;
+public:
+    /* Write one byte (8 bit unsigned integer) */
+    void put(ubyte b);
+    /* Write c-string (bytes array terminated with '\00') */
+    void putCStr(const char* str);
+    /* Write signed 16 bit integer */
+    void putInt16(int16_t val);
+    /* Write signed 32 bit integer */
+    void putInt32(int32_t val);
+    /* Write signed 64 bit integer */
+    void putInt64(int64_t val);
+    /* Write 32 bit floating-point number */
+    void putFloat32(float val);
+    /* Write 64 bit floating-point number */
+    void putFloat64(double val);   
+    
+    /* Write string (uint32 length + bytes) */
+    void put(const std::string& s);
+    /* Write sequence of bytes without any header */
+    void put(const ubyte* arr, size_t size);
+
+    void set(size_t position, ubyte val);
+    void setInt16(size_t position, int16_t val);
+    void setInt32(size_t position, int32_t val);
+    void setInt64(size_t position, int64_t val);
+
+    inline size_t size() const {
+        return buffer.size();
+    }
+    inline const ubyte* data() const {
+        return buffer.data();
+    }
+
+    std::vector<ubyte> build();
+};
+
+/* byteorder: little-endian */
+class ByteReader {
+    const ubyte* data;
+    size_t size;
+    size_t pos;
+public:
+    ByteReader(const ubyte* data, size_t size);
+    ByteReader(const ubyte* data);
+
+    void checkMagic(const char* data, size_t size);
+    /* Read one byte (unsigned 8 bit integer) */
+    ubyte get();
+    /* Read one byte (unsigned 8 bit integer) without pointer move */
+    ubyte peek();
+    /* Read signed 16 bit integer */
+    int16_t getInt16();
+    /* Read signed 32 bit integer */
+    int32_t getInt32();
+    /* Read signed 64 bit integer */
+    int64_t getInt64();
+    /* Read 32 bit floating-point number */
+    float getFloat32();
+    /* Read 64 bit floating-point number */
+    double getFloat64();
+    const char* getCString();
+    std::string getString();
+    bool hasNext() const;
+};
+
+#endif // CODERS_BYTE_UTILS_H_

src/coders/commons.cpp

@@ -1,10 +1,9 @@
 #include "commons.h"
 
 #include <sstream>
+#include <stdexcept>
 #include <math.h>
 
-using std::string;
-
 inline int char2int(int c) {
     if (c >= '0' && c <= '9') {
         return c - '0';
@@ -26,23 +25,24 @@ inline double power(double base, int64_t power) {
     return result;
 }
 
-parsing_error::parsing_error(string message, 
+parsing_error::parsing_error(
-                string filename, 
+    std::string message, 
-                string source, 
+    std::string filename, 
-                uint pos, 
+    std::string source, 
-                uint line, 
+    uint pos, 
-                uint linestart)
+    uint line, 
+    uint linestart)
     : std::runtime_error(message), filename(filename), source(source), 
       pos(pos), line(line), linestart(linestart) {
 }
 
-string parsing_error::errorLog() const {
+std::string parsing_error::errorLog() const {
     std::stringstream ss;
     uint linepos = pos - linestart;
     ss << "parsing error in file '" << filename;
     ss << "' at " << (line+1) << ":" << linepos << ": " << this->what() << "\n";
     size_t end = source.find("\n", linestart);
-    if (end == string::npos) {
+    if (end == std::string::npos) {
         end = source.length();
     }
     ss << source.substr(linestart, end-linestart) << "\n";
@@ -54,7 +54,7 @@ string parsing_error::errorLog() const {
 
 }
 
-string escape_string(string s) {
+std::string escape_string(std::string s) {
     std::stringstream ss;
     ss << '"';
     for (char c : s) {
@@ -124,7 +124,7 @@ char BasicParser::nextChar() {
 void BasicParser::expect(char expected) {
     char c = peek();
     if (c != expected) {
-        throw error("'"+string({expected})+"' expected");
+        throw error("'"+std::string({expected})+"' expected");
     }
     pos++;
 }
@@ -153,7 +153,7 @@ char BasicParser::peek() {
     return source[pos];
 }
 
-string BasicParser::parseName() {
+std::string BasicParser::parseName() {
     char c = peek();
     if (!is_identifier_start(c)) {
         if (c == '"') {
@@ -262,7 +262,7 @@ bool BasicParser::parseNumber(int sign, number_u& out) {
     return true;
 }
 
-string BasicParser::parseString(char quote, bool closeRequired) {
+std::string BasicParser::parseString(char quote, bool closeRequired) {
     std::stringstream ss;
     while (hasNext()) {
         char c = source[pos];
@@ -290,7 +290,8 @@ string BasicParser::parseString(char quote, bool closeRequired) {
                 case '/': ss << '/'; break;
                 case '\n': pos++; continue;
                 default:
-                    throw error("'\\" + string({c}) + "' is an illegal escape");
+                    throw error("'\\" + std::string({c}) + 
+                                "' is an illegal escape");
             }
             continue;
         }

src/coders/json.cpp

@@ -6,17 +6,28 @@
 #include <memory>
 
 #include "commons.h"
+#include "byte_utils.h"
 
 using namespace json;
 
-using std::string;
+const int BJSON_END = 0x0;
-using std::vector;
+const int BJSON_TYPE_DOCUMENT = 0x1;
-using std::unique_ptr;
+const int BJSON_TYPE_LIST = 0x2;
-using std::unordered_map;
+const int BJSON_TYPE_BYTE = 0x3;
-using std::stringstream;
+const int BJSON_TYPE_INT16 = 0x4;
-using std::make_pair;
+const int BJSON_TYPE_INT32 = 0x5;
+const int BJSON_TYPE_INT64 = 0x6;
+const int BJSON_TYPE_NUMBER = 0x7;
+const int BJSON_TYPE_STRING = 0x8;
+const int BJSON_TYPE_BYTES = 0x9;
+const int BJSON_TYPE_FALSE = 0xA;
+const int BJSON_TYPE_TRUE = 0xB;
+const int BJSON_TYPE_NULL = 0xC;
+const int BJSON_TYPE_CDOCUMENT = 0x1F;
 
-inline void newline(stringstream& ss, bool nice, uint indent, const string indentstr) {
+inline void newline(std::stringstream& ss, 
+                    bool nice, uint indent, 
+                    const std::string& indentstr) {
     if (nice) {
         ss << "\n";
         for (uint i = 0; i < indent; i++) {
@@ -27,29 +38,28 @@ inline void newline(stringstream& ss, bool nice, uint indent, const string inden
     }
 }
 
-void stringify(Value* value, 
+void stringify(const Value* value, 
-               stringstream& ss, 
+               std::stringstream& ss, 
                int indent, 
-               string indentstr, 
+               const std::string& indentstr, 
                bool nice);
 
-void stringifyObj(JObject* obj, 
+void stringifyObj(const JObject* obj, 
-               stringstream& ss, 
+               std::stringstream& ss, 
                int indent, 
-               string indentstr, 
+               const std::string& indentstr, 
                bool nice);
 
-#include <iostream>
+void stringify(const Value* value, 
-void stringify(Value* value, 
+               std::stringstream& ss, 
-               stringstream& ss, 
                int indent, 
-               string indentstr, 
+               const std::string& indentstr, 
                bool nice) {
     if (value->type == valtype::object) {
         stringifyObj(value->value.obj, ss, indent, indentstr, nice);
     }
     else if (value->type == valtype::array) {
-        vector<Value*>& list = value->value.arr->values;
+        std::vector<Value*>& list = value->value.arr->values;
         if (list.empty()) {
             ss << "[]";
             return;
@@ -81,7 +91,11 @@ void stringify(Value* value,
     }
 }
 
-void stringifyObj(JObject* obj, stringstream& ss, int indent, string indentstr, bool nice) {
+void stringifyObj(const JObject* obj, 
+                  std::stringstream& ss, 
+                  int indent, 
+                  const std::string& indentstr, 
+                  bool nice) {
     if (obj->map.empty()) {
         ss << "{}";
         return;
@@ -107,12 +121,165 @@ void stringifyObj(JObject* obj, stringstream& ss, int indent, string indentstr,
     ss << '}';
 }
 
-string json::stringify(JObject* obj, bool nice, string indent) {
+std::string json::stringify(
-    stringstream ss;
+        const JObject* obj, 
+        bool nice, 
+        const std::string& indent) {
+    std::stringstream ss;
     stringifyObj(obj, ss, 1, indent, nice);
     return ss.str();
 }
 
+static void to_binary(ByteBuilder& builder, const Value* value) {
+    switch (value->type) {
+        case valtype::object: {
+            std::vector<ubyte> bytes = to_binary(value->value.obj);
+            builder.put(bytes.data(), bytes.size());
+            break;
+        }
+        case valtype::array:
+            builder.put(BJSON_TYPE_LIST);
+            for (Value* element : value->value.arr->values) {
+                to_binary(builder, element);
+            }
+            builder.put(BJSON_END);
+            break;
+        case valtype::integer: {
+            int64_t val = value->value.integer;
+            if (val >= 0 && val <= 255) {
+                builder.put(BJSON_TYPE_BYTE);
+                builder.put(val);
+            } else if (val >= INT16_MIN && val <= INT16_MAX){
+                builder.put(BJSON_TYPE_INT16);
+                builder.putInt16(val);
+            } else if (val >= INT32_MIN && val <= INT32_MAX) {
+                builder.put(BJSON_TYPE_INT32);
+                builder.putInt32(val);
+            } else {
+                builder.put(BJSON_TYPE_INT64);
+                builder.putInt64(val);
+            }
+            break;
+        }
+        case valtype::number:
+            builder.put(BJSON_TYPE_NUMBER);
+            builder.putFloat64(value->value.decimal);
+            break;
+        case valtype::boolean:
+            builder.put(BJSON_TYPE_FALSE + value->value.boolean);
+            break;
+        case valtype::string:
+            builder.put(BJSON_TYPE_STRING);
+            builder.put(*value->value.str);
+            break;
+    }
+}
+
+static JArray* array_from_binary(ByteReader& reader);
+static JObject* object_from_binary(ByteReader& reader);
+
+std::vector<ubyte> json::to_binary(const JObject* obj) {
+    ByteBuilder builder;
+    // type byte
+    builder.put(BJSON_TYPE_DOCUMENT);
+    // document size
+    builder.putInt32(0);
+
+    // writing entries
+    for (auto& entry : obj->map) {
+        builder.putCStr(entry.first.c_str());
+        to_binary(builder, entry.second);
+    }
+    // terminating byte
+    builder.put(BJSON_END);
+
+    // updating document size
+    builder.setInt32(1, builder.size());
+    return builder.build();
+}
+
+static Value* value_from_binary(ByteReader& reader) {
+    ubyte typecode = reader.get();
+    valtype type;
+    valvalue val;
+    switch (typecode) {
+        case BJSON_TYPE_DOCUMENT:
+            type = valtype::object;
+            reader.getInt32();
+            val.obj = object_from_binary(reader);
+            break;
+        case BJSON_TYPE_LIST:
+            type = valtype::array;
+            val.arr = array_from_binary(reader);
+            break;
+        case BJSON_TYPE_BYTE:
+            type = valtype::integer;
+            val.integer = reader.get();
+            break;
+        case BJSON_TYPE_INT16:
+            type = valtype::integer;
+            val.integer = reader.getInt16();
+            break;
+        case BJSON_TYPE_INT32:
+            type = valtype::integer;
+            val.integer = reader.getInt32();
+            break;
+        case BJSON_TYPE_INT64:
+            type = valtype::integer;
+            val.integer = reader.getInt64();
+            break;
+        case BJSON_TYPE_NUMBER:
+            type = valtype::number;
+            val.decimal = reader.getFloat64();
+            break;
+        case BJSON_TYPE_FALSE:
+        case BJSON_TYPE_TRUE:
+            type = valtype::boolean;
+            val.boolean = typecode - BJSON_TYPE_FALSE;
+            break;
+        case BJSON_TYPE_STRING:
+            type = valtype::string;
+            val.str = new std::string(reader.getString());
+            break;
+        default:
+            throw std::runtime_error(
+                  "type "+std::to_string(typecode)+" is not supported");
+    }
+    return new Value(type, val);
+}
+
+static JArray* array_from_binary(ByteReader& reader) {
+    auto array = std::make_unique<JArray>();
+    auto& items = array->values;
+    while (reader.peek() != BJSON_END) {
+        items.push_back(value_from_binary(reader));
+    }
+    reader.get();
+    return array.release();
+}
+
+static JObject* object_from_binary(ByteReader& reader) {
+    auto obj = std::make_unique<JObject>();
+    auto& map = obj->map;
+    while (reader.peek() != BJSON_END) {
+        const char* key = reader.getCString();
+        Value* value = value_from_binary(reader);
+        map.insert(std::make_pair(key, value));
+    }
+    reader.get();
+    return obj.release();
+}
+
+JObject* json::from_binary(const ubyte* src, size_t size) {
+    ByteReader reader(src, size);
+    std::unique_ptr<Value> value (value_from_binary(reader));
+    if (value->type != valtype::object) {
+        throw std::runtime_error("root value is not an object");
+    }
+    JObject* obj = value->value.obj;
+    value->value.obj = nullptr;
+    return obj;
+}
 
 JArray::~JArray() {
     for (auto value : values) {
@@ -168,9 +335,9 @@ bool JArray::flag(size_t index) const {
     return values[index]->value.boolean;
 }
 
-JArray& JArray::put(string value) {
+JArray& JArray::put(std::string value) {
     valvalue val;
-    val.str = new string(value);
+    val.str = new std::string(value);
     values.push_back(new Value(valtype::string, val));
     return *this;
 }
@@ -248,11 +415,11 @@ JObject::~JObject() {
     }
 }
 
-void JObject::str(string key, string& dst) const {
+void JObject::str(std::string key, std::string& dst) const {
     dst = getStr(key, dst);
 }
 
-string JObject::getStr(string key, const string& def) const {
+std::string JObject::getStr(std::string key, const std::string& def) const {
     auto found = map.find(key);
     if (found == map.end())
         return def;
@@ -266,7 +433,7 @@ string JObject::getStr(string key, const string& def) const {
     } 
 }
 
-double JObject::getNum(string key, double def) const {
+double JObject::getNum(std::string key, double def) const {
     auto found = map.find(key);
     if (found == map.end())
         return def;
@@ -280,7 +447,7 @@ double JObject::getNum(string key, double def) const {
     }
 }
 
-int64_t JObject::getInteger(string key, int64_t def) const {
+int64_t JObject::getInteger(std::string key, int64_t def) const {
     auto found = map.find(key);
     if (found == map.end())
         return def;
@@ -294,7 +461,7 @@ int64_t JObject::getInteger(string key, int64_t def) const {
     }
 }
 
-void JObject::num(string key, double& dst) const {
+void JObject::num(std::string key, double& dst) const {
     dst = getNum(key, dst);
 }
 
@@ -346,93 +513,93 @@ void JObject::flag(std::string key, bool& dst) const {
         dst = found->second->value.boolean;
 }
 
-JObject& JObject::put(string key, uint value) {
+JObject& JObject::put(std::string key, uint value) {
     return put(key, (int64_t)value);
 }
 
-JObject& JObject::put(string key, int value) {
+JObject& JObject::put(std::string key, int value) {
     return put(key, (int64_t)value);
 }
 
-JObject& JObject::put(string key, int64_t value) {
+JObject& JObject::put(std::string key, int64_t value) {
     auto found = map.find(key);
     if (found != map.end())  found->second;
     valvalue val;
     val.integer = value;
-    map.insert(make_pair(key, new Value(valtype::integer, val)));
+    map.insert(std::make_pair(key, new Value(valtype::integer, val)));
     return *this;
 }
 
-JObject& JObject::put(string key, uint64_t value) {
+JObject& JObject::put(std::string key, uint64_t value) {
     return put(key, (int64_t)value);
 }
 
-JObject& JObject::put(string key, float value) {
+JObject& JObject::put(std::string key, float value) {
     return put(key, (double)value);
 }
 
-JObject& JObject::put(string key, double value) {
+JObject& JObject::put(std::string key, double value) {
     auto found = map.find(key);
     if (found != map.end()) delete found->second;
     valvalue val;
     val.decimal = value;
-    map.insert(make_pair(key, new Value(valtype::number, val)));
+    map.insert(std::make_pair(key, new Value(valtype::number, val)));
     return *this;
 }
 
-JObject& JObject::put(string key, string value){
+JObject& JObject::put(std::string key, std::string value){
     auto found = map.find(key);
     if (found != map.end()) delete found->second;
     valvalue val;
-    val.str = new string(value);
+    val.str = new std::string(value);
-    map.insert(make_pair(key, new Value(valtype::string, val)));
+    map.insert(std::make_pair(key, new Value(valtype::string, val)));
     return *this;
 }
 
 JObject& JObject::put(std::string key, const char* value) {
-    return put(key, string(value));
+    return put(key, std::string(value));
 }
 
-JObject& JObject::put(string key, JObject* value){
+JObject& JObject::put(std::string key, JObject* value){
     auto found = map.find(key);
     if (found != map.end()) delete found->second;
     valvalue val;
     val.obj = value;
-    map.insert(make_pair(key, new Value(valtype::object, val)));
+    map.insert(std::make_pair(key, new Value(valtype::object, val)));
     return *this;
 }
 
-JObject& JObject::put(string key, JArray* value){
+JObject& JObject::put(std::string key, JArray* value){
     auto found = map.find(key);
     if (found != map.end()) delete found->second;
     valvalue val;
     val.arr = value;
-    map.insert(make_pair(key, new Value(valtype::array, val)));
+    map.insert(std::make_pair(key, new Value(valtype::array, val)));
     return *this;
 }
 
-JObject& JObject::put(string key, bool value){
+JObject& JObject::put(std::string key, bool value){
     auto found = map.find(key);
     if (found != map.end()) delete found->second;
     valvalue val;
     val.boolean = value;
-    map.insert(make_pair(key, new Value(valtype::boolean, val)));
+    map.insert(std::make_pair(key, new Value(valtype::boolean, val)));
     return *this;
 }
 
-JArray& JObject::putArray(string key) {
+JArray& JObject::putArray(std::string key) {
     JArray* arr = new JArray();
     put(key, arr);
     return *arr;
 }
 
-JObject& JObject::putObj(string key) {
+JObject& JObject::putObj(std::string key) {
     JObject* obj = new JObject();
     put(key, obj);
     return *obj;
 }
 
-bool JObject::has(string key) {
+bool JObject::has(std::string key) {
     return map.find(key) != map.end();
 }
 
@@ -449,7 +616,8 @@ Value::~Value() {
     }
 }
 
-Parser::Parser(string filename, string source) : BasicParser(filename, source) {    
+Parser::Parser(std::string filename, std::string source) 
+      : BasicParser(filename, source) {    
 }
 
 JObject* Parser::parse() {
@@ -462,20 +630,20 @@ JObject* Parser::parse() {
 
 JObject* Parser::parseObject() {
     expect('{');
-    unique_ptr<JObject> obj(new JObject());
+    auto obj = std::make_unique<JObject>();
-    unordered_map<string, Value*>& map = obj->map;
+    auto& map = obj->map;
     while (peek() != '}') {
         if (peek() == '#') {
             skipLine();
             continue;
         }
-        string key = parseName();
+        std::string key = parseName();
         char next = peek();
         if (next != ':') {
             throw error("':' expected");
         }
         pos++;
-        map.insert(make_pair(key, parseValue()));
+        map.insert(std::make_pair(key, parseValue()));
         next = peek();
         if (next == ',') {
             pos++;
@@ -491,8 +659,8 @@ JObject* Parser::parseObject() {
 
 JArray* Parser::parseArray() {
     expect('[');
-    unique_ptr<JArray> arr(new JArray());
+    auto arr = std::make_unique<JArray>();
-    vector<Value*>& values = arr->values;
+    auto& values = arr->values;
     while (peek() != ']') {
         if (peek() == '#') {
             skipLine();
@@ -530,7 +698,7 @@ Value* Parser::parseValue() {
         return new Value(type, val);
     }
     if (is_identifier_start(next)) {
-        string literal = parseName();
+        std::string literal = parseName();
         if (literal == "true") {
             val.boolean = true;
             return new Value(valtype::boolean, val);
@@ -568,17 +736,17 @@ Value* Parser::parseValue() {
     }
     if (next == '"' || next == '\'') {
         pos++;
-        val.str = new string(parseString(next));
+        val.str = new std::string(parseString(next));
         return new Value(valtype::string, val);
     }
-    throw error("unexpected character '"+string({next})+"'");
+    throw error("unexpected character '"+std::string({next})+"'");
 }
 
-JObject* json::parse(string filename, string source) {
+JObject* json::parse(std::string filename, std::string source) {
     Parser parser(filename, source);
     return parser.parse();
 }
 
-JObject* json::parse(string source) {
+JObject* json::parse(std::string source) {
     return parse("<string>", source);
 }

src/coders/json.h

@@ -16,7 +16,9 @@ namespace json {
     class JArray;
     class Value;
 
-    extern std::string stringify(JObject* obj, bool nice, std::string indent);
+    extern std::string stringify(const JObject* obj, bool nice, const std::string& indent);
+    extern std::vector<ubyte> to_binary(const JObject* obj);
+    extern JObject* from_binary(const ubyte* src, size_t size);
 
     enum class valtype {
         object, array, number, integer, string, boolean

src/content/ContentLoader.cpp

@@ -87,7 +87,7 @@ void ContentLoader::fixPackIndices() {
     if (modified){
         // rewrite modified json
         std::cout << indexFile << std::endl;
-        files::write_string(indexFile, json::stringify(root.get(), true, "  "));
+        files::write_json(indexFile, root.get());
     }
 }
 

src/files/WorldFiles.cpp

@@ -1,7 +1,6 @@
 #include "WorldFiles.h"
 
 #include "rle.h"
-#include "binary_io.h"
 #include "../window/Camera.h"
 #include "../content/Content.h"
 #include "../objects/Player.h"
@@ -477,7 +476,7 @@ void WorldFiles::writeIndices(const ContentIndices* indices) {
 		items.put(def->name);
 	}
 
-	files::write_string(getIndicesFile(), json::stringify(&root, true, "  "));
+	files::write_json(getIndicesFile(), &root);
 }
 
 void WorldFiles::writeWorldInfo(const World* world) {
@@ -494,7 +493,7 @@ void WorldFiles::writeWorldInfo(const World* world) {
 	timeobj.put("day-time", world->daytime);
 	timeobj.put("day-time-speed", world->daytimeSpeed);
 
-	files::write_string(getWorldFile(), json::stringify(&root, true, "  "));
+	files::write_json(getWorldFile(), &root);
 }
 
 bool WorldFiles::readWorldInfo(World* world) {
@@ -540,7 +539,7 @@ void WorldFiles::writePlayer(Player* player){
 	root.put("flight", player->flight);
 	root.put("noclip", player->noclip);
 
-	files::write_string(getPlayerFile(), json::stringify(&root, true, "  "));
+	files::write_json(getPlayerFile(), &root);
 }
 
 bool WorldFiles::readPlayer(Player* player) {

src/files/binary_io.cpp

@@ -1,165 +0,0 @@
-#include "binary_io.h"
-
-#include <string.h>
-#include <limits>
-#include <stdexcept>
-
-void BinaryWriter::put(ubyte b) {
-    buffer.push_back(b);
-}
-
-void BinaryWriter::putCStr(const char* str) {
-    size_t size = strlen(str)+1;
-    buffer.reserve(buffer.size() + size);
-    for (size_t i = 0; i < size; i++) {
-        buffer.push_back(str[i]);
-    }
-}
-
-void BinaryWriter::put(const std::string& s) {
-    size_t len = s.length();
-    if (len > INT16_MAX) {
-        throw std::domain_error("length > INT16_MAX");
-    }
-    putInt16(len);
-    put((const ubyte*)s.data(), len);
-}
-
-void BinaryWriter::putShortStr(const std::string& s) {
-    size_t len = s.length();
-    if (len > 255) {
-        throw std::domain_error("length > 255");
-    }
-    put(len);
-    put((const ubyte*)s.data(), len);
-}
-
-void BinaryWriter::put(const ubyte* arr, size_t size) {
-    buffer.reserve(buffer.size() + size);
-    for (size_t i = 0; i < size; i++) {
-        buffer.push_back(arr[i]);
-    }
-}
-
-void BinaryWriter::putInt16(int16_t val) {
-    buffer.push_back((char) (val >> 8 & 255));
-    buffer.push_back((char) (val >> 0 & 255));
-}
-
-void BinaryWriter::putInt32(int32_t val) {
-    buffer.reserve(buffer.size() + 4);
-    buffer.push_back((char) (val >> 24 & 255));
-    buffer.push_back((char) (val >> 16 & 255));
-    buffer.push_back((char) (val >> 8 & 255));
-    buffer.push_back((char) (val >> 0 & 255));
-}
-
-void BinaryWriter::putInt64(int64_t val) {
-    buffer.reserve(buffer.size() + 8);
-    buffer.push_back((char) (val >> 56 & 255));
-    buffer.push_back((char) (val >> 48 & 255));
-    buffer.push_back((char) (val >> 40 & 255));
-    buffer.push_back((char) (val >> 32 & 255));
-    buffer.push_back((char) (val >> 24 & 255));
-    buffer.push_back((char) (val >> 16 & 255));
-    buffer.push_back((char) (val >> 8 & 255));
-    buffer.push_back((char) (val >> 0 & 255));
-}
-
-void BinaryWriter::putFloat32(float val) {
-    union {
-        int32_t vali32;
-        float valfloat;
-    } value;
-    value.valfloat = val;
-    putInt32(value.vali32);
-}
-
-BinaryReader::BinaryReader(const ubyte* data, size_t size)
-    : data(data), size(size), pos(0) {
-}
-
-void BinaryReader::checkMagic(const char* data, size_t size) {
-    if (pos + size >= this->size) {
-        throw std::runtime_error("invalid magic number");
-    }
-    for (size_t i = 0; i < size; i++) {
-        if (this->data[pos + i] != (ubyte)data[i]){
-            throw std::runtime_error("invalid magic number");
-        }
-    }
-    pos += size;
-}
-
-ubyte BinaryReader::get() {
-    if (pos == size) {
-        throw std::underflow_error("buffer underflow");
-    }
-    return data[pos++];
-}
-
-int16_t BinaryReader::getInt16() {
-    if (pos+2 > size) {
-        throw std::underflow_error("unexpected end");
-    }
-    pos += 2;
-    return (data[pos - 2] << 8) | 
-           (data[pos - 1]);
-}
-
-int32_t BinaryReader::getInt32() {
-    if (pos+4 > size) {
-        throw std::underflow_error("unexpected end");
-    }
-    pos += 4;
-    return (data[pos - 4] << 24) | 
-           (data[pos - 3] << 16) | 
-           (data[pos - 2] << 8) | 
-           (data[pos - 1]);
-}
-
-int64_t BinaryReader::getInt64() {
-    if (pos+8 > size) {
-        throw std::underflow_error("unexpected end");
-    }
-    pos += 8;
-    return ((int64_t)data[pos - 8] << 56) | 
-           ((int64_t)data[pos - 7] << 48) | 
-           ((int64_t)data[pos - 6] << 40) | 
-           ((int64_t)data[pos - 5] << 32) | 
-           ((int64_t)data[pos - 4] << 24) | 
-           ((int64_t)data[pos - 3] << 16) | 
-           ((int64_t)data[pos - 2] << 8) | 
-           ((int64_t)data[pos - 1]);
-}
-
-float BinaryReader::getFloat32() {
-    union {
-        int32_t vali32;
-        float valfloat;
-    } value;
-    value.vali32 = getInt32();
-    return value.valfloat;
-}
-
-std::string BinaryReader::getString() {
-    uint16_t length = (uint16_t)getInt16();
-    if (pos+length > size) {
-        throw std::underflow_error("unexpected end");
-    }
-    pos += length;
-    return std::string((const char*)(data+pos-length), length);
-}
-
-std::string BinaryReader::getShortString() {
-    ubyte length = get();
-    if (pos+length > size) {
-        throw std::underflow_error("unexpected end");
-    }
-    pos += length;
-    return std::string((const char*)(data+pos-length), length);
-}
-
-bool BinaryReader::hasNext() const {
-    return pos < size;
-}

src/files/binary_io.h

@@ -1,47 +0,0 @@
-#ifndef FILES_BINARY_WRITER_H_
-#define FILES_BINARY_WRITER_H_
-
-#include <string>
-#include <vector>
-#include "../typedefs.h"
-
-class BinaryWriter {
-    std::vector<ubyte> buffer;
-public:
-    void put(ubyte b);
-    void putCStr(const char* str);
-    void putInt16(int16_t val);
-    void putInt32(int32_t val);
-    void putInt64(int64_t val);
-    void putFloat32(float val);
-    void put(const std::string& s);
-    void put(const ubyte* arr, size_t size);
-    void putShortStr(const std::string& s);
-
-    inline size_t size() const {
-        return buffer.size();
-    }
-    inline const ubyte* data() const {
-        return buffer.data();
-    }
-};
-
-class BinaryReader {
-    const ubyte* data;
-    size_t size;
-    size_t pos;
-public:
-    BinaryReader(const ubyte* data, size_t size);
-
-    void checkMagic(const char* data, size_t size);
-    ubyte get();
-    int16_t getInt16();
-    int32_t getInt32();
-    int64_t getInt64();
-    float getFloat32();
-    std::string getString();
-    std::string getShortString();
-    bool hasNext() const;
-};
-
-#endif // FILES_BINARY_WRITER_H_

src/files/files.cpp

@@ -92,16 +92,41 @@ bool files::write_string(fs::path filename, const std::string content) {
 	return true;
 }
 
-json::JObject* files::read_json(fs::path file) {
+bool files::write_json(fs::path filename, const json::JObject* obj, bool nice) {
-	std::string text = files::read_string(file);
+    // -- binary json tests
+    //return write_binary_json(fs::path(filename.u8string()+".bin"), obj);
+    return files::write_string(filename, json::stringify(obj, nice, "  "));
+}
+
+bool files::write_binary_json(fs::path filename, const json::JObject* obj) {
+    std::vector<ubyte> bytes = json::to_binary(obj);
+    return files::write_bytes(filename, (const char*)bytes.data(), bytes.size());
+}
+
+json::JObject* files::read_json(fs::path filename) {
+    // binary json tests
+    // fs::path binfile = fs::path(filename.u8string()+".bin");
+    // if (fs::is_regular_file(binfile)){
+    //     return read_binary_json(binfile);
+    // }
+
+	std::string text = files::read_string(filename);
 	try {
-		return json::parse(file.string(), text);
+		auto obj = json::parse(filename.string(), text);
+        //write_binary_json(binfile, obj);
+        return obj;
 	} catch (const parsing_error& error) {
         std::cerr << error.errorLog() << std::endl;
-        throw std::runtime_error("could not to parse "+file.string());
+        throw std::runtime_error("could not to parse "+filename.string());
     }
 }
 
+json::JObject* files::read_binary_json(fs::path file) {
+    size_t size;
+    std::unique_ptr<char[]> bytes (files::read_bytes(file, size));
+    return json::from_binary((const ubyte*)bytes.get(), size);
+}
+
 std::vector<std::string> files::read_list(fs::path filename) {
 	std::ifstream file(filename);
 	if (!file) {

src/files/files.h

@@ -7,6 +7,8 @@
 #include <filesystem>
 #include "../typedefs.h"
 
+namespace fs = std::filesystem;
+
 namespace json {
     class JObject;
 }
@@ -25,14 +27,18 @@ namespace files {
     };
 
 
-    extern bool write_bytes(std::filesystem::path, const char* data, size_t size);
+    extern bool write_bytes(fs::path, const char* data, size_t size);
-    extern uint append_bytes(std::filesystem::path, const char* data, size_t size);
+    extern uint append_bytes(fs::path, const char* data, size_t size);
-    extern bool read(std::filesystem::path, char* data, size_t size);
+    extern bool write_string(fs::path filename, const std::string content);
-    extern char* read_bytes(std::filesystem::path, size_t& length);
+    extern bool write_json(fs::path filename, const json::JObject* obj, bool nice=true);
-    extern std::string read_string(std::filesystem::path filename);
+    extern bool write_binary_json(fs::path filename, const json::JObject* obj);
-    extern bool write_string(std::filesystem::path filename, const std::string content);
+
-    extern json::JObject* read_json(std::filesystem::path file);
+    extern bool read(fs::path, char* data, size_t size);
-    extern std::vector<std::string> read_list(std::filesystem::path file);
+    extern char* read_bytes(fs::path, size_t& length);
+    extern std::string read_string(fs::path filename);
+    extern json::JObject* read_json(fs::path file);
+    extern json::JObject* read_binary_json(fs::path file);
+    extern std::vector<std::string> read_list(fs::path file);
 }
 
 #endif /* FILES_FILES_H_ */

src/logic/scripting/api_lua.cpp

@@ -14,6 +14,13 @@
 #include "../../lighting/Lighting.h"
 #include "../../logic/BlocksController.h"
 
+inline int lua_pushivec3(lua_State* L, int x, int y, int z) {
+    lua_pushinteger(L, x);
+    lua_pushinteger(L, y);
+    lua_pushinteger(L, z);
+    return 3;
+}
+
 inline void luaL_openlib(lua_State* L, const char* name, const luaL_Reg* libfuncs, int nup) {
     lua_newtable(L);
     luaL_setfuncs(L, libfuncs, nup);
@@ -44,11 +51,6 @@ static const luaL_Reg worldlib [] = {
     {NULL, NULL}
 };
 
-int luaopen_world(lua_State* L) {
-    luaL_openlib(L, "world", worldlib, 0);
-    return 1;
-}
-
 /* == player library ==*/
 static int l_player_get_pos(lua_State* L) {
     int playerid = lua_tointeger(L, 1);
@@ -102,11 +104,6 @@ static const luaL_Reg playerlib [] = {
     {NULL, NULL}
 };
 
-int luaopen_player(lua_State* L) {
-    luaL_openlib(L, "player", playerlib, 0);
-    return 1;
-}
-
 /* == blocks-related functions == */
 static int l_block_name(lua_State* L) {
     int id = lua_tointeger(L, 1);
@@ -158,13 +155,6 @@ static int l_get_block(lua_State* L) {
     return 1;
 }
 
-inline int lua_pushivec3(lua_State* L, int x, int y, int z) {
-    lua_pushinteger(L, x);
-    lua_pushinteger(L, y);
-    lua_pushinteger(L, z);
-    return 3;
-}
-
 static int l_get_block_x(lua_State* L) {
     int x = lua_tointeger(L, 1);
     int y = lua_tointeger(L, 2);
@@ -275,8 +265,8 @@ static int l_is_replaceable_at(lua_State* L) {
                                     lua_setglobal(L, NAME))
 
 void apilua::create_funcs(lua_State* L) {
-    luaopen_world(L);
+    luaL_openlib(L, "world", worldlib, 0);
-    luaopen_player(L);
+    luaL_openlib(L, "player", playerlib, 0);
 
     lua_addfunc(L, l_block_index, "block_index");
     lua_addfunc(L, l_block_name, "block_name");