Memory-mapped game resources (C++/Python)

> Coding, hacking, computer graphics, game dev, and such...
User avatar
fips
Site Admin
Posts: 166
Joined: Wed Nov 12, 2008 9:49 pm
Location: Prague
Contact:

Memory-mapped game resources (C++/Python)

Post by fips »

So far, I've been pretty happy with Google Protocol Buffers when it comes to serializing game resources. However, one can do much better performance-wise if willing to sacrifice some flexibility. What can be faster than to eliminate the process of serialization completely by using the same data structures for both: runtime and storage, then it is possible just to mem-copy your objects around with no additional memory allocations and serialization overhead.

If you know your target platform, you can design such data structures quite easily. In general, it means to get rid of pointers and replace them with offsets, and also be aware of endianness, alignment and padding rules. By using 32-bit offsets, alignment and padding, your data effectively becomes 32-bit wide, perfectly suited for both: 32-bit and 64-bit code running over it.

In the runtime it's convenient to introduce a lightweight wrapper class that maps a BLOB into something more useful with a regular interface (I call the wrappers 'reference types'). Such a wrapper is very cheap to create and copy as it doesn't contain any data (it only refers it). This kind of anti-OOP approach proves to be more and more relevant to me.

On the content side, Python's ctypes module can be conveniently used to transform various intermediate formats into BLOBs with a binary layout matching the runtime. It's where the flexibility of Python meets the power of C++!

The two examples below show how a PNG texture can be transformed into a BLOB (Python) and then manipulated in the runtime (C++11):

VIEW THE CODE BELOW IN FULL-SCREEN (texture_exporter.py)

Code: Select all

#!/usr/bin/env python

# /*
# (c) 2013 +++ Filip Stoklas, aka FipS, http://www.4FipS.com +++
# THIS CODE IS FREE - LICENSED UNDER THE MIT LICENSE
# ARTICLE URL: http://forums.4fips.com/viewtopic.php?f=3&t=1071
# */

import ctypes as ct
import png

class Texture_format8:
    Rgb = 0
    Rgba = 1
    
class c_Blob(ct.Structure):
    _fields_ = [
        ("tag", ct.c_uint8 * 4),
        ("size", ct.c_uint32)
    ]

class c_Block(ct.Structure):
    _fields_ = [
        ("size", ct.c_uint32),
        ("offset", ct.c_uint32)
    ]

class c_Header(ct.Structure):
    _fields_ = [
        ("version", ct.c_uint8),
        ("reserved0", ct.c_uint8),
        ("reserved1", ct.c_uint8),
        ("format", ct.c_uint8),
        ("width", ct.c_uint32),
        ("height", ct.c_uint32),
        ("size", ct.c_uint32)
    ]

def c_Data_factory(size):
    """Parametrized by 'size'."""
    class c_Data(ct.Structure):
        _fields_ = [
            ("data", ct.c_uint8 * size)
        ]
    return c_Data

def c_Texture_factory(c_Data):
    """Parametrized by 'c_Data'."""
    class c_Texture(ct.Structure):
        _fields_ = [
            ("blob", c_Blob),
            ("header_block", c_Block),
            ("data_block", c_Block),
            ("header", c_Header),
            ("data", c_Data)
        ]
    return c_Texture

def create_texture(width, height, format, pixels):
    c_Data = c_Data_factory(len(pixels))
    c_Texture = c_Texture_factory(c_Data)
    return c_Texture(
        c_Blob(tuple([ord(x) for x in "TEX!"]), ct.sizeof(c_Texture)), # blob [tag, size]
        c_Block(ct.sizeof(c_Header), c_Texture.header.offset), # header_block [size, offset]
        c_Block(ct.sizeof(c_Data), c_Texture.data.offset), # data_block [size, offset]
        c_Header(
            1, # version
            0, # reserved0
            0, # reserved1
            format,
            width,
            height,
            len(pixels) # size
        ),
        c_Data(tuple(pixels))
    )

def load_image(fname):
    r = png.Reader(fname)
    width, height, pixels, meta = r.read_flat()
    bitdepth = meta["bitdepth"]
    assert bitdepth == 8
    planes = meta["planes"]
    assert planes == 3 or planes == 4
    format = Texture_format8.Rgb if planes == 3 else Texture_format8.Rgba
    return width, height, format, pixels

def save_texture(texture, fname):
    with open(fname, "wb") as f:
        f.write(texture)

def main():
    width, height, format, pixels = load_image("texture.png")
    texture = create_texture(width, height, format, pixels)
    save_texture(texture, "texture.blob")

main()
VIEW THE CODE BELOW IN FULL-SCREEN (main.cpp)

Code: Select all

/*
(c) 2013 +++ Filip Stoklas, aka FipS, http://www.4FipS.com +++
THIS CODE IS FREE - LICENSED UNDER THE MIT LICENSE
ARTICLE URL: http://forums.4fips.com/viewtopic.php?f=3&t=1071
*/

#include <vector>
#include <memory>
#include <tuple>
#include <cstdint>
#include <cassert>

/// Texture formats (enum of uint8_t, C++11).
enum class Texture_format8 : uint8_t
{
    Rgb,
    Rgba
};

/// General IO PODs (32-bit aligned).
namespace io {
struct Blob
{
    uint8_t tag[4];
    uint32_t size;
};
struct Block
{
    uint32_t size;
    uint32_t offset;
};}

/// Texture related IO PODs (32-bit aligned).
namespace io_texture {
struct Header
{
    uint8_t version;
    uint8_t reserved0;
    uint8_t reserved1;
    Texture_format8 format;
    uint32_t width;
    uint32_t height;
    uint32_t size;
};
struct Texture
{
    io::Blob blob;
    io::Block header_block;
    io::Block data_block;
    // [header]
    // [data]
};}

/// Texture reference (doesn't own the data, it's just a mapping => cheap to create & copy).
class Texture_ref
{
 public:

    Texture_ref(const void *blob)
    {
        // create the mappings:
        using namespace io;
        using namespace io_texture;
        const uint8_t *bytes = static_cast<const uint8_t *>(blob);
        const Texture *texture = reinterpret_cast<const Texture *>(bytes);
        assert(std::make_tuple(texture->blob.tag[0], texture->blob.tag[1], texture->blob.tag[2], texture->blob.tag[3]) == std::make_tuple('T', 'E', 'X', '!'));
        const Header *header = reinterpret_cast<const Header *>(bytes + texture->header_block.offset);
        _version = header->version;
        _format = header->format;
        _width = header->width;
        _height = header->height;
        _data = bytes + texture->data_block.offset;
        _size = header->size;
    }

    uint8_t version() const { return _version; }
    Texture_format8 format() const { return _format; }
    uint32_t width() const { return _width; }
    uint32_t height() const { return _height; }
    const void * data() const { return _data; }
    uint32_t size() const {return _size; }

 private:

    // sorted from bigger to smaller:
    const void *_data;
    uint32_t _width;
    uint32_t _height;
    uint32_t _size;
    uint8_t _version;
    Texture_format8 _format;
};

/// Loads a BLOB and checks its ID (signature).
std::vector<uint8_t> load_blob(const char *fname, std::tuple<uint8_t, uint8_t, uint8_t, uint8_t> id)
{
    auto deleter = [](FILE *f) { if(f) fclose(f); };
    std::unique_ptr<FILE, decltype(deleter)> file(fopen(fname, "rb"), deleter);
    io::Blob blob;
    fread(&blob, sizeof(io::Blob), 1, file.get());
    assert(std::make_tuple(blob.tag[0], blob.tag[1], blob.tag[2], blob.tag[3]) == id);
    std::vector<uint8_t> bytes(blob.size);
    const uint8_t *blob_bytes = reinterpret_cast<const uint8_t *>(&blob);
    std::copy(blob_bytes, blob_bytes + sizeof(io::Blob), bytes.data());
    fread(bytes.data() + sizeof(io::Blob), blob.size - sizeof(io::Blob), 1, file.get());
    return bytes; // move it out (C++11)
}

int main()
{
    const auto blob = load_blob("texture.blob", std::make_tuple('T', 'E', 'X', '!'));
    const Texture_ref tex(blob.data());

    printf(
     "version=%u, format='%s', width=%u, height=%u\n",
     unsigned(tex.version()), tex.format() == Texture_format8::Rgb ? "Rgb" : "Rgba",
     unsigned(tex.width()), unsigned(tex.height())
    );

    printf("data(size=%u): ", unsigned(tex.size()));
    for(auto i = 0; i < tex.size(); ++i)
        printf("%x ", static_cast<const uint8_t *>(tex.data())[i]);
    printf("\n");

    return 0;
}

// Compiled under Visual C++ 2012, output:
// version=1, format='Rgba', width=2, height=2
// data(size=16): ff 0 0 ff 0 ff 0 ff 0 0 ff ff 80 80 80 ff