<repo_name>baldurk/rdoc_serialise_test<file_sep>/main.cpp
#include "serialiser.h"
#include "test_types.h"
#include "xml_codec.h"

#include <time.h>

const char *GetChunkName(uint32_t i)
{
  // dummy function
  return "main()";
}

struct struct1_pure_pod
{
  int x[10000];

};

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, struct1_pure_pod &el)
{
  ser.Serialise("el.x", el.x);
}

INSTANTIATE_SERIALISE_TYPE(struct1_pure_pod);

void perftests()
{
  struct1_pure_pod foo = { };

  double sum = 0.0;

  const uint64_t runs = 10;
  
  WriteSerialiser *ser = new WriteSerialiser(new StreamWriter(10000*10000*4 + 1024));

  for(uint64_t run=0; run < runs; run++)
  {
    ser->GetWriter()->Rewind();

    PerformanceTimer timer;

    for(uint64_t i=0; i < 10000; i++)
      ser->Serialise("foo", foo);

    double d = timer.GetMilliseconds();

    sum += d;

    printf("time %u: %f\n", (uint32_t)run, d);
  }

  printf("avg: %f\n", sum / double(runs));
}

#ifdef WIN32
#include <direct.h>
#define getcwd _getcwd
#endif

#ifndef MAX_PATH
#define MAX_PATH 260
#endif

void real_world_test_read();
void real_world_test_write();

void real_world_test_xml2rdc();
void real_world_test_read2();

void streamio_test();

int main(int argc, char **argv)
{
  char workdir[MAX_PATH+1] = {};
  getcwd(workdir, MAX_PATH);

  srand(time(NULL));

  printf("Working Dir: %s\n\n", workdir);

  printf("%d args:\n", argc);
  for(int i=0; i < argc; i++)
    printf("[%d]: %s\n", i, argv[i]);

  WriteSerialiser *write = new WriteSerialiser(new StreamWriter(StreamWriter::DefaultScratchSize));

  ResourceManager resourcemanager;

  write->SetUserData(&resourcemanager);

  size_t bufferSize = 200*1024;

  int array_of_data[5] = { 0, 1, 2, 3, 4 };

  const char *exts[] = { "VK_foo", "VK_bar" };

  VulkanHandleThingy vulkan; vulkan.handle = 1818;
  ID3DInterface *d3d; d3d = (ID3DInterface *)&write;

  {
    simpleStruct a;
    complexStruct b;
    b.vectorArray.resize(10);
    podStruct c;
    byte *buffer = new byte[bufferSize];
    memset(buffer, 0x54, bufferSize);

    b.numExts = ARRAY_COUNT(exts);
    b.exts = exts;
    b.cstring = (char *)"blah blah !";

    const int *const_array_ptr = array_of_data;
    const int *const_array_ptr2 = array_of_data;
    int *array_ptr = array_of_data;

    const float cfloat = 3.141f;

    ResourceId d;

    WriteSerialiser &ser = *write;

    VulkanHandleThingy multivulkan[8];
    for(int i=0; i < 8; i++) multivulkan[i].handle = rand();

    SCOPED_SERIALISE_CONTEXT(5, 205*1024);
    SERIALISE_ELEMENT(a);
    SERIALISE_ELEMENT(b);
    SERIALISE_ELEMENT(c);
    SERIALISE_ELEMENT(d);
    SERIALISE_ELEMENT_ARRAY(buffer, bufferSize);
    SERIALISE_ELEMENT_ARRAY(const_array_ptr, FIXED_COUNT(5));
    SERIALISE_ELEMENT_ARRAY(const_array_ptr2, FIXED_COUNT(5));
    SERIALISE_ELEMENT_ARRAY(array_ptr, FIXED_COUNT(5));
    SERIALISE_ELEMENT(cfloat);
    SERIALISE_ELEMENT(vulkan);
    SERIALISE_ELEMENT(d3d);
    SERIALISE_ELEMENT(multivulkan);
    
    SAFE_DELETE_ARRAY(buffer);
  }

  StructReadSerialiser *read = new StructReadSerialiser(new StreamReader(write->GetWriter()->GetData(), write->GetWriter()->GetOffset()));
  read->ConfigureStructuredExport(&GetChunkName, false);

  read->SetUserData(&resourcemanager);
  
  {
    simpleStruct a;
    complexStruct b;
    podStruct c;
    ResourceId d;
    byte *buffer = (byte *)0xbaadf00d;

    const int *const_array_ptr = NULL;
    const int *const_array_ptr2 = NULL;
    int *array_ptr = (int *)0xbadf00d;

    float cfloat = 0.0f;

    VulkanHandleThingy multivulkan[8];

    a.Empty();
    b.Empty();
    c.Empty();
    d.innerSecretValue = 0;

    StructReadSerialiser &ser = *read;

    uint32_t chunk = read->BeginChunk(0);
    
    SERIALISE_ELEMENT(a);
    SERIALISE_ELEMENT(b);
    SERIALISE_ELEMENT(c);
    SERIALISE_ELEMENT(d);
    SERIALISE_ELEMENT_ARRAY(buffer, bufferSize);
    SERIALISE_ELEMENT_ARRAY(const_array_ptr, FIXED_COUNT(5));
    SERIALISE_ELEMENT_ARRAY(const_array_ptr2, FIXED_COUNT(5));
    SERIALISE_ELEMENT_ARRAY(array_ptr, FIXED_COUNT(5));
    SERIALISE_ELEMENT(cfloat);
    SERIALISE_ELEMENT(vulkan);
    SERIALISE_ELEMENT(d3d);
    SERIALISE_ELEMENT(multivulkan);

    read->EndChunk();
  }

  std::string xml = Structured2XML(read->GetSerialisedChunks());

  //puts("xml:\n\n");
  //puts(xml.c_str());

  delete write;
  delete read;

  real_world_test_write();
  real_world_test_read();

  real_world_test_xml2rdc();
  real_world_test_read2();

  streamio_test();

  return 0;
}<file_sep>/stringize.h
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2016 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#pragma once

#include <string>
#include <type_traits>

template <typename T>
std::string DoStringize(const T &el);

template <typename T, bool is_pointer = std::is_pointer<T>::value>
struct StringConverter
{
  static std::string Do(const T &el)
  {
    return DoStringize<T>(el);
  }
};

template <typename T>
struct StringConverter<T, true>
{
  static std::string Do(const T &el)
  {
    return DoStringize<void *>(el);
  }
};

template <typename T>
std::string ToStr(const T &el)
{
  return StringConverter<T>::Do(el);
}

// helper macros for common enum switch
#define BEGIN_ENUM_STRINGIZE() \
  switch(el) { default: break;

// stringize the parameter
#define STRINGIZE_ENUM(a) \
  case a: return #a;

#define STRINGIZE_ENUM_NAMED(value, str) \
  case value: return str;

#define END_ENUM_STRINGIZE(type) \
  } return StringFormat::Fmt(#type "<%u>", el);

// helper macros for common bitfield check-and-append
#define BEGIN_BITFIELD_STRINGIZE() \
  std::string ret;

#define STRINGIZE_BITFIELD_NAMED(b) \
  if(el == b) return #b;

#define STRINGIZE_BITFIELD(b, str) \
  if(el == b) return str;

#define STRINGIZE_BITFIELD_BIT(b) \
  if(el & b) ret += " | " #b;

#define STRINGIZE_BITFIELD_BIT_NAMED(b, str) \
  if(el & b) ret += " | " str;

#define END_BITFIELD_STRINGIZE() \
  if(!ret.empty()) { ret = ret.substr(3); } return ret;

template <typename T>
class GetSerialiseTypeName
{
public:
  static const char *Name();
};

#define SERIALISE_ENUM_TYPE(type) \
  template <>                        \
  const char *GetSerialiseTypeName<type>::Name() \
  {                                  \
    return #type;                       \
  }
<file_sep>/test_types.h
#pragma once

#include "serialiser.h"
#include <map>

using std::vector;
using std::map;

struct ResourceId
{
  ResourceId() : innerSecretValue(cur++) {}

  bool operator < (const ResourceId &o) const
  {
    return innerSecretValue < o.innerSecretValue;
  }

  uint64_t innerSecretValue;
  static uint64_t cur;
};

DECLARE_SERIALISE_TYPE(ResourceId);

struct VulkanHandleThingy
{
  uint64_t handle;
};

DECLARE_SERIALISE_TYPE(VulkanHandleThingy);

struct ID3DInterface;

DECLARE_SERIALISE_TYPE(ID3DInterface*);

struct ResourceManager
{
  std::map<ResourceId, ID3DInterface*> d3d;
  std::map<ResourceId, VulkanHandleThingy> vk;
};

struct podStruct
{
  int x, y, width, height;

  podStruct() : x(5), y(5), width(10), height(20) {}

  void Empty()
  {
    x = y = width = height = 0;
  }
};

DECLARE_SERIALISE_TYPE(podStruct);

enum blah
{
  blah_foo,
  blah_bar,
  blah_qux,
};

struct simpleStruct
{
  int foo;
  float bar;
  string hello;

  uint32_t really_a_blah;

  podStruct secret;

  simpleStruct() : foo(5), bar(2.3f), hello("hello"), really_a_blah(blah_qux) {}

  void Empty()
  {
    foo = 0;
    bar = 0.0f;
    hello = "";
    really_a_blah = 999;
    secret.Empty();
  }
};

DECLARE_SERIALISE_TYPE(simpleStruct);

struct complexStruct
{
  simpleStruct basicArray[5];
  vector<simpleStruct> vectorArray;

  simpleStruct secret;

  uint32_t numExts;
  const char **exts;
  char *cstring;

  complexStruct() : numExts(0), exts(NULL), cstring(NULL) {}

  void Empty()
  {
    for(size_t i=0; i < ARRAY_COUNT(basicArray); i++)
      basicArray[i].Empty();
      
    vectorArray.clear();

    secret.Empty();

    numExts = 0;
    exts = NULL;
  }
};

DECLARE_SERIALISE_TYPE(complexStruct);
DECLARE_DESERIALISE_TYPE(complexStruct);
<file_sep>/json_convert.cpp
#include "json_convert.h"
#include "common.h"
#include "string_utils.h"
#include "structured_data.h"

#include "miniz.h"
#include "picojson.h"

static std::string EncodeString(std::string str)
{
  size_t i = 0;
  
  do
  {
    i = str.find_first_of("\"\\\r\n\t", i);
    if(i == std::string::npos)
      break;

    if(str[i] == '\n') str[i] = 'n';
    if(str[i] == '\r') str[i] = 'r';
    if(str[i] == '\t') str[i] = 't';

    str.insert(str.begin()+i, '\\');i += 2;
  } while(true);

  return '"' + str + '"';
}

static std::string EncodeType(const SerialisedType &type)
{
  const char *typeName = "unknown";

  switch(type.basetype)
  {
    case SerialisedType::eChunk:
      typeName = "chunk";
      break;
    case SerialisedType::eStruct:
      typeName = "struct";
      break;
    case SerialisedType::eArray:
      typeName = "array";
      break;
    case SerialisedType::eNull:
      typeName = "null";
      break;
    case SerialisedType::eBuffer:
      typeName = "buffer";
      break;
    case SerialisedType::eString:
      typeName = "string";
      break;
    case SerialisedType::eEnum:
      typeName = "enum";
      break;
    case SerialisedType::eUnsignedInteger:
      typeName = "uint";
      break;
    case SerialisedType::eSignedInteger:
      typeName = "sint";
      break;
    case SerialisedType::eFloat:
      typeName = "float";
      break;
    case SerialisedType::eBoolean:
      typeName = "bool";
      break;
    case SerialisedType::eCharacter:
      typeName = "char";
      break;
  }

  std::string name = (type.name == NULL || type.name[0] == '\0') ? typeName : type.name;
  const char *nullable = (type.flags & SerialisedType::Nullable) ? "true" : "false";

  return StringFormat::Fmt("{ \"baseType\": \"%s\", \"nullable\": %s, \"name\": %s, \"width\": %u }", typeName, nullable, EncodeString(name).c_str(), type.byteSize);
}

static std::string DumpObject(size_t indent, bool isArray, const SerialisedObject &obj, bool humanReadable)
{
  const bool machineReadable = !humanReadable;

  int conciseIndentOffset = (humanReadable ? 0 : 1);

  const std::string ind(indent, '\t');
  const std::string indVal(indent + conciseIndentOffset, '\t');

  std::string ret;

  for(size_t i=0; i < obj.data.children.size(); i++)
  {
    const SerialisedObject &child = *obj.data.children[i];

    if(isArray)
      ret += ind;
    else
      ret += ind + EncodeString(child.name) + ": ";

    bool customString = (child.type.flags & SerialisedType::HasCustomString);

    if(machineReadable)
    {
      ret += "{\n";
      ret += indVal + "\"type\": " + EncodeType(child.type) + ",\n";
      ret += indVal + "\"name\": " + EncodeString(child.name) + ",\n";
      if(customString)
        ret += indVal + "\"string\": " + EncodeString(child.data.str) + ",\n";
      ret += indVal + "\"value\": ";

      customString = false;
    }

    if(child.type.flags & SerialisedType::Hidden)
    {
      ret += "\"hidden object\"";
    }
    else if(child.type.basetype == SerialisedType::eEnum)
    {
      ret += "{ "
        "\"__enum\": " + EncodeString(child.data.str) + ", "
        "\"value\": " + StringFormat::Fmt("%" PRIu64, child.data.basic.u) +
        " }";
    }
    else if(customString)
    {
      ret += EncodeString(child.data.str);
    }
    else
    {
      SerialisedType::BasicType t = child.type.basetype;

      switch(child.type.basetype)
      {
      case SerialisedType::eChunk:
        RDCASSERT("Should not have a chunk object!");
        break;
      case SerialisedType::eNull:
        {
          ret += humanReadable ? StringFormat::Fmt("/* %s */ null", child.type.name) : "null";
          break;
        }
      case SerialisedType::eStruct:
        {
          ret += humanReadable ? StringFormat::Fmt("/* %s, %" PRIu64 " bytes */ {\n", child.type.name, child.type.byteSize) : "[\n";
          ret += DumpObject(indent + 1 + conciseIndentOffset, machineReadable, child, humanReadable);
          ret += indVal + (humanReadable ? "}" : "]");
          break;
        }
      case SerialisedType::eArray:
        {
          ret += "[\n";
          ret += DumpObject(indent + 1 + conciseIndentOffset, true, child, humanReadable);
          ret += indVal + "]";
          break;
        }
      case SerialisedType::eBuffer:
        {
          ret += "{ "
            "\"buffername\": \"" + GetBufferName(child.data.basic.u) + "\", "
            "\"length\": " + StringFormat::Fmt("%" PRIu64, child.type.byteSize) +
            " }";
          break;
        }
      case SerialisedType::eString:
        ret += "\"" + child.data.str + "\"";
        break;
      case SerialisedType::eEnum:
        RDCASSERT("enums should be handled above!");
        break;
      case SerialisedType::eUnsignedInteger:
        ret += StringFormat::Fmt("%" PRIu64, child.data.basic.u);
        break;
      case SerialisedType::eSignedInteger:
        ret += StringFormat::Fmt("%" PRId64, child.data.basic.i);
        break;
      case SerialisedType::eFloat:
        ret += StringFormat::Fmt("%f", child.data.basic.d);
        break;
      case SerialisedType::eBoolean:
        ret += (child.data.basic.b ? "true" : "false");
        break;
      case SerialisedType::eCharacter:
        ret += "\"" + std::string(1, child.data.basic.c) + "\"";
        break;
      }
    }
    
    if(machineReadable)
    {
      ret += "\n" + ind + "}";
    }

    if(i+1 < obj.data.children.size())
      ret += ",\n";
  }

  ret += "\n";

  return ret;
}

std::string SerialisedFile2JSON(SerialisedFile file, bool humanReadable)
{
  const bool machineReadable = !humanReadable;

  std::string ret;

  ret += humanReadable ? "{\n" : "[\n";

  for(size_t i=0; i < file.chunks.size(); i++)
  {
    SerialisedObject *obj = file.chunks[i];
    RDCASSERT(obj->type.basetype == SerialisedType::eChunk);

    if(machineReadable)
    {
      ret += "\t{\n";
      ret += "\t\t\"name\": " + EncodeString(obj->name) + ",\n";
    }
    else
    {
      ret += "\t" + EncodeString(obj->name) + ": {\n";
    }

    if(obj->chunkData)
    {
      ret += "\t\t\"__metadata\": {\n";
      if(obj->chunkData->threadID)
        ret += StringFormat::Fmt("\t\t\t\"threadID\": %u,\n", obj->chunkData->threadID);
      if(obj->chunkData->timestamp)
        ret += StringFormat::Fmt("\t\t\t\"timestamp\": %llu,\n", obj->chunkData->timestamp);
      if(obj->chunkData->duration)
        ret += StringFormat::Fmt("\t\t\t\"duration\": %llu,\n", obj->chunkData->duration);
      if(obj->chunkData->callstack[0])
        ret += "\t\t\t\"callstack\": \"...\",\n";
      ret += StringFormat::Fmt("\t\t\t\"chunkID\": %u\n", obj->chunkData->chunkID);
      ret += "\t\t},\n";
    }

    if(obj->data.children.size() == 1 && obj->data.children[0]->type.flags & SerialisedType::OpaqueChunk)
    {
      ret += "\t\t\"opaque\": { ";
      ret += "\"buffername\": \"" + GetBufferName(obj->data.children[0]->data.basic.u) + "\", ";
      ret += "\"length\": " + StringFormat::Fmt("%" PRIu64, obj->data.children[0]->type.byteSize);
      ret += " }";
      ret += "\n\t}";
    }
    else
    {
      if(machineReadable)
        ret += "\t\t\"contents\": [\n";
      ret += DumpObject(humanReadable ? 2 : 3, machineReadable, *obj, humanReadable);
      ret += humanReadable ? StringFormat::Fmt("\t} /* %" PRIu64 " bytes */", obj->type.byteSize) : "\t\t]";
      if(machineReadable)
        ret += "\n\t}";
    }
    if(i+1 < file.chunks.size()) ret += ",";
    ret += "\n";
  }

  ret += humanReadable ? "}\n" : "]\n";

  return ret;
}

bool has(picojson::object o, const std::string &key)
{
  return o.find(key) != o.end();
}

#include "serialiser.h"

std::vector<SerialisedBuffer> *bufList = NULL;

struct jsonisedStruct
{
  picojson::value *val;
};

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, jsonisedStruct &el)
{
  using namespace picojson;

  RDCASSERT(el.val->is<array>());
  array &structMembers = el.val->get<array>();
  for(size_t i = 0; i < structMembers.size(); i++)
    ser.Serialise("", structMembers[i]);
}

INSTANTIATE_SERIALISE_TYPE(jsonisedStruct);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, picojson::value &el)
{
  using namespace picojson;

  std::vector<SerialisedBuffer> &bufs = *bufList;

  RDCASSERT(el.is<object>());

  object &baseValue = el.get<object>();

  RDCASSERT(has(baseValue, "type"));
  RDCASSERT(has(baseValue, "value"));
  
  picojson::value &t = baseValue["type"];
  picojson::value &v = baseValue["value"];

  RDCASSERT(t.is<object>());

  object &typeData = t.get<object>();

  RDCASSERT(has(typeData, "baseType"));
  RDCASSERT(has(typeData, "nullable"));
  RDCASSERT(has(typeData, "width"));

  picojson::value &baseType = typeData["baseType"];
  picojson::value &nullable = typeData["nullable"];
  picojson::value &width = typeData["width"];

  RDCASSERT(baseType.is<string>());
  RDCASSERT(nullable.is<bool>());
  RDCASSERT(width.is<int64_t>());

  string &baseTypeName = baseType.get<string>();
  bool isNullable = nullable.get<bool>();
  int64_t byteSize = width.get<int64_t>();

  if(baseTypeName == "chunk")
  {
    RDCERR("Did not expect chunk type");
  }
  else if(baseTypeName == "struct")
  {
    // structs are represented as arrays in json to preserve order
    if(isNullable)
    {
      // indirect through a struct so it is serialised as nullable
      jsonisedStruct s = { &v };
      jsonisedStruct *ptr = &s;
      ser.SerialiseNullable("", ptr);
    }
    else
    {
      RDCASSERT(v.is<array>());
      array &structMembers = v.get<array>();
      for(size_t i = 0; i < structMembers.size(); i++)
        ser.Serialise("", structMembers[i]);
    }
  }
  else if(baseTypeName == "array")
  {
    RDCASSERT(v.is<array>());
    array &arrayMembers = v.get<array>();
    
    ser.Serialise("", arrayMembers);
  }
  else if(baseTypeName == "null")
  {
    // any type will do since we're writing a marker that it's NULL
    float *foo = NULL;
    ser.SerialiseNullable("", foo);
  }
  else if(baseTypeName == "buffer")
  {
    RDCASSERT(v.is<object>());
    object &bufferDetails = v.get<object>();

    RDCASSERT(has(bufferDetails, "buffername"));

    size_t fn = (size_t)strtoimax(bufferDetails["buffername"].get<string>().c_str(), NULL, 10);

    if(fn < bufs.size())
    {
      byte *data = bufs[fn].first;
      uint64_t size = bufs[fn].second;
      ser.Serialise("", data, size);
    }
  }
  else if(baseTypeName == "string")
  {
    RDCASSERT(v.is<string>());
    string s = v.get<string>();
    if(isNullable)
    {
      string *ptr = &s;
      ser.SerialiseNullable("", ptr);
    }
    else
    {
      ser.Serialise("", s);
    }
  }
  else if(baseTypeName == "enum")
  {
    RDCASSERT(v.is<object>());
    object &enumdata = v.get<object>();

    RDCASSERT(has(enumdata, "value"));
    v = enumdata["value"];

    RDCASSERT(v.is<int64_t>());
    uint32_t e = (uint32_t)v.get<int64_t>();
    if(isNullable)
    {
      uint32_t *ptr = &e;
      ser.SerialiseNullable("", ptr);
    }
    else
    {
      ser.Serialise("", e);
    }
  }
  else if(baseTypeName == "uint")
  {
    RDCASSERT(v.is<int64_t>());
    uint64_t u = (uint64_t)v.get<int64_t>();
    
    if(byteSize == 1)
    {
      uint8_t u1 = (uint8_t)u;
      if(isNullable)
      {
        uint8_t *ptr = &u1;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", u1);
      }
    }
    else if(byteSize == 2)
    {
      uint16_t u2 = (uint16_t)u;
      if(isNullable)
      {
        uint16_t *ptr = &u2;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", u2);
      }
    }
    else if(byteSize == 4)
    {
      uint32_t u4 = (uint32_t)u;
      if(isNullable)
      {
        uint32_t *ptr = &u4;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", u4);
      }
    }
    else if(byteSize == 8)
    {
      if(isNullable)
      {
        uint64_t *ptr = &u;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", u);
      }
    }
    else
    {
      RDCERR("Unexpected byteSize for %s: %d", baseTypeName.c_str(), (int)byteSize);
    }
  }
  else if(baseTypeName == "sint")
  {
    RDCASSERT(v.is<int64_t>());
    int64_t i = v.get<int64_t>();
    
    if(byteSize == 1)
    {
      int8_t i1 = (int8_t)i;
      if(isNullable)
      {
        int8_t *ptr = &i1;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", i1);
      }
    }
    else if(byteSize == 2)
    {
      int16_t i2 = (int16_t)i;
      if(isNullable)
      {
        int16_t *ptr = &i2;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", i2);
      }
    }
    else if(byteSize == 4)
    {
      int32_t i4 = (int32_t)i;
      if(isNullable)
      {
        int32_t *ptr = &i4;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", i4);
      }
    }
    else if(byteSize == 8)
    {
      if(isNullable)
      {
        int64_t *ptr = &i;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", i);
      }
    }
    else
    {
      RDCERR("Unexpected byteSize for %s: %d", baseTypeName.c_str(), (int)byteSize);
    }
  }
  else if(baseTypeName == "bool")
  {
    RDCASSERT(v.is<bool>());
    bool b = v.get<bool>();
    if(isNullable)
    {
      bool *ptr = &b;
      ser.SerialiseNullable("", ptr);
    }
    else
    {
      ser.Serialise("", b);
    }
  }
  else if(baseTypeName == "float")
  {
    RDCASSERT(v.is<double>());
    double d = v.get<double>();
    
    if(byteSize == 4)
    {
      float f = (float)d;
      if(isNullable)
      {
        float *ptr = &f;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", f);
      }
    }
    else if(byteSize == 8)
    {
      if(isNullable)
      {
        double *ptr = &d;
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", d);
      }
    }
    else
    {
      RDCERR("Unexpected byteSize for %s: %d", baseTypeName.c_str(), (int)byteSize);
    }
  }
  else if(baseTypeName == "char")
  {
    RDCASSERT(v.is<string>());
    string s = v.get<string>();

    if(byteSize == 1)
    {
      if(isNullable)
      {
        char *ptr = &s[0];
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", s[0]);
      }
    }
    else if(byteSize == 2)
    {
      wstring w = L"-"; // convert from s
      if(isNullable)
      {
        wchar_t *ptr = &w[0];
        ser.SerialiseNullable("", ptr);
      }
      else
      {
        ser.Serialise("", w[0]);
      }
    }
    else
    {
      RDCERR("Unexpected byteSize for %s: %d", baseTypeName.c_str(), (int)byteSize);
    }
  }
}

INSTANTIATE_SERIALISE_TYPE(picojson::value);
<file_sep>/Makefile
GCC=gcc
GPP=g++
CLANG=clang++
CFLAGS=-g -O0

CPP_FILES=common.cpp main.cpp serialiser.cpp string_utils.cpp test_types.cpp xml_codec.cpp real.cpp streamio.cpp lz4io.cpp zstdio.cpp
H_FILES=common.h custom_assert.h serialiser.h string_utils.h test_types.h xml_codec.h structured_data.h streamio.h lz4io.h zstdio.h

EXT=lz4/lz4.o miniz.o zstd/common/entropy_common.o zstd/common/error_private.o zstd/common/fse_decompress.o zstd/common/pool.o zstd/common/threading.o zstd/common/xxhash.o zstd/common/zstd_common.o zstd/compress/fse_compress.o zstd/compress/huf_compress.o zstd/compress/zstdmt_compress.o zstd/compress/zstd_compress.o zstd/decompress/huf_decompress.o zstd/decompress/zstd_decompress.o pugixml.o

.PHONY: all
all: gpp_bin clang_bin

pugixml.o: pugixml.cpp
	$(GPP) -c pugixml.cpp -o pugixml.o

%.o : %.c
	$(GCC) -c $< -o $@

gpp_bin: $(CPP_FILES) $(H_FILES) $(EXT)
	$(GPP) -std=c++0x $(CFLAGS) $(CPP_FILES) -Wno-attributes -Wno-narrowing $(EXT) -o gpp_bin
	cp gpp_bin /tmp

clang_bin: $(CPP_FILES) $(H_FILES) libext.a
	$(CLANG) -std=c++0x $(CFLAGS) $(CPP_FILES) $(EXT) -o clang_bin
	cp clang_bin /tmp
<file_sep>/xml_codec.cpp
#include "xml_codec.h"
#include "pugixml.hpp"
#include "miniz.h"

static const char *typeNames[] = {
  "chunk",
  "struct",
  "array",
  "null",
  "buffer",
  "string",
  "enum",
  "uint",
  "int",
  "float",
  "bool",
  "char",
};

template<typename inttype>
std::string GetBufferName(inttype i)
{
  return StringFormat::Fmt("%06u", (uint32_t)i);
}

struct xml_string_writer: pugi::xml_writer
{
  std::string str;

  void write(const void* data, size_t size)
  {
    str.append((const char *)data, size);
  }
};

static void Obj2XML(pugi::xml_node &parent, StructuredObject &child)
{
  pugi::xml_node obj = parent.append_child(typeNames[child.type.basetype]);

  obj.append_attribute("name") = child.name.c_str();

  if(!child.type.name.empty())
    obj.append_attribute("typename") = child.type.name.c_str();

  if(child.type.basetype == SerialisedType::eUnsignedInteger ||
    child.type.basetype == SerialisedType::eSignedInteger ||
    child.type.basetype == SerialisedType::eFloat)
  {
    obj.append_attribute("width") = child.type.byteSize;
  }

  if(child.type.flags & SerialisedType::Hidden)
    obj.append_attribute("hidden") = true;

  if(child.type.flags & SerialisedType::Nullable)
    obj.append_attribute("nullable") = true;

  if(child.type.basetype == SerialisedType::eChunk)
  {
    RDCFATAL("Nested chunks!");
  }
  else if(child.type.basetype == SerialisedType::eNull)
  {
    // redundant
    obj.remove_attribute("nullable");
  }
  else if(child.type.basetype == SerialisedType::eStruct || child.type.basetype == SerialisedType::eArray)
  {
    if(child.type.basetype == SerialisedType::eArray && !child.data.children.empty())
      obj.remove_attribute("typename");

    for(size_t o=0; o < child.data.children.size(); o++)
    {
      Obj2XML(obj, *child.data.children[o]);

      if(child.type.basetype == SerialisedType::eArray)
        obj.last_child().remove_attribute("name");
    }
  }
  else if(child.type.basetype == SerialisedType::eBuffer)
  {
    obj.append_attribute("byteLength") = child.type.byteSize;
    obj.text() = child.data.basic.u;
  }
  else
  {
    if(child.type.flags & SerialisedType::HasCustomString)
    {
      obj.append_attribute("string") = child.data.str.c_str();
    }
    
    switch(child.type.basetype)
    {
      case SerialisedType::eEnum:
      case SerialisedType::eUnsignedInteger:
        obj.text() = child.data.basic.u;
        break;
      case SerialisedType::eSignedInteger:
        obj.text() = child.data.basic.i;
        break;
      case SerialisedType::eString:
        obj.text() = child.data.str.c_str();
        break;
      case SerialisedType::eFloat:
        obj.text() = child.data.basic.d;
        break;
      case SerialisedType::eBoolean:
        obj.text() = child.data.basic.b;
        break;
      case SerialisedType::eCharacter:
      {
        char str[2] = { child.data.basic.c, '\0' };
        obj.text().set(str);
        break;
      }
      default:
        RDCERR("Unexpected case");
    }
  }
}

std::string Structured2XML(const StructuredChunkList &chunks)
{
  pugi::xml_document doc;

  pugi::xml_node xRoot = doc.append_child("rdc");

  for(size_t c=0; c < chunks.size(); c++)
  {
    pugi::xml_node xChunk = xRoot.append_child("chunk");
    StructuredChunk *chunk = chunks[c];

    xChunk.append_attribute("id") = chunk->metadata.chunkID;
    xChunk.append_attribute("name") = chunk->name.c_str();
    if(chunk->metadata.threadID)
      xChunk.append_attribute("threadID") = chunk->metadata.threadID;
    if(chunk->metadata.timestamp)
      xChunk.append_attribute("timestamp") = chunk->metadata.timestamp;
    if(chunk->metadata.duration)
      xChunk.append_attribute("duration") = chunk->metadata.duration;
    if(chunk->metadata.callstack[0])
    {
      pugi::xml_node stack = xChunk.append_child("callstack");

      for(size_t i=0; i < ARRAY_COUNT(chunk->metadata.callstack); i++)
      {
        if(chunk->metadata.callstack[i] == 0)
          break;
        stack.append_child("address").text() = chunk->metadata.callstack[i];
      }
    }
    
    if(chunk->metadata.flags & SerialisedChunkMetaData::OpaqueChunk)
    {
      xChunk.append_attribute("opaque") = true;

      RDCASSERT(!chunk->data.children.empty());
      pugi::xml_node opaque = xChunk.append_child("buffer");
      opaque.append_attribute("byteLength") = chunk->data.children[0]->type.byteSize;
      opaque.text() = chunk->data.children[0]->data.basic.u;
    }
    else
    {
      for(size_t o=0; o < chunk->data.children.size(); o++)
        Obj2XML(xChunk, *chunk->data.children[o]);
    }
  }

  xml_string_writer writer;
  doc.save(writer);

  return writer.str;
}

StructuredObject *XML2Obj(pugi::xml_node &obj)
{
  StructuredObject *ret = new StructuredObject(obj.attribute("name").as_string(), obj.attribute("typename").as_string());

  std::string name = obj.name();
  
  for(size_t i = 0; i < ARRAY_COUNT(typeNames); i++)
  {
    if(name == typeNames[i])
    {
      ret->type.basetype = (SerialisedType::BasicType)i;
      break;
    }
  }

  if(ret->type.basetype == SerialisedType::eUnsignedInteger ||
    ret->type.basetype == SerialisedType::eSignedInteger ||
    ret->type.basetype == SerialisedType::eFloat)
  {
    ret->type.byteSize = obj.attribute("width").as_uint();
  }
  
  if(obj.attribute("hidden"))
    ret->type.flags |= SerialisedType::Hidden;
  
  if(obj.attribute("nullable"))
    ret->type.flags |= SerialisedType::Nullable;

  if(obj.attribute("typename"))
    ret->type.name = obj.attribute("typename").as_string();

  ret->name = obj.attribute("name").as_string();

  if(ret->type.basetype == SerialisedType::eChunk)
  {
    RDCFATAL("Nested chunks!");
  }
  else if(ret->type.basetype == SerialisedType::eNull)
  {
    ret->type.flags |= SerialisedType::Nullable;
  }
  else if(ret->type.basetype == SerialisedType::eStruct || ret->type.basetype == SerialisedType::eArray)
  {
    for(pugi::xml_node child = obj.first_child(); child; child = child.next_sibling())
    {
      ret->data.children.push_back(XML2Obj(child));

      if(ret->type.basetype == SerialisedType::eArray)
        ret->data.children.back()->name = "$el";
    }

    if(ret->type.basetype == SerialisedType::eArray && !ret->data.children.empty())
      ret->type.name = ret->data.children.back()->type.name;
  }
  else if(ret->type.basetype == SerialisedType::eBuffer)
  {
    ret->type.byteSize = obj.attribute("byteLength").as_ullong();
    ret->data.basic.u = obj.text().as_uint();
  }
  else
  {
    if(obj.attribute("string"))
    {
      ret->type.flags |= SerialisedType::HasCustomString;
      ret->data.str = obj.attribute("string").as_string();
    }
    
    switch(ret->type.basetype)
    {
      case SerialisedType::eEnum:
      case SerialisedType::eUnsignedInteger:
        ret->data.basic.u = obj.text().as_ullong();
        break;
      case SerialisedType::eSignedInteger:
        ret->data.basic.i = obj.text().as_llong();
        break;
      case SerialisedType::eString:
        ret->data.str = obj.text().as_string();
        break;
      case SerialisedType::eFloat:
        ret->data.basic.d = obj.text().as_double();
        break;
      case SerialisedType::eBoolean:
        ret->data.basic.b = obj.text().as_bool();
        break;
      case SerialisedType::eCharacter:
        ret->data.basic.c = obj.text().as_string()[0];
        break;
      default:
        RDCERR("Unexpected case");
    }
  }

  return ret;
}

StructuredChunkList XML2Structured(const std::string &xml)
{
  pugi::xml_document doc;
  doc.load_string(xml.c_str());

  pugi::xml_node root = doc.child("rdc");

  if(!root)
  {
    RDCERR("Malformed document, expected rdc node");
    return StructuredChunkList();
  }

  StructuredChunkList ret;

  for(pugi::xml_node xChunk = root.first_child(); xChunk; xChunk = xChunk.next_sibling())
  {
    StructuredChunk *chunk = new StructuredChunk(xChunk.attribute("name").as_string(), "Chunk");
    
    chunk->metadata.chunkID = xChunk.attribute("id").as_uint();
    if(xChunk.attribute("threadID"))
      chunk->metadata.threadID = xChunk.attribute("threadID").as_uint();
    if(xChunk.attribute("timestamp"))
      chunk->metadata.timestamp = xChunk.attribute("timestamp").as_ullong();
    if(xChunk.attribute("duration"))
      chunk->metadata.duration = xChunk.attribute("duration").as_ullong();

    pugi::xml_node callstack = xChunk.child("callstack");
    if(callstack)
    {
      size_t i = 0;
      for(pugi::xml_node address = callstack.first_child(); address; address = address.next_sibling())
      {
        chunk->metadata.callstack[i] = address.text().as_ullong();
        i++;
      }
    }
    
    if(xChunk.attribute("opaque"))
    {
      pugi::xml_node opaque = xChunk.child("buffer");

      chunk->metadata.flags |= SerialisedChunkMetaData::OpaqueChunk;
      
      chunk->data.children.push_back(new StructuredObject("Opaque chunk", "Byte Buffer"));
      chunk->data.children[0]->type.basetype = SerialisedType::eBuffer;
      chunk->data.children[0]->type.byteSize = opaque.attribute("byteLength").as_ullong();
      chunk->data.children[0]->data.basic.u = opaque.text().as_ullong();
    }
    else
    {
      for(pugi::xml_node child = xChunk.first_child(); child; child = child.next_sibling())
        chunk->data.children.push_back(XML2Obj(child));
    }

    ret.push_back(chunk);
  }

  return ret;
}

void Buffers2ZIP(const char *filename, const SerialisedBufferList &buffers)
{
  mz_zip_archive zip;
  memset(&zip, 0, sizeof(zip));

  mz_zip_writer_init_file(&zip, filename, 0);

  for(size_t i = 0; i < buffers.size(); i++)
    mz_zip_writer_add_mem(&zip, GetBufferName(i).c_str(), buffers[i].first, buffers[i].second, MZ_BEST_COMPRESSION);

  mz_zip_writer_finalize_archive(&zip);
  mz_zip_writer_end(&zip);
}

SerialisedBufferList ZIP2Buffers(const char *filename)
{
  SerialisedBufferList ret;

  mz_zip_archive zip;
  memset(&zip, 0, sizeof(zip));

  mz_bool b = mz_zip_reader_init_file(&zip, "in.zip", 0);

  if(b)
  {
    mz_uint numfiles = mz_zip_reader_get_num_files(&zip);

    ret.resize(numfiles);

    for(mz_uint i = 0; i < numfiles; i++)
    {
      mz_zip_archive_file_stat zstat;
      mz_zip_reader_file_stat(&zip, i, &zstat);

      int bufname = atoi(zstat.m_filename);

      size_t sz = 0;

      byte *buf = (byte *)mz_zip_reader_extract_to_heap(&zip, i, &sz, 0);

      if(bufname < ret.size())
      {
        ret[bufname].first = buf;
        ret[bufname].second = (uint64_t)sz;
      }
    }
  }

  mz_zip_reader_end(&zip);

  return ret;
}
<file_sep>/serialiser.h
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2015-2016 <NAME>
 * Copyright (c) 2014 Crytek
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#pragma once

#include <list>
#include <set>
#include <string>
#include <vector>
//#include "api/replay/basic_types.h"
#include "common.h"
#include "structured_data.h"
#include "streamio.h"
#include "stringize.h"
//#include "os/os_specific.h"
//#include "replay/type_helpers.h"

using std::string;

/////////////////////////////////////////////////////////////////////////////
// main serialisation templated function that is specialised for any type
// that needs to be serialised. Implementations will either be a struct type,
// in which case they'll call Serialise() on each of their members, or
// they'll be some custom leaf type (most basic types are specialised in this
// header) which will call SerialiseValue().

template <class SerialiserType, class T>
void DoSerialise(SerialiserType &ser, T &el);

// function to deallocate anything from a serialise. Default impl
// does no deallocation of anything.
template <class T>
void Deserialise(const T &el)
{
}

#define DECLARE_SERIALISE_TYPE(type) \
  template <class SerialiserType> \
  void DoSerialise(SerialiserType &ser, type &el);

#define DECLARE_DESERIALISE_TYPE(type) \
  template <> \
  void Deserialise(const type &el);

// this is definitely the ugliest hack here, but necessary. We don't want to
// have all of our DoSerialise functions defined in global cross-project
// headers and compiled everywhere, we just want to define them in .cpp files.
// However because they are template specializations, if they're not
// instantiated in that same file then the compiler will skip them and not
// include them in the build, so elsewhere we'll get undefined symbol errors.
// This code forces them to be included in a global variable's constructor,
// because it can't optimise out the volatile int check. But in reality it
// will always be 0 so we spend a couple of instructions per type on load.
// It's obviously not ideal, but it's the best compromise.
#define INSTANTIATE_SERIALISE_TYPE(type) \
  template <>                        \
  const char *GetSerialiseTypeName<type>::Name() \
  {                                  \
    return #type;                       \
  } \
  namespace { struct CONCAT(Instantiator, __LINE__) \
  { \
    CONCAT(Instantiator, __LINE__)() \
    { \
      volatile int dummy = 0; type *el = NULL; \
      WriteSerialiser *a = NULL; ReadSerialiser *b = NULL; StructReadSerialiser *c = NULL; \
      void (*do_a)(decltype(*a) &, type &) = NULL; \
      void (*do_b)(decltype(*b) &, type &) = NULL; \
      void (*do_c)(decltype(*c) &, type &) = NULL; \
      if(dummy) \
      { \
        do_a = (decltype(do_a))(&DoSerialise); \
        do_b = (decltype(do_b))(&DoSerialise); \
        do_c = (decltype(do_c))(&DoSerialise); \
      } \
      if(dummy) \
      { \
        do_a(*a, *el); \
        do_b(*b, *el); \
        do_c(*c, *el); \
      } \
    } \
  } CONCAT(dummy, __LINE__); }

typedef const char *(*ChunkLookup)(uint32_t chunkType);

template <class SerialiserType>
class ScopedContext;

// holds the memory, length and type for a given chunk, so that it can be
// passed around and moved between owners before being serialised out
class Chunk
{
public:
  ~Chunk();

  byte *GetData() { return m_Data; }
  uint32_t GetLength() { return m_Length; }
  uint32_t GetChunkType() { return m_ChunkType; }
  bool IsAligned() { return m_AlignedData; }
  bool IsTemporary() { return m_Temporary; }
#if !defined(RELEASE)
  static uint64_t NumLiveChunks() { return m_LiveChunks; }
  static uint64_t TotalMem() { return m_TotalMem; }
#else
  static uint64_t NumLiveChunks() { return 0; }
  static uint64_t TotalMem() { return 0; }
#endif

  // grab current contents of the serialiser into this chunk
  template<class SerialiserType>
  Chunk(SerialiserType &ser, uint32_t chunkType, bool temporary)
  {
    m_Length = (uint32_t)ser.GetWriter()->GetOffset();

    RDCASSERT(ser.GetWriter()->GetOffset() < 0xffffffff);

    m_ChunkType = chunkType;

    m_Temporary = temporary;

    if(ser.HasAlignedData())
    {
      m_Data = AllocAlignedBuffer(m_Length);
      m_AlignedData = true;
    }
    else
    {
      m_Data = new byte[m_Length];
      m_AlignedData = false;
    }

    memcpy(m_Data, ser.GetWriter()->GetData(), m_Length);

    ser.GetWriter()->Rewind();
  }

private:
  // no copy semantics
  Chunk(const Chunk &);
  Chunk &operator=(const Chunk &);
  
  template <class SerialiserType>
  friend class ScopedContext;

  bool m_AlignedData;
  bool m_Temporary;

  uint32_t m_ChunkType;

  uint32_t m_Length;
  byte *m_Data;

#if !defined(RELEASE)
  static int64_t m_LiveChunks, m_MaxChunks, m_TotalMem;
#endif
};

struct SerialiserFlags
{
  enum Flags
  {
    None = 0,
    AllocateMemory = 0x1,
  };
};

enum ContainerError
{
  eContError_None = 0,
  eContError_FileIO,
  eContError_Corrupt,
  eContError_UnsupportedVersion,
};

enum SectionFlags
{
  eSectionFlag_None = 0x0,
  eSectionFlag_ASCIIStored = 0x1,
  eSectionFlag_LZ4Compressed = 0x2,
  eSectionFlag_ZstdCompressed = 0x4,
};

enum SectionType
{
  eSectionType_Unknown = 0,
  eSectionType_FrameCapture,       // renderdoc/internal/framecapture
  eSectionType_ResolveDatabase,    // renderdoc/internal/resolvedb
  eSectionType_FrameBookmarks,     // renderdoc/ui/bookmarks
  eSectionType_Notes,              // renderdoc/ui/notes
  eSectionType_Num,
};

// this class has a few functions. It can be used to serialise chunks - on writing it enforces
// that we only ever write a single chunk, then pull out the data into a Chunk class and erase
// the contents of the serialiser ready to serialise the next (see the RDCASSERT at the start
// of BeginChunk).
//
// We use this functionality for sending and receiving data across the network as well as saving
// out to the capture logfile format.
//
// It's also used on reading where it will contain the stream of chunks that were written out
// to the logfile on capture.
//
// When reading, the Serialiser allocates a window of memory and scans through the file by reading
// data into that window and moving along through the file. The window will expand to accomodate
// whichever is the biggest single element within a chunk that's read (so that you can always
// guarantee
// while reading that the element you're interested in is always in memory).

struct SerialiserType
{
  enum Types
  {
    Writing,
    Reading,
    StructuredReading,
  };
};

class RDOCFile
{
public:
  // opens a file either to write into, or read from
  RDOCFile(const char *path, SerialiserType::Types mode);

  bool HasError();
  ContainerError ErrorCode();

  bool HasSection(SectionType type);
  void *GetSection(SectionType type);
  void *AddSection(SectionType type);
};

struct CompressedFileIO;

template<SerialiserType::Types sertype>
class Serialiser
{
public:
  const bool IsReading() const { return sertype != SerialiserType::Writing; }
  const bool IsWriting() const { return sertype == SerialiserType::Writing; }
  const bool ExportStructure() const { return sertype == SerialiserType::StructuredReading && !m_InternalElement; }
  
  enum ChunkFlags
  {
    ChunkIndexMask = 0x0000ffff,
    ChunkCallstack = 0x00010000,
    ChunkThreadID  = 0x00020000,
    ChunkDuration  = 0x00040000,
    ChunkTimestamp = 0x00080000,
  };

  enum
  {
    BufferAlignment = 64,
  };

  // version number of overall file format or chunk organisation. If the contents/meaning/order of
  // chunks have changed this does not need to be bumped, there are version numbers within each
  // API that interprets the stream that can be bumped.
  static const uint64_t SERIALISE_VERSION = 0x00000032;

  //////////////////////////////////////////
  // Init and error handling

  Serialiser(StreamWriter *writer);
  Serialiser(StreamReader *reader);
  ~Serialiser();

  StreamWriter *GetWriter() { return m_Write; }
  StreamReader *GetReader() { return m_Read; }

  bool HasError() { return m_HasError; }
  ContainerError ErrorCode() { return m_ErrorCode; }

  void SetChunkMetadataRecording(uint32_t flags);

  SerialisedChunkMetaData &ChunkMetadata() { return m_ChunkMetadata; }

  //////////////////////////////////////////
  // Utility functions

  void *GetUserData() { return m_pUserData; }
  void SetUserData(void *userData) { m_pUserData = userData; }
  bool HasAlignedData() { return m_AlignedData; }

  // jumps to the byte after the current chunk, can be called any time after BeginChunk
  void SkipCurrentChunk();

  void InitCallstackResolver();
  bool HasCallstacks() { return m_KnownSections[eSectionType_ResolveDatabase] != NULL; }

  const StructuredChunkList &GetSerialisedChunks()
  {
    return m_StructuredData;
  }

  const SerialisedBufferList &GetSerialisedBuffers()
  {
    return m_StructuredDataBuffers;
  }

  void WriteSerialisedFile(Serialiser &fileser, SerialisedFile file);

  //////////////////////////////////////////
  // Public serialisation interface
  
  void ConfigureStructuredExport(ChunkLookup lookup, bool includeBuffers)
  {
    RDCASSERT(lookup);
    
    if(lookup)
    {
      m_ExportBuffers = includeBuffers;
      m_ChunkLookup = lookup;
    }
  }

  uint32_t BeginChunk(uint32_t chunkID, uint32_t byteLength = 0);
  void EndChunk();

  // Write a chunk to disk
  void Insert(Chunk *el);

  void FlushToDisk();
  
  /////////////////////////////////////////////////////////////////////////////
  /////////////////////////////////////////////////////////////////////////////
  /////////////////////////////////////////////////////////////////////////////
  // Templated serialisation functions

  // Users of the serialisation call Serialise() on the objects they want
  // to serialise
  
  // for structure type members or external use, main entry point

  template <typename T>
  void SerialiseStringify(const T el)
  {
    if(ExportStructure())
    {
      m_StructureStack.back()->data.str = ToStr(el);
      m_StructureStack.back()->type.flags |= SerialisedType::HasCustomString;
    }
  }

  template <class SerialiserType, typename T, bool isEnum = std::is_enum<T>::value>
  struct SerialiseDispatch
  {
    static void Do(SerialiserType &ser, T &el)
    {
      DoSerialise(ser, el);
    }
  };

  template <class SerialiserType, typename T>
  struct SerialiseDispatch<SerialiserType, T, true>
  {
    static void Do(SerialiserType &ser, T &el)
    {
      RDCCOMPILE_ASSERT(sizeof(T) == sizeof(uint32_t), "enum isn't 32-bit");
      ser.SerialiseValue(SerialisedType::eEnum, sizeof(T), el);
      ser.SerialiseStringify(el);
    }
  };

  // serialise an object (either loose, or a structure member).
  template <class T>
  Serialiser &Serialise(const char *name, T &el, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    if(ExportStructure())
    {
      if(m_StructureStack.empty())
      {
        RDCERR("Serialising object outside of chunk context! Call BeginChunk before any Serialise!");
        return *this;
      }

      StructuredObject &current = *m_StructureStack.back();

      current.data.basic.numChildren++;
      current.data.children.push_back(new StructuredObject(name, GetSerialiseTypeName<T>::Name()));
      m_StructureStack.push_back(current.data.children.back());

      StructuredObject &obj = *m_StructureStack.back();
      obj.type.byteSize = sizeof(T);
    }

    SerialiseDispatch<Serialiser, T>::Do(*this, el);
    
    if(ExportStructure())
      m_StructureStack.pop_back();

    return *this;
  }

  // special function for serialising buffers
  Serialiser &Serialise(const char *name, byte *&el, uint64_t &byteSize, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    uint64_t count = byteSize;

    // silently handle NULL buffers
    if(IsWriting() && el == NULL)
      count = 0;

    {
      m_InternalElement = true;
      DoSerialise(*this, count);
      m_InternalElement = false;
    }

    if(IsReading())
      byteSize = count;

    if(ExportStructure())
    {
      if(m_StructureStack.empty())
      {
        RDCERR("Serialising object outside of chunk context! Call BeginChunk before any Serialise!");
        return *this;
      }

      StructuredObject &current = *m_StructureStack.back();

      current.data.basic.numChildren++;
      current.data.children.push_back(new StructuredObject(name, "Byte Buffer"));
      m_StructureStack.push_back(current.data.children.back());

      StructuredObject &obj = *m_StructureStack.back();
      obj.type.basetype = SerialisedType::eBuffer;
      obj.type.byteSize = byteSize;
    }
    
    {
      if(IsWriting())
      {
        // ensure byte alignment
        m_Write->AlignTo<BufferAlignment>();
        
        m_AlignedData = true;
        m_Write->Write(el, byteSize);
      }
      else if(IsReading())
      {
        // ensure byte alignment
        m_Read->AlignTo<BufferAlignment>();
        
        if(flags & SerialiserFlags::AllocateMemory)
          el = AllocAlignedBuffer(byteSize);

        m_Read->Read(el, byteSize);
      }
    }
    
    // if we want byte data associated with SerialisedObject somewhere,
    // here's where we'd also copy byteSize bytes from el elsewhere.

    if(ExportStructure())
    {
      if(m_ExportBuffers)
      {
        StructuredObject &obj = *m_StructureStack.back();

        obj.data.basic.u = m_StructuredDataBuffers.size();

        byte *alloc = new byte[byteSize];
        memcpy(alloc, el, byteSize);

        m_StructuredDataBuffers.push_back(std::make_pair(alloc, byteSize));
      }

      m_StructureStack.pop_back();
    }

    return *this;
  }

  Serialiser &Serialise(const char *name, byte *&el, uint32_t &byteSize, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    uint64_t upcastSize = byteSize;
    Serialise(name, el, byteSize, flags);
    if(IsReading())
      byteSize = (uint32_t)upcastSize;
    return *this;
  }

  Serialiser &Serialise(const char *name, void *&el, uint64_t &byteSize, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    return Serialise(name, (byte*&)el, byteSize, flags);
  }

  Serialiser &Serialise(const char *name, const void *&el, uint64_t &byteSize, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    return Serialise(name, (byte*&)el, byteSize, flags);
  }

  Serialiser &Serialise(const char *name, void *&el, uint32_t &byteSize, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    return Serialise(name, (byte*&)el, byteSize, flags);
  }

  Serialiser &Serialise(const char *name, const void *&el, uint32_t &byteSize, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    return Serialise(name, (byte*&)el, byteSize, flags);
  }

  // serialise a fixed array like foo[4];
  // never needs to allocate, just needs to be iterated
  template <class T, size_t N>
  Serialiser &Serialise(const char *name, T (&el)[N], SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    // for consistency with other arrays, even though this is redundant, we serialise out and in the size
    {
      uint64_t count = N;
      m_InternalElement = true;
      DoSerialise(*this, count);
      m_InternalElement = false;
    }

    if(ExportStructure())
    {
      if(m_StructureStack.empty())
      {
        RDCERR("Serialising object outside of chunk context! Call BeginChunk before any Serialise!");
        return *this;
      }

      StructuredObject &parent = *m_StructureStack.back();
      parent.data.basic.numChildren++;
      parent.data.children.push_back(new StructuredObject(name, GetSerialiseTypeName<T>::Name()));
      m_StructureStack.push_back(parent.data.children.back());
      
      StructuredObject &arr = *m_StructureStack.back();
      arr.type.basetype = SerialisedType::eArray;
      arr.type.byteSize = N;

      arr.data.basic.numChildren = (uint64_t)N;
      arr.data.children.resize(N);
      
      for(size_t i=0; i < N; i++)
      {
        arr.data.children[i] = new StructuredObject("$el", GetSerialiseTypeName<T>::Name());
        m_StructureStack.push_back(arr.data.children[i]);

        StructuredObject &obj = *m_StructureStack.back();
        
        // default to struct. This will be overwritten if appropriate
        obj.type.basetype = SerialisedType::eStruct;
        obj.type.byteSize = sizeof(T);

        SerialiseDispatch<Serialiser, T>::Do(*this, el[i]);

        m_StructureStack.pop_back();
      }

      m_StructureStack.pop_back();
    }
    else
    {
      for(size_t i=0; i < N; i++)
        SerialiseDispatch<Serialiser, T>::Do(*this, el[i]);
    }

    return *this;
  }

  // special function for serialising dynamically sized arrays
  template<class T>
  Serialiser &Serialise(const char *name, T *&el, uint64_t &arrayCount, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    uint64_t count = arrayCount;

    // silently handle NULL arrays
    if(IsWriting() && el == NULL)
      count = 0;

    {
      m_InternalElement = true;
      DoSerialise(*this, count);
      m_InternalElement = false;
    }

    if(IsReading())
      arrayCount = count;

    if(ExportStructure())
    {
      if(m_StructureStack.empty())
      {
        RDCERR("Serialising object outside of chunk context! Call BeginChunk before any Serialise!");
        return *this;
      }

      StructuredObject &parent = *m_StructureStack.back();
      parent.data.basic.numChildren++;
      parent.data.children.push_back(new StructuredObject(name, GetSerialiseTypeName<T>::Name()));
      m_StructureStack.push_back(parent.data.children.back());
      
      StructuredObject &arr = *m_StructureStack.back();
      arr.type.basetype = SerialisedType::eArray;
      arr.type.byteSize = arrayCount;

      arr.data.basic.numChildren = arrayCount;
      arr.data.children.resize(arrayCount);
      
      if(IsReading() && (flags & SerialiserFlags::AllocateMemory))
      {
        if(arrayCount > 0)
          el = new T[arrayCount];
        else
          el = NULL;
      }

      for(uint64_t i=0; el && i < arrayCount; i++)
      {
        arr.data.children[i] = new StructuredObject("$el", GetSerialiseTypeName<T>::Name());
        m_StructureStack.push_back(arr.data.children[i]);

        StructuredObject &obj = *m_StructureStack.back();
        
        // default to struct. This will be overwritten if appropriate
        obj.type.basetype = SerialisedType::eStruct;
        obj.type.byteSize = sizeof(T);

        SerialiseDispatch<Serialiser, T>::Do(*this, el[i]);

        m_StructureStack.pop_back();
      }

      m_StructureStack.pop_back();
    }
    else
    {
      if(IsReading() && (flags & SerialiserFlags::AllocateMemory))
      {
        if(arrayCount > 0)
          el = new T[arrayCount];
        else
          el = NULL;
      }
      
      for(size_t i=0; el && i < arrayCount; i++)
        SerialiseDispatch<Serialiser, T>::Do(*this, el[i]);
    }

    return *this;
  }

  template<class T>
  Serialiser &Serialise(const char *name, T *&el, uint32_t &arrayCount, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    uint64_t upcastCount = arrayCount;
    Serialise(name, el, upcastCount, flags);
    if(IsReading())
      arrayCount = (uint32_t)upcastCount;
    return *this;
  }

  // specialisations for container types
  template <class U>
  Serialiser &Serialise(const char *name, std::vector<U> &el, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    uint64_t size = (uint64_t)el.size();

    {
      m_InternalElement = true;
      DoSerialise(*this, size);
      m_InternalElement = false;
    }

    if(ExportStructure())
    {
      if(m_StructureStack.empty())
      {
        RDCERR("Serialising object outside of chunk context! Call BeginChunk before any Serialise!");
        return *this;
      }

      StructuredObject &parent = *m_StructureStack.back();
      parent.data.basic.numChildren++;
      parent.data.children.push_back(new StructuredObject(name, GetSerialiseTypeName<U>::Name()));
      m_StructureStack.push_back(parent.data.children.back());
      
      StructuredObject &arr = *m_StructureStack.back();
      arr.type.basetype = SerialisedType::eArray;
      arr.type.byteSize = size;

      arr.data.basic.numChildren = size;
      arr.data.children.resize(size);
      
      if(IsReading())
        el.resize(size);

      for(uint64_t i=0; i < size; i++)
      {
        arr.data.children[i] = new StructuredObject("$el", GetSerialiseTypeName<U>::Name());
        m_StructureStack.push_back(arr.data.children[i]);

        StructuredObject &obj = *m_StructureStack.back();
        
        // default to struct. This will be overwritten if appropriate
        obj.type.basetype = SerialisedType::eStruct;
        obj.type.byteSize = sizeof(U);

        SerialiseDispatch<Serialiser, U>::Do(*this, el[i]);

        m_StructureStack.pop_back();
      }

      m_StructureStack.pop_back();
    }
    else
    {
      if(IsReading())
        el.resize(size);

      for(uint64_t i=0; i < size; i++)
        SerialiseDispatch<Serialiser, U>::Do(*this, el[i]);
    }
    
    return *this;
  }

  template <class U>
  Serialiser &Serialise(const char *name, std::list<U> &el, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    RDCFATAL("Not implemented");
    return *this;
  }

  template <class U, class V>
  Serialiser &Serialise(const char *name, std::pair<U, V> &el, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    RDCFATAL("Not implemented");
    return *this;
  }

  // for const types, to cast away the const!
  // caller is responsible for ensuring that on write, it's safe to write into
  // this pointer on the other end
  template<class T>
  Serialiser &Serialise(const char *name, const T &el, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    return Serialise(name, (T&)el, flags);
  }

  // dynamic array variant
  template<class T>
  Serialiser &Serialise(const char *name, const T *&el, uint64_t &arrayCount, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    return Serialise(name, (T *&)el, arrayCount, flags);
  }

  template <class T>
  Serialiser &SerialiseNullable(const char *name, T *&el, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    bool present = (el != NULL);

    {
      m_InternalElement = true;
      DoSerialise(*this, present);
      m_InternalElement = false;
    }

    if(ExportStructure())
    {
      if(m_StructureStack.empty())
      {
        RDCERR("Serialising object outside of chunk context! Call BeginChunk before any Serialise!");
        return *this;
      }

      if(IsReading())
      {
        if(present)
          el = new T;
        else
          el = NULL;
      }
      
      if(el)
      {
        Serialise(name, *el, flags);

        StructuredObject &parent = *m_StructureStack.back();

        StructuredObject &nullable = *parent.data.children.back();

        nullable.type.flags |= SerialisedType::Nullable;
      }
      else
      {
        StructuredObject &parent = *m_StructureStack.back();
        parent.data.basic.numChildren++;
        parent.data.children.push_back(new StructuredObject(name, GetSerialiseTypeName<T>::Name()));

        StructuredObject &nullable = *parent.data.children.back();
        nullable.type.basetype = SerialisedType::eNull;
        nullable.type.byteSize = 0;
        nullable.type.flags |= SerialisedType::Nullable;
      }
    }
    else
    {
      if(el)
        Serialise(name, *el, flags);
    }
    
    return *this;
  }

  template <class T>
  Serialiser &SerialiseNullable(const char *name, const T *&el, SerialiserFlags::Flags flags = SerialiserFlags::None)
  {
    return SerialiseNullable(name, (T *&)el, flags);
  }

  Serialiser &Aligned()
  {
    m_AlignedData = true;
    return *this;
  }

  // these functions can be chained onto the end of a Serialise() call or macro to
  // set additional properties or change things
  Serialiser &Hidden()
  {
    if(ExportStructure() && !m_StructureStack.empty())
    {
      StructuredObject &current = *m_StructureStack.back();

      if(!current.data.children.empty())
        current.data.children.back()->type.flags |= SerialisedType::Hidden;
    }

    return *this;
  }

  Serialiser &Named(const char *name)
  {
    if(ExportStructure() && !m_StructureStack.empty())
    {
      StructuredObject &current = *m_StructureStack.back();

      if(!current.data.children.empty())
        current.data.children.back()->name = name;
    }

    return *this;
  }

  /////////////////////////////////////////////////////////////////////////////

  // for basic/leaf types. Read/written just as byte soup, MUST be plain old data
  template <class T>
  void SerialiseValue(SerialisedType::BasicType type, size_t byteSize, T &el)
  {
    if(IsWriting())
    {
      m_Write->Write(el);
    }
    else if(IsReading())
    {
      m_Read->Read(el);
    }

    if(!ExportStructure())
      return;

    StructuredObject &current = *m_StructureStack.back();

    current.type.basetype = type;
    current.type.byteSize = byteSize;

    switch(type)
    {
      case SerialisedType::eChunk:
      case SerialisedType::eStruct:
      case SerialisedType::eArray:
      case SerialisedType::eBuffer:
      case SerialisedType::eNull:
        RDCFATAL("Cannot call SerialiseValue for type %d!", type);
        break;
      case SerialisedType::eString:
        RDCFATAL("eString should be specialised!");
        break;
      case SerialisedType::eEnum:
      case SerialisedType::eUnsignedInteger:
        if(byteSize == 1)
          current.data.basic.u = (uint64_t)(uint8_t)el;
        else if(byteSize == 2)
          current.data.basic.u = (uint64_t)(uint16_t)el;
        else if(byteSize == 4)
          current.data.basic.u = (uint64_t)(uint32_t)el;
        else if(byteSize == 8)
          current.data.basic.u = (uint64_t)el;
        else
          RDCFATAL("Unsupported unsigned integer byte width: %u", byteSize);
        break;
      case SerialisedType::eSignedInteger:
        if(byteSize == 1)
          current.data.basic.i = (int64_t)(int8_t)el;
        else if(byteSize == 2)
          current.data.basic.i = (int64_t)(int16_t)el;
        else if(byteSize == 4)
          current.data.basic.i = (int64_t)(int32_t)el;
        else if(byteSize == 8)
          current.data.basic.i = (int64_t)el;
        else
          RDCFATAL("Unsupported signed integer byte width: %u", byteSize);
        break;
      case SerialisedType::eFloat:
        if(byteSize == 4)
          current.data.basic.d = (double)(float)el;
        else if(byteSize == 8)
          current.data.basic.d = (double)el;
        else
          RDCFATAL("Unsupported floating point byte width: %u", byteSize);
        break;
      case SerialisedType::eBoolean:
        // co-erce to boolean
        current.data.basic.b = !!(el);
        break;
      case SerialisedType::eCharacter:
        current.data.basic.c = (char)(el);
        break;
    }
  }

  // the only non POD basic type
  void SerialiseValue(SerialisedType::BasicType type, size_t byteSize, std::string &el)
  {
    {
      if(IsReading())
      {
        uint32_t len = 0;
        m_Read->Read(len);
        el.resize(len);
        if(len > 0)
          m_Read->Read(&el[0], len);
      }
      else
      {
        uint32_t len = (uint32_t)el.length();
        m_Write->Write(len);
        m_Write->Write(el.c_str(), len);
      }
    }

    if(ExportStructure())
    {
      StructuredObject &current = *m_StructureStack.back();

      current.type.basetype = type;
      current.type.byteSize = byteSize;
      current.data.str = el;
    }
  }
  
  void SerialiseValue(SerialisedType::BasicType type, size_t byteSize, char *&el)
  {
    {
      if(IsReading())
      {
        uint32_t len = 0;
        m_Read->Read(len);
        string str;
        str.resize(len);
        if(len > 0)
          m_Read->Read(&str[0], len);
        auto it = m_StringDB.insert(str);
        el = (char *)it.first->c_str();
      }
      else
      {
        uint32_t len = (uint32_t)strlen(el);
        m_Write->Write(len);
        m_Write->Write(el, len);
      }
    }

    if(ExportStructure())
    {
      StructuredObject &current = *m_StructureStack.back();

      current.type.basetype = type;
      current.type.byteSize = byteSize;
      current.data.str = el;
    }
  }

  void SerialiseValue(SerialisedType::BasicType type, size_t byteSize, const char *&el)
  {
    SerialiseValue(type, byteSize, (char *&)el);
  }

private:
  // no copies
  Serialiser(const Serialiser &other);

  static void CreateResolver(void *ths);

  // clean out for before constructor and after destructor (and other times probably)
  void Reset()
  {
    m_pUserData = NULL;

    m_ExportBuffers = false;
    m_InternalElement = false;

    m_ChunkMetadata = SerialisedChunkMetaData();

    m_SerVer = SERIALISE_VERSION;

    RDCEraseEl(m_KnownSections);

    m_HasError = false;
    m_ErrorCode = eContError_None;

    m_ChunkLookup = NULL;

    m_AlignedData = false;

    m_ChunkFlags = 0;

    m_LastChunkLen = 0;
    m_LastChunkOffset = 0;
  }

  uint64_t m_SerVer;

  ContainerError m_ErrorCode;
  bool m_HasError;

  void *m_pUserData;

  StreamWriter *m_Write;
  StreamReader *m_Read;

  size_t m_LastChunkLen;
  uint64_t m_LastChunkOffset;
  bool m_AlignedData;
  uint64_t m_ChunkFixup;

  // reading from file:

  struct Section
  {
    Section()
        : type(eSectionType_Unknown), flags(eSectionFlag_None), fileoffset(0), size(0), compressedReader(NULL)
    {
    }
    string name;
    SectionType type;
    SectionFlags flags;

    uint64_t fileoffset;
    uint64_t size;
    std::vector<byte> data;    // some sections can be loaded entirely into memory
    CompressedFileIO *compressedReader;
  };

  // this lists all sections in file order
  std::vector<Section *> m_Sections;

  // this lists known sections, some may be NULL
  Section *m_KnownSections[eSectionType_Num];

  // writing to file
  std::vector<Chunk *> m_Chunks;
  
  bool m_ExportBuffers;
  bool m_InternalElement;
  StructuredChunkList m_StructuredData;
  SerialisedBufferList m_StructuredDataBuffers;
  std::vector<StructuredObject *> m_StructureStack;

  uint32_t m_ChunkFlags;
  SerialisedChunkMetaData m_ChunkMetadata;

  // a database of strings read from the file, useful when serialised structures
  // expect a char* to return and point to static memory
  std::set<string> m_StringDB;

  ChunkLookup m_ChunkLookup;
};

typedef Serialiser<SerialiserType::Writing> WriteSerialiser;
typedef Serialiser<SerialiserType::Reading> ReadSerialiser;
typedef Serialiser<SerialiserType::StructuredReading> StructReadSerialiser;

#define BASIC_TYPE_SERIALISE(typeName, member, type, byteSize) \
  template <class SerialiserType> \
  void DoSerialise(SerialiserType &ser, typeName &el) \
  { \
    ser.SerialiseValue(SerialisedType::type, byteSize, member); \
  }

#define BASIC_TYPE_SERIALISE_STRINGIFY(typeName, member, type, byteSize) \
  template <class SerialiserType> \
  void DoSerialise(SerialiserType &ser, typeName &el) \
  { \
    ser.SerialiseValue(SerialisedType::type, byteSize, member); \
    ser.SerialiseStringify(el); \
  }

BASIC_TYPE_SERIALISE(int64_t, el, eSignedInteger, 8);
BASIC_TYPE_SERIALISE(uint64_t, el, eUnsignedInteger, 8);
BASIC_TYPE_SERIALISE(int32_t, el, eSignedInteger, 4);
BASIC_TYPE_SERIALISE(uint32_t, el, eUnsignedInteger, 4);
BASIC_TYPE_SERIALISE(int16_t, el, eSignedInteger, 2);
BASIC_TYPE_SERIALISE(uint16_t, el, eUnsignedInteger, 2);
BASIC_TYPE_SERIALISE(int8_t, el, eSignedInteger, 1);
BASIC_TYPE_SERIALISE(uint8_t, el, eUnsignedInteger, 1);

BASIC_TYPE_SERIALISE(double, el, eFloat, 8);
BASIC_TYPE_SERIALISE(float, el, eFloat, 4);

BASIC_TYPE_SERIALISE(bool, el, eBoolean, 1);

BASIC_TYPE_SERIALISE(char, el, eCharacter, 1);

BASIC_TYPE_SERIALISE(string, el, eString, 0);
BASIC_TYPE_SERIALISE(char *, el, eString, 0);
BASIC_TYPE_SERIALISE(const char *, el, eString, 0);

template<class SerialiserType>
class ScopedContext
{
public:
  ScopedContext(SerialiserType &s, uint32_t i, uint32_t byteLength = 0)
      : m_Idx(i), m_Ser(s), m_Ended(false)
  {
    m_Ser.BeginChunk(i, byteLength);
  }
  ~ScopedContext()
  {
    if(!m_Ended)
      End();
  }

  Chunk *Get(bool temporary = false)
  {
    End();
    return new Chunk(m_Ser, m_Idx, temporary);
  }

private:
  SerialiserType &m_Ser;
  uint32_t m_Idx;
  bool m_Ended;

  void End()
  {
    RDCASSERT(!m_Ended);

    m_Ser.EndChunk();

    m_Ended = true;
  }
};

template <class T>
struct ScopedDeserialise
{
  ScopedDeserialise(const T &el) : m_El(el) {}
  ~ScopedDeserialise() { Deserialise(m_El); }
  const T &m_El;
};

template<class T>
ScopedDeserialise<T> DeserialiseAutoMaker(const ReadSerialiser &ser, const T &el)
{
  (void)ser;
  return ScopedDeserialise<T>(el);
}

template<class T>
ScopedDeserialise<T> DeserialiseAutoMaker(const StructReadSerialiser &ser, const T &el)
{
  (void)ser;
  return ScopedDeserialise<T>(el);
}

template <class T>
struct ScopedDeserialiseArray
{
  ScopedDeserialiseArray(T **el) : m_El(el) {}
  ~ScopedDeserialiseArray() { delete[] *m_El; }
  T **m_El;
};

template <>
struct ScopedDeserialiseArray<void>
{
  ScopedDeserialiseArray(void **el) : m_El(el) {}
  ~ScopedDeserialiseArray() { FreeAlignedBuffer((byte *)*m_El); }
  void **m_El;
};

template <>
struct ScopedDeserialiseArray<const void>
{
  ScopedDeserialiseArray(const void **el) : m_El(el) {}
  ~ScopedDeserialiseArray() { FreeAlignedBuffer((byte *)*m_El); }
  const void **m_El;
};

template <>
struct ScopedDeserialiseArray<byte>
{
  ScopedDeserialiseArray(byte **el) : m_El(el) {}
  ~ScopedDeserialiseArray() { FreeAlignedBuffer(*m_El); }
  byte **m_El;
};

template<class T>
ScopedDeserialiseArray<T> DeserialiseArrayAutoMaker(const ReadSerialiser &ser, T **el)
{
  (void)ser;
  return ScopedDeserialiseArray<T>(el);
}

template<class T>
ScopedDeserialiseArray<T> DeserialiseArrayAutoMaker(const StructReadSerialiser &ser, T **el)
{
  (void)ser;
  return ScopedDeserialiseArray<T>(el);
}

template <class T>
struct ScopedDeserialiseNullable
{
  ScopedDeserialiseNullable(T **el) : m_El(el) {}
  ~ScopedDeserialiseNullable() { if(*m_El != NULL) { Deserialise(**m_El); delete *m_El; } }
  T **m_El;
};

template<class T>
ScopedDeserialiseNullable<T> DeserialiseNullableAutoMaker(const ReadSerialiser &ser, T **el)
{
  (void)ser;
  return ScopedDeserialiseNullable<T>(el);
}

template<class T>
ScopedDeserialiseNullable<T> DeserialiseNullableAutoMaker(const StructReadSerialiser &ser, T **el)
{
  (void)ser;
  return ScopedDeserialiseNullable<T>(el);
}

template <class T>
struct EmptyClass
{
};

template<class T>
EmptyClass<T> DeserialiseAutoMaker(const WriteSerialiser &ser, const T &el)
{
  (void)ser;
  (void)el;
  return EmptyClass<T>();
}

template<class T>
EmptyClass<T> DeserialiseArrayAutoMaker(const WriteSerialiser &ser, T **el)
{
  (void)ser;
  (void)el;
  return EmptyClass<T>();
}

template<class T>
EmptyClass<T> DeserialiseNullableAutoMaker(const WriteSerialiser &ser, T **el)
{
  (void)ser;
  (void)el;
  return EmptyClass<T>();
}

template<class T>
T *ForceAllocate(const T *garbage)
{
  return new T;
}

// can be overridden to change the name in the helper macros locally
#define GET_SERIALISER (ser)

#define SCOPED_SERIALISE_CONTEXT(...) ScopedContext<WriteSerialiser> scope(GET_SERIALISER, __VA_ARGS__);

// force the data to be aligned
#define SERIALISE_ALIGNED() GET_SERIALISER.Aligned();

// these helper macros are intended for use when serialising objects in chunks
#define SERIALISE_ELEMENT(obj) \
  auto CONCAT(deserialise_, __LINE__) = DeserialiseAutoMaker(GET_SERIALISER, obj); \
  GET_SERIALISER.Serialise(#obj, obj)

// helper macro for when an array has a constant size. Uses a dummy variable to serialise
#define FIXED_COUNT(count) \
  (CONCAT(dummy_array_count, __LINE__) = count,CONCAT(dummy_array_count, __LINE__))

#define SERIALISE_ELEMENT_ARRAY(obj, count) \
  uint64_t CONCAT(dummy_array_count, __LINE__) = 0; \
  auto CONCAT(deserialise_, __LINE__) = DeserialiseArrayAutoMaker(GET_SERIALISER, &obj); \
  GET_SERIALISER.Serialise(#obj, obj, count, SerialiserFlags::AllocateMemory)

#define SERIALISE_ELEMENT_OPT(obj) \
  auto CONCAT(deserialise_, __LINE__) = DeserialiseNullableAutoMaker(GET_SERIALISER, &obj); \
  GET_SERIALISER.SerialiseNullable(#obj, obj)

#define SERIALISE_ELEMENT_LOCAL(obj, inValue)                                      \
  decltype(inValue) obj;                                                           \
  auto CONCAT(deserialise_, __LINE__) = DeserialiseAutoMaker(GET_SERIALISER, obj); \
  if(GET_SERIALISER.IsWriting())                                                   \
    obj = (inValue);                                                               \
  GET_SERIALISER.Serialise(#obj, obj)

// these macros are for use when implementing a DoSerialise function
#define SERIALISE_MEMBER(obj) \
  ser.Serialise(#obj, el.obj)

#define SERIALISE_MEMBER_TYPED(type, obj) \
  ser.template Serialise<type>(#obj, (type &)el.obj)

#define SERIALISE_MEMBER_ARRAY(arrayObj, countObj) \
  ser.Serialise(#arrayObj, el.arrayObj, el.countObj, SerialiserFlags::AllocateMemory)

// a member that is a pointer and could be NULL, so needs a hidden 'present'
// flag serialised out
#define SERIALISE_MEMBER_OPT(obj) \
  ser.SerialiseNullable(#obj, el.obj)
<file_sep>/common.h
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2015-2016 <NAME>
 * Copyright (c) 2014 Crytek
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#pragma once

#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <stdarg.h>
//#include "globalconfig.h"
#include <string>
#include <string.h>

#ifdef WIN32
#define PRIu64 "llu"
#define PRId64 "lld"
#else
#include <cinttypes>
#endif

#ifdef WIN32
#define __PRETTY_FUNCTION_SIGNATURE__ __FUNCSIG__
#define va_copy(dst, src) dst = src
#else
#define __PRETTY_FUNCTION_SIGNATURE__ __PRETTY_FUNCTION__
#define _snprintf snprintf
#endif

namespace StringFormat
{
inline std::string Fmt(const char *format, ...)
{
  va_list args;
  va_start(args, format);

  va_list args2;
  va_copy(args2, args);

  int size = vsnprintf(NULL, 0, format, args2);

  char *buf = new char[size + 1];
  vsnprintf(buf, size + 1, format, args);
  buf[size] = 0;

  va_end(args);
  va_end(args2);

  std::string ret = buf;

  delete[] buf;

  return ret;
}
};

#if defined(__LP64__) || defined(_WIN64) || defined(__x86_64__) || defined(_M_X64) || \
    defined(__ia64) || defined(_M_IA64) || defined(__aarch64__) || defined(__powerpc64__)
#define RDC64BIT 1
#endif

enum
{
  RenderDoc_FirstCaptureNetworkPort = 38920,
  RenderDoc_LastCaptureNetworkPort = RenderDoc_FirstCaptureNetworkPort + 7,
  RenderDoc_ReplayNetworkPort = 39920,
};

#define FORCE_ASSERTS

#define DEBUGBREAK_ON_ERROR_LOG

/////////////////////////////////////////////////
// Utility macros

#ifndef SAFE_DELETE
#define SAFE_DELETE(p) \
  do                   \
  {                    \
    if(p)              \
    {                  \
      delete(p);       \
      (p) = NULL;      \
    }                  \
  } while((void)0, 0)
#endif

#ifndef SAFE_DELETE_ARRAY
#define SAFE_DELETE_ARRAY(p) \
  do                         \
  {                          \
    if(p)                    \
    {                        \
      delete[](p);           \
      (p) = NULL;            \
    }                        \
  } while((void)0, 0)
#endif

#ifndef SAFE_ADDREF
#define SAFE_ADDREF(p) \
  do                   \
  {                    \
    if(p)              \
    {                  \
      (p)->AddRef();   \
    }                  \
  } while((void)0, 0)
#endif

#ifndef SAFE_RELEASE
#define SAFE_RELEASE(p) \
  do                    \
  {                     \
    if(p)               \
    {                   \
      (p)->Release();   \
      (p) = NULL;       \
    }                   \
  } while((void)0, 0)
#define SAFE_RELEASE_NOCLEAR(p) \
  do                            \
  {                             \
    if(p)                       \
    {                           \
      (p)->Release();           \
    }                           \
  } while((void)0, 0)
#endif

#ifndef ARRAY_COUNT
#define ARRAY_COUNT(arr) (sizeof(arr) / sizeof(arr[0]))
#endif

#define RANDF(mn, mx) ((float(rand()) / float(RAND_MAX)) * ((mx) - (mn)) + (mn))

#define STRINGIZE2(a) #a
#define STRINGIZE(a) STRINGIZE2(a)

#define CONCAT2(a, b) a##b
#define CONCAT(a, b) CONCAT2(a, b)

//#include "os/os_specific.h"

typedef uint8_t byte;

#define RDCEraseMem(a, b) memset(a, 0, b)
#define RDCEraseEl(a) memset(&a, 0, sizeof(a))

template <typename T>
T RDCCLAMP(const T &val, const T &mn, const T &mx)
{
  return val < mn ? mn : (val > mx ? mx : val);
}

template <typename T>
T RDCMIN(const T &a, const T &b)
{
  return a < b ? a : b;
}

template <typename T>
T RDCMAX(const T &a, const T &b)
{
  return a > b ? a : b;
}

template <typename T>
inline T AlignUp4(T x)
{
  return (x + 0x3) & (~0x3);
}

template <typename T>
inline T AlignUp16(T x)
{
  return (x + 0xf) & (~0xf);
}

template <typename T>
inline T AlignUp(T x, T a)
{
  return (x + (a - 1)) & (~(a - 1));
}

template <typename T, typename A>
inline T AlignUpPtr(T x, A a)
{
  return (T)AlignUp<uintptr_t>((uintptr_t)x, (uintptr_t)a);
}

#define MAKE_FOURCC(a, b, c, d) \
  (((uint32_t)(d) << 24) | ((uint32_t)(c) << 16) | ((uint32_t)(b) << 8) | (uint32_t)(a))

bool FindDiffRange(void *a, void *b, size_t bufSize, size_t &diffStart, size_t &diffEnd);
uint32_t CalcNumMips(int Width, int Height, int Depth);

byte *AllocAlignedBuffer(size_t size, size_t alignment = 64);
void FreeAlignedBuffer(byte *buf);

/////////////////////////////////////////////////
// Debugging features

#if WIN32

#define FORCE_CRASH() *((int *)NULL) = 0

#else

#define FORCE_CRASH() __builtin_trap()
#define __debugbreak()

#endif

#define RDCDUMP()            \
  do                         \
  {                          \
    FORCE_CRASH();           \
  } while((void)0, 0)

#if !defined(RELEASE) || defined(FORCE_DEBUGBREAK)
#define RDCBREAK()                   \
  do                                 \
  {                                  \
    __debugbreak();                  \
  } while((void)0, 0)
#else
#define RDCBREAK() \
  do               \
  {                \
  } while((void)0, 0)
#endif

#define RDCUNIMPLEMENTED(...)                              \
  do                                                       \
  {                                                        \
    rdclog(RDCLog_Warning, "Unimplemented: " __VA_ARGS__); \
    RDCBREAK();                                            \
  } while((void)0, 0)

//
// Logging
//

enum LogType
{
  RDCLog_First = -1,
  RDCLog_Debug,
  RDCLog_Comment,
  RDCLog_Warning,
  RDCLog_Error,
  RDCLog_Fatal,
  RDCLog_NumTypes,
};

#if defined(STRIP_LOG)
#define RDCLOGFILE(fn) \
  do                   \
  {                    \
  } while((void)0, 0)
#define RDCLOGDELETE() \
  do                   \
  {                    \
  } while((void)0, 0)

#define RDCDEBUG(...) \
  do                  \
  {                   \
  } while((void)0, 0)
#define RDCLOG(...) \
  do                \
  {                 \
  } while((void)0, 0)
#define RDCWARN(...) \
  do                 \
  {                  \
  } while((void)0, 0)
#define RDCERR(...) \
  do                \
  {                 \
  } while((void)0, 0)
#define RDCFATAL(...) \
  do                  \
  {                   \
    RDCDUMP();        \
    exit(0);          \
  } while((void)0, 0)
#define RDCDUMPMSG(message) \
  do                        \
  {                         \
    RDCDUMP();              \
    exit(0);                \
  } while((void)0, 0)
#else
// perform any operations necessary to flush the log
void rdclog_flush();

// actual low-level print to log output streams defined (useful for if we need to print
// fatal error messages from within the more complex log function).
void rdclogprint_int(const char *str);

// printf() style main logger function
void rdclog_int(LogType type, const char *file, unsigned int line, const char *fmt, ...);

#define rdclog(type, ...) rdclog_int(type, __FILE__, __LINE__, __VA_ARGS__)

const char *rdclog_getfilename();
void rdclog_filename(const char *filename);

#define RDCLOGFILE(fn) rdclog_filename(fn)
#define RDCGETLOGFILE() rdclog_getfilename()

#if(!defined(RELEASE) || defined(FORCE_DEBUG_LOGS)) && !defined(STRIP_DEBUG_LOGS)
#define RDCDEBUG(...) rdclog(RDCLog_Debug, __VA_ARGS__)
#else
#define RDCDEBUG(...) \
  do                  \
  {                   \
  } while((void)0, 0)
#endif

#define RDCLOG(...) rdclog(RDCLog_Comment, __VA_ARGS__)
#define RDCWARN(...) rdclog(RDCLog_Warning, __VA_ARGS__)

#if defined(DEBUGBREAK_ON_ERROR_LOG)
#define RDCERR(...)                    \
  do                                   \
  {                                    \
    rdclog(RDCLog_Error, __VA_ARGS__); \
    rdclog_flush();                    \
    RDCBREAK();                        \
  } while((void)0, 0)
#else
#define RDCERR(...) rdclog(RDCLog_Error, __VA_ARGS__)
#endif

#define RDCFATAL(...)                  \
  do                                   \
  {                                    \
    rdclog(RDCLog_Fatal, __VA_ARGS__); \
    rdclog_flush();                    \
    RDCDUMP();                         \
    exit(0);                           \
  } while((void)0, 0)
#define RDCDUMPMSG(message)   \
  do                          \
  {                           \
    rdclogprint_int(message); \
    rdclog_flush();           \
    RDCDUMP();                \
    exit(0);                  \
  } while((void)0, 0)
#endif

//
// Assert
//

#if !defined(RELEASE) || defined(FORCE_ASSERTS)
void rdcassert(const char *msg, const char *file, unsigned int line, const char *func);

// this defines the root macro, RDCASSERTMSG(msg, cond, ...)
// where it will check cond, then print msg (if it's not "") and the values of all values passed via
// varargs.
// the other asserts are defined in terms of that
#include "custom_assert.h"

#else
#define RDCASSERTMSG(cond) \
  do                       \
  {                        \
  } while((void)0, 0)
#endif

#define RDCASSERT(...) RDCASSERTMSG("", __VA_ARGS__)
#define RDCASSERTEQUAL(a, b) RDCASSERTMSG("", a == b, a, b)
#define RDCASSERTNOTEQUAL(a, b) RDCASSERTMSG("", a != b, a, b)

//
// Compile asserts
//

#if defined(STRIP_COMPILE_ASSERTS)
#define RDCCOMPILE_ASSERT(condition, message) \
  do                                          \
  {                                           \
  } while((void)0, 0)
#else
#define RDCCOMPILE_ASSERT(condition, message) static_assert(condition, message)
#endif

#ifdef WIN32

#include <windows.h>

namespace Timing
{
inline double GetTickFrequency()
{
  LARGE_INTEGER li;
  QueryPerformanceFrequency(&li);
  return double(li.QuadPart) / 1000.0;
}

inline uint64_t GetTick()
{
  LARGE_INTEGER li;
  QueryPerformanceCounter(&li);

  return li.QuadPart;
}
};

#else

#include <time.h>
#include <unistd.h>

namespace Timing
{
inline double GetTickFrequency()
{
  return 1000000.0;
}

inline uint64_t GetTick()
{
  timespec ts;
  clock_gettime(CLOCK_MONOTONIC, &ts);
  return uint64_t(ts.tv_sec) * 1000000000ULL + uint32_t(ts.tv_nsec & 0xffffffff);
}
};

#endif

class PerformanceTimer
{
public:
  PerformanceTimer() : m_CounterFrequency(Timing::GetTickFrequency()) { Restart(); }
  double GetMilliseconds() const
  {
    return double(Timing::GetTick() - m_Start) / m_CounterFrequency;
  }

  void Restart() { m_Start = Timing::GetTick(); }
private:
  double m_CounterFrequency;
  uint64_t m_Start;
};

class ScopedTimer
{
public:
  ScopedTimer(const char *file, unsigned int line, const char *fmt, ...)
  {
    m_File = file;
    m_Line = line;

    va_list args;
    va_start(args, fmt);

    char buf[1024];
    buf[1023] = 0;
    vsnprintf(buf, 1023, fmt, args);

    m_Message = buf;

    va_end(args);
  }

  ~ScopedTimer()
  {
    rdclog_int(RDCLog_Comment, m_File, m_Line, "Timer %s - %.3lf ms", m_Message.c_str(),
               m_Timer.GetMilliseconds());
  }

private:
  const char *m_File;
  unsigned int m_Line;
  std::string m_Message;
  PerformanceTimer m_Timer;
};

#define SCOPED_TIMER(...) ScopedTimer CONCAT(timer, __LINE__)(__FILE__, __LINE__, __VA_ARGS__);

#include <vector>

inline std::vector<uint64_t> CollectCallstack_stub()
{
  std::vector<uint64_t> ret;
  ret.resize(5 + rand()%10);

  for(size_t i=0; i < ret.size(); i++)
  {
    uint64_t addr = uint64_t(rand()&0x7fff) << 20;
    addr |= rand()&0x7fff;
    ret[i] = addr;
  }

  return ret;
}

inline uint64_t CollectTimestamp_stub()
{
  static uint64_t time = 0x844674407ULL + ((rand()&0x7fff)<<8);

  time += (rand()&0x7fff)<<6;

  return time;
}

inline uint32_t CollectThreadID_stub()
{
  static uint32_t id = 1000 + rand()&0xff;

  return id;
}<file_sep>/streamio.h
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2017 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#pragma once

#include <vector>
#include <stdio.h>
#include "common.h"

class StreamWriter;
class StreamReader;

class Compressor
{
public:
  Compressor(StreamWriter *write)
    : m_Write(write)
  {
  }
  virtual ~Compressor() {}

  virtual void Write(const void *data, uint64_t numBytes) = 0;
  virtual void Finish() = 0;
protected:
  StreamWriter *m_Write;
};

class Decompressor
{
public:
  Decompressor(StreamReader *read)
    : m_Read(read)
  {
  }
  virtual ~Decompressor() {}

  virtual void Recompress(Compressor *comp) = 0;
  virtual void Read(void *data, uint64_t numBytes) = 0;
protected:
  StreamReader *m_Read;
};


class StreamReader
{
public:
  StreamReader(const void *buffer, size_t bufferSize);
  StreamReader(const std::vector<byte> &buffer);

  StreamReader(FILE *file, size_t fileSize);
  StreamReader(FILE *file);
  StreamReader(Decompressor *decompressor, size_t uncompressedSize);

  ~StreamReader();

  void SetOffset(uint64_t offs);

  inline uint64_t GetOffset()
  {
    return m_BufferHead - m_BufferBase + m_ReadOffset;
  }
  inline uint64_t GetSize()
  {
    return m_InputSize;
  }
  inline bool AtEnd() { return GetOffset() >= GetSize(); }
  
  template<size_t alignment>
  void AlignTo()
  {
    uint64_t offs = GetOffset();
    uint64_t alignedOffs = AlignUp(offs, alignment);

    uint64_t bytesToAlign = alignedOffs - offs;

    if(bytesToAlign > 0)
      Read(NULL, bytesToAlign);
  }

  void Read(void *data, size_t numBytes)
  {
    // if we're reading past the end, error, read nothing (no partial reads) and return
    if(GetOffset() + numBytes > GetSize())
    {
      RDCERR("Reading off the end of the stream");
      m_BufferHead = m_BufferBase + m_BufferSize;
      return;
    }

    // if we're reading from an external source, reserve enough bytes to do the read
    if(m_File || m_Decompressor)
    {
      // This preserves everything from min(m_BufferBase, m_BufferHead - 64) -> end of buffer
      // which will still be in place relative to m_BufferHead.
      // In other words - reservation will keep the aleady-read data that's after the head pointer,
      // as well as up to 64 bytes *behind* the head if it exists.
      if(numBytes > Available())
        Reserve(numBytes);
    }

    // perform the actual copy
    if(data)
      memcpy(data, m_BufferHead, numBytes);

    // advance head
    m_BufferHead += numBytes;
  }

  void SkipBytes(size_t numBytes)
  {
    Read(NULL, numBytes);
  }

  // compile-time constant element to let the compiler inline the memcpy
  template<typename T>
  void Read(T &data)
  {
    Read(&data, sizeof(T));
  }

private:
  inline uint64_t Available()
  {
    return m_BufferSize - (m_BufferHead - m_BufferBase);
  }

  void Reserve(size_t numBytes);
  void ReadFromExternal(uint64_t bufferOffs, size_t length);

  // base of the buffer allocation
  byte *m_BufferBase;

  // where we are currently reading from in the buffer
  byte *m_BufferHead;

  // the size of the buffer (just a window if reading from external source)
  uint64_t m_BufferSize;

  // the total size of the total input. This is how many bytes you can read, regardless
  // of how many bytes might actually be stored on the other side of the source (i.e.
  // this is the uncompressed output size)
  uint64_t m_InputSize;

  // file pointer, if we're reading from a file
  FILE *m_File;

  // the decompressor, if reading from it
  Decompressor *m_Decompressor;

  // the offset in the file/decompressor that corresponds to the start of m_BufferBase
  uint64_t m_ReadOffset;
};

class StreamWriter
{
public:
  StreamWriter(uint64_t initialBufSize);
  StreamWriter(FILE *file);
  StreamWriter(Compressor *compressor);

  static const int DefaultScratchSize = 32 * 1024;

  ~StreamWriter();

  void SetOffset(uint64_t offs)
  {
    if(!m_File && !m_Compressor)
    {
      m_BufferHead = m_BufferBase + offs;
      return;
    }

    RDCERR("Can't seek a file/compressor stream writer");
  }
  
  void Rewind()
  {
    if(!m_File && !m_Compressor)
    {
      m_BufferHead = m_BufferBase;
      m_WriteSize = 0;
      return;
    }

    RDCERR("Can't rewind a file/compressor stream writer");
  }
  
  uint64_t GetOffset() { return m_WriteSize; }
  const byte *GetData() { return m_BufferBase; }
  
  template<size_t alignment>
  void AlignTo()
  {
    if(!m_File && !m_Compressor)
    {
      uint64_t offs = m_BufferHead - m_BufferBase;
      uint64_t alignedOffs = AlignUp(offs, alignment);

      uint64_t bytesToAlign = alignedOffs - offs;

      RDCCOMPILE_ASSERT(alignment <= sizeof(empty), "Empty array is not large enough - increase size to support alignment");

      if(bytesToAlign > 0)
        Write(empty, bytesToAlign);
      return;
    }

    RDCERR("Can't align a file/compressor stream writer");
  }

  void Write(const void *data, size_t numBytes)
  {
    m_WriteSize += numBytes;

    if(!m_File && !m_Compressor)
    {
      // in-memory path

      size_t endByte = numBytes + size_t(m_BufferHead - m_BufferBase);

      // are we about to write outside the buffer? Resize it larger
      if(m_BufferSize < endByte)
      {
        // reallocate to a conservative size, don't 'double and allocate'
        while(m_BufferSize < endByte)
          m_BufferSize += 128 * 1024;

        byte *newBuf = AllocAlignedBuffer((size_t)m_BufferSize);

        size_t curUsed = m_BufferHead - m_BufferBase;

        memcpy(newBuf, m_BufferBase, curUsed);

        FreeAlignedBuffer(m_BufferBase);

        m_BufferBase = newBuf;
        m_BufferHead = newBuf + curUsed;
      }

      // perform the actual copy
      memcpy(m_BufferHead, data, numBytes);

      // advance head
      m_BufferHead += numBytes;
    }
    else if(m_Compressor)
    {
      m_Compressor->Write(data, numBytes);
    }
    else
    {
      RDCASSERT(m_File);
      fwrite(data, 1, numBytes, m_File);
    }
  }

  // compile-time constant amount of data to let the compiler inline the memcpy
  template<typename T>
  void Write(const T &data)
  {
    Write(&data, sizeof(data));
  }

  void Finish()
  {
    if(m_Compressor)
      m_Compressor->Finish();
    else if(m_File)
      fflush(m_File);
  }

private:

  // used for aligned writes
  static const byte empty[128];

  // base of the buffer allocation if we're writing to a buffer
  byte *m_BufferBase;

  // where we are currently writing to in the buffer
  byte *m_BufferHead;

  // the size of the buffer
  uint64_t m_BufferSize;

  // the total size of the file/compressor (ie. how much data flushed through it)
  uint64_t m_WriteSize;

  // file pointer, if we're writing to a file
  FILE *m_File;

  // the compressor, if writing to it
  Compressor *m_Compressor;
};<file_sep>/json_convert.h
#pragma once

#include <string>
#include <vector>

#include "common.h"
#include "serialiser.h"

#define PICOJSON_USE_INT64

#undef min
#undef max

#include "picojson.h"

struct SerialisedFile;
typedef std::pair<byte*,uint64_t> SerialisedBuffer;

std::string SerialisedFile2JSON(SerialisedFile file, bool concise = true);
void Buffers2ZIP(const char *filename, SerialisedFile file);

bool has(picojson::object o, const std::string &key);

extern std::vector<SerialisedBuffer> *bufList;

DECLARE_SERIALISE_TYPE(picojson::value);
<file_sep>/zstdio.cpp
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2017 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#define ZSTD_STATIC_LINKING_ONLY
#include "zstdio.h"

static const size_t zstdBlockSize = 128 * 1024;

ZSTDCompressor::ZSTDCompressor(StreamWriter *write) : Compressor(write)
{
  m_Page = AllocAlignedBuffer(zstdBlockSize);
  m_CompressBuffer = AllocAlignedBuffer(ZSTD_CStreamOutSize());

  m_PageOffset = 0;
  
  m_Stream = ZSTD_createCStream();
}

ZSTDCompressor::~ZSTDCompressor()
{
  ZSTD_freeCStream(m_Stream);

  FreeAlignedBuffer(m_Page);
  FreeAlignedBuffer(m_CompressBuffer);
}

void ZSTDCompressor::Write(const void *data, uint64_t numBytes)
{
  // if we encountered a stream error this will be NULL
  if(!m_CompressBuffer)
    return;

  if(numBytes == 0)
    return;

  // this is largely similar to LZ4Compressor, so check the comments there for more details.
  // The only difference is that the lz4 streaming compression assumes a history of 64kb, where
  // here we use a larger block size but no history must be maintained. This means we can
  // potentially do less copying in the compress-from-source path

  if(m_PageOffset + numBytes <= zstdBlockSize)
  {
    // simplest path, no page wrapping/spanning at all
    memcpy(m_Page + m_PageOffset, data, numBytes);
    m_PageOffset += numBytes;
  }
  else if(numBytes <= 2 * zstdBlockSize)
  {
    // simple path, do through partial copies that span pages and flush as necessary

    const byte *src = (const byte *)data;

    // copy whatever will fit on this page
    {
      uint64_t firstBytes = zstdBlockSize - m_PageOffset;
      memcpy(m_Page + m_PageOffset, src, firstBytes);

      m_PageOffset += firstBytes;
      numBytes -= firstBytes;
      src += firstBytes;
    }
    
    while(numBytes > 0)
    {
      // flush page
      FlushPage();

      // how many bytes can we copy in this page?
      uint64_t partialBytes = RDCMIN(zstdBlockSize, numBytes);
      memcpy(m_Page, src, partialBytes);

      // advance the source pointer, dest offset, and remove the bytes we read
      m_PageOffset += partialBytes;
      numBytes -= partialBytes;
      src += partialBytes;
    }
  }
  else
  {
    // complex path, do a partial copy to finish a page then directly stream through zstd to the writer.
    
    const byte *src = (const byte *)data;

    // copy into the partial page
    {
      uint64_t firstBytes = zstdBlockSize - m_PageOffset;
      memcpy(m_Page + m_PageOffset, src, firstBytes);

      m_PageOffset += firstBytes;
      numBytes -= firstBytes;
      src += firstBytes;

      // flush out to disk
      FlushPage();
    }

    // while we still have at least one whole page remaining, compress & write directly without using the pages
    while(numBytes > zstdBlockSize)
    {
      ZSTD_inBuffer in = {src, zstdBlockSize, 0};
      ZSTD_outBuffer out = {m_CompressBuffer, ZSTD_CStreamOutSize(), 0};

      CompressZSTDFrame(in, out);

      // if there was an error, bail
      if(!m_CompressBuffer)
        return;

      // a bit redundant to write this but it means we can read the entire frame without
      // doing multiple reads
      m_Write->Write((uint32_t)out.pos);
      m_Write->Write(m_CompressBuffer, out.pos);

      numBytes -= zstdBlockSize;
      src += zstdBlockSize;
    }
    
    // write the rest into the page as we know it fits on a page
    memcpy(m_Page + m_PageOffset, src, numBytes);
    m_PageOffset += numBytes;
  }
}

void ZSTDCompressor::Finish()
{
  // This function just writes the current page and closes zstd. Since we assume all blocks are precisely 64kb in size
  // only the last one can be smaller, so we only write a partial page when finishing.
  // Calling Write() after Finish() is illegal

  FlushPage();
}

void ZSTDCompressor::FlushPage()
{
  // if we encountered a stream error this will be NULL
  if(!m_CompressBuffer)
    return;

  ZSTD_inBuffer in = {m_Page, m_PageOffset, 0};
  ZSTD_outBuffer out = {m_CompressBuffer, ZSTD_CStreamOutSize(), 0};

  CompressZSTDFrame(in, out);

  // if there was an error, bail
  if(!m_CompressBuffer)
    return;

  // a bit redundant to write this but it means we can read the entire frame without
  // doing multiple reads
  m_Write->Write((uint32_t)out.pos);
  m_Write->Write(m_CompressBuffer, out.pos);

  // start writing to the start of the page again
  m_PageOffset = 0;
}

void ZSTDCompressor::CompressZSTDFrame(ZSTD_inBuffer &in, ZSTD_outBuffer &out)
{
  size_t err = ZSTD_initCStream(m_Stream, 7);

  if(ZSTD_isError(err))
  {
    RDCERR("Error compressing: %s", ZSTD_getErrorName(err));
    FreeAlignedBuffer(m_Page);
    FreeAlignedBuffer(m_CompressBuffer);
    m_Page = m_CompressBuffer = NULL;
    return;
  }

  // keep calling compressStream until everything is consumed
  while(in.pos < in.size)
  {
    size_t inpos = in.pos;
    size_t outpos = out.pos;

    size_t err = ZSTD_compressStream(m_Stream, &out, &in);

    if(ZSTD_isError(err) || (inpos == in.pos && outpos == out.pos))
    {
      if(ZSTD_isError(err))
        RDCERR("Error compressing: %s", ZSTD_getErrorName(err));
      else
        RDCERR("Error compressing, no progress made");
      FreeAlignedBuffer(m_Page);
      FreeAlignedBuffer(m_CompressBuffer);
      m_Page = m_CompressBuffer = NULL;
      return;
    }
  }

  err = ZSTD_endStream(m_Stream, &out);

  if(ZSTD_isError(err) || err != 0)
  {
    if(ZSTD_isError(err))
      RDCERR("Error compressing: %s", ZSTD_getErrorName(err));
    else
      RDCERR("Error compressing, couldn't end stream");
    FreeAlignedBuffer(m_Page);
    FreeAlignedBuffer(m_CompressBuffer);
    m_Page = m_CompressBuffer = NULL;
    return;
  }
}

ZSTDDecompressor::ZSTDDecompressor(StreamReader *read) : Decompressor(read)
{
  m_Page = AllocAlignedBuffer(zstdBlockSize);
  m_CompressBuffer = AllocAlignedBuffer(ZSTD_DStreamOutSize());

  m_PageOffset = 0;
  m_PageLength = 0;
  
  m_Stream = ZSTD_createDStream();
}

ZSTDDecompressor::~ZSTDDecompressor()
{
  ZSTD_freeDStream(m_Stream);
  FreeAlignedBuffer(m_Page);
  FreeAlignedBuffer(m_CompressBuffer);
}

void ZSTDDecompressor::Recompress(Compressor *comp)
{
  while(!m_Read->AtEnd())
  {
    FillPage();
    comp->Write(m_Page, m_PageLength);
  }
  comp->Finish();
}

void ZSTDDecompressor::Read(void *data, uint64_t numBytes)
{
  // if we encountered a stream error this will be NULL
  if(!m_CompressBuffer)
    return;

  if(numBytes == 0)
    return;

  // this is simpler than the ZstdWriter::Write() implementation.
  // At any point, m_Page contains the current window with uncompressed bytes.
  // If we can satisfy a read from it, then we just memcpy and increment m_PageOffset.
  // When we wrap around, we do a partial memcpy from m_Page, then decompress more bytes.

  // in future we could do an optimisation similar to Write() to reduce copying on large reads.

  // if we already have all the data in-memory, just copy and return
  uint64_t available = m_PageLength - m_PageOffset;

  if(numBytes <= available)
  {
    memcpy(data, m_Page + m_PageOffset, numBytes);
    m_PageOffset += numBytes;
    return;
  }

  byte *dst = (byte *)data;

  // copy what remains in m_Page
  memcpy(dst, m_Page + m_PageOffset, available);

  // adjust what needs to be copied
  dst += available;
  numBytes -= available;

  while(numBytes > 0)
  {
    FillPage();

    // if we can now satisfy the remainder of the read, do so and return
    if(numBytes <= m_PageLength)
    {
      memcpy(dst, m_Page, numBytes);
      m_PageOffset += numBytes;
      return;
    }

    // otherwise copy this page in and continue
    memcpy(dst, m_Page, m_PageLength);
    dst += m_PageLength;
    numBytes -= m_PageLength;
  }
}

void ZSTDDecompressor::FillPage()
{
  uint32_t compSize = 0;

  m_Read->Read(compSize);
  m_Read->Read(m_CompressBuffer, compSize);

  size_t err = ZSTD_initDStream(m_Stream);

  if(ZSTD_isError(err))
  {
    RDCERR("Error decompressing: %s", ZSTD_getErrorName(err));
    FreeAlignedBuffer(m_Page);
    FreeAlignedBuffer(m_CompressBuffer);
    m_Page = m_CompressBuffer = NULL;
    return;
  }

  ZSTD_inBuffer in = {m_CompressBuffer, compSize, 0};
  ZSTD_outBuffer out = {m_Page, ZSTD_DStreamOutSize(), 0};

  // keep calling compressStream until everything is consumed
  while(in.pos < in.size)
  {
    size_t inpos = in.pos;
    size_t outpos = out.pos;

    size_t err = ZSTD_decompressStream(m_Stream, &out, &in);

    if(ZSTD_isError(err) || (inpos == in.pos && outpos == out.pos))
    {
      if(ZSTD_isError(err))
        RDCERR("Error decompressing: %s", ZSTD_getErrorName(err));
      else
        RDCERR("Error decompressing, no progress made");
      FreeAlignedBuffer(m_Page);
      FreeAlignedBuffer(m_CompressBuffer);
      m_Page = m_CompressBuffer = NULL;
      return;
    }
  }

  m_PageOffset = 0;
  m_PageLength = out.pos;
}
<file_sep>/lz4io.cpp
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2017 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#include "lz4io.h"

static const size_t lz4BlockSize = 64 * 1024;

LZ4Compressor::LZ4Compressor(StreamWriter *write) : Compressor(write)
{
  m_Page[0] = AllocAlignedBuffer(lz4BlockSize);
  m_Page[1] = AllocAlignedBuffer(lz4BlockSize);
  m_CompressBuffer = AllocAlignedBuffer(LZ4_COMPRESSBOUND(lz4BlockSize));

  m_PageOffset = 0;
  
  LZ4_resetStream(&m_LZ4Comp);
}

LZ4Compressor::~LZ4Compressor()
{
  FreeAlignedBuffer(m_Page[0]);
  FreeAlignedBuffer(m_Page[1]);
  FreeAlignedBuffer(m_CompressBuffer);
}

void LZ4Compressor::Write(const void *data, uint64_t numBytes)
{
  // if we encountered a stream error this will be NULL
  if(!m_CompressBuffer)
    return;

  if(numBytes == 0)
    return;

  // The basic plan is:
  // Write into page N incrementally until it is completely full. When full, flush it out to lz4 and swap pages.
  // This keeps lz4 happy with 64kb of history each time it compresses.
  // If we are writing some data the crosses the boundary between pages, we write the part that will fit on one page,
  // flush & swap, write the rest into the next page.
  //
  // To reduce the amount of copying, if we are writing some data that is > 3 * page size, then we do something different:
  // if the current page isn't empty (which is almost certain), follow the steps above to copy in the part that will fit.
  // next loop over the remainder of the data, for as many whole-page sizes as we can, flushing each page directly from
  // the source into lz4.
  // when we get to the last < 2 page sizes, we go back to the normal copy and flush. The reason being we can't be using
  // any of the input data as history since it may no longer be around, so we must copy and flush one whole page for
  // history from our buffers and start on the next via copying to our own second page.
  // So worst case we copy (page size - 1) bytes at the start, and (2 * page size - 1) at the end, but the rest is fed
  // directly into lz4 from the input data.

  if(m_PageOffset + numBytes <= lz4BlockSize)
  {
    // simplest path, no page wrapping/spanning at all
    memcpy(m_Page[0] + m_PageOffset, data, numBytes);
    m_PageOffset += numBytes;
  }
  else if(numBytes <= 3 * lz4BlockSize)
  {
    // simple path, do through partial copies that span pages and flush as necessary

    const byte *src = (const byte *)data;

    // copy whatever will fit on this page
    {
      uint64_t firstBytes = lz4BlockSize - m_PageOffset;
      memcpy(m_Page[0] + m_PageOffset, src, firstBytes);

      m_PageOffset += firstBytes;
      numBytes -= firstBytes;
      src += firstBytes;
    }
    
    while(numBytes > 0)
    {
      // flush and swap pages
      FlushPage0();

      // how many bytes can we copy in this page?
      uint64_t partialBytes = RDCMIN(lz4BlockSize, numBytes);
      memcpy(m_Page[0], src, partialBytes);

      // advance the source pointer, dest offset, and remove the bytes we read
      m_PageOffset += partialBytes;
      numBytes -= partialBytes;
      src += partialBytes;
    }
  }
  else
  {
    // complex path, do a partial copy to finish a page then directly stream through lz4 to the writer.
    
    const byte *src = (const byte *)data;

    // copy into the partial page
    {
      uint64_t firstBytes = lz4BlockSize - m_PageOffset;
      memcpy(m_Page[0] + m_PageOffset, src, firstBytes);

      m_PageOffset += firstBytes;
      numBytes -= firstBytes;
      src += firstBytes;

      // flush out to disk
      FlushPage0();
    }

    // while we still have more than two pages remaining, compress & write directly without using the pages
    while(numBytes > lz4BlockSize * 2)
    {
      // compress a block from src into m_CompressBuffer
      int32_t compSize = LZ4_compress_fast_continue(&m_LZ4Comp, (const char *)src,
        (char *)m_CompressBuffer, lz4BlockSize,
        (int)LZ4_COMPRESSBOUND(lz4BlockSize), 1);

      if(compSize < 0)
      {
        RDCERR("Error compressing: %i", compSize);
        FreeAlignedBuffer(m_Page[0]);
        FreeAlignedBuffer(m_Page[1]);
        FreeAlignedBuffer(m_CompressBuffer);
        m_Page[0] = m_Page[1] = m_CompressBuffer = NULL;
        return;
      }

      m_Write->Write(compSize);
      m_Write->Write(m_CompressBuffer, compSize);

      numBytes -= lz4BlockSize;
      src += lz4BlockSize;
    }

    // do a recursive call for the rest, which will go through the simple path above
    Write(src, numBytes);
  }
}

void LZ4Compressor::Finish()
{
  // This function just writes the current page and closes lz4. Since we assume all blocks are precisely 64kb in size
  // only the last one can be smaller, so we only write a partial page when finishing.
  // Calling Write() after Finish() is illegal

  FlushPage0();
}

void LZ4Compressor::FlushPage0()
{
  // if we encountered a stream error this will be NULL
  if(!m_CompressBuffer)
    return;

  // m_PageOffset is the amount written, usually equal to lz4BlockSize except the last block.
  int32_t compSize = LZ4_compress_fast_continue(&m_LZ4Comp, (const char *)m_Page[0],
    (char *)m_CompressBuffer, (int)m_PageOffset,
    (int)LZ4_COMPRESSBOUND(lz4BlockSize), 1);

  if(compSize < 0)
  {
    RDCERR("Error compressing: %i", compSize);
    FreeAlignedBuffer(m_Page[0]);
    FreeAlignedBuffer(m_Page[1]);
    FreeAlignedBuffer(m_CompressBuffer);
    m_Page[0] = m_Page[1] = m_CompressBuffer = NULL;
    return;
  }

  m_Write->Write(compSize);
  m_Write->Write(m_CompressBuffer, compSize);

  // swap pages
  std::swap(m_Page[0], m_Page[1]);

  // start writing to the start of the page again
  m_PageOffset = 0;
}

LZ4Decompressor::LZ4Decompressor(StreamReader *read) : Decompressor(read)
{
  m_Page[0] = AllocAlignedBuffer(lz4BlockSize);
  m_Page[1] = AllocAlignedBuffer(lz4BlockSize);
  m_CompressBuffer = AllocAlignedBuffer(LZ4_COMPRESSBOUND(lz4BlockSize));

  m_PageOffset = 0;
  m_PageLength = 0;
  
  LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0);
}

LZ4Decompressor::~LZ4Decompressor()
{
  FreeAlignedBuffer(m_Page[0]);
  FreeAlignedBuffer(m_Page[1]);
  FreeAlignedBuffer(m_CompressBuffer);
}

void LZ4Decompressor::Recompress(Compressor *comp)
{
  while(!m_Read->AtEnd())
  {
    FillPage0();
    comp->Write(m_Page[0], m_PageLength);
  }
  comp->Finish();
}

void LZ4Decompressor::Read(void *data, uint64_t numBytes)
{
  // if we encountered a stream error this will be NULL
  if(!m_CompressBuffer)
    return;

  if(numBytes == 0)
    return;

  // this is simpler than the LZ4Writer::Write() implementation.
  // At any point, m_Page[0] contains the current window with uncompressed bytes.
  // If we can satisfy a read from it, then we just memcpy and increment m_PageOffset.
  // When we wrap around, we do a partial memcpy from m_Page[0], then swap the pages and
  // decompress some more bytes into m_Page[0]. Thus, m_Page[1] contains the history (if
  // it exists)

  // in future we could do an optimisation similar to Write() to reduce copying on large reads.

  // if we already have all the data in-memory, just copy and return
  uint64_t available = m_PageLength - m_PageOffset;

  if(numBytes <= available)
  {
    memcpy(data, m_Page[0] + m_PageOffset, numBytes);
    m_PageOffset += numBytes;
    return;
  }

  byte *dst = (byte *)data;

  // copy what remains in m_Page[0]
  memcpy(dst, m_Page[0] + m_PageOffset, available);

  // adjust what needs to be copied
  dst += available;
  numBytes -= available;

  while(numBytes > 0)
  {
    FillPage0();

    // if we can now satisfy the remainder of the read, do so and return
    if(numBytes <= m_PageLength)
    {
      memcpy(dst, m_Page[0], numBytes);
      m_PageOffset += numBytes;
      return;
    }

    // otherwise copy this page in and continue
    memcpy(dst, m_Page[0], m_PageLength);
    dst += m_PageLength;
    numBytes -= m_PageLength;
  }
}

void LZ4Decompressor::FillPage0()
{
  // swap pages
  std::swap(m_Page[0], m_Page[1]);

  int32_t compSize = 0;

  m_Read->Read(compSize);
  m_Read->Read(m_CompressBuffer, compSize);

  int32_t decompSize = LZ4_decompress_safe_continue(
    &m_LZ4Decomp, (const char *)m_CompressBuffer, (char *)m_Page[0], compSize, lz4BlockSize);

  if(decompSize < 0)
  {
    RDCERR("Error decompressing: %i", decompSize);
    FreeAlignedBuffer(m_Page[0]);
    FreeAlignedBuffer(m_Page[1]);
    FreeAlignedBuffer(m_CompressBuffer);
    m_Page[0] = m_Page[1] = m_CompressBuffer = NULL;
    return;
  }

  m_PageOffset = 0;
  m_PageLength = decompSize;
}
<file_sep>/structured_data.h
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2016 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#pragma once

#include <stdint.h>
#include <string>
#include <vector>

struct SerialisedType
{
  enum BasicType
  {
    eChunk,
    eStruct,
    eArray,
    eNull,
    eBuffer,
    eString,
    eEnum,
    eUnsignedInteger,
    eSignedInteger,
    eFloat,
    eBoolean,
    eCharacter,
  };

  enum
  {
    HasCustomString = 0x1,
    Hidden = 0x2,
    Nullable = 0x4,
  };

  SerialisedType(const char *n) : name(n), basetype(eStruct), flags(0), byteSize(0) {}

  std::string name;
  BasicType basetype;
  uint32_t flags;
  uint64_t byteSize;

private:
  // don't copy
  SerialisedType(const SerialisedType &);
};

struct SerialisedChunkMetaData
{
  SerialisedChunkMetaData()
  {
    chunkID = 0;
    threadID = 0;
    flags = 0;
    duration = 0;
    timestamp = 0;
    RDCEraseEl(callstack);
  }
  
  enum
  {
    OpaqueChunk = 0x1,
  };

  uint32_t chunkID;
  uint32_t threadID;
  uint64_t flags;
  uint64_t duration;
  uint64_t timestamp;
  uint64_t callstack[32];
};

struct StructuredObject
{
  StructuredObject(const char *n, const char *t)
    : type(t)
  {
    name = n;
    data.basic.u = 0;
  }

  ~StructuredObject()
  {
    for(size_t i=0; i < data.children.size(); i++)
      delete data.children[i];

    data.children.clear();
  }

  std::string name;

  SerialisedType type;

  struct
  {
    union
    {
      uint64_t u;
      int64_t i;
      double d;
      bool b;
      char c;

      // mostly here just for debugging
      uint64_t numChildren;
    } basic;
    std::string str;
    std::vector<StructuredObject*> children;
  } data;

private:
  // don't copy
  StructuredObject(const StructuredObject &);
};

struct StructuredChunk : StructuredObject
{
  StructuredChunk(const char *n, const char *t) : StructuredObject(n, t)
  {
    type.basetype = SerialisedType::eChunk;
  }
  ~StructuredChunk() { }
  SerialisedChunkMetaData metadata;
};

typedef std::pair<byte *,uint64_t> SerialisedBuffer;

typedef std::vector<StructuredChunk *> StructuredChunkList;
typedef std::vector<SerialisedBuffer> SerialisedBufferList;

struct SerialisedFile
{
  SerialisedFile(const StructuredChunkList &c, const SerialisedBufferList &d)
    : chunks(c)
    , data(d)
  {
  }
  const StructuredChunkList &chunks;
  const SerialisedBufferList &data;
};
<file_sep>/real.cpp
#include "serialiser.h"

#include "xml_codec.h"

// for ResourceId stuff already written
#include "test_types.h"

/////////////////////////////////////////////////////////////////
// minimal inlined structs/enums from dx headers

#pragma region headers

#include "official/vulkan.h"

typedef unsigned short      WORD;
typedef int                 INT;
typedef signed int          INT32;
typedef unsigned int        UINT;
typedef unsigned int        UINT32;
typedef long                LONG;
typedef unsigned long       ULONG;
typedef unsigned long       DWORD;
typedef size_t              SIZE_T;

typedef LONG HRESULT;

#define SUCCEEDED(hr)   (((HRESULT)(hr)) >= 0)
#define FAILED(hr)      (((HRESULT)(hr)) < 0)

#define S_OK    ((HRESULT)0L)

struct ID3D11ClassLinkage;
struct ID3D11RenderTargetView;
struct ID3D11InputLayout;
struct ID3D11VertexShader;

#define	D3D11_SDK_VERSION	( 7 )

typedef 
enum D3D_DRIVER_TYPE
    {
        D3D_DRIVER_TYPE_UNKNOWN	= 0,
        D3D_DRIVER_TYPE_HARDWARE	= ( D3D_DRIVER_TYPE_UNKNOWN + 1 ) ,
        D3D_DRIVER_TYPE_REFERENCE	= ( D3D_DRIVER_TYPE_HARDWARE + 1 ) ,
        D3D_DRIVER_TYPE_NULL	= ( D3D_DRIVER_TYPE_REFERENCE + 1 ) ,
        D3D_DRIVER_TYPE_SOFTWARE	= ( D3D_DRIVER_TYPE_NULL + 1 ) ,
        D3D_DRIVER_TYPE_WARP	= ( D3D_DRIVER_TYPE_SOFTWARE + 1 ) 
    } 	D3D_DRIVER_TYPE;

typedef 
enum D3D_FEATURE_LEVEL
    {
        D3D_FEATURE_LEVEL_9_1	= 0x9100,
        D3D_FEATURE_LEVEL_9_2	= 0x9200,
        D3D_FEATURE_LEVEL_9_3	= 0x9300,
        D3D_FEATURE_LEVEL_10_0	= 0xa000,
        D3D_FEATURE_LEVEL_10_1	= 0xa100,
        D3D_FEATURE_LEVEL_11_0	= 0xb000,
        D3D_FEATURE_LEVEL_11_1	= 0xb100,
        D3D_FEATURE_LEVEL_12_0	= 0xc000,
        D3D_FEATURE_LEVEL_12_1	= 0xc100
    } 	D3D_FEATURE_LEVEL;

typedef 
enum D3D11_INPUT_CLASSIFICATION
    {
        D3D11_INPUT_PER_VERTEX_DATA	= 0,
        D3D11_INPUT_PER_INSTANCE_DATA	= 1
    } 	D3D11_INPUT_CLASSIFICATION;


typedef enum DXGI_FORMAT
{
    DXGI_FORMAT_UNKNOWN	                    = 0,
    DXGI_FORMAT_R32G32B32A32_TYPELESS       = 1,
    DXGI_FORMAT_R32G32B32A32_FLOAT          = 2,
    DXGI_FORMAT_R32G32B32A32_UINT           = 3,
    DXGI_FORMAT_R32G32B32A32_SINT           = 4,
    DXGI_FORMAT_R32G32B32_TYPELESS          = 5,
    DXGI_FORMAT_R32G32B32_FLOAT             = 6,
    DXGI_FORMAT_R32G32B32_UINT              = 7,
    DXGI_FORMAT_R32G32B32_SINT              = 8,
    DXGI_FORMAT_R16G16B16A16_TYPELESS       = 9,
    DXGI_FORMAT_R16G16B16A16_FLOAT          = 10,
    DXGI_FORMAT_R16G16B16A16_UNORM          = 11,
    DXGI_FORMAT_R16G16B16A16_UINT           = 12,
    DXGI_FORMAT_R16G16B16A16_SNORM          = 13,
    DXGI_FORMAT_R16G16B16A16_SINT           = 14,
    DXGI_FORMAT_R32G32_TYPELESS             = 15,
    DXGI_FORMAT_R32G32_FLOAT                = 16,
    DXGI_FORMAT_R32G32_UINT                 = 17,
    DXGI_FORMAT_R32G32_SINT                 = 18,
    DXGI_FORMAT_R32G8X24_TYPELESS           = 19,
    DXGI_FORMAT_D32_FLOAT_S8X24_UINT        = 20,
    DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS    = 21,
    DXGI_FORMAT_X32_TYPELESS_G8X24_UINT     = 22,
    DXGI_FORMAT_R10G10B10A2_TYPELESS        = 23,
    DXGI_FORMAT_R10G10B10A2_UNORM           = 24,
    DXGI_FORMAT_R10G10B10A2_UINT            = 25,
    DXGI_FORMAT_R11G11B10_FLOAT             = 26,
    DXGI_FORMAT_R8G8B8A8_TYPELESS           = 27,
    DXGI_FORMAT_R8G8B8A8_UNORM              = 28,
    DXGI_FORMAT_R8G8B8A8_UNORM_SRGB         = 29,
    DXGI_FORMAT_R8G8B8A8_UINT               = 30,
    DXGI_FORMAT_R8G8B8A8_SNORM              = 31,
    DXGI_FORMAT_R8G8B8A8_SINT               = 32,
    DXGI_FORMAT_R16G16_TYPELESS             = 33,
    DXGI_FORMAT_R16G16_FLOAT                = 34,
    DXGI_FORMAT_R16G16_UNORM                = 35,
    DXGI_FORMAT_R16G16_UINT                 = 36,
    DXGI_FORMAT_R16G16_SNORM                = 37,
    DXGI_FORMAT_R16G16_SINT                 = 38,
    DXGI_FORMAT_R32_TYPELESS                = 39,
    DXGI_FORMAT_D32_FLOAT                   = 40,
    DXGI_FORMAT_R32_FLOAT                   = 41,
    DXGI_FORMAT_R32_UINT                    = 42,
    DXGI_FORMAT_R32_SINT                    = 43,
    DXGI_FORMAT_R24G8_TYPELESS              = 44,
    DXGI_FORMAT_D24_UNORM_S8_UINT           = 45,
    DXGI_FORMAT_R24_UNORM_X8_TYPELESS       = 46,
    DXGI_FORMAT_X24_TYPELESS_G8_UINT        = 47,
    DXGI_FORMAT_R8G8_TYPELESS               = 48,
    DXGI_FORMAT_R8G8_UNORM                  = 49,
    DXGI_FORMAT_R8G8_UINT                   = 50,
    DXGI_FORMAT_R8G8_SNORM                  = 51,
    DXGI_FORMAT_R8G8_SINT                   = 52,
    DXGI_FORMAT_R16_TYPELESS                = 53,
    DXGI_FORMAT_R16_FLOAT                   = 54,
    DXGI_FORMAT_D16_UNORM                   = 55,
    DXGI_FORMAT_R16_UNORM                   = 56,
    DXGI_FORMAT_R16_UINT                    = 57,
    DXGI_FORMAT_R16_SNORM                   = 58,
    DXGI_FORMAT_R16_SINT                    = 59,
    DXGI_FORMAT_R8_TYPELESS                 = 60,
    DXGI_FORMAT_R8_UNORM                    = 61,
    DXGI_FORMAT_R8_UINT                     = 62,
    DXGI_FORMAT_R8_SNORM                    = 63,
    DXGI_FORMAT_R8_SINT                     = 64,
    DXGI_FORMAT_A8_UNORM                    = 65,
    DXGI_FORMAT_R1_UNORM                    = 66,
    DXGI_FORMAT_R9G9B9E5_SHAREDEXP          = 67,
    DXGI_FORMAT_R8G8_B8G8_UNORM             = 68,
    DXGI_FORMAT_G8R8_G8B8_UNORM             = 69,
    DXGI_FORMAT_BC1_TYPELESS                = 70,
    DXGI_FORMAT_BC1_UNORM                   = 71,
    DXGI_FORMAT_BC1_UNORM_SRGB              = 72,
    DXGI_FORMAT_BC2_TYPELESS                = 73,
    DXGI_FORMAT_BC2_UNORM                   = 74,
    DXGI_FORMAT_BC2_UNORM_SRGB              = 75,
    DXGI_FORMAT_BC3_TYPELESS                = 76,
    DXGI_FORMAT_BC3_UNORM                   = 77,
    DXGI_FORMAT_BC3_UNORM_SRGB              = 78,
    DXGI_FORMAT_BC4_TYPELESS                = 79,
    DXGI_FORMAT_BC4_UNORM                   = 80,
    DXGI_FORMAT_BC4_SNORM                   = 81,
    DXGI_FORMAT_BC5_TYPELESS                = 82,
    DXGI_FORMAT_BC5_UNORM                   = 83,
    DXGI_FORMAT_BC5_SNORM                   = 84,
    DXGI_FORMAT_B5G6R5_UNORM                = 85,
    DXGI_FORMAT_B5G5R5A1_UNORM              = 86,
    DXGI_FORMAT_B8G8R8A8_UNORM              = 87,
    DXGI_FORMAT_B8G8R8X8_UNORM              = 88,
    DXGI_FORMAT_R10G10B10_XR_BIAS_A2_UNORM  = 89,
    DXGI_FORMAT_B8G8R8A8_TYPELESS           = 90,
    DXGI_FORMAT_B8G8R8A8_UNORM_SRGB         = 91,
    DXGI_FORMAT_B8G8R8X8_TYPELESS           = 92,
    DXGI_FORMAT_B8G8R8X8_UNORM_SRGB         = 93,
    DXGI_FORMAT_BC6H_TYPELESS               = 94,
    DXGI_FORMAT_BC6H_UF16                   = 95,
    DXGI_FORMAT_BC6H_SF16                   = 96,
    DXGI_FORMAT_BC7_TYPELESS                = 97,
    DXGI_FORMAT_BC7_UNORM                   = 98,
    DXGI_FORMAT_BC7_UNORM_SRGB              = 99,
    DXGI_FORMAT_AYUV                        = 100,
    DXGI_FORMAT_Y410                        = 101,
    DXGI_FORMAT_Y416                        = 102,
    DXGI_FORMAT_NV12                        = 103,
    DXGI_FORMAT_P010                        = 104,
    DXGI_FORMAT_P016                        = 105,
    DXGI_FORMAT_420_OPAQUE                  = 106,
    DXGI_FORMAT_YUY2                        = 107,
    DXGI_FORMAT_Y210                        = 108,
    DXGI_FORMAT_Y216                        = 109,
    DXGI_FORMAT_NV11                        = 110,
    DXGI_FORMAT_AI44                        = 111,
    DXGI_FORMAT_IA44                        = 112,
    DXGI_FORMAT_P8                          = 113,
    DXGI_FORMAT_A8P8                        = 114,
    DXGI_FORMAT_B4G4R4A4_UNORM              = 115,

    DXGI_FORMAT_P208                        = 130,
    DXGI_FORMAT_V208                        = 131,
    DXGI_FORMAT_V408                        = 132,


    DXGI_FORMAT_FORCE_UINT                  = 0xffffffff
} DXGI_FORMAT;

typedef struct D3D11_INPUT_ELEMENT_DESC
    {
    const char * SemanticName;
    UINT SemanticIndex;
    DXGI_FORMAT Format;
    UINT InputSlot;
    UINT AlignedByteOffset;
    D3D11_INPUT_CLASSIFICATION InputSlotClass;
    UINT InstanceDataStepRate;
    } 	D3D11_INPUT_ELEMENT_DESC;

typedef struct D3D11_BOX
    {
    UINT left;
    UINT top;
    UINT front;
    UINT right;
    UINT bottom;
    UINT back;
    } 	D3D11_BOX;

#pragma endregion headers

/////////////////////////////////////////////////////////////////
// d3d11_common.cpp / .h

#pragma region d3d11_common.cpp

SERIALISE_ENUM_TYPE(D3D_DRIVER_TYPE);

template <>
string ToStr(const D3D_DRIVER_TYPE &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(D3D_DRIVER_TYPE_UNKNOWN);
    STRINGIZE_ENUM(D3D_DRIVER_TYPE_HARDWARE);
    STRINGIZE_ENUM(D3D_DRIVER_TYPE_REFERENCE);
    STRINGIZE_ENUM(D3D_DRIVER_TYPE_NULL);
    STRINGIZE_ENUM(D3D_DRIVER_TYPE_SOFTWARE);
    STRINGIZE_ENUM(D3D_DRIVER_TYPE_WARP);
  END_ENUM_STRINGIZE(D3D_DRIVER_TYPE);
}

SERIALISE_ENUM_TYPE(D3D_FEATURE_LEVEL);

template <>
string DoStringize(const D3D_FEATURE_LEVEL &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_9_1);
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_9_2);
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_9_3);
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_10_0);
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_10_1);
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_11_0);
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_11_1);
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_12_0);
    STRINGIZE_ENUM(D3D_FEATURE_LEVEL_12_1);
  END_ENUM_STRINGIZE(D3D_FEATURE_LEVEL);
}

SERIALISE_ENUM_TYPE(D3D11_INPUT_CLASSIFICATION);

template <>
string DoStringize(const D3D11_INPUT_CLASSIFICATION &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(D3D11_INPUT_PER_VERTEX_DATA)
    STRINGIZE_ENUM(D3D11_INPUT_PER_INSTANCE_DATA)
  END_ENUM_STRINGIZE(D3D11_INPUT_CLASSIFICATION);
}

SERIALISE_ENUM_TYPE(DXGI_FORMAT);

template <>
string DoStringize(const DXGI_FORMAT &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(DXGI_FORMAT_UNKNOWN)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32B32A32_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32B32A32_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32B32A32_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32B32A32_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32B32_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32B32_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32B32_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32B32_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16B16A16_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16B16A16_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16B16A16_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16B16A16_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16B16A16_SNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16B16A16_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G32_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32G8X24_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_D32_FLOAT_S8X24_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32_FLOAT_X8X24_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_X32_TYPELESS_G8X24_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R10G10B10A2_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R10G10B10A2_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R10G10B10A2_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R11G11B10_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8B8A8_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8B8A8_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8B8A8_UNORM_SRGB)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8B8A8_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8B8A8_SNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8B8A8_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16_SNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R16G16_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_D32_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R32_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R24G8_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_D24_UNORM_S8_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R24_UNORM_X8_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_X24_TYPELESS_G8_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8_SNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R16_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R16_FLOAT)
    STRINGIZE_ENUM(DXGI_FORMAT_D16_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R16_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R16_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R16_SNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R16_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R8_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_R8_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R8_UINT)
    STRINGIZE_ENUM(DXGI_FORMAT_R8_SNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R8_SINT)
    STRINGIZE_ENUM(DXGI_FORMAT_A8_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R1_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R9G9B9E5_SHAREDEXP)
    STRINGIZE_ENUM(DXGI_FORMAT_R8G8_B8G8_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_G8R8_G8B8_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_BC1_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_BC1_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_BC1_UNORM_SRGB)
    STRINGIZE_ENUM(DXGI_FORMAT_BC2_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_BC2_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_BC2_UNORM_SRGB)
    STRINGIZE_ENUM(DXGI_FORMAT_BC3_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_BC3_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_BC3_UNORM_SRGB)
    STRINGIZE_ENUM(DXGI_FORMAT_BC4_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_BC4_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_BC4_SNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_BC5_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_BC5_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_BC5_SNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_B5G6R5_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_B5G5R5A1_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_B8G8R8A8_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_B8G8R8X8_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_R10G10B10_XR_BIAS_A2_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_B8G8R8A8_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_B8G8R8A8_UNORM_SRGB)
    STRINGIZE_ENUM(DXGI_FORMAT_B8G8R8X8_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_B8G8R8X8_UNORM_SRGB)
    STRINGIZE_ENUM(DXGI_FORMAT_BC6H_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_BC6H_UF16)
    STRINGIZE_ENUM(DXGI_FORMAT_BC6H_SF16)
    STRINGIZE_ENUM(DXGI_FORMAT_BC7_TYPELESS)
    STRINGIZE_ENUM(DXGI_FORMAT_BC7_UNORM)
    STRINGIZE_ENUM(DXGI_FORMAT_BC7_UNORM_SRGB)
    // D3D11.1 formats
    STRINGIZE_ENUM(DXGI_FORMAT_AYUV)
    STRINGIZE_ENUM(DXGI_FORMAT_Y410)
    STRINGIZE_ENUM(DXGI_FORMAT_Y416)
    STRINGIZE_ENUM(DXGI_FORMAT_NV12)
    STRINGIZE_ENUM(DXGI_FORMAT_P010)
    STRINGIZE_ENUM(DXGI_FORMAT_P016)
    STRINGIZE_ENUM(DXGI_FORMAT_420_OPAQUE)
    STRINGIZE_ENUM(DXGI_FORMAT_YUY2)
    STRINGIZE_ENUM(DXGI_FORMAT_Y210)
    STRINGIZE_ENUM(DXGI_FORMAT_Y216)
    STRINGIZE_ENUM(DXGI_FORMAT_NV11)
    STRINGIZE_ENUM(DXGI_FORMAT_AI44)
    STRINGIZE_ENUM(DXGI_FORMAT_IA44)
    STRINGIZE_ENUM(DXGI_FORMAT_P8)
    STRINGIZE_ENUM(DXGI_FORMAT_A8P8)
    STRINGIZE_ENUM(DXGI_FORMAT_B4G4R4A4_UNORM)
    // D3D11.2 formats
    STRINGIZE_ENUM(DXGI_FORMAT_P208)
    STRINGIZE_ENUM(DXGI_FORMAT_V208)
    STRINGIZE_ENUM(DXGI_FORMAT_V408)
  END_ENUM_STRINGIZE(DXGI_FORMAT);
}

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, D3D11_INPUT_ELEMENT_DESC &el)
{
  SERIALISE_MEMBER(SemanticName);
  SERIALISE_MEMBER(SemanticIndex);
  SERIALISE_MEMBER(Format);
  SERIALISE_MEMBER(InputSlot);
  SERIALISE_MEMBER(AlignedByteOffset);
  SERIALISE_MEMBER(InputSlotClass);
  SERIALISE_MEMBER(InstanceDataStepRate);
}

INSTANTIATE_SERIALISE_TYPE(D3D11_INPUT_ELEMENT_DESC);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, D3D11_BOX &el)
{
  SERIALISE_MEMBER(left);
  SERIALISE_MEMBER(top);
  SERIALISE_MEMBER(front);
  SERIALISE_MEMBER(right);
  SERIALISE_MEMBER(bottom);
  SERIALISE_MEMBER(back);
}

INSTANTIATE_SERIALISE_TYPE(D3D11_BOX);

#pragma endregion

/////////////////////////////////////////////////////////////////
// core.cpp / .h

#pragma region core.cpp

enum ReplayCreateStatus
{
  eReplayCreate_Success = 0,
  eReplayCreate_UnknownError,
  eReplayCreate_InternalError,
  eReplayCreate_NetworkIOFailed,
  eReplayCreate_FileIOFailed,
  eReplayCreate_FileIncompatibleVersion,
  eReplayCreate_FileCorrupted,
  eReplayCreate_APIUnsupported,
  eReplayCreate_APIInitFailed,
  eReplayCreate_APIIncompatibleVersion,
  eReplayCreate_APIHardwareUnsupported,
};

enum LogState
{
  READING = 0,
  EXECUTING,
  WRITING,
  WRITING_IDLE,
  WRITING_CAPFRAME,
};

enum RDCDriver
{
  RDC_Unknown = 0,
  RDC_D3D11 = 1,
  RDC_OpenGL = 2,
  RDC_Mantle = 3,
  RDC_D3D12 = 4,
  RDC_D3D10 = 5,
  RDC_D3D9 = 6,
  RDC_Image = 7,
  RDC_Vulkan = 8,
  RDC_Custom = 100000,
  RDC_Custom0 = RDC_Custom,
  RDC_Custom1,
  RDC_Custom2,
  RDC_Custom3,
  RDC_Custom4,
  RDC_Custom5,
  RDC_Custom6,
  RDC_Custom7,
  RDC_Custom8,
  RDC_Custom9,
};

SERIALISE_ENUM_TYPE(RDCDriver);

template <>
string DoStringize(const RDCDriver &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM_NAMED(RDC_Unknown, "Unknown");
    STRINGIZE_ENUM_NAMED(RDC_D3D11, "D3D11");
    STRINGIZE_ENUM_NAMED(RDC_OpenGL, "OpenGL");
    STRINGIZE_ENUM_NAMED(RDC_Mantle, "Mantle");
    STRINGIZE_ENUM_NAMED(RDC_D3D12, "D3D12");
    STRINGIZE_ENUM_NAMED(RDC_D3D10, "D3D10");
    STRINGIZE_ENUM_NAMED(RDC_D3D9, "D3D9");
    STRINGIZE_ENUM_NAMED(RDC_Image, "Image");
    STRINGIZE_ENUM_NAMED(RDC_Vulkan, "Vulkan");
  END_ENUM_STRINGIZE(RDCDriver);
}

struct DebugMessage
{
  uint32_t eventID;
  uint32_t messageID;
  string description;
};

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, DebugMessage &el)
{
  SERIALISE_MEMBER(eventID);
  SERIALISE_MEMBER(messageID);
  SERIALISE_MEMBER(description);
}

INSTANTIATE_SERIALISE_TYPE(DebugMessage);

enum ChunkType
{
  // 0 is reserved as a 'null' chunk that is only for debug
  CREATE_PARAMS = 1,
  THUMBNAIL_DATA,
  DRIVER_INIT_PARAMS,
  INITIAL_CONTENTS,

  FIRST_CHUNK_ID = 128,

  CREATE_INPUT_LAYOUT,
  CREATE_VERTEX_SHADER,
  DO_VULKAN_STUFF,
  DEBUG_MESSAGES,
  CLEAR_RTV,
};

struct RDCInitParams
{
  virtual ~RDCInitParams() {}

  virtual void Serialise(WriteSerialiser &ser) = 0;
  virtual void Serialise(ReadSerialiser &ser) = 0;
  virtual void Serialise(StructReadSerialiser &ser) = 0;

  virtual ReplayCreateStatus GetStatus() = 0;
};

// utility macro for derived classes to be able to just make a templated Serialise() function
#define INIT_PARAMS_SERIALISE() \
  template<class SerialiserType> \
  void DoSerialise(SerialiserType &ser); \
  virtual void Serialise(WriteSerialiser &ser) { return DoSerialise(ser); } \
  virtual void Serialise(ReadSerialiser &ser) { return DoSerialise(ser); } \
  virtual void Serialise(StructReadSerialiser &ser) { return DoSerialise(ser); }

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, RDCInitParams &el)
{
  // go through overloaded virtual interface
  el.Serialise(ser);
}

INSTANTIATE_SERIALISE_TYPE(RDCInitParams);

WriteSerialiser *OpenWriteSerialiser(uint32_t frameNum, RDCInitParams &params, byte *thpixels,
                                           size_t thlen, uint32_t thwidth, uint32_t thheight)
{
#if defined(RELEASE)
  const bool debugSerialiser = false;
#else
  const bool debugSerialiser = true;
#endif

  string m_CurrentLogFile = "test.rdc";
  RDCDriver m_CurrentDriver = RDC_Mantle;
  string m_CurrentDriverName = "Mantle";

  FILE *file = fopen(m_CurrentLogFile.c_str(), "wb");

  WriteSerialiser *fileSerialiser = new WriteSerialiser(new StreamWriter(file));
  WriteSerialiser *chunkSerialiser = new WriteSerialiser(new StreamWriter(StreamWriter::DefaultScratchSize));
  WriteSerialiser &ser = *chunkSerialiser;

  {
    SCOPED_SERIALISE_CONTEXT(THUMBNAIL_DATA);

    SERIALISE_ELEMENT_LOCAL(HasThumbnail, thpixels != NULL && thwidth > 0 && thheight > 0);

    if(HasThumbnail)
    {
      SERIALISE_ELEMENT(thwidth).Named("Thumbnail Width");
      SERIALISE_ELEMENT(thheight).Named("Thumbnail Height");
      SERIALISE_ELEMENT_ARRAY(thpixels, thlen).Named("Thumbnail Pixels");
    }

    SERIALISE_ALIGNED();

    fileSerialiser->Insert(scope.Get(true));
  }

  {
    SCOPED_SERIALISE_CONTEXT(CREATE_PARAMS);

    SERIALISE_ELEMENT(m_CurrentDriver).Named("Driver Type");
    SERIALISE_ELEMENT(m_CurrentDriverName).Named("Driver Name");
    SERIALISE_ELEMENT(params).Named("Driver Params");

    SERIALISE_ALIGNED();

    fileSerialiser->Insert(scope.Get(true));
  }

  SAFE_DELETE(chunkSerialiser);

  return fileSerialiser;
}

ReplayCreateStatus FillInitParams(const char *logFile, RDCDriver &driverType, string &driverName, RDCInitParams *params)
{
  FILE *f = fopen(logFile, "rb");
  StreamReader read(f);
  ReadSerialiser ser(&read);

  if(ser.HasError())
    return eReplayCreate_FileIOFailed;

  {
    int chunkType = ser.BeginChunk(0);

    if(chunkType != THUMBNAIL_DATA)
    {
      RDCERR("Malformed logfile '%s', first chunk isn't thumbnail data", logFile);
      return eReplayCreate_FileCorrupted;
    }

    ser.SkipCurrentChunk();

    ser.EndChunk();
  }

  {
    int chunkType = ser.BeginChunk(0);

    if(chunkType != CREATE_PARAMS)
    {
      RDCERR("Malformed logfile '%s', second chunk isn't create params", logFile);
      return eReplayCreate_FileCorrupted;
    }
    
    SERIALISE_ELEMENT(driverType).Named("Driver Type");
    SERIALISE_ELEMENT(driverName).Named("Driver Name");
    
    if(params)
      SERIALISE_ELEMENT(*params).Named("Driver Params");
  }

  // we can just throw away the serialiser, don't need to care about closing/popping contexts
  return eReplayCreate_Success;
}

#pragma endregion

/////////////////////////////////////////////////////////////////
// vk_common.cpp

#pragma region vk_common.cpp

#define SERIALISE_RESOURCE_AS_ID(VulkanType) \
  template <class SerialiserType>                                    \
  void DoSerialise(SerialiserType &ser, VulkanType &el)                                    \
  {                                    \
    ResourceId id;                                    \
                                      \
    if(ser.IsWriting())                                    \
    {                                    \
      id = ResourceId();                        \
    }                                    \
                                      \
    DoSerialise(ser, id);                                    \
                                      \
    if(ser.IsReading())                                    \
    {                                    \
      el = (VulkanType)id.innerSecretValue;                                    \
    }                                    \
  } \
  INSTANTIATE_SERIALISE_TYPE(VulkanType);

SERIALISE_RESOURCE_AS_ID(VkRenderPass);
SERIALISE_RESOURCE_AS_ID(VkPipelineLayout);
SERIALISE_RESOURCE_AS_ID(VkPipeline);
SERIALISE_RESOURCE_AS_ID(VkImageView);
SERIALISE_RESOURCE_AS_ID(VkSampler);
SERIALISE_RESOURCE_AS_ID(VkBuffer);
SERIALISE_RESOURCE_AS_ID(VkBufferView);
SERIALISE_RESOURCE_AS_ID(VkShaderModule);
SERIALISE_RESOURCE_AS_ID(VkDescriptorSet);
SERIALISE_RESOURCE_AS_ID(VkDescriptorSetLayout);

enum VkFlagWithNoBits
{
  FlagWithNoBits_Dummy_Bit = 1,
};

template <>
string DoStringize(const VkFlagWithNoBits &el)
{
  if(el != 0)
    return StringFormat::Fmt("Invalid bits set: %x", el);
  return "";
}

SERIALISE_ENUM_TYPE(VkFlagWithNoBits);

template <>
string DoStringize(const VkShaderStageFlagBits &el)
{
  BEGIN_BITFIELD_STRINGIZE()
  {
    STRINGIZE_BITFIELD_NAMED(VK_SHADER_STAGE_ALL_GRAPHICS);
    STRINGIZE_BITFIELD_NAMED(VK_SHADER_STAGE_ALL);

    STRINGIZE_BITFIELD_BIT(VK_SHADER_STAGE_VERTEX_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SHADER_STAGE_GEOMETRY_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SHADER_STAGE_FRAGMENT_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SHADER_STAGE_COMPUTE_BIT)
  }
  END_BITFIELD_STRINGIZE();
}

SERIALISE_ENUM_TYPE(VkShaderStageFlagBits);

template <>
string DoStringize(const VkImageAspectFlagBits &el)
{
  BEGIN_BITFIELD_STRINGIZE()
    STRINGIZE_BITFIELD_BIT(VK_IMAGE_ASPECT_COLOR_BIT)
    STRINGIZE_BITFIELD_BIT(VK_IMAGE_ASPECT_DEPTH_BIT)
    STRINGIZE_BITFIELD_BIT(VK_IMAGE_ASPECT_STENCIL_BIT)
    STRINGIZE_BITFIELD_BIT(VK_IMAGE_ASPECT_METADATA_BIT)
  END_BITFIELD_STRINGIZE();
}

SERIALISE_ENUM_TYPE(VkImageAspectFlagBits);

template <>
string DoStringize(const VkStructureType &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_APPLICATION_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_SUBMIT_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_FENCE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_BIND_SPARSE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_EVENT_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_TESSELLATION_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_COPY_DESCRIPTOR_SET)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_MEMORY_BARRIER)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_PRESENT_INFO_KHR)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_DISPLAY_MODE_CREATE_INFO_KHR)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_DISPLAY_SURFACE_CREATE_INFO_KHR)
    STRINGIZE_ENUM(VK_STRUCTURE_TYPE_DISPLAY_PRESENT_INFO_KHR)
  END_ENUM_STRINGIZE(VkStructureType);
}

SERIALISE_ENUM_TYPE(VkStructureType);

template <>
string DoStringize(const VkStencilOp &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM_NAMED(VK_STENCIL_OP_KEEP, "KEEP");
    STRINGIZE_ENUM_NAMED(VK_STENCIL_OP_ZERO, "ZERO");
    STRINGIZE_ENUM_NAMED(VK_STENCIL_OP_REPLACE, "REPLACE");
    STRINGIZE_ENUM_NAMED(VK_STENCIL_OP_INCREMENT_AND_CLAMP, "INC_SAT");
    STRINGIZE_ENUM_NAMED(VK_STENCIL_OP_DECREMENT_AND_CLAMP, "DEC_SAT");
    STRINGIZE_ENUM_NAMED(VK_STENCIL_OP_INVERT, "INVERT");
    STRINGIZE_ENUM_NAMED(VK_STENCIL_OP_INCREMENT_AND_WRAP, "INC_WRAP");
    STRINGIZE_ENUM_NAMED(VK_STENCIL_OP_DECREMENT_AND_WRAP, "DEC_WRAP");
  END_ENUM_STRINGIZE(VkStencilOp);
}

SERIALISE_ENUM_TYPE(VkStencilOp);

template <>
string DoStringize(const VkLogicOp &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_CLEAR, "CLEAR");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_AND, "AND");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_AND_REVERSE, "AND_REV");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_COPY, "COPY");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_AND_INVERTED, "AND_INV");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_NO_OP, "NOOP");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_XOR, "XOR");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_OR, "OR");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_NOR, "NOR");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_EQUIVALENT, "EQUIV");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_INVERT, "INVERT");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_OR_REVERSE, "OR_REV");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_COPY_INVERTED, "COPY_INV");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_OR_INVERTED, "OR_INV");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_NAND, "NAND");
    STRINGIZE_ENUM_NAMED(VK_LOGIC_OP_SET, "SET");
  END_ENUM_STRINGIZE(VkLogicOp);
}

SERIALISE_ENUM_TYPE(VkLogicOp);

template <>
string DoStringize(const VkCompareOp &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM_NAMED(VK_COMPARE_OP_NEVER, "NEVER");
    STRINGIZE_ENUM_NAMED(VK_COMPARE_OP_LESS, "LESS");
    STRINGIZE_ENUM_NAMED(VK_COMPARE_OP_EQUAL, "EQUAL");
    STRINGIZE_ENUM_NAMED(VK_COMPARE_OP_LESS_OR_EQUAL, "LESS_EQUAL");
    STRINGIZE_ENUM_NAMED(VK_COMPARE_OP_GREATER, "GREATER");
    STRINGIZE_ENUM_NAMED(VK_COMPARE_OP_NOT_EQUAL, "NOT_EQUAL");
    STRINGIZE_ENUM_NAMED(VK_COMPARE_OP_GREATER_OR_EQUAL, "GREATER_EQUAL");
    STRINGIZE_ENUM_NAMED(VK_COMPARE_OP_ALWAYS, "ALWAYS");
  END_ENUM_STRINGIZE(VkCompareOp);
}

SERIALISE_ENUM_TYPE(VkCompareOp);

template <>
string DoStringize(const VkVertexInputRate &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(VK_VERTEX_INPUT_RATE_VERTEX)
    STRINGIZE_ENUM(VK_VERTEX_INPUT_RATE_INSTANCE)
  END_ENUM_STRINGIZE(VkVertexInputRate);
}

SERIALISE_ENUM_TYPE(VkVertexInputRate);

template <>
string DoStringize(const VkPolygonMode &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(VK_POLYGON_MODE_FILL)
    STRINGIZE_ENUM(VK_POLYGON_MODE_LINE)
    STRINGIZE_ENUM(VK_POLYGON_MODE_POINT)
  END_ENUM_STRINGIZE(VkPolygonMode);
}

SERIALISE_ENUM_TYPE(VkPolygonMode);

template <>
string DoStringize(const VkFrontFace &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(VK_FRONT_FACE_COUNTER_CLOCKWISE)
    STRINGIZE_ENUM(VK_FRONT_FACE_CLOCKWISE)
  END_ENUM_STRINGIZE(VkFrontFace);
}

SERIALISE_ENUM_TYPE(VkFrontFace);

template <>
string DoStringize(const VkBlendFactor &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ZERO, "ZERO");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE, "ONE");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_SRC_COLOR, "SRC_COLOR");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR, "INV_SRC_COLOR");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_DST_COLOR, "DST_COLOR");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR, "INV_DST_COLOR");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_SRC_ALPHA, "SRC_ALPHA");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA, "INV_SRC_ALPHA");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_DST_ALPHA, "DST_ALPHA");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA, "INV_DST_ALPHA");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_CONSTANT_COLOR, "CONST_COLOR");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR, "INV_CONST_COLOR");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_CONSTANT_ALPHA, "CONST_ALPHA");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA, "INV_CONST_ALPHA");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_SRC_ALPHA_SATURATE, "SRC_ALPHA_SAT");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_SRC1_COLOR, "SRC1_COLOR");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR, "INV_SRC1_COLOR");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_SRC1_ALPHA, "SRC1_ALPHA");
    STRINGIZE_ENUM_NAMED(VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA, "INV_SRC1_ALPHA");
  END_ENUM_STRINGIZE(VkBlendFactor);
}

SERIALISE_ENUM_TYPE(VkBlendFactor);

template <>
string DoStringize(const VkBlendOp &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM_NAMED(VK_BLEND_OP_ADD, "ADD");
    STRINGIZE_ENUM_NAMED(VK_BLEND_OP_SUBTRACT, "SUB");
    STRINGIZE_ENUM_NAMED(VK_BLEND_OP_REVERSE_SUBTRACT, "REV_SUB");
    STRINGIZE_ENUM_NAMED(VK_BLEND_OP_MIN, "MIN");
    STRINGIZE_ENUM_NAMED(VK_BLEND_OP_MAX, "MAX");
  END_ENUM_STRINGIZE(VkBlendOp);
}

SERIALISE_ENUM_TYPE(VkBlendOp);

template <>
string DoStringize(const VkPrimitiveTopology &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_POINT_LIST)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_LINE_LIST)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_LINE_STRIP)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY)
    STRINGIZE_ENUM(VK_PRIMITIVE_TOPOLOGY_PATCH_LIST)
  END_ENUM_STRINGIZE(VkPrimitiveTopology);
}

SERIALISE_ENUM_TYPE(VkPrimitiveTopology);

template <>
string DoStringize(const VkDescriptorType &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_SAMPLER)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC)
    STRINGIZE_ENUM(VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT)
  END_ENUM_STRINGIZE(VkDescriptorType);
}

SERIALISE_ENUM_TYPE(VkDescriptorType);

template <>
string DoStringize(const VkFormat &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(VK_FORMAT_UNDEFINED)
    STRINGIZE_ENUM(VK_FORMAT_R4G4_UNORM_PACK8)
    STRINGIZE_ENUM(VK_FORMAT_R4G4B4A4_UNORM_PACK16)
    STRINGIZE_ENUM(VK_FORMAT_B4G4R4A4_UNORM_PACK16)
    STRINGIZE_ENUM(VK_FORMAT_R5G6B5_UNORM_PACK16)
    STRINGIZE_ENUM(VK_FORMAT_B5G6R5_UNORM_PACK16)
    STRINGIZE_ENUM(VK_FORMAT_R5G5B5A1_UNORM_PACK16)
    STRINGIZE_ENUM(VK_FORMAT_B5G5R5A1_UNORM_PACK16)
    STRINGIZE_ENUM(VK_FORMAT_A1R5G5B5_UNORM_PACK16)
    STRINGIZE_ENUM(VK_FORMAT_R8_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_R8_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_R8_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_R8_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_R8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R8_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R8_SRGB)
    STRINGIZE_ENUM(VK_FORMAT_R8G8_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_R8G8_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_R8G8_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_R8G8_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_R8G8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R8G8_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R8G8_SRGB)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8_SRGB)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8_SINT)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8_SRGB)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8A8_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8A8_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8A8_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8A8_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8A8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8A8_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R8G8B8A8_SRGB)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8A8_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8A8_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8A8_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8A8_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8A8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8A8_SINT)
    STRINGIZE_ENUM(VK_FORMAT_B8G8R8A8_SRGB)
    STRINGIZE_ENUM(VK_FORMAT_A8B8G8R8_UNORM_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A8B8G8R8_SNORM_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A8B8G8R8_USCALED_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A8B8G8R8_SSCALED_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A8B8G8R8_UINT_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A8B8G8R8_SINT_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A8B8G8R8_SRGB_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2R10G10B10_UNORM_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2R10G10B10_SNORM_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2R10G10B10_USCALED_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2R10G10B10_SSCALED_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2R10G10B10_UINT_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2R10G10B10_SINT_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2B10G10R10_UNORM_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2B10G10R10_SNORM_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2B10G10R10_USCALED_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2B10G10R10_SSCALED_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2B10G10R10_UINT_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_A2B10G10R10_SINT_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_R16_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_R16_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_R16_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_R16_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_R16_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R16_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R16_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_R16G16_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_R16G16_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_R16G16_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_R16G16_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16A16_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16A16_SNORM)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16A16_USCALED)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16A16_SSCALED)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16A16_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16A16_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R16G16B16A16_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R32_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R32_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R32_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32B32_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32B32_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32B32_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32B32A32_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32B32A32_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R32G32B32A32_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R64_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R64_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R64_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64B64_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64B64_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64B64_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64B64A64_UINT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64B64A64_SINT)
    STRINGIZE_ENUM(VK_FORMAT_R64G64B64A64_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_B10G11R11_UFLOAT_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_E5B9G9R9_UFLOAT_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_D16_UNORM)
    STRINGIZE_ENUM(VK_FORMAT_X8_D24_UNORM_PACK32)
    STRINGIZE_ENUM(VK_FORMAT_D32_SFLOAT)
    STRINGIZE_ENUM(VK_FORMAT_S8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_D16_UNORM_S8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_D24_UNORM_S8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_D32_SFLOAT_S8_UINT)
    STRINGIZE_ENUM(VK_FORMAT_BC1_RGB_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC1_RGB_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC1_RGBA_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC1_RGBA_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC2_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC2_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC3_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC3_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC4_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC4_SNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC5_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC5_SNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC6H_UFLOAT_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC6H_SFLOAT_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC7_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_BC7_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ETC2_R8G8B8_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ETC2_R8G8B8_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ETC2_R8G8B8A1_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ETC2_R8G8B8A1_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ETC2_R8G8B8A8_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ETC2_R8G8B8A8_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_EAC_R11_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_EAC_R11_SNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_EAC_R11G11_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_EAC_R11G11_SNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_4x4_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_4x4_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_5x4_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_5x4_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_5x5_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_5x5_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_6x5_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_6x5_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_6x6_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_6x6_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_8x5_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_8x5_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_8x6_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_8x6_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_8x8_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_8x8_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_10x5_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_10x5_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_10x6_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_10x6_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_10x8_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_10x8_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_10x10_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_10x10_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_12x10_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_12x10_SRGB_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_12x12_UNORM_BLOCK)
    STRINGIZE_ENUM(VK_FORMAT_ASTC_12x12_SRGB_BLOCK)
  END_ENUM_STRINGIZE(VkFormat);
}

SERIALISE_ENUM_TYPE(VkFormat);

template <>
string DoStringize(const VkSampleCountFlagBits &el)
{
  BEGIN_BITFIELD_STRINGIZE()
  {
    STRINGIZE_BITFIELD_BIT(VK_SAMPLE_COUNT_1_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SAMPLE_COUNT_2_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SAMPLE_COUNT_4_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SAMPLE_COUNT_8_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SAMPLE_COUNT_16_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SAMPLE_COUNT_32_BIT)
    STRINGIZE_BITFIELD_BIT(VK_SAMPLE_COUNT_64_BIT)
  }
  END_BITFIELD_STRINGIZE();
}

SERIALISE_ENUM_TYPE(VkSampleCountFlagBits);

template <>
string DoStringize(const VkCullModeFlagBits &el)
{
  BEGIN_BITFIELD_STRINGIZE()
  {
  // technically a bitfield but each combination has a particular meaning
    STRINGIZE_BITFIELD_NAMED(VK_CULL_MODE_NONE)
    STRINGIZE_BITFIELD(VK_CULL_MODE_FRONT_BIT, "VK_CULL_MODE_FRONT")
    STRINGIZE_BITFIELD(VK_CULL_MODE_BACK_BIT, "VK_CULL_MODE_BACK")
    STRINGIZE_BITFIELD_NAMED(VK_CULL_MODE_FRONT_AND_BACK)
  }
  END_BITFIELD_STRINGIZE();
}

SERIALISE_ENUM_TYPE(VkCullModeFlagBits);

template <>
string DoStringize(const VkPipelineCreateFlagBits &el)
{
  BEGIN_BITFIELD_STRINGIZE()
  {
    STRINGIZE_BITFIELD_BIT(VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT)
    STRINGIZE_BITFIELD_BIT(VK_PIPELINE_CREATE_ALLOW_DERIVATIVES_BIT)
    STRINGIZE_BITFIELD_BIT(VK_PIPELINE_CREATE_DERIVATIVE_BIT)
  }
  END_BITFIELD_STRINGIZE();
}

SERIALISE_ENUM_TYPE(VkPipelineCreateFlagBits);

template <>
string DoStringize(const VkDynamicState &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_VIEWPORT)
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_SCISSOR)
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_LINE_WIDTH)
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_DEPTH_BIAS)
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_BLEND_CONSTANTS)
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_DEPTH_BOUNDS)
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK)
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_STENCIL_WRITE_MASK)
    STRINGIZE_ENUM(VK_DYNAMIC_STATE_STENCIL_REFERENCE)
  END_ENUM_STRINGIZE(VkDynamicState);
}

SERIALISE_ENUM_TYPE(VkDynamicState);

template <>
string DoStringize(const VkImageLayout &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_UNDEFINED, "UNDEFINED");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_GENERAL, "GENERAL");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, "COLOR_ATTACHMENT_OPTIMAL");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, "DEPTH_STENCIL_ATTACHMENT_OPTIMAL");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL, "DEPTH_STENCIL_READ_ONLY_OPTIMAL");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, "SHADER_READ_ONLY_OPTIMAL");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, "TRANSFER_SRC_OPTIMAL");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, "TRANSFER_DST_OPTIMAL");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_PREINITIALIZED, "PREINITIALIZED");
    STRINGIZE_ENUM_NAMED(VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, "PRESENT_SRC_KHR");
  END_ENUM_STRINGIZE(VkImageLayout);
}

SERIALISE_ENUM_TYPE(VkImageLayout);

template <class SerialiserType>
static void SerialiseNext(SerialiserType &ser, VkStructureType &sType, const void *&pNext)
{
  SERIALISE_ELEMENT(sType);

  // we don't support any extensions, so pNext must always be NULL
  if(ser.IsReading())
    pNext = NULL;
  else
    RDCASSERT(pNext == NULL);
}

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkExtent2D &el)
{
  SERIALISE_MEMBER(width);
  SERIALISE_MEMBER(height);
}

INSTANTIATE_SERIALISE_TYPE(VkExtent2D);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkExtent3D &el)
{
  SERIALISE_MEMBER(width);
  SERIALISE_MEMBER(height);
  SERIALISE_MEMBER(depth);
}

INSTANTIATE_SERIALISE_TYPE(VkExtent3D);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkOffset2D &el)
{
  SERIALISE_MEMBER(x);
  SERIALISE_MEMBER(y);
}

INSTANTIATE_SERIALISE_TYPE(VkOffset2D);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkOffset3D &el)
{
  SERIALISE_MEMBER(x);
  SERIALISE_MEMBER(y);
  SERIALISE_MEMBER(z);
}

INSTANTIATE_SERIALISE_TYPE(VkOffset3D);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkRect2D &el)
{
  SERIALISE_MEMBER(extent);
  SERIALISE_MEMBER(offset);
}

INSTANTIATE_SERIALISE_TYPE(VkRect2D);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkViewport &el)
{
  SERIALISE_MEMBER(x);
  SERIALISE_MEMBER(y);
  SERIALISE_MEMBER(width);
  SERIALISE_MEMBER(height);
  SERIALISE_MEMBER(minDepth);
  SERIALISE_MEMBER(maxDepth);
}

INSTANTIATE_SERIALISE_TYPE(VkViewport);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineLayoutCreateInfo &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);

  // need to do this one by hand since it's just an array of objects that don't themselves have
  // a Serialise overload
  SERIALISE_MEMBER_ARRAY(pSetLayouts, setLayoutCount);

  // skipped for sanity
  // SERIALISE_MEMBER_ARRAY(pPushConstantRanges, pushConstantRangeCount);
  el.pushConstantRangeCount = 0;
  el.pPushConstantRanges = NULL;
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineLayoutCreateInfo);

template <>
void Deserialise(const VkPipelineLayoutCreateInfo &el)
{
  RDCASSERT(el.pNext == NULL);    // otherwise delete
  delete[] el.pSetLayouts;
  delete[] el.pPushConstantRanges;
}

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkVertexInputBindingDescription &el)
{
  SERIALISE_MEMBER(binding);
  SERIALISE_MEMBER(stride);
  SERIALISE_MEMBER(inputRate);
}

INSTANTIATE_SERIALISE_TYPE(VkVertexInputBindingDescription);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkVertexInputAttributeDescription &el)
{
  SERIALISE_MEMBER(location);
  SERIALISE_MEMBER(binding);
  SERIALISE_MEMBER(format);
  SERIALISE_MEMBER(offset);
}

INSTANTIATE_SERIALISE_TYPE(VkVertexInputAttributeDescription);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineVertexInputStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER_ARRAY(pVertexBindingDescriptions, vertexBindingDescriptionCount);
  SERIALISE_MEMBER_ARRAY(pVertexAttributeDescriptions, vertexAttributeDescriptionCount);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineVertexInputStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineInputAssemblyStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() ||
            el.sType == VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER(topology);
  SERIALISE_MEMBER(primitiveRestartEnable);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineInputAssemblyStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineTessellationStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() ||
            el.sType == VK_STRUCTURE_TYPE_PIPELINE_TESSELLATION_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER(patchControlPoints);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineTessellationStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineViewportStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);
  
  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  
  SERIALISE_MEMBER_ARRAY(pViewports, viewportCount);
  SERIALISE_MEMBER_ARRAY(pScissors, scissorCount);

  // need to serialise these separately afterwards, since viewport count can be
  // positive even with a NULL array, if dynamic state is set
  SERIALISE_MEMBER(viewportCount);
  SERIALISE_MEMBER(scissorCount);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineViewportStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineRasterizationStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() ||
            el.sType == VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER(depthClampEnable);
  SERIALISE_MEMBER(rasterizerDiscardEnable);
  SERIALISE_MEMBER(polygonMode);
  SERIALISE_MEMBER(cullMode);
  SERIALISE_MEMBER(frontFace);
  SERIALISE_MEMBER(depthBiasEnable);
  SERIALISE_MEMBER(depthBiasConstantFactor);
  SERIALISE_MEMBER(depthBiasClamp);
  SERIALISE_MEMBER(depthBiasSlopeFactor);
  SERIALISE_MEMBER(lineWidth);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineRasterizationStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineMultisampleStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER(rasterizationSamples);
  RDCASSERT(el.rasterizationSamples <= VK_SAMPLE_COUNT_32_BIT);
  SERIALISE_MEMBER(sampleShadingEnable);
  SERIALISE_MEMBER(minSampleShading);
  SERIALISE_MEMBER_OPT(pSampleMask);
  SERIALISE_MEMBER(alphaToCoverageEnable);
  SERIALISE_MEMBER(alphaToOneEnable);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineMultisampleStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineColorBlendAttachmentState &el)
{
  SERIALISE_MEMBER(blendEnable);
  SERIALISE_MEMBER(srcColorBlendFactor);
  SERIALISE_MEMBER(dstColorBlendFactor);
  SERIALISE_MEMBER(colorBlendOp);
  SERIALISE_MEMBER(srcAlphaBlendFactor);
  SERIALISE_MEMBER(dstAlphaBlendFactor);
  SERIALISE_MEMBER(alphaBlendOp);
  SERIALISE_MEMBER(colorWriteMask);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineColorBlendAttachmentState);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineColorBlendStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER(logicOpEnable);
  SERIALISE_MEMBER(logicOp);

  SERIALISE_MEMBER_ARRAY(pAttachments, attachmentCount);

  SERIALISE_MEMBER(blendConstants);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineColorBlendStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkStencilOpState &el)
{
  SERIALISE_MEMBER(failOp);
  SERIALISE_MEMBER(passOp);
  SERIALISE_MEMBER(depthFailOp);
  SERIALISE_MEMBER(compareOp);
  SERIALISE_MEMBER(compareMask);
  SERIALISE_MEMBER(writeMask);
  SERIALISE_MEMBER(reference);
}

INSTANTIATE_SERIALISE_TYPE(VkStencilOpState);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkImageSubresourceLayers &el)
{
  SERIALISE_MEMBER_TYPED(VkImageAspectFlagBits, aspectMask);
  SERIALISE_MEMBER(mipLevel);
  SERIALISE_MEMBER(baseArrayLayer);
  SERIALISE_MEMBER(layerCount);
}

INSTANTIATE_SERIALISE_TYPE(VkImageSubresourceLayers);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineShaderStageCreateInfo &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER(stage);
  SERIALISE_MEMBER(module);
  SERIALISE_MEMBER(pName);
  // SERIALISE_MEMBER_OPT(pSpecializationInfo); // skipped for sanity
  el.pSpecializationInfo = NULL;
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineShaderStageCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineDepthStencilStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() ||
            el.sType == VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER(depthTestEnable);
  SERIALISE_MEMBER(depthWriteEnable);
  SERIALISE_MEMBER(depthCompareOp);
  SERIALISE_MEMBER(depthBoundsTestEnable);
  SERIALISE_MEMBER(stencilTestEnable);
  SERIALISE_MEMBER(front);
  SERIALISE_MEMBER(back);
  SERIALISE_MEMBER(minDepthBounds);
  SERIALISE_MEMBER(maxDepthBounds);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineDepthStencilStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkPipelineDynamicStateCreateInfo &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkFlagWithNoBits, flags);
  SERIALISE_MEMBER_ARRAY(pDynamicStates, dynamicStateCount);
}

INSTANTIATE_SERIALISE_TYPE(VkPipelineDynamicStateCreateInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkGraphicsPipelineCreateInfo &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER_TYPED(VkPipelineCreateFlagBits, flags);
  SERIALISE_MEMBER(layout);
  SERIALISE_MEMBER(renderPass);
  SERIALISE_MEMBER(subpass);
  SERIALISE_MEMBER(basePipelineHandle);
  SERIALISE_MEMBER(basePipelineIndex);

  SERIALISE_MEMBER_OPT(pVertexInputState);
  SERIALISE_MEMBER_OPT(pInputAssemblyState);
  SERIALISE_MEMBER_OPT(pTessellationState);
  SERIALISE_MEMBER_OPT(pViewportState);
  SERIALISE_MEMBER_OPT(pRasterizationState);
  SERIALISE_MEMBER_OPT(pMultisampleState);
  SERIALISE_MEMBER_OPT(pDepthStencilState);
  SERIALISE_MEMBER_OPT(pColorBlendState);
  SERIALISE_MEMBER_OPT(pDynamicState);
  SERIALISE_MEMBER_ARRAY(pStages, stageCount);
}

INSTANTIATE_SERIALISE_TYPE(VkGraphicsPipelineCreateInfo);

template <>
void Deserialise(const VkGraphicsPipelineCreateInfo &el)
{
  RDCASSERT(el.pNext == NULL);    // otherwise delete
  if(el.pVertexInputState)
  {
    RDCASSERT(el.pVertexInputState->pNext == NULL);    // otherwise delete
    delete[] el.pVertexInputState->pVertexBindingDescriptions;
    delete[] el.pVertexInputState->pVertexAttributeDescriptions;
    delete el.pVertexInputState;
  }
  if(el.pInputAssemblyState)
  {
    RDCASSERT(el.pInputAssemblyState->pNext == NULL);    // otherwise delete
    delete el.pInputAssemblyState;
  }
  if(el.pTessellationState)
  {
    RDCASSERT(el.pTessellationState->pNext == NULL);    // otherwise delete
    delete el.pTessellationState;
  }
  if(el.pViewportState)
  {
    RDCASSERT(el.pViewportState->pNext == NULL);    // otherwise delete
    if(el.pViewportState->pViewports)
      delete[] el.pViewportState->pViewports;
    if(el.pViewportState->pScissors)
      delete[] el.pViewportState->pScissors;
    delete el.pViewportState;
  }
  if(el.pRasterizationState)
  {
    RDCASSERT(el.pRasterizationState->pNext == NULL);    // otherwise delete
    delete el.pRasterizationState;
  }
  if(el.pMultisampleState)
  {
    RDCASSERT(el.pMultisampleState->pNext == NULL);    // otherwise delete
    delete el.pMultisampleState->pSampleMask;
    delete el.pMultisampleState;
  }
  if(el.pDepthStencilState)
  {
    RDCASSERT(el.pDepthStencilState->pNext == NULL);    // otherwise delete
    delete el.pDepthStencilState;
  }
  if(el.pColorBlendState)
  {
    RDCASSERT(el.pColorBlendState->pNext == NULL);    // otherwise delete
    delete[] el.pColorBlendState->pAttachments;
    delete el.pColorBlendState;
  }
  if(el.pDynamicState)
  {
    RDCASSERT(el.pDynamicState->pNext == NULL);    // otherwise delete
    if(el.pDynamicState->pDynamicStates)
      delete[] el.pDynamicState->pDynamicStates;
    delete el.pDynamicState;
  }
  for(uint32_t i = 0; i < el.stageCount; i++)
  {
    RDCASSERT(el.pStages[i].pNext == NULL);    // otherwise delete
    if(el.pStages[i].pSpecializationInfo)
    {
      delete[](byte *)(el.pStages[i].pSpecializationInfo->pData);
      delete[] el.pStages[i].pSpecializationInfo->pMapEntries;
      delete el.pStages[i].pSpecializationInfo;
    }
  }
  delete[] el.pStages;
}

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkDescriptorImageInfo &el)
{
  SERIALISE_MEMBER(sampler);
  SERIALISE_MEMBER(imageView);
  SERIALISE_MEMBER(imageLayout);
}

INSTANTIATE_SERIALISE_TYPE(VkDescriptorImageInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkDescriptorBufferInfo &el)
{
  SERIALISE_MEMBER(buffer);
  SERIALISE_MEMBER(offset);
  SERIALISE_MEMBER(range);
}

INSTANTIATE_SERIALISE_TYPE(VkDescriptorBufferInfo);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkWriteDescriptorSet &el)
{
  RDCASSERT(ser.IsReading() || el.sType == VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET);
  SerialiseNext(ser, el.sType, el.pNext);

  SERIALISE_MEMBER(dstSet);
  SERIALISE_MEMBER(dstBinding);
  SERIALISE_MEMBER(dstArrayElement);
  SERIALISE_MEMBER(descriptorType);

  if(ser.IsReading())
  {
    el.pImageInfo = NULL;
    el.pBufferInfo = NULL;
    el.pTexelBufferView = NULL;
  }

  // only serialise the array type used, the others are ignored
  if(el.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER ||
     el.descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
     el.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE ||
     el.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE ||
     el.descriptorType == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT)
  {
    SERIALISE_MEMBER_ARRAY(pImageInfo, descriptorCount);
  }
  else if(el.descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER ||
          el.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ||
          el.descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
          el.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC)
  {
    SERIALISE_MEMBER_ARRAY(pBufferInfo, descriptorCount);
  }
  else if(el.descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER ||
          el.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER)
  {
    SERIALISE_MEMBER_ARRAY(pTexelBufferView, descriptorCount);
  }
}

INSTANTIATE_SERIALISE_TYPE(VkWriteDescriptorSet);

template <>
void Deserialise(const VkWriteDescriptorSet &el)
{
  RDCASSERT(el.pNext == NULL);    // otherwise delete
  delete[] el.pImageInfo;
  delete[] el.pBufferInfo;
  delete[] el.pTexelBufferView;
}

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VkImageBlit &el)
{
  SERIALISE_MEMBER(srcSubresource);
  SERIALISE_MEMBER(srcOffsets);
  SERIALISE_MEMBER(dstSubresource);
  SERIALISE_MEMBER(dstOffsets);
}

INSTANTIATE_SERIALISE_TYPE(VkImageBlit);

#pragma endregion

/////////////////////////////////////////////////////////////////
// d3d11_device.cpp / .h

#pragma region d3d11_device.cpp

//   - WrappedID3D11DeviceContext::Serialise_DebugMessages

#define IMPLEMENT_FUNCTION_SERIALISED(ret, func, ...) \
  ret func(__VA_ARGS__);                              \
  template<class SerialiserType>                      \
  bool CONCAT(Serialise_, func(SerialiserType &ser, __VA_ARGS__));

#define IMPLEMENT_FUNCTION_SERIALISED_NOARGS(ret, func) \
  ret func();                              \
  template<class SerialiserType>                      \
  bool CONCAT(Serialise_, func(SerialiserType &ser));

struct FakeD3DVulkanEtc
{
  FakeD3DVulkanEtc() : m_Writer(NULL), m_Reader(NULL), m_State(WRITING) {}

  WriteSerialiser *m_Writer;
  StructReadSerialiser *m_Reader;

  vector<Chunk *> m_Chunks;

  LogState m_State;

  IMPLEMENT_FUNCTION_SERIALISED(virtual HRESULT ,
                                CreateInputLayout, const D3D11_INPUT_ELEMENT_DESC *pInputElementDescs,
                                                  UINT NumElements,
                                                  const void *pShaderBytecodeWithInputSignature,
                                                  SIZE_T BytecodeLength,
                                                  ID3D11InputLayout **ppInputLayout);

  IMPLEMENT_FUNCTION_SERIALISED(virtual HRESULT ,
                                CreateVertexShader, const void *pShaderBytecode, SIZE_T BytecodeLength,
                                                   ID3D11ClassLinkage *pClassLinkage,
                                                   ID3D11VertexShader **ppVertexShader);
  
  IMPLEMENT_FUNCTION_SERIALISED(virtual HRESULT ,
                                ClearRenderTarget, ID3D11RenderTargetView *rtv, UINT foobar,
                                                   const float color[4],
                                                   D3D11_BOX *box);
  
  IMPLEMENT_FUNCTION_SERIALISED(virtual HRESULT ,
                                DoSomeVulkanStuff, uint32_t blah, float foo,
                                VkGraphicsPipelineCreateInfo *pipeInfo,
                                VkWriteDescriptorSet *writeSet,
                                VkPipelineLayoutCreateInfo *layoutInfo);

  vector<DebugMessage> GetDebugMessages()
  {
    DebugMessage foo;
    foo.description = "bl <a> h";
    foo.eventID = 1234;
    foo.messageID = 42;

    return vector<DebugMessage>(1, foo);
  }

  void ReadLog();
};

template <class SerialiserType>
bool FakeD3DVulkanEtc::Serialise_CreateInputLayout(SerialiserType &ser,
    const D3D11_INPUT_ELEMENT_DESC *pInputElementDescs, UINT NumElements,
    const void *pShaderBytecodeWithInputSignature, SIZE_T BytecodeLength,
    ID3D11InputLayout **ppInputLayout)
{
  SERIALISE_ELEMENT_ARRAY(pInputElementDescs, NumElements);
  SERIALISE_ELEMENT_ARRAY(pShaderBytecodeWithInputSignature, BytecodeLength);
  SERIALISE_ELEMENT_LOCAL(pLayout, ResourceId());

  // pretend this is a drawcall, OK
  vector<DebugMessage> msgs = GetDebugMessages();
  SERIALISE_ELEMENT(msgs).Named("Debug Messages");

  ID3D11InputLayout *ret = NULL;
  if(m_State >= WRITING)
  {
    ret = *ppInputLayout;
  }
  else if(m_State == READING)
  {
    // do stuff!
  }

  return true;
}

HRESULT FakeD3DVulkanEtc::CreateInputLayout(const D3D11_INPUT_ELEMENT_DESC *pInputElementDescs,
                                                  UINT NumElements,
                                                  const void *pShaderBytecodeWithInputSignature,
                                                  SIZE_T BytecodeLength,
                                                  ID3D11InputLayout **ppInputLayout)
{
  ID3D11InputLayout *real = NULL;
  ID3D11InputLayout *wrapped = NULL;
  HRESULT ret = S_OK;

  if(SUCCEEDED(ret))
  {
    wrapped = real;

    if(m_Writer)
    {
      WriteSerialiser &ser = *m_Writer;

      SCOPED_SERIALISE_CONTEXT(CREATE_INPUT_LAYOUT);
      Serialise_CreateInputLayout(ser, pInputElementDescs, NumElements,
                                  pShaderBytecodeWithInputSignature, BytecodeLength, &wrapped);

      // do stuff!
      m_Chunks.push_back(scope.Get());
    }
  }

  *ppInputLayout = wrapped;

  return ret;
}

template<class SerialiserType>
bool FakeD3DVulkanEtc::Serialise_CreateVertexShader(SerialiserType &ser, const void *pShaderBytecode,
                                                       SIZE_T BytecodeLength,
                                                       ID3D11ClassLinkage *pClassLinkage,
                                                       ID3D11VertexShader **ppVertexShader)
{
  SERIALISE_ELEMENT_ARRAY(pShaderBytecode, BytecodeLength);
  SERIALISE_ELEMENT_LOCAL(pLinkage, ResourceId());
  SERIALISE_ELEMENT_LOCAL(pShader, ResourceId());

  if(m_State == READING)
  {
    // do stuff!
  }

  return true;
}

HRESULT FakeD3DVulkanEtc::CreateVertexShader(const void *pShaderBytecode, SIZE_T BytecodeLength,
                                                   ID3D11ClassLinkage *pClassLinkage,
                                                   ID3D11VertexShader **ppVertexShader)
{
  ID3D11VertexShader *real = NULL;
  ID3D11VertexShader *wrapped = NULL;
  HRESULT ret = S_OK;

  if(SUCCEEDED(ret))
  {
    wrapped = real;

    if(m_Writer)
    {
      WriteSerialiser &ser = *m_Writer;

      SCOPED_SERIALISE_CONTEXT(CREATE_VERTEX_SHADER);
      Serialise_CreateVertexShader(ser, pShaderBytecode, BytecodeLength, pClassLinkage, &wrapped);

      // do stuff!
      m_Chunks.push_back(scope.Get());
    }
    else
    {
      // do stuff?
    }

    *ppVertexShader = wrapped;
  }

  return ret;
}

template<class SerialiserType>
bool FakeD3DVulkanEtc::Serialise_ClearRenderTarget(SerialiserType &ser, ID3D11RenderTargetView *rtv, UINT foobar, const float color[4], D3D11_BOX *box)
{
  SERIALISE_ELEMENT_LOCAL(RTV, ResourceId());
  SERIALISE_ELEMENT(foobar);
  SERIALISE_ELEMENT_ARRAY(color, FIXED_COUNT(4));
  SERIALISE_ELEMENT_OPT(box);

  if(m_State == READING)
  {
    // do stuff!
  }

  return true;
}

HRESULT FakeD3DVulkanEtc::ClearRenderTarget(ID3D11RenderTargetView *rtv, UINT foobar, const float color[4], D3D11_BOX *box)
{
  HRESULT ret = S_OK;

  if(SUCCEEDED(ret))
  {
    if(m_Writer)
    {
      WriteSerialiser &ser = *m_Writer;

      SCOPED_SERIALISE_CONTEXT(CLEAR_RTV);
      Serialise_ClearRenderTarget(ser, rtv, foobar, color, box);

      // do stuff!
      m_Chunks.push_back(scope.Get());
    }
    else
    {
      // do stuff?
    }
  }

  return ret;
}

template<class SerialiserType>
bool FakeD3DVulkanEtc::Serialise_DoSomeVulkanStuff(SerialiserType &ser, uint32_t blah, float foo,
                                VkGraphicsPipelineCreateInfo *pipeInfo,
                                VkWriteDescriptorSet *writeSet,
                                VkPipelineLayoutCreateInfo *layoutInfo)
{
  SERIALISE_ELEMENT(blah);
  SERIALISE_ELEMENT(foo);
  SERIALISE_ELEMENT_OPT(pipeInfo);
  SERIALISE_ELEMENT_OPT(writeSet);
  SERIALISE_ELEMENT_OPT(layoutInfo);

  return true;
}

HRESULT FakeD3DVulkanEtc::DoSomeVulkanStuff(uint32_t blah, float foo,
                                VkGraphicsPipelineCreateInfo *pipeInfo,
                                VkWriteDescriptorSet *writeSet,
                                VkPipelineLayoutCreateInfo *layoutInfo)
{
  if(m_Writer)
  {
    WriteSerialiser &ser = *m_Writer;

    SCOPED_SERIALISE_CONTEXT(DO_VULKAN_STUFF);
    Serialise_DoSomeVulkanStuff(ser, blah, foo, pipeInfo, writeSet, layoutInfo);

    // do stuff!
    m_Chunks.push_back(scope.Get());
  }

  return S_OK;
}

void FakeD3DVulkanEtc::ReadLog()
{
  StructReadSerialiser &ser = *m_Reader;

  for(;;)
  {
    ChunkType context = (ChunkType)ser.BeginChunk(0);

    if(context == CREATE_INPUT_LAYOUT)
      Serialise_CreateInputLayout(ser, NULL, 0, NULL, 0, NULL);
    else if(context == CREATE_VERTEX_SHADER)
      Serialise_CreateVertexShader(ser, NULL, 0, NULL, NULL);
    else if(context == DO_VULKAN_STUFF)
      Serialise_DoSomeVulkanStuff(ser, 0, 0.0f, NULL, NULL, NULL);
    else if(context == CLEAR_RTV)
      Serialise_ClearRenderTarget(ser, NULL, 0, (float *)NULL, NULL);
    else
      ser.SkipCurrentChunk();

    ser.EndChunk();

    if(ser.GetReader()->AtEnd())
      break;
  }

}

struct D3D11InitParams : public RDCInitParams
{
  D3D11InitParams()
  {
    SerialiseVersion = D3D11_SERIALISE_VERSION;
    DriverType = D3D_DRIVER_TYPE_UNKNOWN;
    Flags = 0;
    SDKVersion = D3D11_SDK_VERSION;
    NumFeatureLevels = 0;
    RDCEraseEl(FeatureLevels);
  }

  INIT_PARAMS_SERIALISE();

  D3D_DRIVER_TYPE DriverType;
  UINT Flags;
  UINT SDKVersion;
  UINT NumFeatureLevels;
  D3D_FEATURE_LEVEL FeatureLevels[16];

  static const uint32_t D3D11_SERIALISE_VERSION = 0x0000009;

  ReplayCreateStatus GetStatus() { return SerialiseVersion < D3D11_SERIALISE_VERSION ? eReplayCreate_APIIncompatibleVersion : eReplayCreate_Success; }

  // version number internal to d3d11 stream
  uint32_t SerialiseVersion;
};

template <class SerialiserType>
void D3D11InitParams::DoSerialise(SerialiserType &ser)
{
  D3D11InitParams &el = *this;

  SERIALISE_MEMBER(SerialiseVersion);
  SERIALISE_MEMBER(DriverType);
  SERIALISE_MEMBER(Flags);
  SERIALISE_MEMBER(SDKVersion);
  SERIALISE_MEMBER(NumFeatureLevels);
  SERIALISE_MEMBER(FeatureLevels);
}

#pragma endregion

/////////////////////////////////////////////////////////////////
// test harness

#pragma region harness

const char *chunklookup(uint32_t idx)
{
  if(idx == CREATE_PARAMS)
    return "Create Params";
  if(idx == THUMBNAIL_DATA)
    return "Thumbnail Data";
  if(idx == DRIVER_INIT_PARAMS)
    return "Driver Init Params";
  if(idx == INITIAL_CONTENTS)
    return "Initial Contents";
  if(idx == CREATE_INPUT_LAYOUT)
    return "CreateInputLayout";
  if(idx == CREATE_VERTEX_SHADER)
    return "CreateVertexShader";
  if(idx == DO_VULKAN_STUFF)
    return "DoVulkanStuff";
  if(idx == CLEAR_RTV)
    return "ClearRenderTarget";
  return "Unknown!";
}

void real_world_test_write()
{
  // volatile just so the constants don't get folded in
  volatile uint32_t frame = 34;
  volatile size_t thlen = frame*frame;
  byte *pixels = new byte[thlen];
  volatile uint32_t w = frame + 5;
  volatile uint32_t h = w - 10;

  pixels[0] = 0x1;
  pixels[1] = 0x2;
  pixels[2] = 0x3;
  pixels[3] = 0x2;
  pixels[4] = 0x1;

  D3D11InitParams init;

  WriteSerialiser *fileWrite = OpenWriteSerialiser(frame, init, (byte *)pixels, thlen, w, h);

  FakeD3DVulkanEtc etc;
  etc.m_Writer = new WriteSerialiser(new StreamWriter(StreamWriter::DefaultScratchSize));
  etc.m_State = WRITING_IDLE;

  etc.m_Writer->SetChunkMetadataRecording(WriteSerialiser::ChunkThreadID | WriteSerialiser::ChunkTimestamp);

  D3D11_INPUT_ELEMENT_DESC inputDesc;

  inputDesc.SemanticName = "POSITION";
  inputDesc.SemanticIndex = 0;
  inputDesc.Format = DXGI_FORMAT_R32G32B32_FLOAT;
  inputDesc.InputSlot = 0;
  inputDesc.InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA;
  inputDesc.AlignedByteOffset = 0;
  inputDesc.InstanceDataStepRate = 0;

  ID3D11InputLayout *ilayout = NULL;
  etc.CreateInputLayout(&inputDesc, 1, (const void *)pixels, thlen, &ilayout);

  ID3D11VertexShader *vs = NULL;
  etc.CreateVertexShader((const void *)pixels, thlen, NULL, &vs);

  ID3D11RenderTargetView *rtv = (ID3D11RenderTargetView *)0xf00baa;
  float magenta[] = { 1.0f, 0.0f, 1.0f, 0.5f };
  etc.ClearRenderTarget(rtv, 1337, magenta, NULL);

  {
    // declare the pipeline creation info and all of its sub-structures
    // these are modified as appropriate for each pipeline we create
    VkPipelineShaderStageCreateInfo stages[2] = {
        {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_VERTEX_BIT,
         VK_NULL_HANDLE, "main", NULL},
        {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, NULL, 0, VK_SHADER_STAGE_FRAGMENT_BIT,
         VK_NULL_HANDLE, "main", NULL},
    };

    VkPipelineVertexInputStateCreateInfo vi = {
        VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
        NULL,
        0,
        0,
        NULL,    // vertex bindings
        0,
        NULL,    // vertex attributes
    };

    VkPipelineInputAssemblyStateCreateInfo ia = {
        VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
        NULL,
        0,
        VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
        false,
    };

    VkRect2D scissor = {{0, 0}, {4096, 4096}};

    VkPipelineViewportStateCreateInfo vp = {
        VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, NULL, 0, 1, NULL, 1, &scissor};

    VkPipelineRasterizationStateCreateInfo rs = {
        VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
        NULL,
        0,
        true,
        false,
        VK_POLYGON_MODE_FILL,
        VK_CULL_MODE_NONE,
        VK_FRONT_FACE_CLOCKWISE,
        false,
        0.0f,
        0.0f,
        0.0f,
        1.0f,
    };

    VkPipelineMultisampleStateCreateInfo msaa = {
        VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
        NULL,
        0,
        VK_SAMPLE_COUNT_1_BIT,
        false,
        0.0f,
        NULL,
        false,
        false,
    };

    VkPipelineDepthStencilStateCreateInfo ds = {
        VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
        NULL,
        0,
        false,
        false,
        VK_COMPARE_OP_ALWAYS,
        false,
        false,
        {VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_ALWAYS, 0, 0, 0},
        {VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_STENCIL_OP_KEEP, VK_COMPARE_OP_ALWAYS, 0, 0, 0},
        0.0f,
        1.0f,
    };

    VkPipelineColorBlendAttachmentState attState = {
        false,
        VK_BLEND_FACTOR_ONE,
        VK_BLEND_FACTOR_ZERO,
        VK_BLEND_OP_ADD,
        VK_BLEND_FACTOR_ONE,
        VK_BLEND_FACTOR_ZERO,
        VK_BLEND_OP_ADD,
        0xf,
    };

    VkPipelineColorBlendStateCreateInfo cb = {
        VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
        NULL,
        0,
        false,
        VK_LOGIC_OP_NO_OP,
        1,
        &attState,
        {1.0f, 1.0f, 1.0f, 1.0f}};

    VkDynamicState dynstates[] = {VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR, VK_DYNAMIC_STATE_LINE_WIDTH};

    VkPipelineDynamicStateCreateInfo dyn = {
        VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
        NULL,
        0,
        ARRAY_COUNT(dynstates),
        dynstates,
    };

    VkGraphicsPipelineCreateInfo pipeInfo = {
        VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
        NULL,
        0,
        2,
        stages,
        &vi,
        &ia,
        NULL,    // tess
        &vp,
        &rs,
        &msaa,
        &ds,
        &cb,
        &dyn,
        VK_NULL_HANDLE,
        VK_NULL_HANDLE,
        0,                 // sub pass
        VK_NULL_HANDLE,    // base pipeline handle
        -1,                // base pipeline index
    };

    VkDescriptorBufferInfo bufinfo = {};
    bufinfo.offset = 1111;

    VkWriteDescriptorSet textSetWrite = 
      {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, NULL, VkDescriptorSet(), 0, 0, 1,
      VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC, NULL, &bufinfo, NULL};
    
    VkPipelineLayoutCreateInfo pipeLayoutInfo = {
        VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
        NULL,
        0,
        4,
        NULL,
        0,
        NULL,    // push constant ranges
    };

    VkDescriptorSetLayout setLayouts[4];

    pipeLayoutInfo.pSetLayouts = setLayouts;

    etc.DoSomeVulkanStuff(32, 4.4f, &pipeInfo, &textSetWrite, &pipeLayoutInfo);
  }

  for(size_t i=0; i < etc.m_Chunks.size(); i++)
    fileWrite->Insert(etc.m_Chunks[i]);

  fileWrite->FlushToDisk();

  delete fileWrite;
  
  for(size_t i=0; i < etc.m_Chunks.size(); i++)
    delete etc.m_Chunks[i];

  delete etc.m_Writer;

  delete[] pixels;
}

void real_world_test_read()
{
  FILE *readfile = fopen("test.rdc", "rb");
  FakeD3DVulkanEtc etc;
  etc.m_Reader = new StructReadSerialiser(new StreamReader(readfile));
  etc.m_Reader->ConfigureStructuredExport(chunklookup, true);
  
  etc.m_State = READING;

  etc.ReadLog();

  std::string xml = Structured2XML(etc.m_Reader->GetSerialisedChunks());

  //printf("result from real serialising:\n\n%s\n\n", xml.c_str());

  FILE *f = fopen("out.xml", "w");
  fwrite(xml.c_str(), 1, xml.size(), f);
  fclose(f);

  Buffers2ZIP("out.zip", etc.m_Reader->GetSerialisedBuffers());

  f = fopen("test.xml", "w");
  fwrite(xml.c_str(), 1, xml.size(), f);
  fclose(f);

  delete etc.m_Reader;
}

void real_world_test_xml2rdc()
{
  FILE *f = fopen("in.xml", "r");

  if(f)
  {
    vector<SerialisedBuffer> bufs;

    fseek(f, 0, SEEK_END);
    int len = ftell(f);
    fseek(f, 0, SEEK_SET);
    char *xml = new char[len+1];
    fread(xml, 1, len, f);
    xml[len] = 0;
    fclose(f);

    StructuredChunkList chunks = XML2Structured(xml);
    SerialisedBufferList buffers = ZIP2Buffers("in.zip");

    delete[] xml;

    FILE *writefile = fopen("converted.rdc", "wb");

    WriteSerialiser *fileSerialiser = new WriteSerialiser(new StreamWriter(writefile));

    WriteSerialiser *chunkSerialiser = new WriteSerialiser(new StreamWriter(StreamWriter::DefaultScratchSize));
    
    chunkSerialiser->WriteSerialisedFile(*fileSerialiser, SerialisedFile(chunks, buffers));

    fileSerialiser->FlushToDisk();

    delete chunkSerialiser;

    delete fileSerialiser;

    for(size_t i = 0; i < buffers.size(); i++)
      free(buffers[i].first);

    for(size_t i = 0; i < chunks.size(); i++)
      delete chunks[i];
  }
}

void real_world_test_read2()
{
  FILE *f = fopen("converted.rdc", "r");

  if(!f)
    return;

  FakeD3DVulkanEtc etc;
  etc.m_Reader = new StructReadSerialiser(new StreamReader(f));

  etc.m_Reader->ConfigureStructuredExport(chunklookup, true);

  etc.m_State = READING;

  etc.ReadLog();

  std::string xml = Structured2XML(etc.m_Reader->GetSerialisedChunks());

  f = fopen("converted.xml", "w");
  fwrite(xml.c_str(), 1, xml.size(), f);
  fclose(f);

  delete etc.m_Reader;
}

#pragma endregion

/////////////////////////////////////////////////////////////////
<file_sep>/xml_codec.h
#pragma once

#include <string>
#include <vector>

#include "common.h"
#include "structured_data.h"

std::string Structured2XML(const StructuredChunkList &chunks);
StructuredChunkList XML2Structured(const std::string &xml);

void Buffers2ZIP(const char *filename, const SerialisedBufferList &buffers);
SerialisedBufferList ZIP2Buffers(const char *filename);
<file_sep>/test_types.cpp
#include "common.h"
#include "test_types.h"

uint64_t ResourceId::cur = 40;

template <>
string ToStr(const blah &el)
{
  BEGIN_ENUM_STRINGIZE();
    STRINGIZE_ENUM(blah_foo);
    STRINGIZE_ENUM(blah_bar);
    STRINGIZE_ENUM(blah_qux);
  END_ENUM_STRINGIZE(blah);
}

SERIALISE_ENUM_TYPE(blah);

BASIC_TYPE_SERIALISE_STRINGIFY(ResourceId, el.innerSecretValue, eUnsignedInteger, 8);

INSTANTIATE_SERIALISE_TYPE(ResourceId);

template <>
string ToStr(const ResourceId &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "Secret_%" PRIu64, el.innerSecretValue);

  return tostrBuf;
}

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, podStruct &el)
{
  SERIALISE_MEMBER(x);
  SERIALISE_MEMBER(y);
  SERIALISE_MEMBER(width);
  SERIALISE_MEMBER(height);
}

INSTANTIATE_SERIALISE_TYPE(podStruct);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, simpleStruct &el)
{
  SERIALISE_MEMBER(foo);
  SERIALISE_MEMBER(bar);
  
  SERIALISE_MEMBER(secret).Hidden();

  SERIALISE_MEMBER(hello);
  SERIALISE_MEMBER_TYPED(blah, really_a_blah);
}

INSTANTIATE_SERIALISE_TYPE(simpleStruct);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, complexStruct &el)
{
  SERIALISE_MEMBER(basicArray);
  SERIALISE_MEMBER(secret).Hidden();
  SERIALISE_MEMBER(vectorArray);
  SERIALISE_MEMBER_ARRAY(exts, numExts);
  SERIALISE_MEMBER(cstring).Named("C-style string");
}

template <>
void Deserialise(const complexStruct &el)
{
  delete[] el.exts;
}

INSTANTIATE_SERIALISE_TYPE(complexStruct);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, VulkanHandleThingy &el)
{
  ResourceManager *rm = (ResourceManager *)ser.GetUserData();

  ResourceId id;

  if(ser.IsWriting())
  {
    id = ResourceId(); // GetResID(el);
    rm->vk.insert(std::make_pair(id, el));
  }

  DoSerialise(ser, id);

  if(ser.IsReading())
  {
    el = rm->vk[id];
  }
}

INSTANTIATE_SERIALISE_TYPE(VulkanHandleThingy);

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, ID3DInterface *&el)
{
  ResourceManager *rm = (ResourceManager *)ser.GetUserData();

  ResourceId id;

  if(ser.IsWriting())
  {
    id = ResourceId(); // GetResID(el);
    rm->d3d.insert(std::make_pair(id, el));
  }

  DoSerialise(ser, id);

  if(ser.IsReading())
  {
    el = rm->d3d[id];
  }
}

INSTANTIATE_SERIALISE_TYPE(ID3DInterface *);<file_sep>/serialiser.cpp
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2015-2016 <NAME>
 * Copyright (c) 2014 Crytek
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#include "serialiser.h"
#include <errno.h>
#include "lz4/lz4.h"
#include "string_utils.h"

#ifdef _MSC_VER
#pragma warning( \
    disable : 4422)    // warning C4422: 'snprintf' : too many arguments passed for format string
                       // false positive as VS is trying to parse renderdoc's custom format strings
#endif

#if !defined(RELEASE)

int64_t Chunk::m_LiveChunks = 0;
int64_t Chunk::m_TotalMem = 0;
int64_t Chunk::m_MaxChunks = 0;

#endif

static const uint32_t MAGIC_HEADER = MAKE_FOURCC('R', 'D', 'O', 'C');

// based on blockStreaming_doubleBuffer.c in lz4 examples
struct CompressedFileIO
{
  // large block size
  static const size_t BlockSize = 64 * 1024;

  CompressedFileIO(FILE *f)
  {
    m_F = f;
    LZ4_resetStream(&m_LZ4Comp);
    LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0);
    m_CompressedSize = m_UncompressedSize = 0;
    m_PageIdx = m_PageOffset = 0;
    m_PageData = 0;

    m_CompressSize = LZ4_COMPRESSBOUND(BlockSize);
    m_CompressBuf = new byte[m_CompressSize];
  }

  ~CompressedFileIO() { SAFE_DELETE_ARRAY(m_CompressBuf); }
  uint32_t GetCompressedSize() { return m_CompressedSize; }
  uint32_t GetUncompressedSize() { return m_UncompressedSize; }
  // write out some data - accumulate into the input pages, then
  // when a page is full call Flush() to flush it out to disk
  void Write(const void *data, size_t len)
  {
    if(data == NULL || len == 0)
      return;

    m_UncompressedSize += (uint32_t)len;

    const byte *src = (const byte *)data;

    size_t remainder = 0;

    // loop continually, writing up to BlockSize out of what remains of data
    do
    {
      remainder = 0;

      // if we're about to copy more than the page, copy only
      // what will fit, then copy the remainder after flushing
      if(m_PageOffset + len > BlockSize)
      {
        remainder = len - (BlockSize - m_PageOffset);
        len = BlockSize - m_PageOffset;
      }

      memcpy(m_InPages[m_PageIdx] + m_PageOffset, src, len);
      m_PageOffset += len;

      if(remainder > 0)
      {
        Flush();    // this will swap the input pages and reset the page offset

        src += len;
        len = remainder;
      }
    } while(remainder > 0);
  }

  // flush out the current page to disk
  void Flush()
  {
    // m_PageOffset is the amount written, usually equal to BlockSize except the last block.
    int32_t compSize = LZ4_compress_fast_continue(&m_LZ4Comp, (const char *)m_InPages[m_PageIdx],
                                                  (char *)m_CompressBuf, (int)m_PageOffset,
                                                  (int)m_CompressSize, 1);

    if(compSize < 0)
    {
      RDCERR("Error compressing: %i", compSize);
      return;
    }

    fwrite(&compSize, sizeof(compSize), 1, m_F);
    fwrite(m_CompressBuf, 1, compSize, m_F);

    m_CompressedSize += compSize + sizeof(int32_t);

    m_PageOffset = 0;
    m_PageIdx = 1 - m_PageIdx;
  }

  // Reset back to 0, only makes sense when reading as writing can't be undone
  void Reset()
  {
    LZ4_setStreamDecode(&m_LZ4Decomp, NULL, 0);
    m_CompressedSize = m_UncompressedSize = 0;
    m_PageIdx = 0;
    m_PageOffset = 0;
  }

  // read out some data - if the input page is empty we fill
  // the next page with data from disk
  void Read(byte *data, size_t len)
  {
    if(data == NULL || len == 0)
      return;

    m_UncompressedSize += (uint32_t)len;

    // loop continually, writing up to BlockSize out of what remains of data
    do
    {
      size_t readamount = len;

      // if we're about to copy more than the page, copy only
      // what will fit, then copy the remainder after refilling
      if(readamount > m_PageData)
        readamount = m_PageData;

      if(readamount > 0)
      {
        memcpy(data, m_InPages[m_PageIdx] + m_PageOffset, readamount);

        m_PageOffset += readamount;
        m_PageData -= readamount;

        data += readamount;
        len -= readamount;
      }

      if(len > 0)
        FillBuffer();    // this will swap the input pages and reset the page offset
    } while(len > 0);
  }

  void FillBuffer()
  {
    int32_t compSize = 0;

    fread(&compSize, sizeof(compSize), 1, m_F);
    fread(m_CompressBuf, 1, compSize, m_F);

    m_CompressedSize += compSize;

    m_PageIdx = 1 - m_PageIdx;

    int32_t decompSize = LZ4_decompress_safe_continue(
        &m_LZ4Decomp, (const char *)m_CompressBuf, (char *)m_InPages[m_PageIdx], compSize, BlockSize);

    if(decompSize < 0)
    {
      RDCERR("Error decompressing: %i", decompSize);
      return;
    }

    m_PageOffset = 0;
    m_PageData = decompSize;
  }

  static void Decompress(byte *destBuf, const byte *srcBuf, size_t len)
  {
    LZ4_streamDecode_t lz4;
    LZ4_setStreamDecode(&lz4, NULL, 0);

    const byte *srcBufEnd = srcBuf + len;

    while(srcBuf + 4 < srcBufEnd)
    {
      const int32_t *compSize = (const int32_t *)srcBuf;
      srcBuf = (const byte *)(compSize + 1);

      if(srcBuf + *compSize > srcBufEnd)
        break;

      int32_t decompSize = LZ4_decompress_safe_continue(&lz4, (const char *)srcBuf, (char *)destBuf,
                                                        *compSize, BlockSize);

      if(decompSize < 0)
        return;

      srcBuf += *compSize;
      destBuf += decompSize;
    }
  }

  LZ4_stream_t m_LZ4Comp;
  LZ4_streamDecode_t m_LZ4Decomp;
  FILE *m_F;
  uint32_t m_CompressedSize, m_UncompressedSize;

  byte m_InPages[2][BlockSize];
  size_t m_PageIdx, m_PageOffset, m_PageData;

  byte *m_CompressBuf;
  size_t m_CompressSize;
};

Chunk::~Chunk()
{
  if(m_AlignedData)
  {
    if(m_Data)
      FreeAlignedBuffer(m_Data);

    m_Data = NULL;
  }
  else
  {
    SAFE_DELETE_ARRAY(m_Data);
  }
}

/*

 -----------------------------
 File format for version 0x31:

 uint64_t MAGIC_HEADER;
 uint64_t version = 0x00000031;
 uint64_t filesize;
 uint64_t resolveDBSize;

 if(resolveDBSize > 0)
 {
   byte resolveDB[resolveDBSize];
   byte paddingTo16Alignment[];
 }

 byte captureData[]; // remainder of the file

 -----------------------------
 File format for version 0x32:

 uint64_t MAGIC_HEADER;
 uint64_t version = 0x00000032;

 1 or more sections:

 Section
 {
   char isASCII = '\0' or 'A'; // indicates the section is ASCII or binary. ASCII allows for easy
 appending by hand/script
   if(isASCII == 'A')
   {
     // ASCII sections are discouraged for tools, but useful for hand-editing by just
     // appending a simple text file
     char newline = '\n';
     char length[]; // length of just section data below, as decimal string
     char newline = '\n';
     char sectionType[]; // section type, see SectionType enum, as decimal string.
     char newline = '\n';
     char sectionName[]; // UTF-8 string name of section.
     char newline = '\n';
     // sectionName is an arbitrary string. If two sections exist with the
     // exact same name, the last one to occur in the file will be taken, the
     // others ignored.

     byte sectiondata[ interpret(length) ]; // section data
   }
   else if(isASCII == '\0')
   {
     byte zero[3]; // pad out the above character with 0 bytes. Reserved for future use
     uint32_t sectionFlags; // section flags - e.g. is compressed or not.
     uint32_t sectionType; // section type enum, see SectionType. Could be eSectionType_Unknown
     uint32_t sectionLength; // byte length of the actual section data
     uint32_t sectionNameLength; // byte length of the string below (minimum 1, for null terminator)
     char sectionName[sectionNameLength]; // UTF-8 string name of section, optional.

     byte sectiondata[length]; // actual contents of the section

     // note: compressed sections will contain the uncompressed length as a uint64_t
     // before the compressed data.
   }
 };

 // remainder of the file is tightly packed/unaligned section structures.
 // The first section must always be the actual frame capture data in
 // binary form
 Section sections[];

*/

struct FileHeader
{
  FileHeader()
  {
    magic = MAGIC_HEADER;
    version = ReadSerialiser::SERIALISE_VERSION;
  }

  uint64_t magic;
  uint64_t version;
};

struct BinarySectionHeader
{
  // 0x0
  byte isASCII;                             
  // 0x0, 0x0, 0x0
  byte zero[3];                             
  // section flags - e.g. is compressed or not.
  SectionFlags sectionFlags;    
  // section type enum, see SectionType. Could be eSectionType_Unknown
  SectionType sectionType;           
  // byte length of the actual section data
  uint32_t sectionLength;    
  // byte length of the string below (could be 0)
  uint32_t sectionNameLength;    
  // actually sectionNameLength, but at least 1 for null terminator
  char name[1];                  

  // char name[sectionNameLength];
  // byte data[sectionLength];
};

#define RETURNCORRUPT(...)           \
  {                                  \
    RDCERR(__VA_ARGS__);             \
    m_ErrorCode = eContError_Corrupt; \
    m_HasError = true;               \
    return;                          \
  }

#if 0 // container file reading
template<>
void ReadSerialiser::Init(const char *path)
{
  Reset();

  m_Filename = path ? path : "";

  FileHeader header;

  {
    m_ReadFileHandle = fopen(m_Filename.c_str(), "rb");

    if(!m_ReadFileHandle)
    {
      RDCERR("Can't open capture file '%s' for read - errno %d", m_Filename.c_str(), errno);
      m_ErrorCode = eContError_FileIO;
      m_HasError = true;
      return;
    }

    RDCDEBUG("Opened capture file for read");

    fread(&header, 1, sizeof(FileHeader), m_ReadFileHandle);

    if(header.magic != MAGIC_HEADER)
    {
      RDCWARN("Invalid capture file. Expected magic %08x, got %08x", MAGIC_HEADER,
              (uint32_t)header.magic);

      m_ErrorCode = eContError_Corrupt;
      m_HasError = true;
      fclose(m_ReadFileHandle);
      m_ReadFileHandle = 0;
      return;
    }

    m_SerVer = header.version;

    if(header.version == 0x00000031)    // backwards compatibility
    {
      uint64_t headerRemainder[2];

      fread(&headerRemainder, 1, sizeof(headerRemainder), m_ReadFileHandle);

      fseek(m_ReadFileHandle, 0, SEEK_END);

      uint64_t realLength = ftell(m_ReadFileHandle);
      if(headerRemainder[0] != realLength)
      {
        RDCERR("Truncated/overlong capture file. Expected length 0x016llx, got 0x016llx",
               headerRemainder[0], realLength);

        m_ErrorCode = eContError_Corrupt;
        m_HasError = true;
        fclose(m_ReadFileHandle);
        m_ReadFileHandle = 0;
        return;
      }

      fseek(m_ReadFileHandle, sizeof(FileHeader) + sizeof(headerRemainder), SEEK_SET);

      size_t resolveDBSize = (size_t)headerRemainder[1];

      if(resolveDBSize > 0)
      {
        Section *resolveDB = new Section();
        resolveDB->type = eSectionType_ResolveDatabase;
        resolveDB->flags = eSectionFlag_None;
        resolveDB->fileoffset = sizeof(FileHeader) + sizeof(headerRemainder);
        resolveDB->name = "renderdoc/internal/resolvedb";
        resolveDB->data.resize(resolveDBSize);

        // read resolve DB entirely into memory
        fread(&resolveDB->data[0], 1, resolveDBSize, m_ReadFileHandle);

        m_Sections.push_back(resolveDB);
        m_KnownSections[eSectionType_ResolveDatabase] = resolveDB;
      }

      // seek to frame capture data
      fseek(m_ReadFileHandle,
                      AlignUp16(sizeof(FileHeader) + sizeof(headerRemainder) + resolveDBSize),
                      SEEK_SET);

      Section *frameCap = new Section();
      frameCap->type = eSectionType_FrameCapture;
      frameCap->flags = eSectionFlag_None;
      frameCap->fileoffset = ftell(m_ReadFileHandle);
      frameCap->name = "renderdoc/internal/framecapture";
      frameCap->size = realLength - frameCap->fileoffset;

      m_Sections.push_back(frameCap);
      m_KnownSections[eSectionType_FrameCapture] = frameCap;
    }
    else if(header.version == SERIALISE_VERSION)
    {
      while(!feof(m_ReadFileHandle))
      {
        BinarySectionHeader sectionHeader = {0};
        byte *reading = (byte *)&sectionHeader;

        fread(reading, 1, 1, m_ReadFileHandle);
        reading++;

        if(feof(m_ReadFileHandle))
          break;

        if(sectionHeader.isASCII == 'A')
        {
          // ASCII section
          char c = 0;
          fread(&c, 1, 1, m_ReadFileHandle);
          if(c != '\n')
            RETURNCORRUPT("Invalid ASCII data section '%hhx'", c);

          if(feof(m_ReadFileHandle))
            RETURNCORRUPT("Invalid truncated ASCII data section");

          uint64_t length = 0;

          c = '0';

          while(!feof(m_ReadFileHandle) && c != '\n')
          {
            c = '0';
            fread(&c, 1, 1, m_ReadFileHandle);

            if(c == '\n')
              break;

            length *= 10;
            length += int(c - '0');
          }

          if(feof(m_ReadFileHandle))
            RETURNCORRUPT("Invalid truncated ASCII data section");

          union
          {
            uint32_t u32;
            SectionType t;
          } type;

          type.u32 = 0;

          c = '0';

          while(!feof(m_ReadFileHandle) && c != '\n')
          {
            c = '0';
            fread(&c, 1, 1, m_ReadFileHandle);

            if(c == '\n')
              break;

            type.u32 *= 10;
            type.u32 += int(c - '0');
          }

          if(feof(m_ReadFileHandle))
            RETURNCORRUPT("Invalid truncated ASCII data section");

          string name;

          c = 0;

          while(!feof(m_ReadFileHandle) && c != '\n')
          {
            c = 0;
            fread(&c, 1, 1, m_ReadFileHandle);

            if(c == 0 || c == '\n')
              break;

            name.push_back(c);
          }

          if(feof(m_ReadFileHandle))
            RETURNCORRUPT("Invalid truncated ASCII data section");

          Section *sect = new Section();
          sect->flags = eSectionFlag_ASCIIStored;
          sect->type = type.t;
          sect->name = name;
          sect->size = length;
          sect->data.resize((size_t)length);
          sect->fileoffset = ftell(m_ReadFileHandle);

          fread(&sect->data[0], 1, (size_t)length, m_ReadFileHandle);

          if(sect->type != eSectionType_Unknown && sect->type < eSectionType_Num)
            m_KnownSections[sect->type] = sect;
          m_Sections.push_back(sect);
        }
        else if(sectionHeader.isASCII == 0x0)
        {
          fread(reading, 1, offsetof(BinarySectionHeader, name) - 1, m_ReadFileHandle);

          Section *sect = new Section();
          sect->flags = sectionHeader.sectionFlags;
          sect->type = sectionHeader.sectionType;
          sect->name.resize(sectionHeader.sectionNameLength - 1);
          sect->size = sectionHeader.sectionLength;

          fread(&sect->name[0], 1, sectionHeader.sectionNameLength - 1, m_ReadFileHandle);
          char nullterm = 0;
          fread(&nullterm, 1, 1, m_ReadFileHandle);

          sect->fileoffset = ftell(m_ReadFileHandle);

          if(sect->flags & eSectionFlag_LZ4Compressed)
          {
            sect->compressedReader = new CompressedFileIO(m_ReadFileHandle);
            fread(&sect->size, 1, sizeof(uint64_t), m_ReadFileHandle);

            sect->fileoffset += sizeof(uint64_t);
          }

          if(sect->type != eSectionType_Unknown && sect->type < eSectionType_Num)
            m_KnownSections[sect->type] = sect;
          m_Sections.push_back(sect);

          // if section isn't frame capture data and is small enough, read it all into memory now,
          // otherwise skip
          if(sect->type != eSectionType_FrameCapture && sectionHeader.sectionLength < 4 * 1024 * 1024)
          {
            sect->data.resize(sectionHeader.sectionLength);
            fread(&sect->data[0], 1, sectionHeader.sectionLength, m_ReadFileHandle);
          }
          else
          {
            fseek(m_ReadFileHandle, sectionHeader.sectionLength, SEEK_CUR);
          }
        }
        else
        {
          RETURNCORRUPT("Unrecognised section type '%hhx'", sectionHeader.isASCII);
        }
      }
    }
    else
    {
      RDCERR(
          "Capture file from wrong version. This program is logfile version %" PRIu64 ", file is logfile "
          "version %" PRIu64,
          SERIALISE_VERSION, header.version);

      m_ErrorCode = eContError_UnsupportedVersion;
      m_HasError = true;
      fclose(m_ReadFileHandle);
      m_ReadFileHandle = 0;
      return;
    }

    if(m_KnownSections[eSectionType_FrameCapture] == NULL)
    {
      RDCERR("Capture file doesn't have a frame capture");

      m_ErrorCode = eContError_Corrupt;
      m_HasError = true;
      fclose(m_ReadFileHandle);
      m_ReadFileHandle = 0;
      return;
    }

    m_BufferSize = m_KnownSections[eSectionType_FrameCapture]->size;
    m_CurrentBufferSize = (size_t)RDCMIN(m_BufferSize, (uint64_t)64 * 1024);
    m_BufferHead = m_Buffer = AllocAlignedBuffer(m_CurrentBufferSize);
    m_ReadOffset = 0;

    fseek(m_ReadFileHandle, m_KnownSections[eSectionType_FrameCapture]->fileoffset,
                    SEEK_SET);

    // read initial buffer of data
    ReadFromFile(0, m_CurrentBufferSize);
  }
}
#endif

/////////////////////////////////////////////////////////////
// Read Serialiser functions

template<>
ReadSerialiser::Serialiser(StreamReader *reader)
{
  // ensure binary sizes of enums
  RDCCOMPILE_ASSERT(sizeof(SectionFlags) == sizeof(uint32_t), "Section flags not in uint32");
  RDCCOMPILE_ASSERT(sizeof(SectionType) == sizeof(uint32_t), "Section type not in uint32");

  RDCCOMPILE_ASSERT(offsetof(BinarySectionHeader, name) == sizeof(uint32_t) * 5,
                    "BinarySectionHeader size has changed or contains padding");

  Reset();

  m_Read = reader;
  m_Write = NULL;
}

template<>
ReadSerialiser::~Serialiser()
{
  for(auto it = m_StructuredData.begin(); it != m_StructuredData.end(); ++it)
    delete *it;

  m_StructuredData.clear();

  for(auto it = m_StructuredDataBuffers.begin(); it != m_StructuredDataBuffers.end(); ++it)
    delete[] it->first;

  m_StructuredDataBuffers.clear();
  
  for(size_t i = 0; i < m_Sections.size(); i++)
  {
    SAFE_DELETE(m_Sections[i]->compressedReader);
    SAFE_DELETE(m_Sections[i]);
  }

  RDCASSERT(m_Chunks.empty());
}

template<>
uint32_t ReadSerialiser::BeginChunk(uint32_t, uint32_t)
{
  uint32_t chunkID = 0;

  m_ChunkMetadata = SerialisedChunkMetaData();

  {
    uint32_t c = 0;
    m_Read->Read(c);

    // chunk index 0 is not allowed in normal situations.
    // allows us to indicate some control bytes
    while(c == 0)
    {
      uint8_t controlByte;
      m_Read->Read(controlByte);

      if(controlByte == 0x0)
      {
        // padding
        uint8_t padLength;
        m_Read->Read(padLength);

        // might have padded with these 5 control bytes,
        // so a pad length of 0 IS VALID.
        if(padLength > 0)
        {
          m_Read->SkipBytes(padLength);
        }
      }
      else
      {
        RDCERR("Unexpected control byte: %x", (uint32_t)controlByte);
      }

      m_Read->Read(c);
    }

    chunkID = c & ChunkIndexMask;

    /////////////////

    m_ChunkMetadata.chunkID = chunkID;

    if(c & ChunkCallstack)
    {
      size_t sz = sizeof(m_ChunkMetadata.callstack);
      m_Read->Read(m_ChunkMetadata.callstack, sz);
    }

    if(c & ChunkThreadID)
      m_Read->Read(m_ChunkMetadata.threadID);

    if(c & ChunkDuration)
      m_Read->Read(m_ChunkMetadata.duration);

    if(c & ChunkTimestamp)
      m_Read->Read(m_ChunkMetadata.timestamp);

    uint32_t chunkSize = 0xbeebfeed;
    m_Read->Read(chunkSize);

    m_LastChunkLen = chunkSize;
    m_LastChunkOffset = m_Read->GetOffset();
  }

  return chunkID;
}

template<>
void ReadSerialiser::SkipCurrentChunk()
{
  uint64_t readBytes = m_Read->GetOffset() - m_LastChunkOffset;

  if(readBytes > m_LastChunkLen)
  {
    RDCERR("Can't skip current chunk outside of {BeginChunk, EndChunk}");
    return;
  }

  m_Read->SkipBytes(m_LastChunkLen - readBytes);
}

template<>
void ReadSerialiser::EndChunk()
{
  // this will be a no-op if the last chunk length was accurate. If it was a
  // conservative estimate of the length then we'll skip some padding bytes
  SkipCurrentChunk();
}

/////////////////////////////////////////////////////////////
// Structured Read Serialiser functions

template<>
StructReadSerialiser::Serialiser(StreamReader *reader)
{
  // ensure binary sizes of enums
  RDCCOMPILE_ASSERT(sizeof(SectionFlags) == sizeof(uint32_t), "Section flags not in uint32");
  RDCCOMPILE_ASSERT(sizeof(SectionType) == sizeof(uint32_t), "Section type not in uint32");

  RDCCOMPILE_ASSERT(offsetof(BinarySectionHeader, name) == sizeof(uint32_t) * 5,
                    "BinarySectionHeader size has changed or contains padding");

  Reset();

  m_Read = reader;
  m_Write = NULL;
}

template<>
StructReadSerialiser::~Serialiser()
{
  for(auto it = m_StructuredData.begin(); it != m_StructuredData.end(); ++it)
    delete *it;

  m_StructuredData.clear();

  for(auto it = m_StructuredDataBuffers.begin(); it != m_StructuredDataBuffers.end(); ++it)
    delete[] it->first;

  m_StructuredDataBuffers.clear();
  
  for(size_t i = 0; i < m_Sections.size(); i++)
  {
    SAFE_DELETE(m_Sections[i]->compressedReader);
    SAFE_DELETE(m_Sections[i]);
  }

  RDCASSERT(m_Chunks.empty());
}

template<>
uint32_t StructReadSerialiser::BeginChunk(uint32_t, uint32_t)
{
  uint32_t chunkID = 0;

  m_ChunkMetadata = SerialisedChunkMetaData();

  {
    uint32_t c = 0;
    m_Read->Read(c);

    // chunk index 0 is not allowed in normal situations.
    // allows us to indicate some control bytes
    while(c == 0)
    {
      uint8_t controlByte;
      m_Read->Read(controlByte);

      if(controlByte == 0x0)
      {
        // padding
        uint8_t padLength;
        m_Read->Read(padLength);

        // might have padded with these 5 control bytes,
        // so a pad length of 0 IS VALID.
        if(padLength > 0)
        {
          m_Read->SkipBytes(padLength);
        }
      }
      else
      {
        RDCERR("Unexpected control byte: %x", (uint32_t)controlByte);
      }

      m_Read->Read(c);
    }

    chunkID = c & ChunkIndexMask;

    /////////////////

    m_ChunkMetadata.chunkID = chunkID;

    if(c & ChunkCallstack)
    {
      size_t sz = sizeof(m_ChunkMetadata.callstack);
      m_Read->Read(m_ChunkMetadata.callstack, sz);
    }

    if(c & ChunkThreadID)
      m_Read->Read(m_ChunkMetadata.threadID);

    if(c & ChunkDuration)
      m_Read->Read(m_ChunkMetadata.duration);

    if(c & ChunkTimestamp)
      m_Read->Read(m_ChunkMetadata.timestamp);

    uint32_t chunkSize = 0xbeebfeed;
    m_Read->Read(chunkSize);

    m_LastChunkLen = chunkSize;
    m_LastChunkOffset = m_Read->GetOffset();
  }

  if(ExportStructure())
  {
    const char *name = m_ChunkLookup ? m_ChunkLookup(chunkID) : NULL;

    if(!name)
      name = "<Unknown Chunk>";

    StructuredChunk *chunk = new StructuredChunk(name, "Chunk");
    chunk->metadata = m_ChunkMetadata;

    m_StructuredData.push_back(chunk);
    m_StructureStack.push_back(chunk);

    m_InternalElement = false;
  }

  return chunkID;
}

template<>
void StructReadSerialiser::SkipCurrentChunk()
{
  if(ExportStructure())
  {
    RDCASSERTMSG("Skipping chunk after we've begun serialising!", m_StructureStack.size() == 1, m_StructureStack.size());

    StructuredObject &current = *m_StructureStack.back();

    current.data.basic.numChildren++;
    current.data.children.push_back(new StructuredObject("Opaque chunk", "Byte Buffer"));

    StructuredObject &obj = *current.data.children.back();
    obj.type.basetype = SerialisedType::eBuffer;
    obj.type.byteSize = m_LastChunkLen;

    if(m_StructureStack.size() == 1)
    {
      StructuredChunk *chunk = (StructuredChunk *)m_StructureStack.back();
      chunk->metadata.flags |= SerialisedChunkMetaData::OpaqueChunk;
    }
  }

  {
    uint64_t readBytes = m_Read->GetOffset() - m_LastChunkOffset;

    if(readBytes > m_LastChunkLen)
    {
      RDCERR("Can't skip current chunk outside of {BeginChunk, EndChunk}");
      return;
    }

    if(readBytes > 0)
    {
      RDCWARN("Partially consumed bytes at SkipCurrentChunk - blob data will be truncated");
    }

    uint64_t chunkBytes = m_LastChunkLen - readBytes;

    if(ExportStructure() && m_ExportBuffers)
    {
      StructuredObject &current = *m_StructureStack.back();

      StructuredObject &obj = *current.data.children.back();

      obj.data.basic.u = m_StructuredDataBuffers.size();

      byte *alloc = new byte[chunkBytes];
      m_Read->Read(alloc, chunkBytes);

      m_StructuredDataBuffers.push_back(std::make_pair(alloc, chunkBytes));
    }
    else
    {
      m_Read->SkipBytes(chunkBytes);
    }
  }
}

template<>
void StructReadSerialiser::EndChunk()
{
  if(ExportStructure())
  {
    RDCASSERTMSG("Object Stack is imbalanced!", m_StructureStack.size() <= 1, m_StructureStack.size());

    if(!m_StructureStack.empty())
    {
      m_StructureStack.back()->type.byteSize = m_LastChunkLen;
      m_StructureStack.pop_back();
    }
  }
  
  // this will be a no-op if the last chunk length was accurate. If it was a
  // conservative estimate of the length then we'll skip some padding bytes
  uint64_t readBytes = m_Read->GetOffset() - m_LastChunkOffset;
  m_Read->SkipBytes(m_LastChunkLen - readBytes);
}

template<>
WriteSerialiser::Serialiser(StreamWriter *writer)
{
  Reset();

  m_Write = writer;
  m_Read = NULL;
}

template<>
WriteSerialiser::~Serialiser()
{
  for(auto it = m_StructuredData.begin(); it != m_StructuredData.end(); ++it)
    delete *it;

  m_StructuredData.clear();

  for(auto it = m_StructuredDataBuffers.begin(); it != m_StructuredDataBuffers.end(); ++it)
    delete[] it->first;

  m_StructuredDataBuffers.clear();
  
  RDCASSERT(m_Sections.empty());

  for(size_t i = 0; i < m_Chunks.size(); i++)
  {
    if(m_Chunks[i]->IsTemporary())
      SAFE_DELETE(m_Chunks[i]);
  }

  m_Chunks.clear();
}

/////////////////////////////////////////////////////////////
// generic

template<>
void WriteSerialiser::Insert(Chunk *chunk)
{
  m_Chunks.push_back(chunk);
}

template<>
void WriteSerialiser::FlushToDisk()
{
#if 0 // container file writing
  if(m_Filename != "" && !m_HasError)
  {
    RDCDEBUG("writing capture files");

    FILE *binFile = fopen(m_Filename.c_str(), "w+b");

    if(!binFile)
    {
      RDCERR("Can't open capture file '%s' for write, errno %d", m_Filename.c_str(), errno);
      m_ErrorCode = eContError_FileIO;
      m_HasError = true;
      return;
    }

    RDCDEBUG("Opened capture file for write");

    FileHeader header;    // automagically initialised with correct data

    // write header
    fwrite(&header, 1, sizeof(FileHeader), binFile);

    static const byte padding[BufferAlignment] = {0};

    uint64_t compressedSizeOffset = 0;
    uint64_t uncompressedSizeOffset = 0;

    // write frame capture section header
    {
      const char sectionName[] = "renderdoc/internal/framecapture";

      BinarySectionHeader section = {0};
      section.isASCII = 0;                                // redundant but explicit
      section.sectionNameLength = sizeof(sectionName);    // includes null terminator
      section.sectionType = eSectionType_FrameCapture;
      section.sectionFlags = eSectionFlag_LZ4Compressed;
      section.sectionLength =
          0;    // will be fixed up later, to avoid having to compress everything into memory

      compressedSizeOffset = ftell(binFile) + offsetof(BinarySectionHeader, sectionLength);

      fwrite(&section, 1, offsetof(BinarySectionHeader, name), binFile);
      fwrite(sectionName, 1, sizeof(sectionName), binFile);

      uint64_t len = 0;    // will be fixed up later
      uncompressedSizeOffset = ftell(binFile);
      fwrite(&len, 1, sizeof(uint64_t), binFile);
    }

    CompressedFileIO fwriter(binFile);

    // track offset so we can add padding. The padding is relative
    // to the start of the decompressed buffer, so we start it from 0
    uint64_t offs = 0;
    uint64_t alignedoffs = 0;

    // write frame capture contents
    for(size_t i = 0; i < m_Chunks.size(); i++)
    {
      Chunk *chunk = m_Chunks[i];

      alignedoffs = AlignUp(offs, (uint64_t)BufferAlignment);

      if(offs != alignedoffs && chunk->IsAligned())
      {
        uint32_t chunkID = 0;    // write a '0' chunk that indicates special behaviour
        fwriter.Write(&chunkID, sizeof(chunkID));
        offs += sizeof(chunkID);

        uint8_t controlByte = 0;    // control byte 0 indicates padding
        fwriter.Write(&controlByte, sizeof(controlByte));
        offs += sizeof(controlByte);

        offs++;    // we will have to write out a byte indicating how much padding exists, so add 1
        alignedoffs = AlignUp(offs, (uint64_t)BufferAlignment);

        RDCCOMPILE_ASSERT(BufferAlignment < 0x100,
                          "Buffer alignment must be less than 256");    // with a byte at most
                                                                        // indicating how many bytes
                                                                        // to pad,
        // this is our maximal representable alignment

        uint8_t padLength = (alignedoffs - offs) & 0xff;
        fwriter.Write(&padLength, sizeof(padLength));

        // we might have padded with the control bytes, so only write some bytes if we need to
        if(padLength > 0)
        {
          fwriter.Write(padding, size_t(alignedoffs - offs));
          offs += alignedoffs - offs;
        }
      }

      fwriter.Write(chunk->GetData(), chunk->GetLength());

      offs += chunk->GetLength();

      if(chunk->IsTemporary())
        SAFE_DELETE(chunk);
    }

    fwriter.Flush();

    m_Chunks.clear();

    // fixup section size
    {
      uint32_t compsize = 0;
      uint64_t uncompsize = 0;

      uint64_t curoffs = ftell(binFile);

      fseek(binFile, compressedSizeOffset, SEEK_SET);

      compsize = fwriter.GetCompressedSize();
      fwrite(&compsize, 1, sizeof(compsize), binFile);

      fseek(binFile, uncompressedSizeOffset, SEEK_SET);

      uncompsize = fwriter.GetUncompressedSize();
      fwrite(&uncompsize, 1, sizeof(uncompsize), binFile);

      fseek(binFile, curoffs, SEEK_SET);

      RDCLOG("Compressed frame capture data from %u to %u", fwriter.GetUncompressedSize(),
             fwriter.GetCompressedSize());
    }

    fclose(binFile);
  }
#endif
}

template<>
void WriteSerialiser::SetChunkMetadataRecording(uint32_t flags)
{
  // cannot change this mid-chunk
  RDCASSERT(m_Write->GetOffset() == 0);

  m_ChunkFlags = flags;
}

template<>
uint32_t WriteSerialiser::BeginChunk(uint32_t chunkID, uint32_t byteLength)
{
  {
    // if writing, we should be writing into the start of the serialiser. A serialiser should
    // only ever have one actively constructing chunk in it.
    RDCASSERT(m_Write->GetOffset() == 0);

    // chunk index needs to be valid
    RDCASSERT(chunkID > 0);

    {
      uint32_t c = chunkID & ChunkIndexMask;
      RDCASSERT(chunkID <= ChunkIndexMask);

      c |= m_ChunkFlags;

      m_ChunkMetadata.chunkID = chunkID;

      /////////////////

      m_Write->Write(c);

      if(c & ChunkCallstack)
      {
        if(m_ChunkMetadata.callstack[0] == 0)
        {
          vector<uint64_t> cs = CollectCallstack_stub();
          memcpy(m_ChunkMetadata.callstack, &cs[0], sizeof(uint64_t)*cs.size());
        }

        m_Write->Write(m_ChunkMetadata.callstack, sizeof(m_ChunkMetadata.callstack));
      }

      if(c & ChunkThreadID)
      {
        if(m_ChunkMetadata.threadID == 0)
          m_ChunkMetadata.threadID = CollectThreadID_stub();

        m_Write->Write(m_ChunkMetadata.threadID);
      }

      if(c & ChunkDuration)
        m_Write->Write(m_ChunkMetadata.duration);

      if(c & ChunkTimestamp)
      {
        if(m_ChunkMetadata.timestamp == 0)
          m_ChunkMetadata.timestamp = CollectTimestamp_stub();

        m_Write->Write(m_ChunkMetadata.timestamp);
      }

      if(byteLength > 0)
      {
        // write length, assuming it is an upper bound
        m_ChunkFixup = 0;
        m_Write->Write(byteLength);
        m_LastChunkOffset = m_Write->GetOffset();
        m_LastChunkLen = byteLength;
      }
      else
      {      
        // length will be fixed up in EndChunk
        uint32_t chunkSize = 0xbeebfeed;
        m_ChunkFixup = m_Write->GetOffset();
        m_Write->Write(chunkSize);
      }
    }
  }

  return chunkID;
}

template<>
void WriteSerialiser::EndChunk()
{
  if(m_ChunkFixup != 0)
  {
    // fix up the chunk header
    uint64_t chunkOffset = m_ChunkFixup;
    m_ChunkFixup = 0;

    uint64_t curOffset = m_Write->GetOffset();

    RDCASSERT(curOffset > chunkOffset);

    uint64_t chunkLength = (curOffset - chunkOffset) - sizeof(uint32_t);

    RDCASSERT(chunkLength < 0xffffffff);

    uint32_t chunklen = (uint32_t)chunkLength;

    m_Write->SetOffset(chunkOffset);
    m_Write->Write(chunklen);
    m_Write->SetOffset(curOffset);
  }
  else if(m_Write->GetOffset() - m_LastChunkOffset < m_LastChunkLen)
  {
    uint64_t numPadBytes = m_LastChunkLen - (m_Write->GetOffset() - m_LastChunkOffset);

    // need to write some padding bytes so that the length is accurate
    for(uint64_t i=0; i < numPadBytes; i++)
    {
      byte padByte = 0xbb;
      m_Write->Write(padByte);
    }
  }

  m_ChunkMetadata = SerialisedChunkMetaData();
}

template<>
void WriteSerialiser::WriteSerialisedFile(WriteSerialiser &fileser, SerialisedFile file)
{
  m_StructuredDataBuffers = file.data;

  for(size_t i = 0; i < file.chunks.size(); i++)
  {
    const StructuredChunk &chunk = *file.chunks[i];

    m_ChunkMetadata = chunk.metadata;

    m_ChunkFlags = 0;

    if(m_ChunkMetadata.callstack[0] != 0)
      m_ChunkFlags |= ChunkCallstack;

    if(m_ChunkMetadata.threadID != 0)
      m_ChunkFlags |= ChunkThreadID;

    if(m_ChunkMetadata.duration != 0)
      m_ChunkFlags |= ChunkDuration;

    if(m_ChunkMetadata.timestamp != 0)
      m_ChunkFlags |= ChunkTimestamp;
    
    BeginChunk(m_ChunkMetadata.chunkID);

    if(chunk.metadata.flags & SerialisedChunkMetaData::OpaqueChunk)
    {
      RDCASSERT(chunk.data.children.size() == 1);

      uint64_t bufID = chunk.data.children[0]->data.basic.u;
      byte *ptr = m_StructuredDataBuffers[bufID].first;
      uint64_t len = m_StructuredDataBuffers[bufID].second;
      
      // the information about whether a chunk is aligned or not is
      // lost and can't be easily retrieved. It is safer to just
      // always align opaque chunks. The only downside is that this
      // could mean unmodified logs won't be bitwise identical
      m_AlignedData = true;
      m_Write->Write(ptr, len);
    }
    else
    {
      for(size_t o = 0; o < chunk.data.children.size(); o++)
      {
        // note, we don't need names because we aren't exporting structured data
        Serialise("", chunk.data.children[o]);
      }
    }

    EndChunk();

    fileser.Insert(new Chunk(*this, m_ChunkMetadata.chunkID, true));
  }

  // clear it again so we don't try and deallocate any of the memory,
  // because this function doesn't own it.
  m_StructuredDataBuffers.clear();
}

template <>
const char *GetSerialiseTypeName<StructuredObject *>::Name()
{
  return "";
}

template <class SerialiserType>
void DoSerialise(SerialiserType &ser, StructuredObject *el)
{
  if(el->type.flags & SerialisedType::Nullable)
  {
    bool present = el->type.basetype != SerialisedType::eNull;
    ser.Serialise("", present);
  }

  const SerialisedBufferList &buffers = ser.GetSerialisedBuffers();

  switch(el->type.basetype)
  {
    case SerialisedType::eChunk:
      RDCERR("Unexpected chunk inside object!");
      break;
    case SerialisedType::eStruct:
      for(size_t o = 0; o < el->data.children.size(); o++)
        ser.Serialise("", el->data.children[o]);
        break;
    case SerialisedType::eArray:
      ser.Serialise("", el->data.children);
      break;
    case SerialisedType::eNull:
      // nothing to do, we serialised present flag above
      RDCASSERT(el->type.flags & SerialisedType::Nullable);
      break;
    case SerialisedType::eBuffer:
    {
      uint64_t bufID = el->data.basic.u;
      byte *buf = buffers[bufID].first;
      uint64_t size = buffers[bufID].second;
      ser.Serialise("", buf, size);
      break;
    }
    case SerialisedType::eString:
      ser.Serialise("", el->data.str);
      break;
    case SerialisedType::eEnum:
    {
      uint32_t e = (uint32_t)el->data.basic.u;
      ser.Serialise("", e);
      break;
    }
    case SerialisedType::eBoolean:
      ser.Serialise("", el->data.basic.b);
      break;
    case SerialisedType::eCharacter:
      ser.Serialise("", el->data.basic.c);
      break;
    case SerialisedType::eUnsignedInteger:
      if(el->type.byteSize == 1)
      {
        uint8_t u = uint8_t(el->data.basic.u);
        ser.Serialise("", u);
      }
      else if(el->type.byteSize == 2)
      {
        uint16_t u = uint16_t(el->data.basic.u);
        ser.Serialise("", u);
      }
      else if(el->type.byteSize == 4)
      {
        uint32_t u = uint32_t(el->data.basic.u);
        ser.Serialise("", u);
      }
      else if(el->type.byteSize == 8)
      {
        ser.Serialise("", el->data.basic.u);
      }
      else
      {
        RDCERR("Unexpeted integer size %u", el->type.byteSize);
      }
      break;
    case SerialisedType::eSignedInteger:
      if(el->type.byteSize == 1)
      {
        int8_t i = int8_t(el->data.basic.i);
        ser.Serialise("", i);
      }
      else if(el->type.byteSize == 2)
      {
        int16_t i = int16_t(el->data.basic.i);
        ser.Serialise("", i);
      }
      else if(el->type.byteSize == 4)
      {
        int32_t i = int32_t(el->data.basic.i);
        ser.Serialise("", i);
      }
      else if(el->type.byteSize == 8)
      {
        ser.Serialise("", el->data.basic.i);
      }
      else
      {
        RDCERR("Unexpeted integer size %u", el->type.byteSize);
      }
      break;
    case SerialisedType::eFloat:
      if(el->type.byteSize == 4)
      {
        float f = float(el->data.basic.d);
        ser.Serialise("", f);
      }
      else if(el->type.byteSize == 8)
      {
        ser.Serialise("", el->data.basic.d);
      }
      else
      {
        RDCERR("Unexpeted float size %u", el->type.byteSize);
      }
      break;
  }
}

/////////////////////////////////////////////////////////////
// Basic types

#define BASIC_TYPE_SERIALISE_NAME(typeName) \
  template <>                        \
  const char *GetSerialiseTypeName<typeName>::Name() \
  {                                  \
    return #typeName;                       \
  }

BASIC_TYPE_SERIALISE_NAME(int64_t);
BASIC_TYPE_SERIALISE_NAME(uint64_t);
BASIC_TYPE_SERIALISE_NAME(int32_t);
BASIC_TYPE_SERIALISE_NAME(uint32_t);
BASIC_TYPE_SERIALISE_NAME(int16_t);
BASIC_TYPE_SERIALISE_NAME(uint16_t);
BASIC_TYPE_SERIALISE_NAME(int8_t);
BASIC_TYPE_SERIALISE_NAME(uint8_t);

BASIC_TYPE_SERIALISE_NAME(double);
BASIC_TYPE_SERIALISE_NAME(float);

BASIC_TYPE_SERIALISE_NAME(bool);

BASIC_TYPE_SERIALISE_NAME(wchar_t);
BASIC_TYPE_SERIALISE_NAME(char);

BASIC_TYPE_SERIALISE_NAME(string);
BASIC_TYPE_SERIALISE_NAME(char *);
BASIC_TYPE_SERIALISE_NAME(const char *);

template <>
string DoStringize(void *const &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "0x%p", el);

  return tostrBuf;
}

template <>
string DoStringize(const int64_t &el)
{
  char tostrBuf[256] = {0};

  _snprintf(tostrBuf, 255, "%" PRId64, el);

  return tostrBuf;
}

#if defined(RENDERDOC_PLATFORM_APPLE)
template <>
string DoStringize(const size_t &el)
{
  char tostrBuf[256] = {0};

  _snprintf(tostrBuf, 255, "%" PRIu64, (uint64_t)el);

  return tostrBuf;
}
#endif

template <>
string DoStringize(const uint64_t &el)
{
  char tostrBuf[256] = {0};

  _snprintf(tostrBuf, 255, "%" PRIu64, el);

  return tostrBuf;
}

template <>
string DoStringize(const uint32_t &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "%u", el);

  return tostrBuf;
}

template <>
string DoStringize(const char &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "'%c'", el);

  return tostrBuf;
}

template <>
string DoStringize(const wchar_t &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "'%lc'", el);

  return tostrBuf;
}

template <>
string DoStringize(const byte &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "%u", (uint32_t)el);

  return tostrBuf;
}

template <>
string DoStringize(const uint16_t &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "%04d", el);

  return tostrBuf;
}

template <>
string DoStringize(const int &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "%d", el);

  return tostrBuf;
}

template <>
string DoStringize(const short &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "%04d", el);

  return tostrBuf;
}

template <>
string DoStringize(const float &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "%0.4f", el);

  return tostrBuf;
}

template <>
string DoStringize(const double &el)
{
  char tostrBuf[256] = {0};
  _snprintf(tostrBuf, 255, "%0.4lf", el);

  return tostrBuf;
}

template <>
string DoStringize(const bool &el)
{
  if(el)
    return "True";

  return "False";
}
<file_sep>/streamio.cpp
/******************************************************************************
 * The MIT License (MIT)
 *
 * Copyright (c) 2017 <NAME>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 ******************************************************************************/

#include "streamio.h"

static const uint64_t initialBufferSize = 64 * 1024;
const byte StreamWriter::empty[128] = {};

StreamReader::StreamReader(const void *buffer, size_t bufferSize)
{
  m_InputSize = m_BufferSize = bufferSize;
  m_BufferHead = m_BufferBase = AllocAlignedBuffer(m_BufferSize);

  memcpy(m_BufferHead, buffer, m_BufferSize);

  m_File = NULL;
  m_Decompressor = NULL;
  m_ReadOffset = 0;
}

StreamReader::StreamReader(const std::vector<byte> &buffer)
{
  m_InputSize = m_BufferSize = buffer.size();
  m_BufferHead = m_BufferBase = AllocAlignedBuffer(m_BufferSize);

  memcpy(m_BufferHead, buffer.data(), m_BufferSize);

  m_File = NULL;
  m_Decompressor = NULL;
  m_ReadOffset = 0;
}

StreamReader::StreamReader(FILE *file, size_t fileSize)
{
  m_File = file;
  m_Decompressor = NULL;
  m_ReadOffset = 0;
  m_InputSize = fileSize;

  m_BufferSize = initialBufferSize;
  m_BufferHead = m_BufferBase = AllocAlignedBuffer(m_BufferSize);

  ReadFromExternal(0, m_BufferSize);
}

StreamReader::StreamReader(FILE *file)
{
  m_File = file;
  m_Decompressor = NULL;
  m_ReadOffset = 0;

  fseek(file, 0, SEEK_END);

  m_InputSize = ftell(file);

  fseek(file, 0, SEEK_SET);

  m_BufferSize = initialBufferSize;
  m_BufferHead = m_BufferBase = AllocAlignedBuffer(m_BufferSize);

  ReadFromExternal(0, m_BufferSize);
}

StreamReader::StreamReader(Decompressor *decompressor, size_t uncompressedSize)
{
  m_File = NULL;
  m_Decompressor = decompressor;
  m_ReadOffset = 0;
  m_InputSize = uncompressedSize;
  
  m_BufferSize = initialBufferSize;
  m_BufferHead = m_BufferBase = AllocAlignedBuffer(m_BufferSize);

  ReadFromExternal(0, RDCMIN(uncompressedSize, m_BufferSize));
}

StreamReader::~StreamReader()
{
  FreeAlignedBuffer(m_BufferBase);
}

void StreamReader::SetOffset(uint64_t offs)
{
  if(m_File || m_Decompressor)
  {
    RDCERR("File and decompress stream readers do not support seeking");
    return;
  }

  m_BufferHead = m_BufferBase + offs;
}

void StreamReader::Reserve(size_t numBytes)
{
  RDCASSERT(m_File || m_Decompressor);

  // store old buffer and the read data, so we can move it into the new buffer
  byte *oldBuffer = m_BufferBase;

  // always keep at least a certain window behind what we read.
  size_t backwardsWindow = RDCMIN((size_t)64, size_t(m_BufferHead - m_BufferBase));

  byte *currentData = m_BufferHead - backwardsWindow;
  size_t currentDataSize = m_BufferSize - (m_BufferHead - m_BufferBase) + backwardsWindow;

  size_t BufferOffset = m_BufferHead - m_BufferBase;

  // if we are reading more than our current buffer size, expand the buffer size
  if(numBytes + backwardsWindow > m_BufferSize)
  {
    // very conservative resizing - don't do "double and add" - to avoid
    // a 1GB buffer being read and needing to allocate 2GB. The cost is we
    // will reallocate a bit more often
    m_BufferSize = numBytes + backwardsWindow;
    m_BufferBase = AllocAlignedBuffer(m_BufferSize);
  }

  // move the unread data into the buffer
  memmove(m_BufferBase, currentData, currentDataSize);

  if(BufferOffset > backwardsWindow)
  {
    m_ReadOffset += BufferOffset - backwardsWindow;
    m_BufferHead = m_BufferBase + backwardsWindow;
  }
  else
  {
    m_BufferHead = m_BufferBase + BufferOffset;
  }

  // if there's anything left of the file to read in, do so now
  ReadFromExternal(currentDataSize, RDCMIN(m_BufferSize - currentDataSize,
                                       size_t(m_InputSize - m_ReadOffset - currentDataSize)));

  if(oldBuffer != m_BufferBase)
    FreeAlignedBuffer(oldBuffer);
}

void StreamReader::ReadFromExternal(uint64_t bufferOffs, size_t length)
{
  if(m_Decompressor)
  {
    m_Decompressor->Read(m_BufferBase + bufferOffs, length);
  }
  else
  {
    RDCASSERT(m_File);
    fread(m_BufferBase + bufferOffs, 1, length, m_File);
  }
}

StreamWriter::StreamWriter(uint64_t initialBufSize)
{
  m_BufferSize = initialBufSize;
  m_BufferBase = m_BufferHead = AllocAlignedBuffer(initialBufSize);

  m_WriteSize = 0;
  m_File = NULL;
  m_Compressor = NULL;
}

StreamWriter::StreamWriter(FILE *file)
{
  m_BufferSize = 0;
  m_BufferBase = m_BufferHead = NULL;

  m_WriteSize = 0;
  m_File = file;
  m_Compressor = NULL;
}

StreamWriter::StreamWriter(Compressor *compressor)
{
  m_BufferSize = 0;
  m_BufferBase = m_BufferHead = NULL;

  m_WriteSize = 0;
  m_File = NULL;
  m_Compressor = compressor;
}

StreamWriter::~StreamWriter()
{
  FreeAlignedBuffer(m_BufferBase);
}



#include "lz4io.h"
#include "zstdio.h"

void streamio_test()
{
  {
    StreamWriter writer(512);

    writer.Write<int>(512);
    writer.Write<char>('c');
    writer.Write<bool>(true);
    writer.Write<float>(5.0f);
    writer.AlignTo<64>();
    writer.Write<double>(64.0);

    unsigned char buffer[20] = {};
    memset(buffer, 0xbb, sizeof(buffer));
    writer.Write(buffer, sizeof(buffer));

    RDCASSERT(writer.GetOffset() == 64 + 8 + 20);

    writer.Finish();

    StreamReader reader(writer.GetData(), (size_t)writer.GetOffset());

    writer.Rewind();

    RDCASSERT(writer.GetOffset() == 0);
    
    int i = 0;
    char c = '\0';
    bool b = false;
    float f = 0.0f;
    double d = 0.0;
    memset(buffer, 0, sizeof(buffer));

    reader.Read(i);
    reader.Read(c);
    reader.Read(b);
    reader.Read(f);
    reader.AlignTo<64>();
    reader.Read(d);
    reader.Read(buffer, sizeof(buffer));

    RDCASSERT(i == 512);
    RDCASSERT(c == 'c');
    RDCASSERT(b == true);
    RDCASSERT(f == 5.0f);
    RDCASSERT(d == 64.0);
    for(size_t i=0; i < sizeof(buffer); i++)
      RDCASSERT(buffer[i] == 0xbb);
  }

  {
    // the actual destination buffer
    StreamWriter compressedWriter(512);

    // compressor into the buffer
    LZ4Compressor lz4Compressor(&compressedWriter);

    // writer into the compressor
    StreamWriter lz4Writer(&lz4Compressor);

    struct Dummy
    {
      int i[100];
      float f;
      double d;
      char str[200];
      bool b;
    } dummy = {};

    for(int i=0; i < 100; i++)
      dummy.i[i] = i * 100;

    strcpy(dummy.str, "Foo bar writer into the compressor// compressor into the bufferLZ4Compressor lz4Compressor");
    dummy.b = true;
    dummy.d = 3.1415926535;
    dummy.f = 420.69f;

    for(int i=0; i < 1000; i++)
      lz4Writer.Write(dummy);

    lz4Writer.Write<int>(0xb33b1337);
    lz4Writer.Write<int>(0xb33b1337);
    lz4Writer.Write<int>(0xb33b1337);

    char *hugebuf = new char[1024*1024*128];
    for(int i=0; i < 1024*1024; i++)
      memset(hugebuf + i*128, (rand()>>8)&0xff, 128);

    lz4Writer.Write(hugebuf, 1024*1024*128);

    lz4Writer.Finish();

    uint64_t uncomp = lz4Writer.GetOffset();
    uint64_t comp = compressedWriter.GetOffset();
    
    // reader of compressed data
    StreamReader compressedReader(compressedWriter.GetData(), (size_t)compressedWriter.GetOffset());

    // decompressor
    LZ4Decompressor decompressor(&compressedReader);

    StreamReader lz4Reader(&decompressor, uncomp);

    for(int i=0; i < 1000; i++)
    {
      Dummy read = {};

      lz4Reader.Read(read);
      RDCASSERT(memcmp(&read, &dummy, sizeof(read)) == 0);
    }

    int marker = 0;
    lz4Reader.Read(marker); RDCASSERT(marker == 0xb33b1337);
    lz4Reader.Read(marker); RDCASSERT(marker == 0xb33b1337);
    lz4Reader.Read(marker); RDCASSERT(marker == 0xb33b1337);
    
    char *hugebuf2 = new char[1024*1024*128];
    lz4Reader.Read(hugebuf2, 1024*1024*128);
    
    RDCASSERT(memcmp(hugebuf, hugebuf2, 1024*1024*128) == 0);

    StreamWriter recompressedWriter(512);
    ZSTDCompressor zstdCompressor(&recompressedWriter);
    
    StreamReader compressedReader2(compressedWriter.GetData(), (size_t)compressedWriter.GetOffset());
    LZ4Decompressor decompressor2(&compressedReader2);

    decompressor2.Recompress(&zstdCompressor);
    
    uint64_t comp2 = recompressedWriter.GetOffset();
    
    // reader of compressed data
    StreamReader recompressedReader(recompressedWriter.GetData(), (size_t)recompressedWriter.GetOffset());

    // decompressor
    ZSTDDecompressor zstdDecompressor(&recompressedReader);

    StreamReader zstdReader(&zstdDecompressor, uncomp);

    for(int i=0; i < 1000; i++)
    {
      Dummy read = {};

      zstdReader.Read(read);
      RDCASSERT(memcmp(&read, &dummy, sizeof(read)) == 0);
    }

    zstdReader.Read(marker); RDCASSERT(marker == 0xb33b1337);
    zstdReader.Read(marker); RDCASSERT(marker == 0xb33b1337);
    zstdReader.Read(marker); RDCASSERT(marker == 0xb33b1337);
    
    char *hugebuf3 = new char[1024*1024*128];
    zstdReader.Read(hugebuf3, 1024*1024*128);
    
    RDCASSERT(memcmp(hugebuf, hugebuf3, 1024*1024*128) == 0);

    delete[] hugebuf;
    delete[] hugebuf2;
    delete[] hugebuf3;
  }
}