text:
/*----------------------------------------------------------------------------*/
/* schema.c: Schema Python class and related types.                           */
/*----------------------------------------------------------------------------*/

#include "schema.h"

#include "structmember.h"

#include "avro.h"
#include "bufferrange.h"
#include "common.h"
#include "protocol.h"
#include "record.h"

/*----------------------------------------------------------------------------*/

/* Forward declarations for schema validation and I/O functions. */

static Schema* validate_schema(Schema* schema);
static PyObject* read_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max);
static PyObject* prepare_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size);
static int write_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value);

/*----------------------------------------------------------------------------*/

/* Schema class implementation. */

/* Internal function that returns a tuple containing the values necessary to
   reconstruct a Schema. Used for implementing __repr__. */
static PyObject* _Schema_repr_object(Schema* self)
{
    PyObject* tuple = NULL;

    Py_ssize_t field_count;
    Py_ssize_t count;

    field_count = PyTuple_GET_SIZE(self->fields);
    count = (self->name != Py_None ? 1 : 0) + (self->default_value != Py_None ? 1 : 0) + (field_count > 0 ? 1 : 0);

    if (count == 0)
    {
        Py_INCREF(self->data_type_name);
        return self->data_type_name;
    }

    tuple = PyTuple_New(count + 1);
    CHECK(tuple, error)
    count = 0;

    if (self->name != Py_None)
    {
        Py_INCREF(self->name);
        PyTuple_SET_ITEM(tuple, count, self->name);
        ++count;
    }

    Py_INCREF(self->data_type_name);
    PyTuple_SET_ITEM(tuple, count, self->data_type_name);
    ++count;

    if (self->default_value != Py_None)
    {
        Py_INCREF(self->default_value);
        PyTuple_SET_ITEM(tuple, count, self->default_value);
        ++count;
    }

    if (field_count > 0)
    {
        PyObject* field_list;
        Py_ssize_t i;

        field_list = PyList_New(field_count);
        CHECK(field_list, error)
        PyTuple_SET_ITEM(tuple, count, field_list);

        for (i = 0; i < field_count; ++i)
        {
            PyObject* field_repr = _Schema_repr_object((Schema*)PyTuple_GET_ITEM(self->fields, i));
            CHECK(field_repr, error)
            PyList_SET_ITEM(field_list, i, field_repr);
        }
    }

    return tuple;

error:
    Py_XDECREF(tuple);
    return NULL;
}

/* Python Schema object deallocator. */
static void Schema_dealloc(Schema* self)
{
    Py_XDECREF(self->data_type_name);
    Py_XDECREF(self->name);
    Py_XDECREF(self->default_value);
    Py_XDECREF(self->fields);
    Py_TYPE(self)->tp_free((PyObject*)self);
}

/* Python Schema.decode method. Decodes an object from an Avro-encoded binary
   buffer according to the schema definition.

   Parameters:
       buffer (buffer)
           The buffer containing the Avro-encoded binary data.

       range (BufferRange, optional)
           Range of bytes within the buffer containing the object. If not
           specified, the entire buffer is used. If the object's data does
           not take up the entire range, any extra data is ignored.

    Returns:
        The decoded object. */
static PyObject* Schema_decode(Schema* self, PyObject* args, PyObject* kwargs)
{
    Py_buffer buffer = { NULL };

    PyObject* arg_range = NULL;
    static char* keywords[] = { "buffer", "range", NULL };

    PyObject* result;

    uint8_t* pos;
    uint8_t* max;

    CHECK(PyArg_ParseTupleAndKeywords(args, kwargs, "s*|O", keywords, &buffer, &arg_range), error)

    if (arg_range)
    {
        Py_ssize_t start;
        Py_ssize_t length;

        CHECK_STRING(BufferRange_check(arg_range), PyExc_TypeError, "range must be BufferRange", error)
        start = ((BufferRange*)arg_range)->start;
        CHECK_STRING(start >= 0 && start <= buffer.len, PyExc_ValueError, "start index out of range", error)
        pos = (uint8_t*)buffer.buf + start;
        length = ((BufferRange*)arg_range)->length;
        CHECK_STRING(length >= 0 && start + length <= buffer.len, PyExc_ValueError, "length out of range", error)
        max = pos + length;
    }
    else
    {
        pos = (uint8_t*)buffer.buf;
        max = pos + buffer.len;
    }

    result = read_schema(self, (uint8_t*)buffer.buf, &pos, max);
    PyBuffer_Release(&buffer);
    return result;

error:
    if (buffer.buf)
    {
        PyBuffer_Release(&buffer);
    }

    return NULL;
}

/* Python Schema.encode method. Encodes an object into Avro-encoded binary
   format according to the schema definition.

   Parameters:
       value (object):
           The object to encode. Must conform to the schema definition.

   Returns:
       Bytes (Python 3) or str (Python 2) containing the Avro-encoded binary
       data. */
static PyObject* Schema_encode(Schema* self, PyObject* value)
{
    PyObject* prepared = NULL;
    PyObject* result = NULL;

    PyObject* path = NULL;
    Py_ssize_t size = 0;

    uint8_t* pos;
    uint8_t* max;

    /* First, prepare the value. This validates that the value is valid
       according to the schema's data type, does any necessary data type
       conversions, and precomputes the required buffer size. */

    prepared = prepare_schema(self, value, &path, &size);

    if (!prepared)
    {
        if (path)
        {
            prefix_exception(path);
            Py_DECREF(path);
            goto error;
        }
    }

    /* Allocate a buffer of the correct size to hold the result. */

    #if PY_MAJOR_VERSION >= 3
        result = PyBytes_FromStringAndSize(NULL, size);
        CHECK(result, error)
        pos = (uint8_t*)PyBytes_AS_STRING(result);
    #else
        result = PyString_FromStringAndSize(NULL, size);
        CHECK(result, error)
        pos = (uint8_t*)PyString_AS_STRING(result);
    #endif

    max = pos + size;

    /* Encode the prepared value into the buffer. */

    CHECK(write_schema(self, &pos, max, prepared), error)

    Py_DECREF(prepared);
    return result;

error:
    Py_XDECREF(prepared);
    Py_XDECREF(result);
    return NULL;
}

/* Python Schema object constructor.

   Parameters:
       name (unicode, optional)
           Name of the field within a parent record schema that this schema
           represents. Defaults to None.

       data_type (unicode)
           Name of the data type of the schema. Must be a valid data type.

       default_value (object, optional)
           The default value of the field within a parent record schema that
           this schema represents. Defaults to None. If provided, must be a
           valid value for the data type of this schema.

       fields (sequence of Schema or unicode, optional)
           Schema objects for fields that are part of this schema. Unicode
           strings can also be used if a field only requires a data type to
           be specified (without a name, default value, or subfields). Defaults
           to None. Requirements vary based on the data type of this schema. */
static Schema* Schema_new(PyTypeObject* type, PyObject* args, PyObject* kwargs)
{
    PyObject* name = NULL;
    PyObject* data_type_name = NULL;
    PyObject* default_value = NULL;
    PyObject* field_seq = NULL;
    PyObject* fields = NULL;

    PyObject* arg_0;
    PyObject* arg_1 = NULL;
    PyObject* arg_2 = NULL;
    PyObject* arg_3 = NULL;
    PyObject* arg_data_type;
    static char* keywords[] = { "name", "data_type", "default_value", "fields", NULL };

    SchemaDataType data_type;
    Schema* self;

    ProtocolState* state = GET_STATE();
    CHECK(state, error)

    CHECK(PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOO", keywords, &arg_0, &arg_1, &arg_2, &arg_3), error);

    if (!kwargs)
    {
        /* Since all parameters except the second (data_type) are optional, if
           keywords are not used, it isn't possible to know which parameter is
           which based on order alone. So analyze what was provided to find
           the data_type. */

        if (!arg_1)
        {
            /* Only one parameter was passed, so it must be data_type. */

            arg_data_type = arg_0;
            arg_0 = NULL;
        }
        else if (!arg_3 && !IS_STRING(arg_1))
        {
            /* Two or three parameters were passed. The second one is not a
               string, so it can't be data_type, which means name wasn't
               passed and the first parameter must be data_type. */

            arg_data_type = arg_0;
            arg_0 = NULL;

            /* Since name, which would normally be the first parameter, was
               not passed, the second and third position parameters (arg_1 and
               arg_2), if present, actually belong in positions three and four
               (arg_2 and arg_3), so bump them forward. */

            if (arg_2)
            {
                arg_3 = arg_2;
            }

            arg_2 = arg_1;
            arg_1 = NULL;
        }
        else
        {
            /* All four parameters were passed, or two or three were passed and
               the second one is a string. In either case, it can be assumed
               that a name was provided as the first parameter and data_type is
               the second.

               (The combination of data_type, default_value and optionally
               fields could in theory also result in the second parameter being
               a string, but this combination is not supported, since
               default_value is only used for record fields, which must have a
               name anyway.) */

            arg_data_type = arg_1;
            arg_1 = NULL;
        }
    }
    else
    {
        /* Since keywords were used, the keyword decoder will have put the
           parameters in the correct order, and no special detection is
           required. */

        arg_data_type = arg_1;
        CHECK_STRING(arg_data_type, PyExc_TypeError, "Required argument 'data_type' (pos 2) not found", error)
    }

    /* Look up data type by name in the schema data type name list. */

    CHECK_STRING(IS_STRING(arg_data_type), PyExc_TypeError, "data type must be string", error)
    data_type_name = TO_STRING(arg_data_type);
    CHECK(data_type_name, error)
    data_type = lookup_string(data_type_name, state->schema_data_type_names, SDT_MAX);
    CHECK_OBJECT(data_type != SDT_MAX, PyExc_ValueError, format_string("unknown data type %S", data_type_name), error)

    if (!kwargs)
    {
        /* Keywords were not used, so at this point, the third parameter
           position (arg_2) could be either default_value or fields. If both
           the third and fourth positions are present, then the third parameter
           must be default_value. Otherwise, if the data_type requires fields,
           assume the third position is fields, and bump it to the fourth
           position. */

        if (arg_2 && !arg_3 && (data_type == SDT_NULLABLE || data_type == SDT_ARRAY || data_type == SDT_MAP || data_type == SDT_RECORD))
        {
            arg_3 = arg_2;
            arg_2 = NULL;
        }
    }

    /* At this point, the first parameter position (arg_0) is name, the
       third (arg_2) is default_value, and the fourth (arg_3) is fields. */

    if (arg_0 && arg_0 != Py_None)
    {
        CHECK_STRING(IS_STRING(arg_0), PyExc_TypeError, "name must be string", error)
        name = TO_STRING(arg_0);
        CHECK(name, error)
    }
    else
    {
        Py_INCREF(Py_None);
        name = Py_None;
    }

    if (arg_2)
    {
        Py_INCREF(arg_2);
        default_value = arg_2;
    }
    else
    {
        Py_INCREF(Py_None);
        default_value = Py_None;
    }

    if (arg_3)
    {
        Py_ssize_t field_count;
        Py_ssize_t i;

        field_seq = PySequence_Fast(arg_3, "fields must be iterable");
        CHECK(field_seq, error)
        field_count = PySequence_Fast_GET_SIZE(field_seq);
        fields = PyTuple_New(field_count);
        CHECK(fields, error)

        for (i = 0; i < field_count; ++i)
        {
            PyObject* item = PySequence_Fast_GET_ITEM(field_seq, i);
            PyObject* field;

            if (PyTuple_Check(item))
            {
                /* The field is a tuple, so assume it contains the parameters
                   for the Schema constructor and call it recursively. */

                field = PyObject_Call((PyObject*)&Schema_type, item, NULL);
                CHECK(field, error)
            }
            else if (IS_STRING(item))
            {
                /* The field is a string, so assume it contains a data type.
                   Create a tuple out of it, and call the Schema constructor
                   recursively. */

                PyObject* temp = PyTuple_Pack(1, item);
                field = PyObject_Call((PyObject*)&Schema_type, temp, NULL);
                Py_DECREF(temp);
                CHECK(field, error)
            }
            else
            {
                /* The field is not a tuple or string, so it must be a
                   Schema. */

                CHECK_STRING(PyObject_TypeCheck(item, &Schema_type), PyExc_TypeError, "field must be Schema", error)
                Py_INCREF(item);
                field = item;
            }

            PyTuple_SET_ITEM(fields, i, field);
        }

        Py_CLEAR(field_seq);
    }
    else
    {
        fields = PyTuple_New(0);
        CHECK(fields, error)
    }

    self = (Schema*)type->tp_alloc(type, 0);
    CHECK(self, error)

    self->name = name;
    self->data_type_name = data_type_name;
    self->default_value = default_value;
    self->fields = fields;
    self->data_type = data_type;

    /* Make sure the resulting Schema object is valid. */

    return validate_schema(self);

error:
    Py_XDECREF(name);
    Py_XDECREF(data_type_name);
    Py_XDECREF(default_value);
    Py_XDECREF(field_seq);
    Py_XDECREF(fields);
    return NULL;
}

/* Python Schema.__repr__ method. */
static PyObject* Schema_repr(Schema* self)
{
    return generic_repr((PyObject*)self, (reprfunc)_Schema_repr_object);
}

/* Python Schema rich compare function (supports == and != operators). */
static PyObject* Schema_richcompare(PyObject* a, PyObject* b, int op)
{
    PyObject* result;
    int eq;

    result = generic_richcompare(&Schema_type, a, b, op);

    if (result != (PyObject*)&Schema_type)
    {
        return result;
    }

    eq = ((Schema*)a)->data_type == ((Schema*)b)->data_type;

    if (eq)
    {
        eq = PyObject_RichCompareBool(((Schema*)a)->name, ((Schema*)b)->name, Py_EQ);
        CHECK(eq != -1, error)

        if (eq)
        {
            eq = PyObject_RichCompareBool(((Schema*)a)->default_value, ((Schema*)b)->default_value, Py_EQ);
            CHECK(eq != -1, error)

            if (eq)
            {
                eq = PyObject_RichCompareBool(((Schema*)a)->fields, ((Schema*)b)->fields, Py_EQ);
                CHECK(eq != -1, error)
            }
        }
    }

    result = eq ? (op == Py_EQ ? Py_True : Py_False) : (op == Py_EQ ? Py_False : Py_True);
    Py_INCREF(result);
    return result;

error:
    return NULL;
}

static PyMemberDef Schema_members[] =
{
    { "name", T_OBJECT_EX, offsetof(Schema, name), READONLY, NULL },
    { "data_type", T_OBJECT_EX, offsetof(Schema, data_type_name), READONLY, NULL },
    { "default_value", T_OBJECT_EX, offsetof(Schema, default_value), READONLY, NULL },
    { "fields", T_OBJECT_EX, offsetof(Schema, fields), READONLY, NULL },
    { NULL }
};

static PyMethodDef Schema_methods[] =
{
    { "decode", (PyCFunction)Schema_decode, METH_VARARGS | METH_KEYWORDS, NULL },
    { "encode", (PyCFunction)Schema_encode, METH_O, NULL },
    { NULL }
};

PyTypeObject Schema_type =
{
    PyVarObject_HEAD_INIT(NULL, 0)
    "kinetica.protocol.Schema",      /* tp_name */
    sizeof(Schema),                  /* tp_basicsize */
    0,                               /* tp_itemsize */
    (destructor)Schema_dealloc,      /* tp_dealloc */
    0,                               /* tp_print */
    0,                               /* tp_getattr */
    0,                               /* tp_setattr */
    0,                               /* tp_compare */
    (reprfunc)Schema_repr,           /* tp_repr */
    0,                               /* tp_as_number */
    0,                               /* tp_as_sequence */
    0,                               /* tp_as_mapping */
    0,                               /* tp_hash */
    0,                               /* tp_call */
    0,                               /* tp_str */
    0,                               /* tp_getattro */
    0,                               /* tp_setattro */
    0,                               /* tp_as_buffer */
    Py_TPFLAGS_DEFAULT,              /* tp_flags */
    0,                               /* tp_doc */
    0,                               /* tp_traverse */
    0,                               /* tp_clear */
    (richcmpfunc)Schema_richcompare, /* tp_richcompare */
    0,                               /* tp_weaklistoffset */
    0,                               /* tp_iter */
    0,                               /* tp_iternext */
    Schema_methods,                  /* tp_methods */
    Schema_members,                  /* tp_members */
    0,                               /* tp_getset */
    0,                               /* tp_base */
    0,                               /* tp_dict */
    0,                               /* tp_descr_get */
    0,                               /* tp_descr_set */
    0,                               /* tp_dictoffset */
    0,                               /* tp_init */
    0,                               /* tp_alloc */
    (newfunc)Schema_new,             /* tp_new */
};

/*----------------------------------------------------------------------------*/

/* Schema validation functions. */

/* Function type for a function that validates a Schema object with a specific
   data type. Returns 1 if the Schema is valid, or 0 (with exception set) if
   not. */
typedef int (*ValidateSchemaTypeFunc)(Schema*);

/* Forward declaration of validation function dispatch table. Each entry in
   the table contains a pointer to the Schema validation function for one
   data type. */
static ValidateSchemaTypeFunc validate_schema_types[SDT_MAX];

/* Validation function for Schemas with data types that have no fields. */
static int validate_zero_field_schema(Schema* schema)
{
    CHECK_OBJECT(PyTuple_GET_SIZE(schema->fields) == 0, PyExc_ValueError, format_string("%U must not have fields", schema->data_type_name), error)
    return 1;

error:
    return 0;
}

/* Validation function for Schemas with data types that require one field. */
static int validate_one_field_schema(Schema* schema)
{
    CHECK_OBJECT(PyTuple_GET_SIZE(schema->fields) == 1, PyExc_ValueError, format_string("%U must have exactly one field", schema->data_type_name), error)
    return 1;

error:
    return 0;
}

/* Validation type for Schemas with record data type. */
static int validate_record_schema(Schema* schema)
{
    PyObject* name_set;

    Py_ssize_t field_count;

    Py_ssize_t i;

    name_set = PySet_New(NULL);
    CHECK(name_set, error)

    field_count = PyTuple_GET_SIZE(schema->fields);
    CHECK_STRING(field_count > 0, PyExc_ValueError, "record must have at least one field", error)

    for (i = 0; i < field_count; ++i)
    {
        Schema* field;
        int r;

        field = (Schema*)PyTuple_GET_ITEM(schema->fields, i);
        CHECK_OBJECT(field->name != Py_None, PyExc_ValueError, format_string("record field %zd must have name", i), error)
        r = PySet_Contains(name_set, field->name);
        CHECK(r >= 0, error)
        CHECK_OBJECT(!r, PyExc_ValueError, format_string("duplicate record field name %S", field->name), error)
        CHECK(PySet_Add(name_set, field->name) == 0, error)
    }

    Py_DECREF(name_set);
    return 1;

error:
    Py_XDECREF(name_set);
    return 0;
}

/* Validation function dispatch table. */
static ValidateSchemaTypeFunc validate_schema_types[SDT_MAX] =
{
    validate_one_field_schema,  /* SDT_NULLABLE */
    validate_zero_field_schema, /* SDT_BOOLEAN */
    validate_zero_field_schema, /* SDT_BYTES */
    validate_zero_field_schema, /* SDT_DOUBLE */
    validate_zero_field_schema, /* SDT_FLOAT */
    validate_zero_field_schema, /* SDT_INT */
    validate_zero_field_schema, /* SDT_LONG */
    validate_zero_field_schema, /* SDT_STRING */
    validate_one_field_schema,  /* SDT_ARRAY */
    validate_one_field_schema,  /* SDT_MAP */
    validate_record_schema,     /* SDT_RECORD */
    validate_zero_field_schema, /* SDT_OBJECT */
    validate_zero_field_schema  /* SDT_OBJECT_ARRAY */
};

/* Internal function to validate a Schema object. Returns the object passed in
   if it is valid, otherwise returns NULL with an exception set. */
static Schema* validate_schema(Schema* schema)
{
    /* Dispatch the correct validation function based on the data type. */

    if (validate_schema_types[schema->data_type](schema))
    {
        /* The Schema appears to be valid, but default_value, if present,
           must be checked independently. */

        if (schema->default_value != Py_None)
        {
            /* A default value was provided. Attempt to prepare the default
               value for encoding, which will validate that it is valid
               according to the schema's data type. */

            PyObject* path = NULL;
            Py_ssize_t size = 0;
            PyObject* temp = prepare_schema(schema, schema->default_value, &path, &size);

            if (!temp)
            {
                if (path)
                {
                    prefix_exception(path);
                    Py_DECREF(path);
                }

                path = format_string_safe("invalid default value");

                if (path)
                {
                    prefix_exception(path);
                    Py_DECREF(path);
                }

                return NULL;
            }

            /* Discard the prepared value (the prepare was only for
               validation). */

            Py_DECREF(temp);
        }

        return schema;
    }
    else
    {
        Py_DECREF(schema);
        return NULL;
    }
}

/*----------------------------------------------------------------------------*/

/* Schema reading functions. */

/* Function type for a function that reads an object from an Avro-encoded
   binary buffer using a Schema object with a specific data type for decoding.
   All functions of this type take these parameters:

   schema: The Schema object to use for decoding.

   buf: Pointer to the start of the buffer containing the Avro-encoded binary
        object data. For object and object_array type Schemas, any returned
        BufferRange objects will contain start indices relative to this buffer.

   pos: Pointer to a pointer to the start of the Avro-encoded binary data for
        the object being read within the buffer. This will be updated to point
        to the position immediately following the object data on return. If an
        error occurs, this may point to an arbitrary location between the
        initial value and max on return.

    max: Pointer to the end of the buffer containing the Avro-encoded binary
         object data (must be >= *pos, and may be beyond the end of the object
         data). If this is reached before the object data is completely read,
         an exception is set.

    If successful, the object read is returned; otherwise NULL is returned and
    an exception is set. */
typedef PyObject* (*ReadSchemaTypeFunc)(Schema*, uint8_t*, uint8_t**, uint8_t*);

/* Forward declaration of reading function dispatch table. Each entry in the
   table contains a pointer to the Schema reading function for one data type. */
static ReadSchemaTypeFunc read_schema_types[SDT_MAX];

/* Reading function for nullable Schemas. */
static PyObject* read_nullable_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    PY_LONG_LONG is_null;

    CHECK(handle_read_error(read_long(pos, max, &is_null)), error)

    if (is_null == 1)
    {
        Py_RETURN_NONE;
    }
    else if (is_null == 0)
    {
        return read_schema((Schema*)PyTuple_GET_ITEM(schema->fields, 0), buf, pos, max);
    }
    else
    {
        CHECK(handle_read_error(ERR_OVERFLOW), error)
    }

error:
    return NULL;
}

/* Reading function for boolean Schemas. */
static PyObject* read_boolean_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    char value = 0;

    CHECK(handle_read_error(read_boolean(pos, max, &value)), error)

    if (value)
    {
        Py_RETURN_TRUE;
    }
    else
    {
        Py_RETURN_FALSE;
    }

error:
    return NULL;
}

/* Reading function for bytes Schemas. */
static PyObject* read_bytes_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    PyObject* result = NULL;

    Py_ssize_t len;

    CHECK(handle_read_error(read_bytes_len(pos, max, &len)), error)

    #if PY_MAJOR_VERSION >= 3
        result = PyBytes_FromStringAndSize(NULL, len);
        read_bytes_data(pos, max, (uint8_t*)PyBytes_AS_STRING(result), len);
    #else
        result = PyString_FromStringAndSize(NULL, len);
        read_bytes_data(pos, max, (uint8_t*)PyString_AS_STRING(result), len);
    #endif

    return result;

error:
    Py_XDECREF(result);
    return NULL;
}

/* Reading function for double Schemas. */
static PyObject* read_double_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    double value = 0.0;

    CHECK(handle_read_error(read_double(pos, max, &value)), error)
    return PyFloat_FromDouble(value);

error:
    return NULL;
}

/* reading function for float Schemas. */
static PyObject* read_float_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    float value = 0.0f;

    CHECK(handle_read_error(read_float(pos, max, &value)), error)
    return PyFloat_FromDouble((double)value);

error:
    return NULL;
}

/* Reading function for int Schemas. */
static PyObject* read_int_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    long value;

    CHECK(handle_read_error(read_int(pos, max, &value)), error)

    #if PY_MAJOR_VERSION >= 3
        return PyLong_FromLong((long)value);
    #else
        return PyInt_FromLong((long)value);
    #endif

error:
    return NULL;
}

/* Reading function for long Schemas. */
static PyObject* read_long_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    PY_LONG_LONG value;

    CHECK(handle_read_error(read_long(pos, max, &value)), error)
    return PyLong_FromLongLong((PY_LONG_LONG)value);

error:
    return NULL;
}

/* Reading function for string Schemas. */
static PyObject* read_string_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    Py_ssize_t len;
    char* data;
    PyObject* result;

    CHECK(handle_read_error(read_bytes_len(pos, max, &len)), error)
    data = (char*)MALLOC(len);
    CHECK(handle_read_error(data ? ERR_NONE : ERR_OOM), error)
    read_bytes_data(pos, max, (uint8_t*)data, len);
    result = PyUnicode_FromStringAndSize(data, len);
    free(data);
    return result;

error:
    return NULL;
}

/* Reading function for array Schemas. */
static PyObject* read_array_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    PyObject* result = NULL;

    Schema* field;
    Py_ssize_t count;

    Py_ssize_t block_count = 0;
    Py_ssize_t i;

    field = (Schema*)PyTuple_GET_ITEM(schema->fields, 0);
    CHECK(handle_read_error(read_size(pos, max, &block_count)), error)
    count = (block_count >= 0) ? block_count : -block_count;
    result = PyList_New(count);
    CHECK(result, error)
    i = 0;

    while (1)
    {
        if (block_count == 0)
        {
            break;
        }

        if (block_count < 0)
        {
            PY_LONG_LONG size;

            CHECK(handle_read_error(read_long(pos, max, &size)), error)
            block_count = -block_count;
        }

        if (i != 0)
        {
            PyObject* temp;
            Py_ssize_t j;

            count = i + block_count;
            temp = PyList_New(count);
            CHECK(temp, error)

            for (j = 0; j < i; ++j)
            {
                PyList_SET_ITEM(temp, j, PyList_GET_ITEM(result, j));
                PyList_SET_ITEM(result, j, NULL);
            }

            Py_DECREF(result);
            result = temp;
        }

        while (block_count > 0)
        {
            PyObject* item = read_schema(field, buf, pos, max);
            CHECK(item, error)
            PyList_SET_ITEM(result, i, item);
            ++i;
            --block_count;
        }

        CHECK(handle_read_error(read_size(pos, max, &block_count)), error)
    }

    return result;

error:
    Py_XDECREF(result);
    return NULL;
}

/* Reading function for map Schemas. */
static PyObject* read_map_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    PyObject* result = NULL;
    PyObject* key = NULL;
    PyObject* value = NULL;

    Schema* field;

    result = PyDict_New();
    CHECK(result, error)
    field = (Schema*)PyTuple_GET_ITEM(schema->fields, 0);

    while (1)
    {
        Py_ssize_t count = 0;

        CHECK(handle_read_error(read_size(pos, max, &count)), error)

        if (count == 0)
        {
            break;
        }

        if (count < 0)
        {
            PY_LONG_LONG size;

            CHECK(handle_read_error(read_long(pos, max, &size)), error)
            count = -count;
        }

        while (count > 0)
        {
            key = read_string_schema(schema, buf, pos, max);
            CHECK(key, error)
            value = read_schema(field, buf, pos, max);
            CHECK(value, error)
            CHECK(PyDict_SetItem(result, key, value) == 0, error)
            Py_DECREF(key);
            Py_CLEAR(value);
            --count;
        }
    }

    return result;

error:
    Py_XDECREF(result);
    Py_XDECREF(key);
    Py_XDECREF(value);
    return NULL;
}

/* Reading function for record Schemas. */
static PyObject* read_record_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    PyObject* result;

    Py_ssize_t field_count;

    Py_ssize_t i;

    result = PyDict_New();
    CHECK(result, error)
    field_count = PyTuple_GET_SIZE(schema->fields);

    for (i = 0; i < field_count; ++i)
    {
        Schema* field;
        PyObject* value;
        int r;

        field = (Schema*)PyTuple_GET_ITEM(schema->fields, i);
        value = read_schema(field, buf, pos, max);
        CHECK(value, error)
        r = PyDict_SetItem(result, field->name, value);
        Py_DECREF(value);
        CHECK(r == 0, error)
    }

    return result;

error:
    Py_XDECREF(result);
    return NULL;
}

/* Reading function for object Schemas. */
static PyObject* read_object_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    Py_ssize_t len = 0;
    PyObject* result;

    CHECK(handle_read_error(read_bytes_len(pos, max, &len)), error)
    result = BufferRange_create((Py_ssize_t)(*pos - buf), len);
    CHECK(result, error)
    *pos += len;
    return result;

error:
    return NULL;
}

/* Reading function for object_array Schemas. */
static PyObject* read_object_array_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    PyObject* result = NULL;

    Py_ssize_t count;

    Py_ssize_t block_count = 0;
    Py_ssize_t i;

    CHECK(handle_read_error(read_size(pos, max, &block_count)), error)
    count = (block_count >= 0) ? block_count : -block_count;
    result = PyList_New(count);
    CHECK(result, error)
    i = 0;

    while (1)
    {
        if (block_count == 0)
        {
            break;
        }

        if (block_count < 0)
        {
            PY_LONG_LONG size;

            CHECK(handle_read_error(read_long(pos, max, &size)), error)
            block_count = -block_count;
        }

        if (i != 0)
        {
            PyObject* temp;
            Py_ssize_t j;

            count = i + block_count;
            temp = PyList_New(count);
            CHECK(temp, error)

            for (j = 0; j < i; ++j)
            {
                PyList_SET_ITEM(temp, j, PyList_GET_ITEM(result, j));
                PyList_SET_ITEM(result, j, NULL);
            }

            Py_DECREF(result);
            result = temp;
        }

        while (block_count > 0)
        {
            PyObject* item = read_object_schema(schema, buf, pos, max);
            CHECK(item, error)
            PyList_SET_ITEM(result, i, item);
            ++i;
            --block_count;
        }

        CHECK(handle_read_error(read_size(pos, max, &block_count)), error)
    }

    return result;

error:
    Py_XDECREF(result);
    return NULL;
}

/* Schema reading function dispatch table. */
static ReadSchemaTypeFunc read_schema_types[SDT_MAX] =
{
    read_nullable_schema,    /* SDT_NULLABLE */
    read_boolean_schema,     /* SDT_BOOLEAN */
    read_bytes_schema,       /* SDT_BYTES */
    read_double_schema,      /* SDT_DOUBLE */
    read_float_schema,       /* SDT_FLOAT */
    read_int_schema,         /* SDT_INT */
    read_long_schema,        /* SDT_LONG */
    read_string_schema,      /* SDT_STRING */
    read_array_schema,       /* SDT_ARRAY */
    read_map_schema,         /* SDT_MAP */
    read_record_schema,      /* SDT_RECORD */
    read_object_schema,      /* SDT_OBJECT */
    read_object_array_schema /* SDT_OBJECT_ARRAY */
};

/* Internal function to read an object from an Avro-encoded binary buffer using
   a Schema object for decoding. The function signature is the same as for
   ReadSchemaTypeFunc above. */
static PyObject* read_schema(Schema* schema, uint8_t* buf, uint8_t** pos, uint8_t* max)
{
    /* Dispatch the correct reading function based on the data type. */
    return read_schema_types[schema->data_type](schema, buf, pos, max);
}

/*----------------------------------------------------------------------------*/

/* Schema preparation functions.

   Preparation is the first step of the two-step process for writing an object
   into a buffer in Avro-encoded binary form using a Schema. It validates that
   the object is valid according to the schema's data type, does any necessary
   data type conversions, precomputes the required buffer size, and results in
   a new "prepared" object that is passed into the second step (write). */

/* Function type for a function that prepares an object for writing using a
   Schema object with a specific data type. All functions of this type take
   these parameters:

   schema: The Schema object to use for preparation.

   value: The object to prepare.

   path: Pointer to a variable that, on error, will be populated with a unicode
         object containing a description of the path through the schema to the
         place where the error occurred. (For example, "array item 10 of value
         of record field foo".) This can then be used as the basis for an
         exception message.

   size: Pointer to a Py_ssize_t that will be populated with the required
         buffer size to hold the Avro-encoded binary form of the object.

   The return value on success is the prepared object to pass to the
   write_schema function, or NULL on error. Note that the prepared object is a
   different object than the one passed into the value parameter and must be
   freed separately; the caller is also responsible for any object returned via
   the path parameter on error. */
typedef PyObject* (*PrepareSchemaTypeFunc)(Schema*, PyObject*, PyObject**, Py_ssize_t*);

/* Forward declaration of preparation function dispatch table. Each entry in
   the table contains a pointer to the Schema preparation function for one data
   type. */
static PrepareSchemaTypeFunc prepare_schema_types[SDT_MAX];

/* Preparation function for nullable Schemas. */
static PyObject* prepare_nullable_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    *size += 1;

    if (value == Py_None)
    {
        Py_RETURN_NONE;
    }
    else
    {
        return prepare_schema((Schema*)PyTuple_GET_ITEM(schema->fields, 0), value, path, size);
    }
}

/* Preparation function for boolean Schemas. */
static PyObject* prepare_boolean_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    int result = PyObject_IsTrue(value);
    CHECK(result != -1, error)
    *size += 1;

    if (result)
    {
        Py_RETURN_TRUE;
    }
    else
    {
        Py_RETURN_FALSE;
    }

error:
    return NULL;
}

/* Preparation function for bytes schemas. */
static PyObject* prepare_bytes_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result;
    Py_ssize_t len;

    #if PY_MAJOR_VERSION >= 3
        result = PyObject_Bytes(value);
        CHECK(result, error)
        len = PyBytes_GET_SIZE(result);
    #else
        result = PyObject_Str(value);
        CHECK(result, error)
        len = PyString_GET_SIZE(result);
    #endif

    *size += size_long((PY_LONG_LONG)len) + len;
    return result;

error:
    return NULL;
}

/* Preparation function for double schemas. */
static PyObject* prepare_double_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result = PyNumber_Float(value);
    CHECK(result, error)
    *size += 8;
    return result;

error:
    return NULL;
}

/* Preparation function for float schemas. */
static PyObject* prepare_float_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result = PyNumber_Float(value);
    CHECK(result, error)
    *size += 4;
    return result;

error:
    return NULL;
}

/* Preparation function for int schemas. */
static PyObject* prepare_int_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result;

    #if PY_MAJOR_VERSION >= 3
        long temp;

        result = PyNumber_Long(value);
        CHECK(result, error)
        temp = PyLong_AsLong(result);
        CHECK(temp != -1 || !PyErr_Occurred(), error)
        CHECK_STRING(temp >= INT32_MIN && temp <= INT32_MAX, PyExc_OverflowError, "value out of range", error)
    #else
        long temp;

        result = PyNumber_Int(value);
        CHECK(result, error)
        CHECK_STRING(PyInt_Check(result), PyExc_ValueError, "value out of range", error)
        temp = PyInt_AS_LONG(result);
        CHECK_STRING(temp >= INT32_MIN && temp <= INT32_MAX, PyExc_OverflowError, "value out of range", error)
    #endif

    *size += size_long((PY_LONG_LONG)temp);
    return result;

error:
    Py_XDECREF(result);
    return NULL;
}

/* Preparation function for long schemas. */
static PyObject* prepare_long_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result;

    PY_LONG_LONG temp;

    result = PyNumber_Long(value);
    CHECK(result, error)
    temp = PyLong_AsLongLong(result);
    CHECK(temp != -1 || !PyErr_Occurred(), error)
    CHECK_STRING(temp >= INT64_MIN && temp <= INT64_MAX, PyExc_OverflowError, "value out of range", error)
    *size += size_long(temp);
    return result;

error:
    Py_XDECREF(result);
    return NULL;
}

/* Preparation function for string schemas. */
static PyObject* prepare_string_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* string;
    PyObject* result;
    Py_ssize_t len;

    #if PY_MAJOR_VERSION >= 3
        string = PyObject_Str(value);
        CHECK(string, error)
        result = PyUnicode_AsUTF8String(string);
        Py_DECREF(string);
        CHECK(result, error)
        len = PyBytes_GET_SIZE(result);
    #else
        string = PyObject_Unicode(value);
        CHECK(string, error)
        result = PyUnicode_AsUTF8String(string);
        Py_DECREF(string);
        CHECK(result, error)
        len = PyString_GET_SIZE(result);
    #endif

    *size += size_long((PY_LONG_LONG)len) + len;
    return result;

error:
    return NULL;
}

/* Preparation function for array schemas. */
static PyObject* prepare_array_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result = NULL;
    PyObject* value_seq;

    Py_ssize_t count;
    Schema* field;

    Py_ssize_t i;

    value_seq = PySequence_Fast(value, "value must be iterable");
    CHECK(value_seq, error)
    count = PySequence_Fast_GET_SIZE(value_seq);
    result = PyList_New(count);
    CHECK(result, error)
    field = (Schema*)PyTuple_GET_ITEM(schema->fields, 0);

    for (i = 0; i < count; ++i)
    {
        PyObject* item = prepare_schema(field, PySequence_Fast_GET_ITEM(value_seq, i), path, size);
        CHECK(item, item_error)
        PyList_SET_ITEM(result, i, item);
    }

    *size += size_long((PY_LONG_LONG)count) + (count > 0 ? 1 : 0);
    Py_DECREF(value_seq);
    return result;

item_error:
    if (*path)
    {
        PyObject* temp = format_string_safe("%S of array item %zd", *path, i);
        Py_DECREF(*path);
        *path = temp;
    }
    else
    {
        *path = format_string_safe("array item %zd", i);
    }

error:
    Py_XDECREF(result);
    Py_XDECREF(value_seq);
    return NULL;
}

/* Preparation function for map schemas. */
static PyObject* prepare_map_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result = NULL;
    PyObject* items = NULL;
    PyObject* items_seq = NULL;

    Py_ssize_t count;
    Schema* field;
    PyObject* key;

    Py_ssize_t i;

    CHECK_STRING(PyMapping_Check(value), PyExc_TypeError, "value must be mapping", error)
    items = PyMapping_Items(value);
    CHECK(items, error);
    items_seq = PySequence_Fast(items, "value contains invalid mapping");
    CHECK(items_seq, error);
    count = PySequence_Fast_GET_SIZE(items_seq);
    result = PyList_New(count * 2);
    CHECK(result, error);
    field = (Schema*)PyTuple_GET_ITEM(schema->fields, 0);
    key = NULL;

    for (i = 0; i < count; ++i)
    {
        PyObject* item;
        PyObject* prepared_key;
        PyObject* value;

        item = PySequence_Fast_GET_ITEM(items_seq, i);
        CHECK_STRING(PyTuple_Check(item) && PyTuple_GET_SIZE(item) == 2, PyExc_TypeError, "value contains invalid mapping", error)
        key = PyTuple_GET_ITEM(item, 0);
        prepared_key = prepare_string_schema(schema, key, path, size);
        CHECK(prepared_key, key_error)
        key = prepared_key;
        PyList_SET_ITEM(result, i * 2, key);
        value = prepare_schema(field, PyTuple_GET_ITEM(item, 1), path, size);
        CHECK(value, value_error)
        PyList_SET_ITEM(result, i * 2 + 1, value);
    }

    *size += size_long((PY_LONG_LONG)count) + (count > 0 ? 1 : 0);
    Py_DECREF(items);
    Py_DECREF(items_seq);
    return result;

key_error:
    if (*path)
    {
        Py_DECREF(*path);
    }

    *path = format_string_safe("map key %S", key);
    goto error;

value_error:
    if (*path)
    {
        PyObject* temp = format_string_safe("%S of value of map key %S", *path, key);
        Py_DECREF(*path);
        *path = temp;
    }
    else
    {
        *path = format_string_safe("value of map key %S", key);
    }

error:
    Py_XDECREF(result);
    Py_XDECREF(items);
    Py_XDECREF(items_seq);
    return NULL;
}

/* Preparation function for record schemas. */
static PyObject* prepare_record_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result = NULL;

    Py_ssize_t field_count;
    Py_ssize_t count;
    PyObject* key;

    Py_ssize_t i;

    CHECK_STRING(PyMapping_Check(value), PyExc_TypeError, "value must be mapping", error)
    field_count = PyTuple_GET_SIZE(schema->fields);
    result = PyList_New(field_count);
    CHECK(result, error)
    count = PyMapping_Length(value);
    CHECK(count >= 0, error)

    for (i = 0; i < field_count; ++i)
    {
        Schema* field;
        PyObject* prepared_field_value;

        field = (Schema*)PyTuple_GET_ITEM(schema->fields, i);
        key = field->name;

        if (PyMapping_HasKey(value, key))
        {
            PyObject* field_value = PyObject_GetItem(value, key);
            CHECK(field_value, key_error)

            if (field_value == Py_None)
            {
                Py_DECREF(field_value);
                CHECK_STRING(field->data_type == SDT_NULLABLE || field->default_value != Py_None, PyExc_ValueError, "required", key_error)
                prepared_field_value = prepare_schema(field, field->default_value, path, size);
            }
            else
            {
                prepared_field_value = prepare_schema(field, field_value, path, size);
                Py_DECREF(field_value);
            }

            --count;
        }
        else
        {
            CHECK_STRING(field->data_type == SDT_NULLABLE || field->default_value != Py_None, PyExc_ValueError, "not found", key_error)
            prepared_field_value = prepare_schema(field, field->default_value, path, size);
        }

        CHECK(prepared_field_value, value_error)
        PyList_SET_ITEM(result, i, prepared_field_value);
    }

    CHECK_STRING(count == 0, PyExc_ValueError, "extraneous fields provided", error)
    return result;

key_error:
    if (*path)
    {
        Py_DECREF(*path);
    }

    *path = format_string_safe("record field %S", key);
    goto error;

value_error:
    if (*path)
    {
        PyObject* temp = format_string_safe("%S of value of record field %S", *path, key);
        Py_DECREF(*path);
        *path = temp;
    }
    else
    {
        *path = format_string_safe("value of record field %S", key);
    }

error:
    Py_XDECREF(result);
    return NULL;
}

/* Preparation function for object schemas. */
static PyObject* prepare_object_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result = NULL;

    Py_ssize_t tuple_size;
    PyObject* type;
    PyObject* object;

    CHECK_STRING(PyTuple_Check(value), PyExc_TypeError, "value must be tuple", error)
    tuple_size = PyTuple_GET_SIZE(value);

    if (tuple_size == 0)
    {
        Py_INCREF(value);
        *size += 1;
        return value;
    }

    CHECK_STRING(tuple_size, PyExc_TypeError, "value must contain type and object", error)
    result = PyTuple_New(2);
    CHECK(result, error)
    type = PyTuple_GET_ITEM(value, 0);
    Py_INCREF(type);
    PyTuple_SET_ITEM(result, 0, type);
    object = PyTuple_GET_ITEM(value, 1);

    if (PyObject_TypeCheck(type, &Schema_type))
    {
        PyObject* object_tuple;
        Py_ssize_t object_size;

        object_tuple = PyTuple_New(2);
        CHECK(object_tuple, error)
        PyTuple_SET_ITEM(result, 1, object_tuple);
        object_size = 0;
        object = prepare_schema((Schema*)type, object, path, &object_size);
        CHECK(object, object_error)
        PyTuple_SET_ITEM(object_tuple, 1, object);
        object = PyLong_FromSsize_t(object_size);
        CHECK(object, error)
        PyTuple_SET_ITEM(object_tuple, 0, object);
        *size += object_size + size_long((PY_LONG_LONG)object_size);
    }
    else
    {
        Py_ssize_t record_size;

        CHECK_STRING(RecordType_check(type), PyExc_TypeError, "type must be Schema or RecordType", error)
        CHECK_STRING(Record_check(object), PyExc_TypeError, "object must be Record", object_error)
        CHECK_STRING(((Record*)object)->type == (RecordType*)type, PyExc_ValueError, "object has incorrect RecordType", object_error)
        record_size = size_record((Record*)object);
        *size += record_size + size_long((PY_LONG_LONG)record_size);
        Py_INCREF(object);
        PyTuple_SET_ITEM(result, 1, object);
    }

    return result;

object_error:
    if (*path)
    {
        PyObject* temp = format_string_safe("%S of object", *path);
        Py_DECREF(*path);
        *path = temp;
    }
    else
    {
        *path = format_string_safe("object");
    }

error:
    Py_XDECREF(result);
    return NULL;
}

/* Preparation function for object_array Schemas. */
static PyObject* prepare_object_array_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    PyObject* result = NULL;
    PyObject* object_seq = NULL;

    Py_ssize_t tuple_size;
    PyObject* type;
    char is_schema;
    Py_ssize_t count;
    PyObject* prepared;

    Py_ssize_t i;

    CHECK_STRING(PyTuple_Check(value), PyExc_TypeError, "value must be tuple", error)
    tuple_size = PyTuple_GET_SIZE(value);

    if (tuple_size == 0)
    {
        Py_INCREF(value);
        *size += 1;
        return value;
    }

    CHECK_STRING(tuple_size == 2, PyExc_TypeError, "value must contain type and object list", error)
    type = PyTuple_GET_ITEM(value, 0);
    is_schema = PyObject_TypeCheck(type, &Schema_type);

    if (!is_schema)
    {
        CHECK_STRING(RecordType_check(type), PyExc_TypeError, "type must be Schema or RecordType", error)
    }

    result = PyTuple_New(2);
    CHECK(result, error)
    Py_INCREF(type);
    PyTuple_SET_ITEM(result, 0, type);
    object_seq = PySequence_Fast(PyTuple_GET_ITEM(value, 1), "object list must be iterable");
    CHECK(object_seq, error)
    count = PySequence_Fast_GET_SIZE(object_seq);
    prepared = PyList_New(count);
    CHECK(prepared, error)
    PyTuple_SET_ITEM(result, 1, prepared);

    if (is_schema)
    {
        for (i = 0; i < count; ++i)
        {
            PyObject* object_tuple;
            PyObject* object;
            Py_ssize_t object_size;

            object_tuple = PyTuple_New(2);
            CHECK(object_tuple, error)
            PyList_SET_ITEM(prepared, i, object_tuple);
            object_size = 0;
            object = prepare_schema((Schema*)type, PySequence_Fast_GET_ITEM(object_seq, i), path, &object_size);
            CHECK(object, object_error)
            PyTuple_SET_ITEM(object_tuple, 1, object);
            object = PyLong_FromSsize_t(object_size);
            CHECK(object, error)
            PyTuple_SET_ITEM(object_tuple, 0, object);
            *size += object_size + size_long((PY_LONG_LONG)object_size);
        }
    }
    else
    {
        for (i = 0; i < count; ++i)
        {
            PyObject* object;
            Py_ssize_t record_size;

            object = PySequence_Fast_GET_ITEM(object_seq, i);
            CHECK_STRING(Record_check(object), PyExc_TypeError, "object must be Record", object_error)
            CHECK_STRING(((Record*)object)->type == (RecordType*)type, PyExc_ValueError, "object has incorrect RecordType", object_error)
            Py_INCREF(object);
            PyList_SET_ITEM(prepared, i, object);
            record_size = size_record((Record*)object);
            *size += record_size + size_long((PY_LONG_LONG)record_size);
        }
    }

    *size += size_long((PY_LONG_LONG)count) + (count > 0 ? 1 : 0);
    Py_DECREF(object_seq);
    return result;

object_error:
    if (*path)
    {
        PyObject* temp = format_string_safe("%S of array object %zd", *path, i);
        Py_DECREF(*path);
        *path = temp;
    }
    else
    {
        *path = format_string_safe("array object %zd", i);
    }

error:
    Py_XDECREF(result);
    Py_XDECREF(object_seq);
    return NULL;
}

/* Preparation function dispatch table. */
static PrepareSchemaTypeFunc prepare_schema_types[SDT_MAX] =
{
    prepare_nullable_schema,    /* SDT_NULLABLE */
    prepare_boolean_schema,     /* SDT_BOOLEAN */
    prepare_bytes_schema,       /* SDT_BYTES */
    prepare_double_schema,      /* SDT_DOUBLE */
    prepare_float_schema,       /* SDT_FLOAT */
    prepare_int_schema,         /* SDT_INT */
    prepare_long_schema,        /* SDT_LONG */
    prepare_string_schema,      /* SDT_STRING */
    prepare_array_schema,       /* SDT_ARRAY */
    prepare_map_schema,         /* SDT_MAP */
    prepare_record_schema,      /* SDT_RECORD */
    prepare_object_schema,      /* SDT_OBJECT */
    prepare_object_array_schema /* SDT_OBJECT_ARRAY */
};

/* Internal function to prepare an object for writing using a Schema object.
   The function signature is the same as for PrepareSchemaTypeFunc above. */
static PyObject* prepare_schema(Schema* schema, PyObject* value, PyObject** path, Py_ssize_t* size)
{
    return prepare_schema_types[schema->data_type](schema, value, path, size);
}

/*----------------------------------------------------------------------------*/

/* Schema writing functions.

   Writing is the second step of the two-step process for writing an object
   into a buffer in Avro-encoded binary form using a Schema. It requires a
   previously "prepared" form of the object being written, and that an
   appropriately-sized buffer has already been allocated. */

/* Function type for a function that writes a prepared object in Avro-encoded
   binary form into a buffer using a Schema object with a specific data type
   for encoding. All functions of this type take these parameters:

   schema: The Schema object to use for encoding.

   pos: Pointer to a pointer to the position within the buffer to write the
        object. This will be updated on a successful write to point to the
        position immediately following the object data on return. If an error
        occurs, this may point to an arbitrary location between the initial
        value and max on return.

   max: Pointer to the end of the buffer (must be >= *pos). If this is reached
        before the object data is completely written, an exception is set.

   value: The prepared object to write, previously returned by prepare_schema.

   Returns 1 on successful, or 0 (with exception set) if not. */
typedef int (*WriteSchemaTypeFunc)(Schema*, uint8_t**, uint8_t*, PyObject*);

/* Forward declaration of writing function dispatch table. Each entry in the
   table contains a pointer to the Schema writing function for one data type. */
static WriteSchemaTypeFunc write_schema_types[SDT_MAX];

/* Writing function for nullable Schemas. */
static int write_nullable_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    if (value == Py_None)
    {
        return handle_write_error(write_long(pos, max, 1));
    }
    else
    {
        CHECK(handle_write_error(write_long(pos, max, 0)), error)
        return write_schema((Schema*)PyTuple_GET_ITEM(schema->fields, 0), pos, max, value);
    }

error:
    return 0;
}

/* Writing function for boolean Schemas. */
static int write_boolean_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    if (value == Py_True)
    {
        return handle_write_error(write_boolean(pos, max, 1));
    }
    else
    {
        return handle_write_error(write_boolean(pos, max, 0));
    }
}

/* Writing function for bytes Schemas. */
static int write_bytes_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    #if PY_MAJOR_VERSION >= 3
        return handle_write_error(write_bytes(pos, max, (uint8_t*)PyBytes_AS_STRING(value), PyBytes_GET_SIZE(value)));
    #else
        return handle_write_error(write_bytes(pos, max, (uint8_t*)PyString_AS_STRING(value), PyString_GET_SIZE(value)));
    #endif
}

/* Writing function for double Schemas. */
static int write_double_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    return handle_write_error(write_double(pos, max, PyFloat_AS_DOUBLE(value)));
}

/* Writing function for float Schemas. */
static int write_float_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    return handle_write_error(write_float(pos, max, (float)PyFloat_AS_DOUBLE(value)));
}

/* Writing function for int Schemas. */
static int write_int_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    #if PY_MAJOR_VERSION >= 3
        return handle_write_error(write_int(pos, max, PyLong_AsLong(value)));
    #else
        return handle_write_error(write_int(pos, max, PyInt_AS_LONG(value)));
    #endif
}

/* Writing function for long Schemas. */
static int write_long_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    return handle_write_error(write_long(pos, max, PyLong_AsLongLong(value)));
}

/* Writing function for array Schemas. */
static int write_array_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    Py_ssize_t count;
    Schema* field;

    Py_ssize_t i;

    count = PyList_GET_SIZE(value);
    CHECK(handle_write_error(write_size(pos, max, count)), error)

    if (count == 0)
    {
        return 1;
    }

    field = (Schema*)PyTuple_GET_ITEM(schema->fields, 0);

    for (i = 0; i < count; ++i)
    {
        CHECK(write_schema(field, pos, max, PyList_GET_ITEM(value, i)), error)
    }

    return handle_write_error(write_size(pos, max, 0));

error:
    return 0;
}

/* Writing function for map Schemas. */
static int write_map_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    Py_ssize_t count;
    Schema* field;

    Py_ssize_t i;

    count = PyList_GET_SIZE(value);
    CHECK(handle_write_error(write_size(pos, max, count / 2)), error)

    if (count == 0)
    {
        return 1;
    }

    field = (Schema*)PyTuple_GET_ITEM(schema->fields, 0);

    for (i = 0; i < count; i+= 2)
    {
        CHECK(write_bytes_schema(schema, pos, max, PyList_GET_ITEM(value, i)), error)
        CHECK(write_schema(field, pos, max, PyList_GET_ITEM(value, i + 1)), error)
    }

    return handle_write_error(write_size(pos, max, 0));

error:
    return 0;
}

/* Writing function for record Schemas. */
static int write_record_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    Py_ssize_t field_count;

    Py_ssize_t i;

    field_count = PyTuple_GET_SIZE(schema->fields);

    for (i = 0; i < field_count; ++i)
    {
        CHECK(write_schema((Schema*)PyTuple_GET_ITEM(schema->fields, i), pos, max, PyList_GET_ITEM(value, i)), error)
    }

    return 1;

error:
    return 0;
}

/* Writing function for object Schemas. */
static int write_object_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    PyObject* type;

    if (PyTuple_GET_SIZE(value) == 0)
    {
        CHECK(handle_write_error(write_size(pos, max, 0)), error);
        return 1;
    }

    type = PyTuple_GET_ITEM(value, 0);

    if (PyObject_TypeCheck(type, &Schema_type))
    {
        PyObject* object_tuple = PyTuple_GET_ITEM(value, 1);
        CHECK(handle_write_error(write_size(pos, max, PyLong_AsSsize_t(PyTuple_GET_ITEM(object_tuple, 0)))), error)
        return write_schema((Schema*)type, pos, max, PyTuple_GET_ITEM(object_tuple, 1));
    }
    else
    {
        Record* record = (Record*)PyTuple_GET_ITEM(value, 1);
        CHECK(handle_write_error(write_size(pos, max, size_record(record))), error)
        return handle_write_error(write_record(pos, max, record));
    }

error:
    return 0;
}

/* Writing function for object_array Schemas. */
static int write_object_array_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    Record** records = NULL;

    PyObject* type;
    PyObject* objects;
    Py_ssize_t count;

    Py_ssize_t i;

    if (PyTuple_GET_SIZE(value) == 0)
    {
        CHECK(handle_write_error(write_size(pos, max, 0)), error)
        return 1;
    }

    type = PyTuple_GET_ITEM(value, 0);
    objects = PyTuple_GET_ITEM(value, 1);
    count = PyList_GET_SIZE(objects);
    CHECK(handle_write_error(write_size(pos, max, count)), error)

    if (count == 0)
    {
        return 1;
    }

    if (PyObject_TypeCheck(type, &Schema_type))
    {
        for (i = 0; i < count; ++i)
        {
            PyObject* object_tuple = PyList_GET_ITEM(objects, i);
            CHECK(handle_write_error(write_size(pos, max, PyLong_AsSsize_t(PyTuple_GET_ITEM(object_tuple, 0)))), error)
            CHECK(write_schema((Schema*)type, pos, max, PyTuple_GET_ITEM(object_tuple, 1)), error)
        }
    }
    else
    {
        AvroErrorCode error;

        records = PyMem_New(Record*, count);
        CHECK_NONE(records, PyExc_MemoryError, error)

        for (i = 0; i < count; ++i)
        {
            records[i] = (Record*)PyList_GET_ITEM(objects, i);
        }

        Py_BEGIN_ALLOW_THREADS

        for (i = 0; i < count; ++i)
        {
            Record* record = records[i];
            Py_ssize_t size = size_record(record);
            error = write_size(pos, max, size);

            if (error != ERR_NONE)
            {
                break;
            }

            error = write_record(pos, max, record);

            if (error != ERR_NONE)
            {
                break;
            }
        }

        Py_END_ALLOW_THREADS;

        CHECK(handle_write_error(error), error)
        PyMem_Free(records);
        records = NULL;
    }

    CHECK(handle_write_error(write_size(pos, max, 0)), error)
    return 1;

error:
    PyMem_Free(records);
    return 0;
}

/* Schema writing function dispatch table. */
static WriteSchemaTypeFunc write_schema_types[SDT_MAX] =
{
    write_nullable_schema,    /* SDT_NULLABLE */
    write_boolean_schema,     /* SDT_BOOLEAN */
    write_bytes_schema,       /* SDT_BYTES */
    write_double_schema,      /* SDT_DOUBLE */
    write_float_schema,       /* SDT_FLOAT */
    write_int_schema,         /* SDT_INT */
    write_long_schema,        /* SDT_LONG */
    write_bytes_schema,       /* SDT_STRING */
    write_array_schema,       /* SDT_ARRAY */
    write_map_schema,         /* SDT_MAP */
    write_record_schema,      /* SDT_RECORD */
    write_object_schema,      /* SDT_OBJECT */
    write_object_array_schema /* SDT_OBJECT_ARRAY */
};

/* Internal function to write a prepared object in Avro-encoded binary form
   into a buffer using a Schema object for encoding. The function signature is
   the same as for WriteSchemaTypeFunc above. */
static int write_schema(Schema* schema, uint8_t** pos, uint8_t* max, PyObject* value)
{
    return write_schema_types[schema->data_type](schema, pos, max, value);
}

/*----------------------------------------------------------------------------*/

/* Schema data type names. Used to populate schema_data_type_names tuple in
   module state during initialization. */
static char* schema_data_type_names[SDT_MAX] =
{
    "nullable",
    "boolean",
    "bytes",
    "double",
    "float",
    "int",
    "long",
    "string",
    "array",
    "map",
    "record",
    "object",
    "object_array"
};

/* File initialization: called during module initialization. */
int init_schema(PyObject* module)
{
    ProtocolState* state;

    int i;

    state = GET_STATE_MODULE(module);
    CHECK(state, error)

    CHECK(PyType_Ready(&Schema_type) == 0, error)

    state->schema_data_type_names = PyTuple_New(SDT_MAX);
    CHECK(state->schema_data_type_names, error)

    for (i = 0; i < SDT_MAX; ++i)
    {
        PyObject* name = PyUnicode_FromString(schema_data_type_names[i]);
        CHECK(name, error)
        CHECK(PyTuple_SetItem(state->schema_data_type_names, i, name) == 0, error)
    }

    Py_INCREF(&Schema_type);
    CHECK(PyModule_AddObject(module, "Schema", (PyObject*)&Schema_type) == 0, error)

    return 1;

error:
    return 0;
}


Despite growing implementation and acceptance of biometrical identification methods like fingerprints, palm prints, iris scan, voice recognition and face recognition, passwords are still widely used. At this point in time, their usage actually still outweighs that of all biometrics combined.
There is little doubt that the day will come when biometrics will take over personal identification for good and passwords will become a thing of the past. We are not there yet; the password concept is not giving up easily. Why are not biometrics completely taking over this rather outdated, flawed thing?
All new high-end smartphones that feature biometrical identification for unlocking still sport a password system, just in case the first system fails! This says it all. No method is perfect. Biometrical sensors sometime fail to detect the pattern they are supposed to read, and we, as human beings, are all prone to forgetting that password at the most crucial moment.
The list of the strengths and weaknesses of each system is long, and a comprehensive comparison of all biometrics and passwords would be the subject of a technical article in a specialised high-tech magazine. Passwords, however, present two major, undeniable advantages: people have been using them for ages, long before the digital era, and they do not require any additional technical equipment. In a password's validation process there are no physical sensors or sophisticated software that could fail – a perfect case of "simple is beautiful".
In a world heavily relying on technology, the extreme simplicity of the password concept gives comfort to all those who are not particularly technically-minded, which is still the case of a large number of users.
Apart from the understood and annoying fact that you have to remember them, the main weakness of passwords is the poor security element that is often associated with them, and it comes from the way people use them, despite repeated recommendations from the IT community.
Since the advent of personal computing in the early 1980s, consumers have been given simple, basic rules, first about how to create a password, and second about how to be careful storing it, not disclosing it, and not saving on a computer or device that many other people use.
At the risk of over repeating it or perhaps even irritating those who do not need to be reminded: do not make passwords that are shorter than eight or ten characters. Combine capitals, small letters, numbers and special signs. Before anything else, the golden rule is not to use any known pattern, number, name or series that makes any sense or that follows a logic of some kind.
People who do not obey that golden rule unconsciously want to create a password that they can remember. But this is precisely the point, it is not supposed to be easily remembered. It is meant to be "weird", totally meaningless and complex. Like for example "hG6TwMQ#8 per centm$". Most if not all people are aware of that golden rule but about 10 to 15 per cent of the population still does not apply it. Otherwise how to explain that email boxes are still hacked every day?
My Samsung's Galaxy phone has fingerprint and face recognition for unlocking it. The first method works fine most of the time, though not all the time. The second works, well…. from time to time. In all cases, however, and although I mainly use the fingerprint method to unlock it, the handset's Android operating system requests the entry of a password about once or twice a week — totally rejecting and bypassing the fingerprint and face recognition methods. This is not a flaw and not an error I would be making, this is just how the concept is built. It still counts on passwords every now and then, as an extra precaution.
There are several applications where eye iris scan has been adopted for many years now for unequivocal and perfectly safe personal identification. These include security control at airports and border points in general, and at a few banks (at ATM and counter). Iris scan is infinitely superior and safer than any other method. Experience shows, however, that choosing biometrics instead of passwords is being decided on a case-by-case basis —until further news or development.


After the fall of Saddam Hussein, the United States successfully pressed creditors to write off much of Iraq's external debt. As political change again sweeps across the Middle East, the issue of so-called "odious" debt is back.
NEW YORK – After Saddam Hussein's fall, the United States successfully pressed creditors to write off much of Iraq's external debt. Senior American officials, including Paul Wolfowitz, later President of the World Bank, argued that the Iraqi people should not be saddled with obligations that the dictator contracted in order to enrich himself and oppress his subjects. Citing a long-standing doctrine in international law, advocates of a write-off claimed that Iraq's debt was "odious." As a result, the creditors were no longer protected under global legal rules.
As political change again sweeps across the Middle East, the issue of odious debt is back. But all debt that was contracted by a previous oppressive regime cannot, for that reason alone, be classified as "odious." The question is this: how much of the money went to meritorious development projects, and how much went instead to prop up the regime and line its leaders' pockets?
In the case of Egypt, for example, Minister of Economic Cooperation Faiza Abu al-Naga suggested in January that Mubarak himself was directing the country's foreign-loan policy and "oversee[ing] the entire process." Even if true, there may be nothing legally wrong with such an arrangement. After all, lending and aid to Egypt by the US and other Western powers have long been entangled with geopolitics, and using the tools of finance to back a loyal ally in an unstable region is not odious or illegitimate per se.
Robert Howse is a professor at New York University School of Law.
Ruti Teitel is the Ernst C. Stiefel Professor of Comparative Law at New York Law School, a lifetime member of the Council on Foreign Relations, and the author of Globalizing Transitional Justice. Her forthcoming book from Oxford University Press is Righting Our Global Wrongs: The President and Justice for American Empire.


text:
Битва при Фетанге () — битва между Надир-шахом и Ильбарс-ханом II, правителем Хивинского ханства, произошла в рамках похода Надира в Среднюю Азию.

История
Когда Надир был в Индии, воины Хивинского ханства периодически нападали и грабили Хорасан. В ответ на это наследный принц Надир-шаха в Хорасане Реза Кули-мирза двинулся в Среднюю Азию и одержал победу. Но Надир не дал ему продолжить свои атаки. Потому что его целью было вернуться и провести большую кампанию под своим командованием.

Вернувшись после великой победы в Индии, Надир двинулся в Среднюю Азию. Его целью были бухарский судья Абульфаз-хан и хивинский судья Ильбарс-хан. В конце 1740 года Абульфаз-хан потерпел поражение. Абульфаз-хан, заявивший, что подчинится Надиру после того, как тот потерпит поражение, был поставлен во главе Туркестанского княжества. Хотя такое предложение было сделано Ильбарсу-хану, он отклонил это предложение.

Первичные столкновения
Пока Надир-шах был ещё в Бухаре, он написал Ильбарсу-хану и попросил его следовать за ним как Абульфаз-хан, но это предложение было отклонено. Ильбарс-хан также решил отправить в Чарджо 30-тысячный отряд узбекской, аральской, ямутской и туркменской конницы под предводительством Мухаммадали-хана Кидила. Пусть эта группа быстрее группы армий Надир-шаха достигнет Чарджо, разрушит переправу через Амударью и тем затруднит переправу войск Надир-шаха через реку. Ильбарс-хан был уверен, что рассеет войска Надир-шаха с его армией под предводительством Мухаммадали-хана и заставит его отступить. Когда эта группа войск вышла на расстояние 6 миль от Чарджо, Надир-шах узнал об обстановке. Было ясно, что захват переправы через Аму-Дарью у Чарджо осложнит поход Надир-шаха к Хорезме. Поэтому Надир-шах оставил груз и припасы войска и ускорил движение к Чарджо со своей группой опытных воинов. Оставленный багаж и припасы должны были продолжить движение с сопровождающим отрядом и присоединиться к Надир-шаху в Чарджо. Рзагулу Мирзе было поручено руководство этим отрядом. В отряде под предводительством Надир-шаха было 20 000 воинов. Кроме того, Надир-шах направил послание Махаммадреза-хану Гырхли, бейларбею Мерва, чтобы тот присоединился к нему с трехтысячным мервским отрядом, назначенным для охраны вокруг Чарджо. Поскольку жители Мерва постоянно враждовали с туркменскими или узбекскими племенами, Надир Шах считал полезным использовать против этих племен отряд Мерва.

Надир-шаху также было известно, что у Ильбарс-хана сильная и боеспособная армия. Поэтому при движении к Чарджо обеспечивал прикрытие войск с флангов, тыла и фронта, а также компактное движение войск. Со временем Надир-шах достиг Чарджо, называемого рекой Аму-Дарья, и расположился лагерем на другом берегу. Через сутки подошли остальные силы и нагнали его. Достигнув Чарджо, Надир-шах согласился на возвращение Рзагулу Мирзы. Завершив необходимые организационные работы в Чарджо, Надир-шах двинулся со своим войском в место, называемое долиной Девабойну. Перед уходом он перегруппировал войско, и вся армия была разделена на четыре большие группы. За этой группой наступала основная часть войск. Две другие группы сформировали группы фланговой охраны. Кроме того, отряд в 6000 человек двигался вдоль берега и обеспечивал охрану корабля, перевозившего припасы в реке. Когда они достигли долины Девабойну, Надир-шах приказал войскам остановиться, и по его приказу армейский обоз был собран и для его охраны был выделен особый отряд. Прибытие Надир-шаха в долину Девабойну совпадает с последними днями октября 1740 года.

В сражении между передовыми частями обеих армий победу одержали части Надира.

Битва
Противоположные стороны сошлись лицом к лицу возле замка Фетанк. Перед началом боя обе стороны сгруппировали свои силы. Были определены центр, правый и левый фланги, засады и резервные силы, каждый из которых был возложен на опытных генералов. Группа из 6000 воинов Ямута была одной из первых, кто присоединился к битве в качестве колесных мастеров войск Ильбарса. Они отрезали пути движения войскам Надир-шаха и вызвали обострение боя. Воины Ямута проявили такую решимость, что смогли сломить сопротивление войск Надир-шаха на двух направлениях фронта и продвинуться вперед. Чтобы продвижение ямутов не стало опасным, сам Надир-шах немедленно бросился на поле боя и выдвинулся навстречу войскам ямутов. Надир-шаха сопровождала конница Афшар, Марв и Атак, которые всегда подвергались мародерским набегам ямутской и туркестанской конницы и искали случая отомстить ямутам. Этим шагом Надир-шах создал силовое преимущество на том направлении, где ямуты временно добились успеха, и очень скоро это силовое преимущество превратилось в военное преимущество. Ямутам, рвавшимся вперед, был нанесен серьёзный удар, и в их рядах поднялась волна. Напряжение на поле боя усилилось с вступлением в бой Надир-шаха. В целом, хотя реальную картину того, что происходило вокруг крепости Фетанк, создать невозможно, известно, что успех, достигнутый Надир-шахом на том направлении, на котором он участвовал в сражении, сказался на общем результате. Встретив упорное сопротивление Надир-шаха, ямутские воины были вынуждены отступить, их действия отразились на других отрядах, и Ильбарс-хану пришлось в следующем столкновении бежать назад, беспомощному перед храбростью войск Надир-шаха. Многие были убиты и взяты в плен во время преследования его войск. Ильбарс-хан спас свои войска от уничтожения, когда они подошли к крепости Хезарасб.

Итог
В результате этой битвы Надир разбил войска Ильбарс-хана. Крепость Хезаресб, где позже укрылся Ильбарс-хан, и крепость Ханега были захвачены Надиром. Наконец Ильбарс-хан, истощивший все свои силы, был схвачен и казнен Надиром.

Примечания

Литература

Сражения по алфавиту
Военная история
История Центральной Азии
Походы Надир Шаха Афшара
Сражения 1740 года
Хивинское ханство

redpajama_set_name: RedPajamaWikipedia


Unfortunately, the general public still believes that PII and credit card data are the primary targets of cyber criminals. While this information is still stolen on a regular basis, it is important for the public to recognize a bigger picture and threats posed by criminals around the world.
Most criminals are probably not specifically targeting you or your company, but without an effective security posture it is easy to be singled out. Mass scanning tools and distributed systems make finding security weaknesses easier for criminals than ever before.
Ransoms - Cyber criminals infect a single machine or entire organization's network with malware that encrypts all the data, making it unreadable without the key. This technique has proven highly effective for attackers and the threat continues to grow.
Distributed Computing Power - Systems with weak security are often targeted in order to use the computing power at the will of the cyber criminal. When an attacker can execute software they've installed on thousands of machines, they can perform DDoS (Distributed Denial of Service) attacks on servers resulting in major outages. They can also use crypto currency mining malware to make a profit using other people's computing resources.
Financial Market Manipulation - News of a breach can cause a significant drop in the value of a company. Cyber crime rings have been known to short stock of a company they've attacked, shortly before releasing news of the breach.
Sell News of Breach - Reporters and media outlets are often happy to pay for news of a breach if it means they can be the first to report it to the public.
Blackmail - It is common for an attacker to steal a customer list or other proprietary information, then demand a large sum of money from the compromised company. The cyber criminal threatens to sell the data or contact the customer list to diminish brand credibility.
Email Access for Spear Phishing - Attackers gaining access to a company or individual's email system will often use that system to facilitate wire fraud or other malicious activities.
While a few cyber criminals focus on hacktivism and bragging rights, the vast majority are focused on generating income from targets of opportunity. It is important to realize there are many ways for criminals to profit from cyber attacks and they don't need to involve PII or credit card data.
When's the last time your company did a vulnerability scan? We offer a complimentary scan and consultation below!


text:
Абдураззаков — фамилия; имеет женскую форму Абдураззакова.
 Абдураззаков, Убайдулла Аббасович (1932—2014) — государственный деятель и дипломат Республики Узбекистан.
 Абдураззаков, Хабибулло Абдураззакович (род. 1937) — советский и таджикский актёр и режиссёр, Заслуженный артист Таджикской ССР.


text:
Wozo Marketing Group Serves as a News Outlet and Resource Center - WBOC-TV 16, Delmarvas News Leader, FOX 21 -
Marketing Group Serves as News Outlet, Resource Center for Eastern Shore's Haitian Community
Tuesday, December 17, 2019 3:25 PM EST
By Rodaris Richardson
rrichardson@wboc.com
SALISBURY, Md.- Wozo Marketing Group is located in downtown Salisbury, but works as a news outlet and resource center for the entire Haitian community on the Eastern Shore.
"We cover the news, we also explain how they can live in a different community in a different country, to respect the law and to know their rights," said Wozo Marketin Group CEO Roosevelt Pierre.
The group covers all type of news, including local, national and international. It airs six days a week, and it provides access to its news and radio talk shows via Facebook, YouTube and through it's app.
However, Wozo's mission is more than that.
"We consider ourselves as a mentor, as a guide for the community, because every single show we have here we help people to better understand what is the purpose to live in the United States and the community," said Mivida Domercant, Wozo's office manager.
Wozo also works as a resource center, and has been dedicated to serving Delmarva since 2015, providing for the needs of all minority groups, not just the Haitian community.
In 2020, Wozo plans to expand its scope.
"We will let them know and go deeper into how someone can move into the Eastern Shore, and what they need to do," added Pierre.
But Wozo's ultimate goal remains the same.
"We want to cut deep in what's going on in Delmarva, so that the Haitian community has a voice," said Isaac Marcelin, Wozo's CFO.
Marketing Group Serves as News Outlet, Resource Center for Eastern Shore's Haitian CommunityMore>>
Delmarvawide News
Most Popular StoriesMost Popular StoriesMore>>
Huge Crowd Has Their Say Over Wind Farms off Maryland and Delaware
Posted: Saturday, January 18 2020 6:33 PM EST2020-01-18 23:33:24 GMT
Updated: Saturday, January 18 2020 6:34 PM EST2020-01-18 23:34:51 GMT
A standing room only crowd descended in Ocean City to hear and be heard on the issue of wind power off the coasts of Maryland and Delaware. The Ocean City Fire Department estimates there were 1,850 people in attendance.More
DSP: Suspect Arrested in Bridgeville Assault
Photo: Delaware State Police
The Delaware State Police have arrested 48-year-old Kenneth E. Greene of Laurel, who is wanted on a felony assault charge.More
Former Greensboro Police Chief Sentenced for Malfeasance in Office
Posted: Friday, January 17 2020 3:11 PM EST2020-01-17 20:11:08 GMT
Updated: Friday, January 17 2020 5:30 PM EST2020-01-17 22:30:22 GMT
The former police chief of a Caroline County, Md. town where a black teenager died in custody has been sentenced in an unrelated case.
Most Popular VideosMost Popular VideosMore>>
A standing room only crowd descended in Ocean City to hear and be heard on the issue of wind power off the coasts of Maryland and Delaware. The Ocean City Fire Department estimates there were 1,850 people in attendance.
WBOC Evening Weather: January 18, 2020
The Delaware State Police have arrested 48-year-old Kenneth E. Greene of Laurel, who is wanted on a felony assault charge.
Maryland NewsMaryland NewsMore>>
Wind Turbines Get Fresh Look
The controversial proposal to put wind turbines off the coast of Ocean City will be heard before the Maryland Public Service Commission Saturday.More
The controversial proposal to put wind turbines off the coast of Ocean City will be heard before the Maryland Public Service Commission Saturday. More
Delmar Man Convicted of Armed Robbery
A Delmar man has been found guilty of armed robbery after a two-day trial in Wicomico County.More
A Delmar, Md. man has been found guilty of armed robbery after a two-day trial in Wicomico County. More
Traffic Advisory Issued for Fruitland Police Officer's Funeral
Spencer Wiersberg
The Wicomico County Sheriff's Office says it is expecting a very large procession on Saturday to honor Fruitland police officer Spencer Wiersberg, who passed away last weekend after a battle with bone cancer.More
Courthouse to be Named After Late US Rep. Elijah Cummings
Posted: Friday, January 17 2020 12:55 PM EST2020-01-17 17:55:30 GMT
Updated: Friday, January 17 2020 12:55 PM EST2020-01-17 17:55:30 GMT
The city of Baltimore will formally name a courthouse after the late U.S. Rep. Elijah Cummings in the first half of 2020.More
The city of Baltimore will formally name a courthouse after the late U.S. Rep. Elijah Cummings in the first half of 2020. More
Hebron Business Cited After Failing Alcohol Compliance Check
A routine compliance check conducted this week by the Wicomico County Alcohol Task Force resulted in a Hebron business being cited for underage alcohol sales.More
Maryland Democratic Party Announces Director's Resignation
Posted: Friday, January 17 2020 8:52 AM EST2020-01-17 13:52:12 GMT
Updated: Friday, January 17 2020 8:52 AM EST2020-01-17 13:52:12 GMT
The Maryland Democratic Party announced on Thursday that its executive director has resigned.More
The Maryland Democratic Party announced on Thursday that its executive director has resigned. More
Perdue Foundation Donates $1M to Help Combat Hunger
Posted: Thursday, January 16 2020 3:36 PM EST2020-01-16 20:36:23 GMT
Updated: Thursday, January 16 2020 3:38 PM EST2020-01-16 20:38:22 GMT
Salisbury, Maryland-based Perdue Farms announced Thursday a Franklin P. and Arthur W. Perdue Foundation-funded $1 million donation to support its Feeding America-affiliated food bank partners in celebration of the poultry company's 100th anniversary.More
Maryland Governor Proposes Tax Relief for Retirees
Maryland Gov. speaks about his tax reduction plan at a Thursday, Jan. 16, 2020, press conference. (Photo: Gov. Larry Hogan Facebook page)
Maryland would cut state income taxes for retirees under a proposal announced Thursday by Gov. Larry Hogan.More
Maryland would cut state income taxes for retirees under a proposal announced Thursday by Gov. Larry Hogan. More

redpajama_set_name: RedPajamaCommonCrawl


text:
You are here: » Networks and ordination analyses reveal the stream commun...
Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment
Research output: Contribution to journal/Conference contribution in journal/Contribution to newspaper › Journal article › Research › peer-review
Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment. / Qu, Xiaodong; Peng, Wenqi; Liu, Yang et al.
In: Ecological Indicators, Vol. 101, 06.2019, p. 501-511.
Qu, X, Peng, W, Liu, Y, Zhang, M, Ren, Z, Wu, N & Liu, X 2019, 'Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment', Ecological Indicators, vol. 101, pp. 501-511. https://doi.org/10.1016/j.ecolind.2019.01.030
Qu, X., Peng, W., Liu, Y., Zhang, M., Ren, Z., Wu, N., & Liu, X. (2019). Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment. Ecological Indicators, 101, 501-511. https://doi.org/10.1016/j.ecolind.2019.01.030
Qu X, Peng W, Liu Y, Zhang M, Ren Z, Wu N, Liu X. 2019. Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment. Ecological Indicators. 101:501-511. https://doi.org/10.1016/j.ecolind.2019.01.030
Qu, Xiaodong et al. "Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment". Ecological Indicators. 2019, 101. 501-511. https://doi.org/10.1016/j.ecolind.2019.01.030
Qu X, Peng W, Liu Y, Zhang M, Ren Z, Wu N et al. Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment. Ecological Indicators. 2019 Jun;101:501-511. doi: 10.1016/j.ecolind.2019.01.030
Qu, Xiaodong ; Peng, Wenqi ; Liu, Yang et al. / Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment. In: Ecological Indicators. 2019 ; Vol. 101. pp. 501-511.
@article{e6a18aeb157e478fa1b21e0f4b795bb2,
title = "Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment",
abstract = "Rivers are perhaps the most altered ecosystem due to the human disturbances of water contamination, habitat destruction, hydrological regime change, overexploitation and exotic species introduction. Knowledge about the integrated effects of both natural environmental gradients and anthropogenic disturbances on multiple aquatic organism at a watershed scale is limited. In this study, the physical, chemical, hydrological and biological data were collected in May 2009 and 2010 at 281 sampling sites spread out wide stress gradients in the Hun-Tai River. We conducted canonical correspondence analysis (CCA) and network analysis to reveal the aquatic assemblage structures of fish, macroinvertebrates and benthic algae under the steep gradients of nutrient concentration and multiple environmental parameters. The results showed that habitat quality, elevation, water depth, stream width, dissolved oxygen, conductivity, TN, TP, and N:P ratio were identified as the significant parameters on the community compositions of fish, macroinvertebrate and benthic algae. In contrast to the historical effects of organic pollution, the eutrophication became the currently dominant human influencing factor on aquatic organisms in the Hun-Tai River. The nutrient parameters of nitrogen and phosphorus, and stoichiometric of N:P (mainly influenced by TP) were all identified as the significant parameters in the CCA biplots of fish, macroinvertebrate and benthic algae. Fish community was closely related with stream hydromorphological and nutrients parameters, however, macroinvertebrate and periphyton communities were driven by morphometry, chemistry and nutrients parameters. Network analysis was used to reveal the taxa associations and their correlations with nutrient parameters. By calculating the topological parameters of the networks, both the benthic algae and macroinvertebrate assemblages showed higher values of network centralization, heterogeneity, and average numbers of neighbors than fish in the network, those which usually indicated the higher association among taxa and stability of community structures. Huigobio chinssuensis, Cobitis granoei, and Ctenogobius brunneus were the core fish taxa, the subfamily Tanypodiinae, Epeorus and Ephemerella were the core macroinvertebrate taxa, and genera of Diatom, Fragilaria, Achnanthes, Navicula, Cymbella, and Nitzschia were the core benthic algae taxa in the network. A few taxa of fish showed significantly negative correlations with enrichment of nutrients and increasing the N:P ratio, and significantly positive correlations only existed with N:P ratio. In contrast to the fish network, macroinvertebrate and benthic algae taxa showed stronger and broader negative correlations to that of enrichment and N:P ratios in the network. Meanwhile, the positive correlations also broadly existed in the benthic algae network. This finding offers new clues of specific taxa of macroinvertebrate and benthic algae could be used as the indicator of stream eutrophication. This study provided new insights into integrated application of ordination method and network analysis in structuring the assemblage composition. And the network analysis also improve our understanding of individual interactions in the community and their relations to environmental stressors at the species level.",
keywords = "Network analysis, Nutrient enrichment, Stoichiometry N:P ratio, The Hun-Tai River",
author = "Xiaodong Qu and Wenqi Peng and Yang Liu and Min Zhang and Ze Ren and Naicheng Wu and Xiaobo Liu",
doi = "10.1016/j.ecolind.2019.01.030",
journal = "Ecological Indicators",
T1 - Networks and ordination analyses reveal the stream community structures of fish, macroinvertebrate and benthic algae, and their responses to nutrient enrichment
AU - Qu, Xiaodong
AU - Peng, Wenqi
AU - Liu, Yang
AU - Zhang, Min
AU - Ren, Ze
AU - Wu, Naicheng
AU - Liu, Xiaobo
N2 - Rivers are perhaps the most altered ecosystem due to the human disturbances of water contamination, habitat destruction, hydrological regime change, overexploitation and exotic species introduction. Knowledge about the integrated effects of both natural environmental gradients and anthropogenic disturbances on multiple aquatic organism at a watershed scale is limited. In this study, the physical, chemical, hydrological and biological data were collected in May 2009 and 2010 at 281 sampling sites spread out wide stress gradients in the Hun-Tai River. We conducted canonical correspondence analysis (CCA) and network analysis to reveal the aquatic assemblage structures of fish, macroinvertebrates and benthic algae under the steep gradients of nutrient concentration and multiple environmental parameters. The results showed that habitat quality, elevation, water depth, stream width, dissolved oxygen, conductivity, TN, TP, and N:P ratio were identified as the significant parameters on the community compositions of fish, macroinvertebrate and benthic algae. In contrast to the historical effects of organic pollution, the eutrophication became the currently dominant human influencing factor on aquatic organisms in the Hun-Tai River. The nutrient parameters of nitrogen and phosphorus, and stoichiometric of N:P (mainly influenced by TP) were all identified as the significant parameters in the CCA biplots of fish, macroinvertebrate and benthic algae. Fish community was closely related with stream hydromorphological and nutrients parameters, however, macroinvertebrate and periphyton communities were driven by morphometry, chemistry and nutrients parameters. Network analysis was used to reveal the taxa associations and their correlations with nutrient parameters. By calculating the topological parameters of the networks, both the benthic algae and macroinvertebrate assemblages showed higher values of network centralization, heterogeneity, and average numbers of neighbors than fish in the network, those which usually indicated the higher association among taxa and stability of community structures. Huigobio chinssuensis, Cobitis granoei, and Ctenogobius brunneus were the core fish taxa, the subfamily Tanypodiinae, Epeorus and Ephemerella were the core macroinvertebrate taxa, and genera of Diatom, Fragilaria, Achnanthes, Navicula, Cymbella, and Nitzschia were the core benthic algae taxa in the network. A few taxa of fish showed significantly negative correlations with enrichment of nutrients and increasing the N:P ratio, and significantly positive correlations only existed with N:P ratio. In contrast to the fish network, macroinvertebrate and benthic algae taxa showed stronger and broader negative correlations to that of enrichment and N:P ratios in the network. Meanwhile, the positive correlations also broadly existed in the benthic algae network. This finding offers new clues of specific taxa of macroinvertebrate and benthic algae could be used as the indicator of stream eutrophication. This study provided new insights into integrated application of ordination method and network analysis in structuring the assemblage composition. And the network analysis also improve our understanding of individual interactions in the community and their relations to environmental stressors at the species level.
AB - Rivers are perhaps the most altered ecosystem due to the human disturbances of water contamination, habitat destruction, hydrological regime change, overexploitation and exotic species introduction. Knowledge about the integrated effects of both natural environmental gradients and anthropogenic disturbances on multiple aquatic organism at a watershed scale is limited. In this study, the physical, chemical, hydrological and biological data were collected in May 2009 and 2010 at 281 sampling sites spread out wide stress gradients in the Hun-Tai River. We conducted canonical correspondence analysis (CCA) and network analysis to reveal the aquatic assemblage structures of fish, macroinvertebrates and benthic algae under the steep gradients of nutrient concentration and multiple environmental parameters. The results showed that habitat quality, elevation, water depth, stream width, dissolved oxygen, conductivity, TN, TP, and N:P ratio were identified as the significant parameters on the community compositions of fish, macroinvertebrate and benthic algae. In contrast to the historical effects of organic pollution, the eutrophication became the currently dominant human influencing factor on aquatic organisms in the Hun-Tai River. The nutrient parameters of nitrogen and phosphorus, and stoichiometric of N:P (mainly influenced by TP) were all identified as the significant parameters in the CCA biplots of fish, macroinvertebrate and benthic algae. Fish community was closely related with stream hydromorphological and nutrients parameters, however, macroinvertebrate and periphyton communities were driven by morphometry, chemistry and nutrients parameters. Network analysis was used to reveal the taxa associations and their correlations with nutrient parameters. By calculating the topological parameters of the networks, both the benthic algae and macroinvertebrate assemblages showed higher values of network centralization, heterogeneity, and average numbers of neighbors than fish in the network, those which usually indicated the higher association among taxa and stability of community structures. Huigobio chinssuensis, Cobitis granoei, and Ctenogobius brunneus were the core fish taxa, the subfamily Tanypodiinae, Epeorus and Ephemerella were the core macroinvertebrate taxa, and genera of Diatom, Fragilaria, Achnanthes, Navicula, Cymbella, and Nitzschia were the core benthic algae taxa in the network. A few taxa of fish showed significantly negative correlations with enrichment of nutrients and increasing the N:P ratio, and significantly positive correlations only existed with N:P ratio. In contrast to the fish network, macroinvertebrate and benthic algae taxa showed stronger and broader negative correlations to that of enrichment and N:P ratios in the network. Meanwhile, the positive correlations also broadly existed in the benthic algae network. This finding offers new clues of specific taxa of macroinvertebrate and benthic algae could be used as the indicator of stream eutrophication. This study provided new insights into integrated application of ordination method and network analysis in structuring the assemblage composition. And the network analysis also improve our understanding of individual interactions in the community and their relations to environmental stressors at the species level.
KW - Nutrient enrichment
KW - Stoichiometry N:P ratio
KW - The Hun-Tai River
U2 - 10.1016/j.ecolind.2019.01.030
DO - 10.1016/j.ecolind.2019.01.030
JO - Ecological Indicators
JF - Ecological Indicators


We understand that our reputation is directly linked to our clients success and that is why we continually strive for excellence. we offer a tailored personalised service and believe in keeping out clients updated throughout the real estate lifecycle - whether you're new to the property market, buying, selling or investing we can provide the expertise and experience to make it a smooth and enjoyable process.
We recently engaged Kylie as a Buyer's Agent to search for a suitable property to buy in Toowoomba.
We have had the pleasure of knowing Kylie when she was involved in selling our house a few years ago.
We want to thank you, Kylie, for being a true professional with outstanding work ethics and values.


I have sometimes wondered why it [Einstein's theory of relativity] made such a great stir both among the general public and among philosophers.
I suppose it is this, that it meant the dethronement of time as a rigid tyrant imposed on us from outside, a liberation from the unbreakable rule of before and after. For indeed time is our most severe master by ostensibly restricting the existence of each of us to narrow limits – seventy or eighty years, as the Pentateuch has it.
To be allowed to play about with such a master's programme, believed unassailable until then, to play about with it albeit in a small way, seems to be a great relief, it seems to encourage the thought that the whole timetable is probably not quite as serious as it appears at first sight.