text:
package com.fasterxml.jackson.databind.util;

import java.io.*;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.TreeMap;

import com.fasterxml.jackson.core.*;
import com.fasterxml.jackson.core.base.ParserMinimalBase;
import com.fasterxml.jackson.core.json.JsonReadContext;
import com.fasterxml.jackson.core.json.JsonWriteContext;
import com.fasterxml.jackson.core.util.ByteArrayBuilder;
import com.fasterxml.jackson.databind.*;

/**
 * Utility class used for efficient storage of {@link JsonToken}
 * sequences, needed for temporary buffering.
 * Space efficient for different sequence lengths (especially so for smaller
 * ones; but not significantly less efficient for larger), highly efficient
 * for linear iteration and appending. Implemented as segmented/chunked
 * linked list of tokens; only modifications are via appends.
 *

 * Note that before version 2.0, this class was located in the "core"
 * bundle, not data-binding; but since it was only used by data binding,
 * was moved here to reduce size of core package
 */
public class TokenBuffer
/* Won't use JsonGeneratorBase, to minimize overhead for validity
 * checking
 */
    extends JsonGenerator
{
    protected final static int DEFAULT_GENERATOR_FEATURES = JsonGenerator.Feature.collectDefaults();

    /*
    /**********************************************************
    /* Configuration
    /**********************************************************
     */

    /**
     * Object codec to use for stream-based object
     * conversion through parser/generator interfaces. If null,
     * such methods can not be used.
     */
    protected ObjectCodec _objectCodec;

    /**
     * Bit flag composed of bits that indicate which
     * {@link com.fasterxml.jackson.core.JsonGenerator.Feature}s
     * are enabled.
     *

     * NOTE: most features have no effect on this class
     */
    protected int _generatorFeatures;

    protected boolean _closed;

    /**
     * @since 2.3
     */
    protected boolean _hasNativeTypeIds;

    /**
     * @since 2.3
     */
    protected boolean _hasNativeObjectIds;

    /**
     * @since 2.3
     */
    protected boolean _mayHaveNativeIds;

    /*
    /**********************************************************
    /* Token buffering state
    /**********************************************************
     */

    /**
     * First segment, for contents this buffer has
     */
    protected Segment _first;

    /**
     * Last segment of this buffer, one that is used
     * for appending more tokens
     */
    protected Segment _last;

    /**
     * Offset within last segment,
     */
    protected int _appendAt;

    /**
     * If native type ids supported, this is the id for following
     * value (or first token of one) to be written.
     */
    protected Object _typeId;

    /**
     * If native object ids supported, this is the id for following
     * value (or first token of one) to be written.
     */
    protected Object _objectId;

    /**
     * Do we currnetly have a native type or object id buffered?
     */
    protected boolean _hasNativeId = false;

    /*
    /**********************************************************
    /* Output state
    /**********************************************************
     */

    protected JsonWriteContext _writeContext;

    /*
    /**********************************************************
    /* Life-cycle
    /**********************************************************
     */

    /**
     * @param codec Object codec to use for stream-based object
     *   conversion through parser/generator interfaces. If null,
     *   such methods can not be used.
     *
     * @deprecated since 2.3 preferred variant is one that takes {@link JsonParser} or additional boolean parameter.
     */
    @Deprecated
    public TokenBuffer(ObjectCodec codec) {
        this(codec, false);
    }

    /**
     * @param codec Object codec to use for stream-based object
     *   conversion through parser/generator interfaces. If null,
     *   such methods can not be used.
     * @param hasNativeIds Whether resulting {@link JsonParser} (if created)
     *   is considered to support native type and object ids
     */
    public TokenBuffer(ObjectCodec codec, boolean hasNativeIds)
    {
        _objectCodec = codec;
        _generatorFeatures = DEFAULT_GENERATOR_FEATURES;
        _writeContext = JsonWriteContext.createRootContext(null);
        // at first we have just one segment
        _first = _last = new Segment();
        _appendAt = 0;
        _hasNativeTypeIds = hasNativeIds;
        _hasNativeObjectIds = hasNativeIds;

        _mayHaveNativeIds = _hasNativeTypeIds | _hasNativeObjectIds;
    }

    /**
     * @since 2.3
     */
    public TokenBuffer(JsonParser jp)
    {
        _objectCodec = jp.getCodec();
        _generatorFeatures = DEFAULT_GENERATOR_FEATURES;
        _writeContext = JsonWriteContext.createRootContext(null);
        // at first we have just one segment
        _first = _last = new Segment();
        _appendAt = 0;
        _hasNativeTypeIds = jp.canReadTypeId();
        _hasNativeObjectIds = jp.canReadObjectId();
        _mayHaveNativeIds = _hasNativeTypeIds | _hasNativeObjectIds;
    }

    @Override
    public Version version() {
        return com.fasterxml.jackson.databind.cfg.PackageVersion.VERSION;
    }

    /**
     * Method used to create a {@link JsonParser} that can read contents
     * stored in this buffer. Will use default _objectCodec for
     * object conversions.
     *

     * Note: instances are not synchronized, that is, they are not thread-safe
     * if there are concurrent appends to the underlying buffer.
     *
     * @return Parser that can be used for reading contents stored in this buffer
     */
    public JsonParser asParser()
    {
        return asParser(_objectCodec);
    }

    /**
     * Method used to create a {@link JsonParser} that can read contents
     * stored in this buffer.
     *

     * Note: instances are not synchronized, that is, they are not thread-safe
     * if there are concurrent appends to the underlying buffer.
     *
     * @param codec Object codec to use for stream-based object
     *   conversion through parser/generator interfaces. If null,
     *   such methods can not be used.
     *
     * @return Parser that can be used for reading contents stored in this buffer
     */
    public JsonParser asParser(ObjectCodec codec)
    {
        return new Parser(_first, codec, _hasNativeTypeIds, _hasNativeObjectIds);
    }

    /**
     * @param src Parser to use for accessing source information
     *    like location, configured codec
     */
    public JsonParser asParser(JsonParser src)
    {
        Parser p = new Parser(_first, src.getCodec(), _hasNativeTypeIds, _hasNativeObjectIds);
        p.setLocation(src.getTokenLocation());
        return p;
    }

    /*
    /**********************************************************
    /* Additional accessors
    /**********************************************************
     */

    public JsonToken firstToken() {
        if (_first != null) {
            return _first.type(0);
        }
        return null;
    }

    /*
    /**********************************************************
    /* Other custom methods not needed for implementing interfaces
    /**********************************************************
     */

    /**
     * Helper method that will append contents of given buffer into this
     * buffer.
     * Not particularly optimized; can be made faster if there is need.
     *
     * @return This buffer
     */
    @SuppressWarnings("resource")
    public TokenBuffer append(TokenBuffer other)
        throws IOException, JsonGenerationException
    {
        // Important? If source has native ids, need to store
        if (!_hasNativeTypeIds) {
            _hasNativeTypeIds = other.canWriteTypeId();
        }
        if (!_hasNativeObjectIds) {
            _hasNativeObjectIds = other.canWriteObjectId();
        }
        _mayHaveNativeIds = _hasNativeTypeIds | _hasNativeObjectIds;

        JsonParser jp = other.asParser();
        while (jp.nextToken() != null) {
            copyCurrentStructure(jp);
        }
        return this;
    }

    /**
     * Helper method that will write all contents of this buffer
     * using given {@link JsonGenerator}.
     *

     * Note: this method would be enough to implement
     * JsonSerializer  for TokenBuffer type;
     * but we can not have upwards
     * references (from core to mapper package); and as such we also
     * can not take second argument.
     */
    public void serialize(JsonGenerator jgen)
        throws IOException, JsonGenerationException
    {
        Segment segment = _first;
        int ptr = -1;

        final boolean checkIds = _mayHaveNativeIds;
        boolean hasIds = checkIds && (segment.hasIds());

        while (true) {
            if (++ptr >= Segment.TOKENS_PER_SEGMENT) {
                ptr = 0;
                segment = segment.next();
                if (segment == null) break;
                hasIds = checkIds && (segment.hasIds());
            }
            JsonToken t = segment.type(ptr);
            if (t == null) break;

            if (hasIds) {
                Object id = segment.findObjectId(ptr);
                if (id != null) {
                    jgen.writeObjectId(id);
                }
                id = segment.findTypeId(ptr);
                if (id != null) {
                    jgen.writeTypeId(id);
                }
            }

            // Note: copied from 'copyCurrentEvent'...
            switch (t) {
            case START_OBJECT:
                jgen.writeStartObject();
                break;
            case END_OBJECT:
                jgen.writeEndObject();
                break;
            case START_ARRAY:
                jgen.writeStartArray();
                break;
            case END_ARRAY:
                jgen.writeEndArray();
                break;
            case FIELD_NAME:
            {
                // 13-Dec-2010, tatu: Maybe we should start using different type tokens to reduce casting?
                Object ob = segment.get(ptr);
                if (ob instanceof SerializableString) {
                    jgen.writeFieldName((SerializableString) ob);
                } else {
                    jgen.writeFieldName((String) ob);
                }
            }
                break;
            case VALUE_STRING:
                {
                    Object ob = segment.get(ptr);
                    if (ob instanceof SerializableString) {
                        jgen.writeString((SerializableString) ob);
                    } else {
                        jgen.writeString((String) ob);
                    }
                }
                break;
            case VALUE_NUMBER_INT:
                {
                    Object n = segment.get(ptr);
                    if (n instanceof Integer) {
                        jgen.writeNumber((Integer) n);
                    } else if (n instanceof BigInteger) {
                        jgen.writeNumber((BigInteger) n);
                    } else if (n instanceof Long) {
                        jgen.writeNumber((Long) n);
                    } else if (n instanceof Short) {
                        jgen.writeNumber((Short) n);
                    } else {
                        jgen.writeNumber(((Number) n).intValue());
                    }
                }
                break;
            case VALUE_NUMBER_FLOAT:
                {
                    Object n = segment.get(ptr);
                    if (n instanceof Double) {
                        jgen.writeNumber(((Double) n).doubleValue());
                    } else if (n instanceof BigDecimal) {
                        jgen.writeNumber((BigDecimal) n);
                    } else if (n instanceof Float) {
                        jgen.writeNumber(((Float) n).floatValue());
                    } else if (n == null) {
                        jgen.writeNull();
                    } else if (n instanceof String) {
                        jgen.writeNumber((String) n);
                    } else {
                        throw new JsonGenerationException("Unrecognized value type for VALUE_NUMBER_FLOAT: "+n.getClass().getName()+", can not serialize");
                    }
                }
                break;
            case VALUE_TRUE:
                jgen.writeBoolean(true);
                break;
            case VALUE_FALSE:
                jgen.writeBoolean(false);
                break;
            case VALUE_NULL:
                jgen.writeNull();
                break;
            case VALUE_EMBEDDED_OBJECT:
                jgen.writeObject(segment.get(ptr));
                break;
            default:
                throw new RuntimeException("Internal error: should never end up through this code path");
            }
        }
    }

    /**
     * Helper method used by standard deserializer.
     *
     * @since 2.3
     */
    public TokenBuffer deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException
    {
        if (jp.getCurrentTokenId() != JsonToken.FIELD_NAME.id()) {
            copyCurrentStructure(jp);
            return this;
        }
        /* 28-Oct-2014, tatu: As per #592, need to support a special case of starting from
         *    FIELD_NAME, which is taken to mean that we are missing START_OBJECT, but need
         *    to assume one did exist.
         */
        JsonToken t;
        writeStartObject();
        do {
            copyCurrentStructure(jp);
        } while ((t = jp.nextToken()) == JsonToken.FIELD_NAME);
        if (t != JsonToken.END_OBJECT) {
            throw ctxt.mappingException("Expected END_OBJECT after copying contents of a JsonParser into TokenBuffer, got "+t);
        }
        writeEndObject();
        return this;
    }

    @Override
    @SuppressWarnings("resource")
    public String toString()
    {
        // Let's print up to 100 first tokens...
        final int MAX_COUNT = 100;

        StringBuilder sb = new StringBuilder();
        sb.append("[TokenBuffer: ");

        /*
sb.append("NativeTypeIds=").append(_hasNativeTypeIds).append(",");
sb.append("NativeObjectIds=").append(_hasNativeObjectIds).append(",");
*/

        JsonParser jp = asParser();
        int count = 0;
        final boolean hasNativeIds = _hasNativeTypeIds || _hasNativeObjectIds;

        while (true) {
            JsonToken t;
            try {
                t = jp.nextToken();
                if (t == null) break;

                if (hasNativeIds) {
                    _appendNativeIds(sb);
                }

                if (count < MAX_COUNT) {
                    if (count > 0) {
                        sb.append(", ");
                    }
                    sb.append(t.toString());
                    if (t == JsonToken.FIELD_NAME) {
                        sb.append('(');
                        sb.append(jp.getCurrentName());
                        sb.append(')');
                    }
                }
            } catch (IOException ioe) { // should never occur
                throw new IllegalStateException(ioe);
            }
            ++count;
        }

        if (count >= MAX_COUNT) {
            sb.append(" ... (truncated ").append(count-MAX_COUNT).append(" entries)");
        }
        sb.append(']');
        return sb.toString();
    }

    private final void _appendNativeIds(StringBuilder sb)
    {
        Object objectId = _last.findObjectId(_appendAt-1);
        if (objectId != null) {
            sb.append("[objectId=").append(String.valueOf(objectId)).append(']');
        }
        Object typeId = _last.findTypeId(_appendAt-1);
        if (typeId != null) {
            sb.append("[typeId=").append(String.valueOf(typeId)).append(']');
        }
    }

    /*
    /**********************************************************
    /* JsonGenerator implementation: configuration
    /**********************************************************
     */

    @Override
    public JsonGenerator enable(Feature f) {
        _generatorFeatures |= f.getMask();
        return this;
    }

    @Override
    public JsonGenerator disable(Feature f) {
        _generatorFeatures &= ~f.getMask();
        return this;
    }

    //public JsonGenerator configure(SerializationFeature f, boolean state) { }

    @Override
    public boolean isEnabled(Feature f) {
        return (_generatorFeatures & f.getMask()) != 0;
    }

    @Override
    public int getFeatureMask() {
        return _generatorFeatures;
    }

    @Override
    public JsonGenerator setFeatureMask(int mask) {
        _generatorFeatures = mask;
        return this;
    }

    @Override
    public JsonGenerator useDefaultPrettyPrinter() {
        // No-op: we don't indent
        return this;
    }

    @Override
    public JsonGenerator setCodec(ObjectCodec oc) {
        _objectCodec = oc;
        return this;
    }

    @Override
    public ObjectCodec getCodec() { return _objectCodec; }

    @Override
    public final JsonWriteContext getOutputContext() { return _writeContext; }

    /*
    /**********************************************************
    /* JsonGenerator implementation: capability introspection
    /**********************************************************
     */

    /**
     * Since we can efficiently store byte[], yes.
     */
    @Override
    public boolean canWriteBinaryNatively() {
        return true;
    }

    /*
    /**********************************************************
    /* JsonGenerator implementation: low-level output handling
    /**********************************************************
     */

    @Override
    public void flush() throws IOException { /* NOP */ }

    @Override
    public void close() throws IOException {
        _closed = true;
    }

    @Override
    public boolean isClosed() { return _closed; }

    /*
    /**********************************************************
    /* JsonGenerator implementation: write methods, structural
    /**********************************************************
     */

    @Override
    public final void writeStartArray()
        throws IOException, JsonGenerationException
    {
        _append(JsonToken.START_ARRAY);
        _writeContext = _writeContext.createChildArrayContext();
    }

    @Override
    public final void writeEndArray()
        throws IOException, JsonGenerationException
    {
        _append(JsonToken.END_ARRAY);
        // Let's allow unbalanced tho... i.e. not run out of root level, ever
        JsonWriteContext c = _writeContext.getParent();
        if (c != null) {
            _writeContext = c;
        }
    }

    @Override
    public final void writeStartObject()
        throws IOException, JsonGenerationException
    {
        _append(JsonToken.START_OBJECT);
        _writeContext = _writeContext.createChildObjectContext();
    }

    @Override
    public final void writeEndObject()
        throws IOException, JsonGenerationException
    {
        _append(JsonToken.END_OBJECT);
        // Let's allow unbalanced tho... i.e. not run out of root level, ever
        JsonWriteContext c = _writeContext.getParent();
        if (c != null) {
            _writeContext = c;
        }
    }

    @Override
    public final void writeFieldName(String name)
        throws IOException, JsonGenerationException
    {
        _append(JsonToken.FIELD_NAME, name);
        _writeContext.writeFieldName(name);
    }

    @Override
    public void writeFieldName(SerializableString name)
        throws IOException, JsonGenerationException
    {
        _append(JsonToken.FIELD_NAME, name);
        _writeContext.writeFieldName(name.getValue());
    }

    /*
    /**********************************************************
    /* JsonGenerator implementation: write methods, textual
    /**********************************************************
     */

    @Override
    public void writeString(String text) throws IOException,JsonGenerationException {
        if (text == null) {
            writeNull();
        } else {
            _append(JsonToken.VALUE_STRING, text);
        }
    }

    @Override
    public void writeString(char[] text, int offset, int len) throws IOException, JsonGenerationException {
        writeString(new String(text, offset, len));
    }

    @Override
    public void writeString(SerializableString text) throws IOException, JsonGenerationException {
        if (text == null) {
            writeNull();
        } else {
            _append(JsonToken.VALUE_STRING, text);
        }
    }

    @Override
    public void writeRawUTF8String(byte[] text, int offset, int length)
        throws IOException, JsonGenerationException
    {
        // could add support for buffering if we really want it...
        _reportUnsupportedOperation();
    }

    @Override
    public void writeUTF8String(byte[] text, int offset, int length)
        throws IOException, JsonGenerationException
    {
        // could add support for buffering if we really want it...
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRaw(String text) throws IOException, JsonGenerationException {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRaw(String text, int offset, int len) throws IOException, JsonGenerationException {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRaw(SerializableString text) throws IOException, JsonGenerationException {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRaw(char[] text, int offset, int len) throws IOException, JsonGenerationException {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRaw(char c) throws IOException, JsonGenerationException {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRawValue(String text) throws IOException, JsonGenerationException {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRawValue(String text, int offset, int len) throws IOException, JsonGenerationException {
        _reportUnsupportedOperation();
    }

    @Override
    public void writeRawValue(char[] text, int offset, int len) throws IOException, JsonGenerationException {
        _reportUnsupportedOperation();
    }

    /*
    /**********************************************************
    /* JsonGenerator implementation: write methods, primitive types
    /**********************************************************
     */

    @Override
    public void writeNumber(short i) throws IOException, JsonGenerationException {
        _append(JsonToken.VALUE_NUMBER_INT, Short.valueOf(i));
    }

    @Override
    public void writeNumber(int i) throws IOException, JsonGenerationException {
        _append(JsonToken.VALUE_NUMBER_INT, Integer.valueOf(i));
    }

    @Override
    public void writeNumber(long l) throws IOException, JsonGenerationException {
        _append(JsonToken.VALUE_NUMBER_INT, Long.valueOf(l));
    }

    @Override
    public void writeNumber(double d) throws IOException,JsonGenerationException {
        _append(JsonToken.VALUE_NUMBER_FLOAT, Double.valueOf(d));
    }

    @Override
    public void writeNumber(float f) throws IOException, JsonGenerationException {
        _append(JsonToken.VALUE_NUMBER_FLOAT, Float.valueOf(f));
    }

    @Override
    public void writeNumber(BigDecimal dec) throws IOException,JsonGenerationException {
        if (dec == null) {
            writeNull();
        } else {
            _append(JsonToken.VALUE_NUMBER_FLOAT, dec);
        }
    }

    @Override
    public void writeNumber(BigInteger v) throws IOException, JsonGenerationException {
        if (v == null) {
            writeNull();
        } else {
            _append(JsonToken.VALUE_NUMBER_INT, v);
        }
    }

    @Override
    public void writeNumber(String encodedValue) throws IOException, JsonGenerationException {
        /* 03-Dec-2010, tatu: related to [JACKSON-423], should try to keep as numeric
         *   identity as long as possible
         */
        _append(JsonToken.VALUE_NUMBER_FLOAT, encodedValue);
    }

    @Override
    public void writeBoolean(boolean state) throws IOException,JsonGenerationException {
        _append(state ? JsonToken.VALUE_TRUE : JsonToken.VALUE_FALSE);
    }

    @Override
    public void writeNull() throws IOException, JsonGenerationException {
        _append(JsonToken.VALUE_NULL);
    }

    /*
    /***********************************************************
    /* JsonGenerator implementation: write methods for POJOs/trees
    /***********************************************************
     */

    @Override
    public void writeObject(Object value) throws IOException
    {
        if (value == null) {
            writeNull();
            return;
        }
        Class<?> raw = value.getClass();
        if (raw == byte[].class) {
            _append(JsonToken.VALUE_EMBEDDED_OBJECT, value);
            return;
        } else if (_objectCodec == null) {
            /* 28-May-2014, tatu: Tricky choice here; if no codec, should we
             *   err out, or just embed? For now, do latter.
             */
//          throw new JsonMappingException("No ObjectCodec configured for TokenBuffer, writeObject() called");
            _append(JsonToken.VALUE_EMBEDDED_OBJECT, value);
        } else {
            _objectCodec.writeValue(this, value);
        }
    }

    @Override
    public void writeTree(TreeNode node) throws IOException
    {
        if (node == null) {
            writeNull();
            return;
        }

        if (_objectCodec == null) {
            // as with 'writeObject()', is codec optional?
            _append(JsonToken.VALUE_EMBEDDED_OBJECT, node);
        } else {
            _objectCodec.writeTree(this, node);
        }
    }

    /*
    /***********************************************************
    /* JsonGenerator implementation; binary
    /***********************************************************
     */

    @Override
    public void writeBinary(Base64Variant b64variant, byte[] data, int offset, int len)
        throws IOException, JsonGenerationException
    {
        /* 31-Dec-2009, tatu: can do this using multiple alternatives; but for
         *   now, let's try to limit number of conversions.
         *   The only (?) tricky thing is that of whether to preserve variant,
         *   seems pointless, so let's not worry about it unless there's some
         *   compelling reason to.
         */
        byte[] copy = new byte[len];
        System.arraycopy(data, offset, copy, 0, len);
        writeObject(copy);
    }

    /**
     * Although we could support this method, it does not necessarily make
     * sense: we can not make good use of streaming because buffer must
     * hold all the data. Because of this, currently this will simply
     * throw {@link UnsupportedOperationException}
     */
    @Override
    public int writeBinary(Base64Variant b64variant, InputStream data, int dataLength) {
        throw new UnsupportedOperationException();
    }

    /*
    /***********************************************************
    /* JsonGenerator implementation: native ids
    /***********************************************************
     */

    @Override
    public boolean canWriteTypeId() {
        return _hasNativeTypeIds;
    }

    @Override
    public boolean canWriteObjectId() {
        return _hasNativeObjectIds;
    }

    @Override
    public void writeTypeId(Object id) {
        _typeId = id;
        _hasNativeId = true;
    }

    @Override
    public void writeObjectId(Object id) {
        _objectId = id;
        _hasNativeId = true;
    }

    /*
    /**********************************************************
    /* JsonGenerator implementation; pass-through copy
    /**********************************************************
     */

    @Override
    public void copyCurrentEvent(JsonParser jp) throws IOException, JsonProcessingException
    {
        if (_mayHaveNativeIds) {
            _checkNativeIds(jp);
        }
        switch (jp.getCurrentToken()) {
        case START_OBJECT:
            writeStartObject();
            break;
        case END_OBJECT:
            writeEndObject();
            break;
        case START_ARRAY:
            writeStartArray();
            break;
        case END_ARRAY:
            writeEndArray();
            break;
        case FIELD_NAME:
            writeFieldName(jp.getCurrentName());
            break;
        case VALUE_STRING:
            if (jp.hasTextCharacters()) {
                writeString(jp.getTextCharacters(), jp.getTextOffset(), jp.getTextLength());
            } else {
                writeString(jp.getText());
            }
            break;
        case VALUE_NUMBER_INT:
            switch (jp.getNumberType()) {
            case INT:
                writeNumber(jp.getIntValue());
                break;
            case BIG_INTEGER:
                writeNumber(jp.getBigIntegerValue());
                break;
            default:
                writeNumber(jp.getLongValue());
            }
            break;
        case VALUE_NUMBER_FLOAT:
            switch (jp.getNumberType()) {
            case BIG_DECIMAL:
                writeNumber(jp.getDecimalValue());
                break;
            case FLOAT:
                writeNumber(jp.getFloatValue());
                break;
            default:
                writeNumber(jp.getDoubleValue());
            }
            break;
        case VALUE_TRUE:
            writeBoolean(true);
            break;
        case VALUE_FALSE:
            writeBoolean(false);
            break;
        case VALUE_NULL:
            writeNull();
            break;
        case VALUE_EMBEDDED_OBJECT:
            writeObject(jp.getEmbeddedObject());
            break;
        default:
            throw new RuntimeException("Internal error: should never end up through this code path");
        }
    }

    @Override
    public void copyCurrentStructure(JsonParser jp) throws IOException, JsonProcessingException
    {
        JsonToken t = jp.getCurrentToken();

        // Let's handle field-name separately first
        if (t == JsonToken.FIELD_NAME) {
            if (_mayHaveNativeIds) {
                _checkNativeIds(jp);
            }
            writeFieldName(jp.getCurrentName());
            t = jp.nextToken();
            // fall-through to copy the associated value
        }

        if (_mayHaveNativeIds) {
            _checkNativeIds(jp);
        }

        switch (t) {
        case START_ARRAY:
            writeStartArray();
            while (jp.nextToken() != JsonToken.END_ARRAY) {
                copyCurrentStructure(jp);
            }
            writeEndArray();
            break;
        case START_OBJECT:
            writeStartObject();
            while (jp.nextToken() != JsonToken.END_OBJECT) {
                copyCurrentStructure(jp);
            }
            writeEndObject();
            break;
        default: // others are simple:
            copyCurrentEvent(jp);
        }
    }

    private final void _checkNativeIds(JsonParser jp) throws IOException, JsonProcessingException
    {
        if ((_typeId = jp.getTypeId()) != null) {
            _hasNativeId = true;
        }
        if ((_objectId = jp.getObjectId()) != null) {
            _hasNativeId = true;
        }
    }

    /*
    /**********************************************************
    /* Internal methods
    /**********************************************************
     */

    protected final void _append(JsonToken type)
    {
        Segment next = _hasNativeId
                ? _last.append(_appendAt, type, _objectId, _typeId)
                : _last.append(_appendAt, type);
        if (next == null) {
            ++_appendAt;
        } else {
            _last = next;
            _appendAt = 1; // since we added first at 0
        }
    }

    protected final void _append(JsonToken type, Object value)
    {
        Segment next = _hasNativeId
                ? _last.append(_appendAt, type, value, _objectId, _typeId)
                : _last.append(_appendAt, type, value);
        if (next == null) {
            ++_appendAt;
        } else {
            _last = next;
            _appendAt = 1;
        }
    }

    protected final void _appendRaw(int rawType, Object value)
    {
        Segment next = _hasNativeId
                ? _last.appendRaw(_appendAt, rawType, value, _objectId, _typeId)
                : _last.appendRaw(_appendAt, rawType, value);
        if (next == null) {
            ++_appendAt;
        } else {
            _last = next;
            _appendAt = 1;
        }
    }

    @Override
    protected void _reportUnsupportedOperation() {
        throw new UnsupportedOperationException("Called operation not supported for TokenBuffer");
    }

    /*
    /**********************************************************
    /* Supporting classes
    /**********************************************************
     */

    protected final static class Parser
        extends ParserMinimalBase
    {
        /*
        /**********************************************************
        /* Configuration
        /**********************************************************
         */

        protected ObjectCodec _codec;

        /**
         * @since 2.3
         */
        protected final boolean _hasNativeTypeIds;

        /**
         * @since 2.3
         */
        protected final boolean _hasNativeObjectIds;

        protected final boolean _hasNativeIds;

        /*
        /**********************************************************
        /* Parsing state
        /**********************************************************
         */

        /**
         * Currently active segment
         */
        protected Segment _segment;

        /**
         * Pointer to current token within current segment
         */
        protected int _segmentPtr;

        /**
         * Information about parser context, context in which
         * the next token is to be parsed (root, array, object).
         */
        protected JsonReadContext _parsingContext;

        protected boolean _closed;

        protected transient ByteArrayBuilder _byteBuilder;

        protected JsonLocation _location = null;

        /*
        /**********************************************************
        /* Construction, init
        /**********************************************************
         */

        public Parser(Segment firstSeg, ObjectCodec codec,
                boolean hasNativeTypeIds,
                boolean hasNativeObjectIds)
        {
            super(0);
            _segment = firstSeg;
            _segmentPtr = -1; // not yet read
            _codec = codec;
            _parsingContext = JsonReadContext.createRootContext(null);
            _hasNativeTypeIds = hasNativeTypeIds;
            _hasNativeObjectIds = hasNativeObjectIds;
            _hasNativeIds = (hasNativeTypeIds | hasNativeObjectIds);
        }

        public void setLocation(JsonLocation l) {
            _location = l;
        }

        @Override
        public ObjectCodec getCodec() { return _codec; }

        @Override
        public void setCodec(ObjectCodec c) { _codec = c; }

        @Override
        public Version version() {
            return com.fasterxml.jackson.databind.cfg.PackageVersion.VERSION;
        }

        /*
        /**********************************************************
        /* Extended API beyond JsonParser
        /**********************************************************
         */

        public JsonToken peekNextToken()
            throws IOException, JsonParseException
        {
            // closed? nothing more to peek, either
            if (_closed) return null;
            Segment seg = _segment;
            int ptr = _segmentPtr+1;
            if (ptr >= Segment.TOKENS_PER_SEGMENT) {
                ptr = 0;
                seg = (seg == null) ? null : seg.next();
            }
            return (seg == null) ? null : seg.type(ptr);
        }

        /*
        /**********************************************************
        /* Closeable implementation
        /**********************************************************
         */

        @Override
        public void close() throws IOException {
            if (!_closed) {
                _closed = true;
            }
        }

        /*
        /**********************************************************
        /* Public API, traversal
        /**********************************************************
         */

        @Override
        public JsonToken nextToken() throws IOException, JsonParseException
        {
            // If we are closed, nothing more to do
            if (_closed || (_segment == null)) return null;

            // Ok, then: any more tokens?
            if (++_segmentPtr >= Segment.TOKENS_PER_SEGMENT) {
                _segmentPtr = 0;
                _segment = _segment.next();
                if (_segment == null) {
                    return null;
                }
            }
            _currToken = _segment.type(_segmentPtr);
            // Field name? Need to update context
            if (_currToken == JsonToken.FIELD_NAME) {
                Object ob = _currentObject();
                String name = (ob instanceof String) ? ((String) ob) : ob.toString();
                _parsingContext.setCurrentName(name);
            } else if (_currToken == JsonToken.START_OBJECT) {
                _parsingContext = _parsingContext.createChildObjectContext(-1, -1);
            } else if (_currToken == JsonToken.START_ARRAY) {
                _parsingContext = _parsingContext.createChildArrayContext(-1, -1);
            } else if (_currToken == JsonToken.END_OBJECT
                    || _currToken == JsonToken.END_ARRAY) {
                // Closing JSON Object/Array? Close matching context
                _parsingContext = _parsingContext.getParent();
                // but allow unbalanced cases too (more close markers)
                if (_parsingContext == null) {
                    _parsingContext = JsonReadContext.createRootContext(null);
                }
            }
            return _currToken;
        }

        @Override
        public boolean isClosed() { return _closed; }

        /*
        /**********************************************************
        /* Public API, token accessors
        /**********************************************************
         */

        @Override
        public JsonStreamContext getParsingContext() { return _parsingContext; }

        @Override
        public JsonLocation getTokenLocation() { return getCurrentLocation(); }

        @Override
        public JsonLocation getCurrentLocation() {
            return (_location == null) ? JsonLocation.NA : _location;
        }

        @Override
        public String getCurrentName() { return _parsingContext.getCurrentName(); }

        @Override
        public void overrideCurrentName(String name)
        {
            // Simple, but need to look for START_OBJECT/ARRAY's "off-by-one" thing:
            JsonReadContext ctxt = _parsingContext;
            if (_currToken == JsonToken.START_OBJECT || _currToken == JsonToken.START_ARRAY) {
                ctxt = ctxt.getParent();
            }
            try {
                ctxt.setCurrentName(name);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }

        /*
        /**********************************************************
        /* Public API, access to token information, text
        /**********************************************************
         */

        @Override
        public String getText()
        {
            // common cases first:
            if (_currToken == JsonToken.VALUE_STRING
                    || _currToken == JsonToken.FIELD_NAME) {
                Object ob = _currentObject();
                if (ob instanceof String) {
                    return (String) ob;
                }
                return (ob == null) ? null : ob.toString();
            }
            if (_currToken == null) {
                return null;
            }
            switch (_currToken) {
            case VALUE_NUMBER_INT:
            case VALUE_NUMBER_FLOAT:
                Object ob = _currentObject();
                return (ob == null) ? null : ob.toString();
            default:
            	return _currToken.asString();
            }
        }

        @Override
        public char[] getTextCharacters() {
            String str = getText();
            return (str == null) ? null : str.toCharArray();
        }

        @Override
        public int getTextLength() {
            String str = getText();
            return (str == null) ? 0 : str.length();
        }

        @Override
        public int getTextOffset() { return 0; }

        @Override
        public boolean hasTextCharacters() {
            // We never have raw buffer available, so:
            return false;
        }

        /*
        /**********************************************************
        /* Public API, access to token information, numeric
        /**********************************************************
         */

        @Override
        public BigInteger getBigIntegerValue() throws IOException, JsonParseException
        {
            Number n = getNumberValue();
            if (n instanceof BigInteger) {
                return (BigInteger) n;
            }
            if (getNumberType() == NumberType.BIG_DECIMAL) {
                return ((BigDecimal) n).toBigInteger();
            }
            // int/long is simple, but let's also just truncate float/double:
            return BigInteger.valueOf(n.longValue());
        }

        @Override
        public BigDecimal getDecimalValue() throws IOException, JsonParseException
        {
            Number n = getNumberValue();
            if (n instanceof BigDecimal) {
                return (BigDecimal) n;
            }
            switch (getNumberType()) {
            case INT:
            case LONG:
                return BigDecimal.valueOf(n.longValue());
            case BIG_INTEGER:
                return new BigDecimal((BigInteger) n);
            default:
            }
            // float or double
            return BigDecimal.valueOf(n.doubleValue());
        }

        @Override
        public double getDoubleValue() throws IOException, JsonParseException {
            return getNumberValue().doubleValue();
        }

        @Override
        public float getFloatValue() throws IOException, JsonParseException {
            return getNumberValue().floatValue();
        }

        @Override
        public int getIntValue() throws IOException, JsonParseException
        {
            // optimize common case:
            if (_currToken == JsonToken.VALUE_NUMBER_INT) {
                return ((Number) _currentObject()).intValue();
            }
            return getNumberValue().intValue();
        }

        @Override
        public long getLongValue() throws IOException, JsonParseException {
            return getNumberValue().longValue();
        }

        @Override
        public NumberType getNumberType() throws IOException, JsonParseException
        {
            Number n = getNumberValue();
            if (n instanceof Integer) return NumberType.INT;
            if (n instanceof Long) return NumberType.LONG;
            if (n instanceof Double) return NumberType.DOUBLE;
            if (n instanceof BigDecimal) return NumberType.BIG_DECIMAL;
            if (n instanceof BigInteger) return NumberType.BIG_INTEGER;
            if (n instanceof Float) return NumberType.FLOAT;
            if (n instanceof Short) return NumberType.INT;       // should be SHORT
            return null;
        }

        @Override
        public final Number getNumberValue() throws IOException, JsonParseException {
            _checkIsNumber();
            Object value = _currentObject();
            if (value instanceof Number) {
                return (Number) value;
            }
            // Difficult to really support numbers-as-Strings; but let's try.
            // NOTE: no access to DeserializationConfig, unfortunately, so can not
            // try to determine Double/BigDecimal preference...
            if (value instanceof String) {
                String str = (String) value;
                if (str.indexOf('.') >= 0) {
                    return Double.parseDouble(str);
                }
                return Long.parseLong(str);
            }
            if (value == null) {
                return null;
            }
            throw new IllegalStateException("Internal error: entry should be a Number, but is of type "
                    +value.getClass().getName());
        }

        /*
        /**********************************************************
        /* Public API, access to token information, other
        /**********************************************************
         */

        @Override
        public Object getEmbeddedObject()
        {
            if (_currToken == JsonToken.VALUE_EMBEDDED_OBJECT) {
                return _currentObject();
            }
            return null;
        }

        @Override
        @SuppressWarnings("resource")
        public byte[] getBinaryValue(Base64Variant b64variant) throws IOException, JsonParseException
        {
            // First: maybe we some special types?
            if (_currToken == JsonToken.VALUE_EMBEDDED_OBJECT) {
                // Embedded byte array would work nicely...
                Object ob = _currentObject();
                if (ob instanceof byte[]) {
                    return (byte[]) ob;
                }
                // fall through to error case
            }
            if (_currToken != JsonToken.VALUE_STRING) {
                throw _constructError("Current token ("+_currToken+") not VALUE_STRING (or VALUE_EMBEDDED_OBJECT with byte[]), can not access as binary");
            }
            final String str = getText();
            if (str == null) {
                return null;
            }
            ByteArrayBuilder builder = _byteBuilder;
            if (builder == null) {
                _byteBuilder = builder = new ByteArrayBuilder(100);
            } else {
                _byteBuilder.reset();
            }
            _decodeBase64(str, builder, b64variant);
            return builder.toByteArray();
        }

        @Override
        public int readBinaryValue(Base64Variant b64variant, OutputStream out)
            throws IOException, JsonParseException
        {
            byte[] data = getBinaryValue(b64variant);
            if (data != null) {
                out.write(data, 0, data.length);
                return data.length;
            }
            return 0;
        }

        /*
        /**********************************************************
        /* Public API, native ids
        /**********************************************************
         */

        @Override
        public boolean canReadObjectId() {
            return _hasNativeObjectIds;
        }

        @Override
        public boolean canReadTypeId() {
            return _hasNativeTypeIds;
        }

        @Override
        public Object getTypeId() {
            return _segment.findTypeId(_segmentPtr);
        }

        @Override
        public Object getObjectId() {
            return _segment.findObjectId(_segmentPtr);
        }

        /*
        /**********************************************************
        /* Internal methods
        /**********************************************************
         */

        protected final Object _currentObject() {
            return _segment.get(_segmentPtr);
        }

        protected final void _checkIsNumber() throws JsonParseException
        {
            if (_currToken == null || !_currToken.isNumeric()) {
                throw _constructError("Current token ("+_currToken+") not numeric, can not use numeric value accessors");
            }
        }

        @Override
        protected void _handleEOF() throws JsonParseException {
            _throwInternal();
        }
    }

    /**
     * Individual segment of TokenBuffer that can store up to 16 tokens
     * (limited by 4 bits per token type marker requirement).
     * Current implementation uses fixed length array; could alternatively
     * use 16 distinct fields and switch statement (slightly more efficient
     * storage, slightly slower access)
     */
    protected final static class Segment
    {
        public final static int TOKENS_PER_SEGMENT = 16;

        /**
         * Static array used for fast conversion between token markers and
         * matching {@link JsonToken} instances
         */
        private final static JsonToken[] TOKEN_TYPES_BY_INDEX;
        static {
            // ... here we know that there are <= 15 values in JsonToken enum
            TOKEN_TYPES_BY_INDEX = new JsonToken[16];
            JsonToken[] t = JsonToken.values();
            // and reserve entry 0 for "not available"
            System.arraycopy(t, 1, TOKEN_TYPES_BY_INDEX, 1, Math.min(15, t.length - 1));
        }

        // // // Linking

        protected Segment _next;

        // // // State

        /**
         * Bit field used to store types of buffered tokens; 4 bits per token.
         * Value 0 is reserved for "not in use"
         */
        protected long _tokenTypes;

        // Actual tokens

        protected final Object[] _tokens = new Object[TOKENS_PER_SEGMENT];

        /**
         * Lazily constructed Map for storing native type and object ids, if any
         */
        protected TreeMap<Integer,Object> _nativeIds;

        public Segment() { }

        // // // Accessors

        public JsonToken type(int index)
        {
            long l = _tokenTypes;
            if (index > 0) {
                l >>= (index << 2);
            }
            int ix = ((int) l) & 0xF;
            return TOKEN_TYPES_BY_INDEX[ix];
        }

        public int rawType(int index)
        {
            long l = _tokenTypes;
            if (index > 0) {
                l >>= (index << 2);
            }
            int ix = ((int) l) & 0xF;
            return ix;
        }

        public Object get(int index) {
            return _tokens[index];
        }

        public Segment next() { return _next; }

        /**
         * Accessor for checking whether this segment may have native
         * type or object ids.
         */
        public boolean hasIds() {
            return _nativeIds != null;
        }

        // // // Mutators

        public Segment append(int index, JsonToken tokenType)
        {
            if (index < TOKENS_PER_SEGMENT) {
                set(index, tokenType);
                return null;
            }
            _next = new Segment();
            _next.set(0, tokenType);
            return _next;
        }

        public Segment append(int index, JsonToken tokenType,
                Object objectId, Object typeId)
        {
            if (index < TOKENS_PER_SEGMENT) {
                set(index, tokenType, objectId, typeId);
                return null;
            }
            _next = new Segment();
            _next.set(0, tokenType, objectId, typeId);
            return _next;
        }

        public Segment append(int index, JsonToken tokenType, Object value)
        {
            if (index < TOKENS_PER_SEGMENT) {
                set(index, tokenType, value);
                return null;
            }
            _next = new Segment();
            _next.set(0, tokenType, value);
            return _next;
        }

        public Segment append(int index, JsonToken tokenType, Object value,
                Object objectId, Object typeId)
        {
            if (index < TOKENS_PER_SEGMENT) {
                set(index, tokenType, value, objectId, typeId);
                return null;
            }
            _next = new Segment();
            _next.set(0, tokenType, value, objectId, typeId);
            return _next;
        }

        public Segment appendRaw(int index, int rawTokenType, Object value)
        {
            if (index < TOKENS_PER_SEGMENT) {
                set(index, rawTokenType, value);
                return null;
            }
            _next = new Segment();
            _next.set(0, rawTokenType, value);
            return _next;
        }

        public Segment appendRaw(int index, int rawTokenType, Object value,
                Object objectId, Object typeId)
        {
            if (index < TOKENS_PER_SEGMENT) {
                set(index, rawTokenType, value, objectId, typeId);
                return null;
            }
            _next = new Segment();
            _next.set(0, rawTokenType, value, objectId, typeId);
            return _next;
        }

        private void set(int index, JsonToken tokenType)
        {
            /* Assumption here is that there are no overwrites, just appends;
             * and so no masking is needed (nor explicit setting of null)
             */
            long typeCode = tokenType.ordinal();
            if (index > 0) {
                typeCode <<= (index << 2);
            }
            _tokenTypes |= typeCode;
        }

        private void set(int index, JsonToken tokenType,
                Object objectId, Object typeId)
        {
            long typeCode = tokenType.ordinal();
            if (index > 0) {
                typeCode <<= (index << 2);
            }
            _tokenTypes |= typeCode;
            assignNativeIds(index, objectId, typeId);
        }

        private void set(int index, JsonToken tokenType, Object value)
        {
            _tokens[index] = value;
            long typeCode = tokenType.ordinal();
            if (index > 0) {
                typeCode <<= (index << 2);
            }
            _tokenTypes |= typeCode;
        }

        private void set(int index, JsonToken tokenType, Object value,
                Object objectId, Object typeId)
        {
            _tokens[index] = value;
            long typeCode = tokenType.ordinal();
            if (index > 0) {
                typeCode <<= (index << 2);
            }
            _tokenTypes |= typeCode;
            assignNativeIds(index, objectId, typeId);
        }

        private void set(int index, int rawTokenType, Object value)
        {
            _tokens[index] = value;
            long typeCode = (long) rawTokenType;
            if (index > 0) {
                typeCode <<= (index << 2);
            }
            _tokenTypes |= typeCode;
        }

        private void set(int index, int rawTokenType, Object value, Object objectId, Object typeId)
        {
            _tokens[index] = value;
            long typeCode = (long) rawTokenType;
            if (index > 0) {
                typeCode <<= (index << 2);
            }
            _tokenTypes |= typeCode;
            assignNativeIds(index, objectId, typeId);
        }

        private final void assignNativeIds(int index, Object objectId, Object typeId)
        {
            if (_nativeIds == null) {
                _nativeIds = new TreeMap<Integer,Object>();
            }
            if (objectId != null) {
                _nativeIds.put(_objectIdIndex(index), objectId);
            }
            if (typeId != null) {
                _nativeIds.put(_typeIdIndex(index), typeId);
            }
        }

        /**
         * @since 2.3
         */
        public Object findObjectId(int index) {
            return (_nativeIds == null) ? null : _nativeIds.get(_objectIdIndex(index));
        }

        /**
         * @since 2.3
         */
        public Object findTypeId(int index) {
            return (_nativeIds == null) ? null : _nativeIds.get(_typeIdIndex(index));
        }

        private final int _typeIdIndex(int i) { return i+i; }
        private final int _objectIdIndex(int i) { return i+i+1; }
    }
}


text:
import Date from '../components/date'
import Layout from '../components/layout'
import Link from 'next/link'
import { GetStaticProps } from 'next'
import { getSortedPostsData } from '../lib/posts'

const title = 'Blog'

export default function Blog({
  allPostsData
}: {
  allPostsData: {
    id: string
    slug: string
    title: string
    date: string
  }[]
}) {
  return (
    <Layout title={title}>

{title}

        {allPostsData.map(({ id, slug, title, date }) => (

            <Link href={`/posts/${slug}`}>
              {title}
            </Link>

            <Date dateString={date} />

        ))}

    </Layout>
  )
}

export const getStaticProps: GetStaticProps = async () => {
  const allPostsData = getSortedPostsData()

  return {
    props: {
      allPostsData
    }
  }
}

redpajama_set_name: RedPajamaGithub


text:
package org.ebayopensource.turmeric.tools.codegen.protobuf;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

public class FileUtil {

public static String readProtoFileToString(File fileLocation){
		final Logger logger = Logger.getLogger("");

		StringBuffer sb = new StringBuffer();
		FileReader fr  = null;
		BufferedReader br =null;
		if(fileLocation != null){

			try {

				fr = new FileReader(fileLocation);
				br = new BufferedReader(fr);

				String line = null;
				do{
					line = br.readLine();
					if(line != null){

						sb.append(line.trim());

					}

				}while(line != null);

			} catch (FileNotFoundException e) {
				logger.log(Level.FINE, "", e);
			} catch (IOException e) {
				logger.log(Level.FINE, "", e);
			}finally{
				if(fr != null && br != null){

					try {
						fr.close();
						br.close();
					} catch (IOException e) {
						// TODO Auto-generated catch block
						logger.log(Level.FINE, "", e);
					}
				}

			}

		}else{
			return null;
		}
		return sb.toString();
	}

public static List<String> readFileAsLines(File fileLocation){
	final Logger logger = Logger.getLogger("");

	List<String> lines = new ArrayList<String>();
	FileReader fr  = null;
	BufferedReader br =null;
	if(fileLocation != null){

		try {

			fr = new FileReader(fileLocation);
			br = new BufferedReader(fr);

			String line = null;
			do{
				line = br.readLine();
				if(line != null){

					lines.add(line.trim());

				}

			}while(line != null);

		} catch (FileNotFoundException e) {

			logger.log(Level.FINE, "", e);
		} catch (IOException e) {

			logger.log(Level.FINE, "", e);
		}finally{
			if(fr != null && br != null){

				try {
					fr.close();
					br.close();
				} catch (IOException e) {
					logger.log(Level.FINE, "", e);
				}
			}

		}

	}else{
		return null;
	}
	return lines;
}

}


text:
#include "system.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <limits.h>
#ifdef WIN32
#include <io.h>
#else /* WIN32 */
#include <sys/file.h>
#include <dirent.h>
#include <pwd.h>
#endif /* WIN32 */
#include <fcntl.h>
#include "zlib.h"
#include "lz4.h"
#include "lz4hc.h"

/* !!! UNLESS you enjoy spending endless days tracking down race conditions
   !!! involving the garbage collector, please note: DEACTIVATE and
   !!! REACTIVATE or LOCKandDEACTIVATE and REACTIVATEandLOCK should be used
   !!! around operations that can block.  While deactivated, the process
   !!! MUST NOT touch any unlocked Scheme objects (ptrs) or allocate any
   !!! new Scheme objects.  It helps to bracket only small pieces of code
   !!! with DEACTIVATE/REACTIVATE or LOCKandDEACTIVATE/REACTIVATE_and_LOCK. */
#ifdef PTHREADS
/* assume the scheme wrapper has us in a critical section */
#define DEACTIVATE(tc) if (DISABLECOUNT(tc) == FIX(1)) { deactivate_thread(tc); }
#define REACTIVATE(tc) if (DISABLECOUNT(tc) == FIX(1)) { reactivate_thread(tc); }
#define LOCKandDEACTIVATE(tc,bv) if (DISABLECOUNT(tc) == FIX(1)) { Slock_object(bv); deactivate_thread(tc); }
#define REACTIVATEandUNLOCK(tc,bv) if (DISABLECOUNT(tc) == FIX(1)) { reactivate_thread(tc); Sunlock_object(bv); }
#else /* PTHREADS */
#define DEACTIVATE(tc)
#define REACTIVATE(tc)
#define LOCKandDEACTIVATE(tc,bv)
#define REACTIVATEandUNLOCK(tc,bv)
#endif /* PTHREADS */

/* locally defined functions */
static ptr new_open_output_fd_helper(const char *filename, INT mode,
             INT flags, INT no_create, INT no_fail, INT no_truncate,
             INT append, INT lock, INT replace, INT compressed);
static INT lockfile(INT fd);
static int is_valid_zlib_length(iptr count);
static int is_valid_lz4_length(iptr count);

/*
 not_ok_is_fatal: !ok definitely implies error, so ignore glzerror
 ok: whether the result of body seems to be ok
 flag: will be set when an error is detected and cleared if no error
 fd: the glzFile object to call glzerror on
 body: the operation we are checking the error on
*/
#ifdef EINTR
/* like FD_EINTR_GUARD and GZ_EINTR_GUARD but ignores EINTR.
   used for calls to close so we don't close a file descriptor that
   might already have been reallocated by a different thread */
#define FD_GUARD(ok,flag,body)                          \
  do { body;                                            \
    flag = !(ok) && errno != EINTR;                     \
  } while (0)
#define GZ_GUARD(not_ok_is_fatal,ok,flag,fd,body) \
  do { body;                                            \
    if (ok) { flag = 0; }                               \
    else {                                              \
      INT errnum;                                       \
      S_glzerror((fd),&errnum);                         \
      S_glzclearerr((fd));                              \
      if (errnum == Z_ERRNO) {                          \
        flag = errno != EINTR;                          \
      } else {                                          \
        flag = not_ok_is_fatal || errnum != Z_OK;       \
        errno = 0;                                      \
      }                                                 \
    }                                                   \
  } while (0)
/* like FD_GUARD and GZ_GUARD but spins on EINTR */
#define FD_EINTR_GUARD(ok,flag,body)                    \
  do { body;                                            \
    if (ok) { flag = 0; break; }                        \
    else if (errno != EINTR) { flag = 1; break; }       \
  } while (1)
#define GZ_EINTR_GUARD(not_ok_is_fatal,ok,flag,fd,body) \
  do { body;                                            \
    if (ok) { flag = 0; break; }                        \
    else {                                              \
      INT errnum;                                       \
      S_glzerror((fd),&errnum);                         \
      S_glzclearerr((fd));                              \
      if (errnum == Z_ERRNO) {                          \
        if (errno != EINTR) { flag = 1; break; }        \
      } else {                                          \
        flag = not_ok_is_fatal || errnum != Z_OK;       \
        errno = 0;                                      \
        break;                                          \
      }                                                 \
    }                                                   \
  } while (1)
#else /* EINTR */
#define FD_GUARD(ok,flag,body) do { body; flag = !(ok); } while (0)
#define GZ_GUARD(not_ok_is_fatal,ok,flag,fd,body)       \
  do { body;                                            \
    if (ok) { flag = 0; }                               \
    else {                                              \
      INT errnum;                                       \
      S_glzerror((fd),&errnum);                         \
      S_glzclearerr((fd));                              \
      if (errnum == Z_ERRNO) { flag = 1; }              \
      else {                                            \
        flag = not_ok_is_fatal || errnum != Z_OK;       \
        errno = 0;                                      \
      }                                                 \
    }                                                   \
  } while (0)
#define FD_EINTR_GUARD FD_GUARD
#define GZ_EINTR_GUARD GZ_GUARD
#endif /* EINTR */

#ifndef O_BINARY
#define O_BINARY 0
#endif /* O_BINARY */

/* These functions are intended for use immediately upon opening
 * (lockfile) fd.  They need to be redesigned for general-purpose
 * locking. */
#ifdef FLOCK
static INT lockfile(INT fd) { return FLOCK(fd, LOCK_EX); }
#endif
#ifdef LOCKF
static INT lockfile(INT fd) { return lockf(fd, F_LOCK, (off_t)0); }
#endif

#define MAKE_GZXFILE(x) Sinteger((iptr)x)
#define GZXFILE_GZFILE(x) ((glzFile)Sinteger_value(x))

INT S_gzxfile_fd(ptr x) {
  return GZXFILE_GZFILE(x)->fd;
}

glzFile S_gzxfile_gzfile(ptr x) {
  return GZXFILE_GZFILE(x);
}

ptr S_new_open_input_fd(const char *infilename, IBOOL compressed) {
  char *filename;
  INT saved_errno = 0;
  INT fd, dupfd, error, result, ok, flag;
  glzFile file;
#ifdef PTHREADS
  ptr tc = get_thread_context();
#endif

  filename = S_malloc_pathname(infilename);

  /* NB: don't use infilename, which might point into a Scheme string, after this point */
  DEACTIVATE(tc)
  FD_EINTR_GUARD(fd>=0, error, fd=OPEN(filename,O_BINARY|O_RDONLY,0));
  saved_errno = errno;
  REACTIVATE(tc)

  /* NB: don't use free'd filename after this point */
  free(filename);

  if (error) {
    ptr str = S_strerror(saved_errno);
    switch (saved_errno) {
      case EACCES:
        return Scons(FIX(OPEN_ERROR_PROTECTION), str);
      case ENOENT:
        return Scons(FIX(OPEN_ERROR_EXISTSNOT), str);
      default:
        return Scons(FIX(OPEN_ERROR_OTHER), str);
    }
  }

  if (!compressed) {
    return MAKE_FD(fd);
  }

  if ((dupfd = DUP(fd)) == -1) {
    ptr str = S_strerror(errno);
    FD_GUARD(result == 0, error, result = CLOSE(fd));
    return Scons(FIX(OPEN_ERROR_OTHER), str);
  }

  DEACTIVATE(tc) /* glzdopen_input reads the magic word from the file */
  if ((file = S_glzdopen_input(dupfd)) == Z_NULL) {
    REACTIVATE(tc)
    FD_GUARD(result == 0, error, result = CLOSE(fd));
    FD_GUARD(result == 0, error, result = CLOSE(dupfd));
    return Scons(FIX(OPEN_ERROR_OTHER), Sstring("unable to allocate compression state (too many open files?)"));
  }

  compressed = !S_glzdirect(file);
  REACTIVATE(tc)

  if (compressed) {
    FD_GUARD(result == 0, error, result = CLOSE(fd));
    /* box indicates compressed */
    return Sbox(MAKE_GZXFILE(file));
  }

  GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, file, ok = S_glzclose(file));
  if (flag) {} /* make the compiler happy */
  if (LSEEK(fd, 0, SEEK_SET) != 0) { /* glzdopen and glzdirect might not leave fd at position 0 */
    FD_GUARD(result == 0, error, result = CLOSE(fd));
    return Scons(FIX(OPEN_ERROR_OTHER),Sstring("unable to reset after reading header bytes"));
  }
  return MAKE_FD(fd);
}

ptr S_compress_input_fd(INT fd, I64 pos) {
  INT dupfd, error, result, ok, flag; IBOOL compressed;
  glzFile file;
#ifdef PTHREADS
  ptr tc = get_thread_context();
#endif

  if ((dupfd = DUP(fd)) == -1) {
    return S_strerror(errno);
  }

  DEACTIVATE(tc)
  if ((file = S_glzdopen_input(dupfd)) == Z_NULL) {
    REACTIVATE(tc)
    FD_GUARD(result == 0, error, result = CLOSE(dupfd));
    return Sstring("unable to allocate compression state (too many open files?)");
  }

  compressed = !S_glzdirect(file);
  REACTIVATE(tc)

  if (compressed) {
    FD_GUARD(result == 0, error, result = CLOSE(fd));
    if (error) {} /* make the compiler happy */
    /* box indicates compressed */
    return Sbox(MAKE_GZXFILE(file));
  }

  GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, file, ok = S_glzclose(file));
  if (flag) {} /* make the compiler happy */
  if (LSEEK(fd, pos, SEEK_SET) != pos) { /* glzdirect does not leave fd at same position */
    return Sstring("unable to reset after reading header bytes");
  }
  return MAKE_FD(fd);
}

ptr S_compress_output_fd(INT fd) {
  glzFile file;
  ptr tc = get_thread_context();

  file = S_glzdopen_output(fd, (INT)UNFIX(COMPRESSFORMAT(tc)), (INT)UNFIX(COMPRESSLEVEL(tc)));

  if (file == Z_NULL)
    return Sstring("unable to allocate compression state (too many open files?)");

  /* box indicates compressed */
  return Sbox(MAKE_GZXFILE(file));
}

static ptr new_open_output_fd_helper(
  const char *infilename, INT mode, INT flags,
  IBOOL no_create, IBOOL no_fail, IBOOL no_truncate,
  IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed) {
  char *filename;
  INT saved_errno = 0;
  iptr error;
  INT fd, result;
  ptr tc = get_thread_context();

  flags |=
    (no_create ? 0 : O_CREAT) |
    ((no_fail || no_create) ? 0 : O_EXCL) |
    (no_truncate ? 0 : O_TRUNC) |
    ((!append) ? 0 : O_APPEND);

  filename = S_malloc_pathname(infilename);

  if (replace && UNLINK(filename) != 0 && errno != ENOENT) {
    ptr str = S_strerror(errno);
    switch (errno) {
      case EACCES:
        return Scons(FIX(OPEN_ERROR_PROTECTION), str);
      default:
        return Scons(FIX(OPEN_ERROR_OTHER), str);
    }
  }

  /* NB: don't use infilename, which might point into a Scheme string, after this point */
  DEACTIVATE(tc)
  FD_EINTR_GUARD(fd >= 0, error, fd = OPEN(filename, flags, mode));
  saved_errno = errno;
  REACTIVATE(tc)

  /* NB: don't use free'd filename after this point */
  free(filename);

  if (error) {
    ptr str = S_strerror(saved_errno);
    switch (saved_errno) {
      case EACCES:
        return Scons(FIX(OPEN_ERROR_PROTECTION), str);
      case EEXIST:
        return Scons(FIX(OPEN_ERROR_EXISTS), str);
      case ENOENT:
        return Scons(FIX(OPEN_ERROR_EXISTSNOT), str);
      default:
        return Scons(FIX(OPEN_ERROR_OTHER), str);
    }
  }

  if (lock) {
    DEACTIVATE(tc)
    error = lockfile(fd);
    saved_errno = errno;
    REACTIVATE(tc)
    if (error) {
      FD_GUARD(result == 0, error, result = CLOSE(fd));
      return Scons(FIX(OPEN_ERROR_OTHER), S_strerror(saved_errno));
    }
  }

  if (!compressed) {
    return MAKE_FD(fd);
  }

  glzFile file;
  file = S_glzdopen_output(fd, (INT)UNFIX(COMPRESSFORMAT(tc)), (INT)UNFIX(COMPRESSLEVEL(tc)));
  if (file == Z_NULL) {
    FD_GUARD(result == 0, error, result = CLOSE(fd));
    return Scons(FIX(OPEN_ERROR_OTHER), Sstring("unable to allocate compression state"));
  }

  return MAKE_GZXFILE(file);
}

ptr S_new_open_output_fd(
  const char *filename, INT mode,
  IBOOL no_create, IBOOL no_fail, IBOOL no_truncate,
  IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed) {
  return new_open_output_fd_helper(
    filename, mode, O_BINARY | O_WRONLY,
    no_create, no_fail, no_truncate,
    append, lock, replace, compressed);
}

ptr S_new_open_input_output_fd(
  const char *filename, INT mode,
  IBOOL no_create, IBOOL no_fail, IBOOL no_truncate,
  IBOOL append, IBOOL lock, IBOOL replace, IBOOL compressed) {
  if (compressed)
    return Sstring("compressed input/output files not supported");
  else
    return new_open_output_fd_helper(
      filename, mode, O_BINARY | O_RDWR,
      no_create, no_fail, no_truncate,
      append, lock, replace, 0);
}

ptr S_close_fd(ptr file, IBOOL gzflag) {
  INT saved_errno = 0;
  INT ok, flag;
  INT fd = gzflag ? 0 : GET_FD(file);
  glzFile gzfile = gzflag ? GZXFILE_GZFILE(file) : NULL;
#ifdef PTHREADS
  ptr tc = get_thread_context();
#endif

 /* refuse to close stdin, stdout, and stderr fds */
  if (!gzflag && fd <= 2) return Strue;

 /* file is not locked; do not reference after deactivating thread! */
  file = (ptr)-1;

 /* NOTE: close automatically releases locks so we don't to call unlock*/
  DEACTIVATE(tc)
  if (!gzflag) {
    FD_GUARD(ok == 0, flag, ok = CLOSE(fd));
  } else {
   /* zlib 1.2.1 returns Z_BUF_ERROR when closing an empty file opened for reading */
    GZ_GUARD(1, ok == 0 || ok == Z_BUF_ERROR, flag, gzfile, ok = S_glzclose(gzfile));
  }
  saved_errno = errno;
  REACTIVATE(tc)

  if (!flag) {
    return Strue;
  }

  if (gzflag && saved_errno == 0) {
    return Sstring("compression failed");
  }

  return S_strerror(saved_errno);
}

#define GZ_IO_SIZE_T unsigned int

#ifdef WIN32
#define IO_SIZE_T unsigned int
static HANDLE hStdin = NULL;
static iptr read_console(char* buf, unsigned size) {
  static char u8buf[1024];
  static int u8i = 0;
  static int u8n = 0;
  iptr n = 0;
  do {
    for (; size > 0 && u8n > 0; size--, u8n--, n++)
      *buf++ = u8buf[u8i++];
    if (n == 0 && size > 0) {
      wchar_t wbuf[256];
      DWORD wn;
      if (!ReadConsoleW(hStdin, wbuf, 256, &wn, NULL) || wn == 0)
        return 0;
      u8n = WideCharToMultiByte(CP_UTF8, 0, wbuf, wn, u8buf, 1024, NULL, NULL);
      u8i = 0;
    }
  } while (n == 0);
  return n;
}
#else /* WIN32 */
#define IO_SIZE_T size_t
#endif /* WIN32 */

/* Returns string on error, #!eof on end-of-file and integer-count otherwise */
ptr S_bytevector_read(ptr file, ptr bv, iptr start, iptr count, IBOOL gzflag) {
  INT saved_errno = 0;
  ptr tc = get_thread_context();
  iptr m, flag = 0;
  INT fd = gzflag ? 0 : GET_FD(file);
  glzFile gzfile = gzflag ? GZXFILE_GZFILE(file) : NULL;

 /* file is not locked; do not reference after deactivating thread! */
  file = (ptr)-1;

#if (iptr_bits > 32)
  if ((WIN32 || gzflag) && (unsigned int)count != count) count = 0xffffffff;
#endif

  LOCKandDEACTIVATE(tc, bv)
#ifdef CHECK_FOR_ROSETTA
  /* If we are running on Apple Silicon under Rosetta 2 translation, work around
     a bug (present in 11.2.3 at least) in its handling of memory page protection
     bits.  One of the tasks that Rosetta handles is to appropriately twiddle the
     execute and write bits based on what's happening to the memory in order to
     preserve the illusion that the pages have RWX permissions, whereas Apple
     Silicon enforces a W^X (write XOR execute) model.  For some reason, this
     bit-twiddling sometimes fails when the bytevector passed to `read` extends
     onto a page that's currently R-X, causing the `read` to fail with EFAULT
     ("bad address").  By writing to each subsequent page, we force Rosetta to
     do the right magic to the protection bits.  (Or at least it makes the error
     go away and all the mats pass.)
     */
  if (is_rosetta) {
    for (iptr idx = start+count; idx > start; idx -= S_pagesize) {
      volatile octet b = BVIT(bv,idx);
      BVIT(bv,idx) = b;
    }
  }
#endif

#ifdef WIN32
  if (!gzflag && fd == 0 && hStdin != NULL) {
    DWORD error_code;
    SetConsoleCtrlHandler(NULL, TRUE);
    SetLastError(0);
    m = read_console(&BVIT(bv,start), (IO_SIZE_T)count);
    error_code = GetLastError();
    if (m == 0 && error_code == 0x3e3) {
      /* Guard against Windows calling the ConsoleCtrlHandler after we
       * turn it back on by waiting a bit. */
      Sleep(1);
#ifdef PTHREADS
      /* threaded io.ss doesn't handle interrupts because
       * with-tc-mutex disables them, so bail out. */
      SetConsoleCtrlHandler(NULL, FALSE);
      REACTIVATEandUNLOCK(tc, bv)
      S_noncontinuable_interrupt();
#else
      KEYBOARDINTERRUPTPENDING(tc) = Strue;
      SOMETHINGPENDING(tc) = Strue;
#endif
    }
    SetConsoleCtrlHandler(NULL, FALSE);
  } else
#endif /* WIN32 */
  {
    if (!gzflag) {
      FD_EINTR_GUARD(
        m >= 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)), flag,
        m = READ(fd,&BVIT(bv,start),(IO_SIZE_T)count));
    } else {
      GZ_EINTR_GUARD(
        1, m >= 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)),
        flag, gzfile,
        m = S_glzread(gzfile, &BVIT(bv,start), (GZ_IO_SIZE_T)count));
    }
  }
  saved_errno = errno;
  REACTIVATEandUNLOCK(tc, bv)

  if (Sboolean_value(KEYBOARDINTERRUPTPENDING(tc))) {
    return Sstring("interrupt");
  }

  if (!flag) {
    return m == 0 ? Seof_object : FIX(m);
  }

  if (saved_errno == EAGAIN) {
    return FIX(0);
  }

  return S_strerror(saved_errno);
}

/* Returns:
    string on error, including if not supported,
    n when read,
    0 on non-blocking and
    #!eof otherwise */
ptr S_bytevector_read_nb(ptr file, ptr bv, iptr start, iptr count, IBOOL gzflag) {
#ifdef WIN32
  HANDLE h;

 /* assume compressed files are always ready */
  if (gzflag) return FIX(1);

  if ((h = (HANDLE)_get_osfhandle(GET_FD(file))) != INVALID_HANDLE_VALUE) {
    switch (GetFileType(h)) {
      case FILE_TYPE_CHAR:
       /* if h is hStdin, PeekConsoleInput can tell us if a key down event
          is waiting, but if it's not a newline, we can't be sure that
          a read will succeed.  so PeekConsoleInput is basically useless
          for our purposes. */
        break;
      case FILE_TYPE_PIPE: {
        DWORD bytes;
        if (PeekNamedPipe(h, NULL, 0, NULL, &bytes, NULL) && bytes == 0) return FIX(0);
       /* try the read on error or if bytes > 0 */
        return S_bytevector_read(file, bv, start, count, gzflag);
      }
      default: {
        if (WaitForSingleObject(h, 0) == WAIT_TIMEOUT) return FIX(0);
       /* try the read on error or if bytes > 0 */
        return S_bytevector_read(file, bv, start, count, gzflag);
      }
    }
  }

  return Sstring("cannot determine ready status");
#else /* WIN32 */
  INT fcntl_flags;
  ptr result;
  INT fd;

 /* assume compressed files are always ready */
  if (gzflag) return FIX(1);

  fd = GET_FD(file);

 /* set NOBLOCK for nonblocking read */
  fcntl_flags = fcntl(fd, F_GETFL, 0);
  if (!(fcntl_flags & NOBLOCK)) (void) fcntl(fd, F_SETFL, fcntl_flags | NOBLOCK);

  result = S_bytevector_read(file, bv, start, count, gzflag);

 /* reset NOBLOCK for normal blocking read */
  if (!(fcntl_flags & NOBLOCK)) (void) fcntl(fd, F_SETFL, fcntl_flags);

  return result;
#endif /* WIN32 */
}

ptr S_bytevector_write(ptr file, ptr bv, iptr start, iptr count, IBOOL gzflag) {
  iptr i, s, c;
  ptr tc = get_thread_context();
  INT flag = 0, saved_errno = 0;
  INT fd = gzflag ? 0 : GET_FD(file);
  glzFile gzfile = gzflag ? GZXFILE_GZFILE(file) : NULL;

  for (s = start, c = count; c > 0; s += i, c -= i) {
    iptr cx = c;

#if (iptr_bits > 32)
  if ((WIN32 || gzflag) && (unsigned int)cx != cx) cx = 0xffffffff;
#endif

  /* if we could know that fd is nonblocking, we wouldn't need to deactivate.
     we could test ioctl, but some other thread could change it before we actually
     get around to writing. */
    LOCKandDEACTIVATE(tc, bv)
    if (gzflag) {
     /* strangely, gzwrite returns 0 on error */
      GZ_EINTR_GUARD(
        i < 0, i > 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)),
        flag, gzfile,
        i = S_glzwrite(gzfile, &BVIT(bv,s), (GZ_IO_SIZE_T)cx));
    } else {
      FD_EINTR_GUARD(i >= 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)),
                     flag, i = WRITE(fd, &BVIT(bv,s), (IO_SIZE_T)cx));
    }
    saved_errno = errno;
    REACTIVATEandUNLOCK(tc, bv)

    if (flag) {
      if (saved_errno == EAGAIN) { flag = 0; }
      break;
    }

   /* we escape from loop if keyboard interrupt is pending, but this won't
      do much good until we fix up the interrupt protocol to guarantee
      that the interrupt handler is actually called */
    if (Sboolean_value(KEYBOARDINTERRUPTPENDING(tc))) {
      if (i >= 0) s += i;
      break;
    }
  }

  if (!flag) {
    return FIX(s - start);
  }

  if (saved_errno == EAGAIN) {
    return FIX(0);
  }

  if (gzflag && saved_errno == 0) {
    return Sstring("compression failed");
  }

  return S_strerror(saved_errno);
}

/* S_put_byte is a simplified version of S_bytevector_write for writing one
   byte on unbuffered ports */
ptr S_put_byte(ptr file, INT byte, IBOOL gzflag) {
  iptr i;
  ptr tc = get_thread_context();
  INT flag = 0, saved_errno = 0;
  INT fd = gzflag ? 0 : GET_FD(file);
  glzFile gzfile = gzflag ? GZXFILE_GZFILE(file) : NULL;
  octet buf[1];

  buf[0] = (octet)byte;

  DEACTIVATE(tc)
  if (gzflag) {
   /* strangely, gzwrite returns 0 on error */
    GZ_EINTR_GUARD(
      i < 0, i > 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)),
      flag, gzfile,
      i = S_glzwrite(gzfile, buf, 1));
  } else {
    FD_EINTR_GUARD(i >= 0 || Sboolean_value(KEYBOARDINTERRUPTPENDING(tc)),
                   flag, i = WRITE(fd, buf, 1));
  }
  saved_errno = errno;
  REACTIVATE(tc)

  if (flag) {
    if (saved_errno == EAGAIN) { flag = 0; }
  }

  if (!flag) {
    return FIX(i);
  }

  if (saved_errno == EAGAIN) {
    return FIX(0);
  }

  if (gzflag && saved_errno == 0) {
    return Sstring("compression failed");
  }

  return S_strerror(saved_errno);
}

ptr S_get_fd_pos(ptr file, IBOOL gzflag) {
  errno = 0;
  if (gzflag) {
    z_off_t offset = S_glzseek(GZXFILE_GZFILE(file), 0, SEEK_CUR);
    if (offset != -1) return Sinteger64(offset);
  } else {
    OFF_T offset = LSEEK(GET_FD(file), 0, SEEK_CUR);
    if (offset != -1) return Sinteger64(offset);
  }
  if (gzflag && errno == 0) return Sstring("compression failed");
  return S_strerror(errno);
}

/* assume wrapper ensures 0 <= pos <= 2^63-1 */
ptr S_set_fd_pos(ptr file, ptr pos, IBOOL gzflag) {
  I64 offset64 = S_int64_value("set-file-position", pos);

  if (gzflag) {
    z_off_t offset = (z_off_t)offset64;
    if (sizeof(z_off_t) != sizeof(I64))
      if (offset != offset64) return Sstring("invalid position");
    errno = 0;
    if (S_glzseek(GZXFILE_GZFILE(file),offset,SEEK_SET) == offset) return Strue;
    if (errno == 0) return Sstring("compression failed");
    return S_strerror(errno);
  } else {
    OFF_T offset = (OFF_T)offset64;
    if (sizeof(OFF_T) != sizeof(I64))
      if (offset != offset64) return Sstring("invalid position");
    if (LSEEK(GET_FD(file), offset, SEEK_SET) == offset) return Strue;
    return S_strerror(errno);
  }
}

ptr S_get_fd_non_blocking(ptr file, IBOOL gzflag) {
#ifdef WIN32
  return Sfalse;
#else /* WIN32 */
  INT fcntl_flags;

  if (gzflag) return Sfalse;

  fcntl_flags = fcntl(GET_FD(file), F_GETFL, 0);

  if (fcntl_flags == -1) {
    return S_strerror(errno);
  }

  return Sboolean(NOBLOCK & fcntl_flags);
#endif /* WIN32 */
}

ptr S_set_fd_non_blocking(ptr file, IBOOL x, IBOOL gzflag) {
#ifdef WIN32
  return Sstring("unsupported");
#else /* WIN32 */
  iptr fd;
  INT fcntl_flags;

  if (gzflag) {
    if (x) return Sstring("Compressed non-blocking ports not supported");
    else return Strue;
  }

  fd = GET_FD(file);
  fcntl_flags = fcntl(fd, F_GETFL, 0);

  if (fcntl_flags == -1) {
    return S_strerror(errno);
  }

  if (x) {
    if (fcntl_flags & NOBLOCK) {
      return Strue;
    }
    if (0 == fcntl(fd, F_SETFL, fcntl_flags | NOBLOCK)) {
      return Strue;
    }
    return S_strerror(errno);
  } else {
    if (!(fcntl_flags & NOBLOCK)) {
      return Strue;
    }
    if (0 == fcntl(fd, F_SETFL, fcntl_flags & ~NOBLOCK)) {
      return Strue;
    }
    return S_strerror(errno);
  }
#endif /* WIN32 */
}

ptr S_get_fd_length(ptr file, IBOOL gzflag) {
  struct STATBUF statbuf;

  if (gzflag) return Sstring("Not supported on compressed files");

  if (FSTAT(GET_FD(file), &statbuf) == 0) {
    return Sinteger64(statbuf.st_size);
  }

  return S_strerror(errno);
}

ptr S_set_fd_length(ptr file, ptr length, IBOOL gzflag) {
  INT fd, ok, flag = 0;
  I64 len64; off_t len;
#ifdef PTHREADS
  ptr tc = get_thread_context();
#endif

  if (gzflag) return Sstring("Not supported on compressed files");

  len64 = S_int64_value("set-file-length", length);
  len = (off_t)len64;
  if (sizeof(off_t) != sizeof(I64))
    if (len != len64) return Sstring("invalid length");

  fd = GET_FD(file);
  DEACTIVATE(tc)
  FD_EINTR_GUARD(ok == 0, flag, ok = ftruncate(fd, len));
  REACTIVATE(tc)

  return flag ? S_strerror(errno) : Strue;
}

void S_new_io_init(void) {
  if (S_boot_time) {
    S_set_symbol_value(S_intern((const unsigned char *)"$c-bufsiz"), Sinteger(SBUFSIZ));
  }
#ifdef WIN32
  { /* Get the console input handle for reading Unicode characters */
    HANDLE h;
    DWORD mode;
    if ((h = GetStdHandle(STD_INPUT_HANDLE)) != INVALID_HANDLE_VALUE
        && GetConsoleMode(h, &mode))
      hStdin = h;
  }
 /* transcoder, if any, does its own cr, lf translations */
  _setmode(_fileno(stdin), O_BINARY);
  _setmode(_fileno(stdout), O_BINARY);
  _setmode(_fileno(stderr), O_BINARY);
  /* Set the console output to handle UTF-8 */
  SetConsoleOutputCP(CP_UTF8);
#endif /* WIN32 */
}

static int is_valid_zlib_length(iptr count) {
  /* A zlib `uLong` may be the same as `unsigned long`,
     which is not as big as `iptr` on 64-bit Windows. */
  return count == (iptr)(uLong)count;
}

static int is_valid_lz4_length(iptr len) {
  return (len <= LZ4_MAX_INPUT_SIZE);
}

/* Accept `iptr` because we expect it to represent a bytevector size,
   which always fits in `iptr`. Return `uptr`, because the result might
   not fit in `iptr`. */
uptr S_bytevector_compress_size(iptr s_count, INT compress_format) {
  switch (compress_format) {
    case COMPRESS_GZIP:
      if (is_valid_zlib_length(s_count))
        return compressBound((uLong)s_count);
      else {
        /* Compression will report "source too long" */
        return 0;
      }
    case COMPRESS_LZ4:
      if (is_valid_lz4_length(s_count))
        return LZ4_compressBound((uLong)s_count);
      else {
        /* Compression will report "source too long" */
        return 0;
      }
    default:
      S_error1("S_bytevector_compress_size", "unexpected compress format ~s", FIX(compress_format));
      return 0;
  }
}

ptr S_bytevector_compress(ptr dest_bv, iptr d_start, iptr d_count,
                          ptr src_bv, iptr s_start, iptr s_count,
                          INT compress_format) {
  ptr tc = get_thread_context();
  int compress_level = (INT)UNFIX(COMPRESSLEVEL(tc));

  /* On error, an message-template string with ~s for the bytevector */
  switch (compress_format) {
    case COMPRESS_GZIP:
      {
        int r;
        uLong destLen;

        if (!is_valid_zlib_length(s_count))
          return Sstring("source bytevector ~s is too large");

        destLen = (uLong)d_count;

        r = compress2(&BVIT(dest_bv, d_start), &destLen, &BVIT(src_bv, s_start), (uLong)s_count, S_zlib_compress_level(compress_level));

        if (r == Z_OK)
          return FIX(destLen);
        else if (r == Z_BUF_ERROR)
          return Sstring("destination bytevector is too small for compressed form of ~s");
        else
          return Sstring("internal error compressing ~s");
      }
    case COMPRESS_LZ4:
      {
        int destLen;

        if (!is_valid_lz4_length(s_count))
          return Sstring("source bytevector ~s is too large");

        if (compress_level == COMPRESS_MIN) {
          destLen = LZ4_compress_default((char *)&BVIT(src_bv, s_start), (char *)&BVIT(dest_bv, d_start), (int)s_count, (int)d_count);
        } else {
          destLen = LZ4_compress_HC((char *)&BVIT(src_bv, s_start), (char *)&BVIT(dest_bv, d_start), (int)s_count, (int)d_count, S_lz4_compress_level(compress_level));
        }

        if (destLen > 0)
          return Sfixnum(destLen);
        else
          return Sstring("compression failed for ~s");
      }
    default:
      S_error1("S_bytevector_compress", "unexpected compress format ~s", FIX(compress_format));
      return Sfalse;
  }
}

ptr S_bytevector_uncompress(ptr dest_bv, iptr d_start, iptr d_count,
                            ptr src_bv, iptr s_start, iptr s_count,
                            INT compress_format) {
  /* On error, an message-template string with ~s for the bytevector */
  switch (compress_format) {
    case COMPRESS_GZIP:
      {
        int r;
        uLongf destLen;

        if (!is_valid_zlib_length(d_count))
          return Sstring("expected result size of uncompressed source ~s is too large");

        destLen = (uLongf)d_count;

        r = uncompress(&BVIT(dest_bv, d_start), &destLen, &BVIT(src_bv, s_start), (uLong)s_count);

        if (r == Z_OK)
          return FIX(destLen);
        else if (r == Z_BUF_ERROR)
          return Sstring("uncompressed ~s is larger than expected size");
        else if (r == Z_DATA_ERROR)
          return Sstring("invalid data in source bytevector ~s");
        else
          return Sstring("internal error uncompressing ~s");
      }
    case COMPRESS_LZ4:
      {
        int r;

        if (!is_valid_lz4_length(d_count))
          return Sstring("expected result size of uncompressed source ~s is too large");

        r = LZ4_decompress_safe((char *)&BVIT(src_bv, s_start), (char *)&BVIT(dest_bv, d_start), (int)s_count, (int)d_count);

        if (r >= 0)
          return Sfixnum(r);
        else
          return Sstring("internal error uncompressing ~s");
      }
    default:
      return Sstring("unexpected compress format ~s");
  }
}


text:
"""Camera control on a surface.

Camera is just a pygame surface, so treat it as such!

A camera allows you to have a viewport which is smaller than the size
of the surface you're hoping to pan that viewport around on. Think
about looking at a big painting through a paper towel tube.

Cameras have an output resolution it scales its viewport to, plus a
behavior which governs how a Camera pans on its associated surface.

"""

import pygame

class CameraOutOfBounds(Exception):
    """The subsurface, which is the camera's view,
    goes beyond the surface the camera's viewing.

    Camera has been moved so that its area exceeds
    the surface it uses to make a subsurface.

    Attributes:
        camera (Camera):

    """

    def __init__(self, camera):
        super(CameraOutOfBounds, self).__init__(camera)
        self.camera = camera

class CameraBehavior(object):
    """How a camera moves. How it handles boundaries,
    character movement, etc.

    This CameraBehavior, the default, keeps the focal rectangle
    in the top left of the camera view.

    You'll want to inherit this class when creating
    a CameraBehavior, including overriding the
    move method.

    """

    @staticmethod
    def move(camera, focal_rectangle):
        """Move the camera, keeping the focal rectangle in
        the top left of the camera view.

        This method should be overridden in a child class.

        Arguments:
            camera (sappho.camera.Camera): Associated
                Sappho camera object to control.
            focal_rectangle (pygame.Rect): Rectangle which
                is used to possibly adjust camera position.

        """

        scroll_position = focal_rectangle.topleft
        camera.view_rect.topleft = scroll_position

class CameraCenterBehavior(CameraBehavior):
    """A camera behavior that centers the
    focal rectangle on the screen.

    Will not cause Camera.update_state() to raise
    CameraOutOfBounds, because the move logic
    prevents such from occuring!

    """

    @staticmethod
    def move(camera, focal_rectangle):
        """Move the camera, keeping the focal rectangle
        in the center of the screen where possible.

        Arguments:
            camera (sappho.camera.Camera):
            focal_rectangle (pygame.Rect): Rectangle which
                is used to possibly adjust camera position.

        """

        new_view_rect = camera.view_rect.copy()
        new_view_rect.center = focal_rectangle.center

        # Make sure the camera isn't centered in a way that
        # exceeds the environment's dimensions (don't go off
        # the map!)
        if new_view_rect.left < 0:
            new_view_rect.left = 0

        if new_view_rect.top < 0:
            new_view_rect.top = 0

        if new_view_rect.bottom > camera.source_resolution[1]:
            new_view_rect.bottom = camera.source_resolution[1]

        if new_view_rect.right > camera.source_resolution[0]:
            new_view_rect.right = camera.source_resolution[0]

        camera.view_rect = new_view_rect

class Camera(pygame.surface.Surface):
    """Surface that acts as a scrollable view, with optional scaling
    onto another surface.

    Attributes:
        source_resolution (tuple[int, int]): Maximum size of the
            environment being portrayed. If you have a map with many
            inconsistently sized layers, this should be the size of
            all of those layers flattened onto a single new layer.
            Anything beyond this size will not be on camera.
        output_resolution (tuple[int, int]): Resolution to scale up the
            view of the surface to
        view_rect (pygame.Rect): Rectangle area of this camera's view,
            which is used to create the subsurface, which is scaled
            to output_resolution and blit to self/camera.
        behavior (CameraBehavior): The initial behavior to use for this
            Camera. The :py:class:`CameraBehavior <sappho.CameraBehavior>`
            that this Camera uses to control movement.

    """

    def __init__(self, source_resolution, output_resolution,
                 view_resolution, behavior=None):

        """Create a Camera!

        Arguments:
            view_resolution (tuple[int, int]): used to create
                view_rect attribute.

        """

        super(Camera, self).__init__(output_resolution)

        self.source_surface = pygame.surface.Surface(source_resolution,
                                                     pygame.SRCALPHA)
        self.source_resolution = source_resolution
        self.output_resolution = output_resolution
        self.view_rect = pygame.Rect((0, 0), view_resolution)
        self.behavior = behavior or CameraBehavior()

    def update_state(self, timedelta):
        """Update the Camera to point to the
        current scroll position.

        Gets the "view" from the current scroll position
        and camera resolution. That view is used to create
        a subsurface of the "source subsurface," scaled to
        the target_resolution. The new subsurface is then
        blit to the camera (which is a surface, itself!).

        Arguments:
            timedelta (None): This isn't used at all, but it
                is included as an argument for consistency.

        """

        try:
            subsurface = self.source_surface.subsurface(self.view_rect)
        except ValueError:
            raise CameraOutOfBounds(self)

        scaled_surface = pygame.transform.scale(subsurface,
                                                self.output_resolution)

        # Blit the scaled surface to this camera (which is also a surface)
        super(Camera, self).blit(scaled_surface, (0, 0))

    def scroll_to(self, focal_rectangle):
        """Scroll to the given focal rectangle using the current behavior.

        Parameters:
            focal_rectangle (pygame.Rect): Rectangle to possibly update
                the view position to using the camera's current behavior

        """

        self.behavior.move(self, focal_rectangle)