<repo_name>Qix-/starfuse<file_sep>/starfuse/pak/sbbf03.py
"""Implementation of the StarBound block file v2/3 storage"""

import logging
import struct

from starfuse.fs.mapped_file import MappedFile

log = logging.getLogger(__name__)


class InvalidMagic(Exception):
    """A block file has an invalid magic string"""
    def __init__(self, path):
        super(InvalidMagic, self).__init__('a block file has an invalid magic string: %s' % path)


class SBBF03(MappedFile):
    """Implements a StarBound block file v3 store that is backed by a file

    Can also be used to read in v2.

    It's worth noting that the memory regions in this class are mapped and not
    read-in."""
    def __init__(self, path, page_count, read_only=False):
        super(SBBF03, self).__init__(path, page_count, read_only=read_only)

        self._header_size = 0
        self._block_size = 0

        self.header = None
        self.user_header = None
        self.blocks = dict()

        self.__load(path)

    def __del__(self):
        self.close()

    def block_region(self, bid):
        """Gets a block region given the block ID"""
        base_offset = self._header_size + (self._block_size * bid)
        return self.region(offset=base_offset, size=self._block_size)

    @property
    def block_count(self):
        block_region_size = len(self) - self._header_size
        return block_region_size // self._block_size

    def __load(self, path):
        log.debug('loading SBBF03 block file: %s', path)
        region = self.region(0, 32)

        # magic constant
        magic = region.read(6)
        if magic not in [b'SBBF03', b'SBBF02']:
            raise InvalidMagic(path)
        log.debug('block file has valid magic constant: %s', magic)

        # get the header and block size
        # this is all we need to actually read from the file before we start mmap-ing.
        # this is because we want to be able to mmap the header as well, and all we need to know
        # are the header sizes and block sizes.
        (self._header_size, self._block_size) = struct.unpack('>ii', region.read(8))
        log.debug('header_size=%d, block_size=%d', self._header_size, self._block_size)

        # calculate number of blocks
        log.debug('block count: %d', self.block_count)

        # map header
        self.header = self.region(offset=0, size=self._header_size)
        self.user_header = self.header.region(0x20)

        # map user header
        self.user_header = self.header.region(offset=0x20)
        log.debug('mapped headers successfully')
<file_sep>/starfuse/main.py
"""StarFuse, the StarBound packfile FUSE filesystem

This program allows you to mount StarBound PAK files as a directory,
granting you the ability to work directly within the PAK file instead
of having to unpack/re-pack it over and over again.
"""

import logging
import starfuse.config as config
from fuse import FUSE
from starfuse.fusepak import FusePAK

log = logging.getLogger(__name__)


def main():
    """StarFuse entry point"""
    log.info('starting StarFuse')
    log.info('mounting pakfile %s as %s', config.pak_file, ('read-only' if config.read_only else 'read/write'))
    pak = FusePAK(config.pak_file, page_count=config.page_count, read_only=config.read_only)
    log.info('mounting on %s', config.mount_dir)
    FUSE(pak, config.mount_dir, foreground=True)
<file_sep>/starfuse/pak/btreedb4.py
"""
BTreeDB4 implementation

Heavily influenced by Blixt's starfuse-py utilities,
this implementation's sine qua non.
"""
import binascii
import bisect
import io
import struct
import logging

import starfuse.pak.sbon as sbon
from starfuse.pak.sbbf03 import SBBF03

# Override range with xrange when running Python 2.x.
try:
    range = xrange
except:
    pass

log = logging.getLogger(__name__)


class BTreeKeyError(Exception):
    def __init__(self, key):
        super(BTreeKeyError, self).__init__(key)


def keyed(fn):
    def wrapper(*args, **kwargs):
        raw_key = False
        key = None
        try:
            if 'raw_key' in kwargs:
                raw_key = kwargs['raw_key']
                del kwargs['raw_key']

            self = args[0]
            key = args[1]

            if not raw_key:
                key = self.encode_key(key)

            assert len(key) == self.key_size, 'Invalid key length'

            nargs = (self, key) + args[2:]
            return fn(*nargs, **kwargs)
        except (BTreeKeyError, KeyError):
            hex_key = binascii.hexlify(key)
            if raw_key:
                raise BTreeKeyError(hex_key)
            else:
                raise BTreeKeyError('%s (%s)' % (hex_key, args[1]))
    return wrapper


class BTreeDB4(SBBF03):
    """A B-tree database format on top of the SBBF03 block format.

    Note: The developers of this format probably intended for the underlying
    file format to be arbitrary, but this database has pretty strong
    connections to SBBF02 right now so it's been implemented as inheriting from
    that file format. In the future we may want to split away from the
    inheritance chain and instead use the SBBF02 file as an API.
    """
    def __init__(self, path, page_count, read_only=False):
        super(BTreeDB4, self).__init__(path, page_count, read_only=False)

        self.key_size = None

        # Set this attribute to True to make reading more forgiving.
        self.repair = False

        self.alternate_root_node = None
        self.root_node = None
        self.root_node_is_leaf = None

        self.__load()

    def encode_key(self, key):
        """Can be overridden to encode a key before looking for it in the
        database (for example if the key needs to be hashed)."""
        return key

    def block(self, index):
        """Gets a block object given the specified index"""
        region = self.block_region(index)

        signature = bytes(region.read(2))

        if signature in _block_types:
            return _block_types[signature](self, index, region)

        if signature is not b'\0\0':
            raise Exception('Invalid signature detected: %s', signature)

        return None

    @keyed
    def _leaf_for_key(self, key):
        """Returns the binary data for the provided key."""

        block = self.block(self.root_node)
        assert block is not None, 'Root block is None'

        # Scan down the B-tree until we reach a leaf.
        while isinstance(block, BTreeIndex):
            block_number = block.block_for_key(key)
            block = self.block(block_number)
        assert isinstance(block, BTreeLeaf), 'Did not reach a leaf'

        return block

    @keyed
    def file_contents(self, key):
        leaf = self._leaf_for_key(key, raw_key=True)
        stream = LeafReader(self, leaf)

        # The number of keys is read on-demand because only leaves pointed to
        # by an index contain this number (others just contain arbitrary data).
        num_keys, = struct.unpack('>i', stream.read(4))
        assert num_keys < 1000, 'Leaf had unexpectedly high number of keys'
        for i in range(num_keys):
            cur_key = stream.read(self.key_size)
            # TODO do some smarter parsing here (read_bytes is easy to recreate
            # using regions)
            value = sbon.read_bytes(stream)

            if cur_key == key:
                return value

        raise BTreeKeyError(key)

    @keyed
    def file_size(self, key):
        leaf = self._leaf_for_key(key, raw_key=True)
        stream = LeafReader(self, leaf)

        # The number of keys is read on-demand because only leaves pointed to
        # by an index contain this number (others just contain arbitrary data).
        num_keys, = struct.unpack('>i', stream.read(4))
        assert num_keys < 1000, 'Leaf had unexpectedly high number of keys'
        for i in range(num_keys):
            cur_key = stream.read(self.key_size)
            # TODO do much better streaming here when LeafReader is refactored
            size = sbon.read_varlen_number(stream)
            stream.read(size)

            if cur_key == key:
                return size

        raise BTreeKeyError(key)

    def __load(self):
        stream = self.user_header

        # Require that the format of the content is BTreeDB4.
        db_format = sbon.read_fixlen_string(stream, 12)
        assert db_format == 'BTreeDB4', 'Expected binary tree database'
        log.debug('found BTreeDB4 header successfully')

        # Name of the database.
        self.identifier = sbon.read_fixlen_string(stream, 12)
        log.info('database name: %s', self.identifier)

        fields = struct.unpack('>i?xi?xxxi?', stream.read(19))
        self.key_size = fields[0]
        log.debug('key size=%d', self.key_size)

        # Whether to use the alternate root node index.
        self.alternate_root_node = fields[1]
        if self.alternate_root_node:
            self.root_node, self.root_node_is_leaf = fields[4:6]
            self.other_root_node, self.other_root_node_is_leaf = fields[2:4]
        else:
            self.root_node, self.root_node_is_leaf = fields[2:4]
            self.other_root_node, self.other_root_node_is_leaf = fields[4:6]
        log.debug('loaded root nodes: root=%d isleaf=%r', self.root_node, self.root_node_is_leaf)


class BTreeBlock(object):
    def __init__(self, btree, index, region):
        self._btree = btree
        self.index = index
        self.block_region = region


class BTreeIndex(BTreeBlock):
    SIGNATURE = b'II'

    __slots__ = ['keys', 'level', 'num_keys', 'values']

    def __init__(self, btree, index, region):
        super(BTreeIndex, self).__init__(btree, index, region)
        self.level, self.num_keys, left_block = struct.unpack('>Bii', self.block_region.read(9))

        self.keys = []
        self.values = [left_block]

        for i in range(self.num_keys):
            key = self.block_region.read(btree.key_size)
            block, = struct.unpack('>i', self.block_region.read(4))

            self.keys.append(key)
            self.values.append(block)

    def __str__(self):
        return 'Index(level={}, num_keys={})'.format(self.level, self.num_keys)

    def block_for_key(self, key):
        i = bisect.bisect_right(self.keys, key)
        return self.values[i]


class BTreeLeaf(BTreeBlock):
    SIGNATURE = b'LL'

    __slots__ = ['data', 'next_block']

    def __init__(self, btree, index, region):
        super(BTreeLeaf, self).__init__(btree, index, region)
        # Substract 6 for signature and next_block.
        self.data = self.block_region.read(btree._block_size - 6)

        value, = struct.unpack('>i', self.block_region.read(4))
        self.next_block = value if value != -1 else None

    def __str__(self):
        return 'Leaf(next_block={})'.format(self.next_block)


class BTreeFree(BTreeBlock):
    SIGNATURE = b'FF'

    __slots__ = ['next_free_block']

    def __init__(self, btree, index, region):
        super(BTreeFree, self).__init__(btree, index, region)
        self.raw_data = self.block_region.region()
        value, = struct.unpack('>i', self.raw_data[:4])
        self.next_free_block = value if value != -1 else None

    def __str__(self):
        return 'Free(next_free_block={})'.format(self.next_free_block)


class BTreeRestoredLeaf(BTreeLeaf):
    def __init__(self, free_block):
        assert isinstance(free_block, BTreeFree), 'Expected free block'
        self.data = free_block.raw_data[:-4]

        value, = struct.unpack('>i', free_block.raw_data[-4:])
        self.next_block = value if value != -1 else None

    def __str__(self):
        return 'RestoredLeaf(next_block={})'.format(self.next_block)


class LeafReader(object):
    """A pseudo-reader that will cross over block boundaries if necessary."""
    __slots__ = ['_file', '_leaf', '_offset', '_visited']

    def __init__(self, file, leaf):
        assert isinstance(file, BTreeDB4), 'File is not a BTreeDB4 instance'
        assert isinstance(leaf, BTreeLeaf), 'Leaf is not a BTreeLeaf instance'

        self._file = file
        self._leaf = leaf
        self._offset = 0
        self._visited = [leaf.index]

    def read(self, length):
        offset = self._offset

        if offset + length <= len(self._leaf.data):
            self._offset += length
            return self._leaf.data[offset:offset + length]

        buffer = io.BytesIO()

        # If the file is in repair mode, make the buffer available globally.
        if self._file.repair:
            LeafReader.last_buffer = buffer

        # Exhaust current leaf.
        num_read = buffer.write(self._leaf.data[offset:])
        length -= num_read

        # Keep moving onto the next leaf until we have read the desired amount.
        while length > 0:
            next_block = self._leaf.next_block

            assert next_block is not None, 'Tried to read too far'
            assert next_block not in self._visited, 'Tried to read visited block'
            self._visited.append(next_block)

            self._leaf = self._file.block(next_block)
            if self._file.repair and isinstance(self._leaf, BTreeFree):
                self._leaf = BTreeRestoredLeaf(self._leaf)

            assert isinstance(self._leaf, BTreeLeaf), \
                'Leaf pointed to non-leaf %s after reading %d byte(s)' % (next_block, buffer.tell())

            num_read = buffer.write(self._leaf.data[:length])
            length -= num_read

        # The new offset will be how much was read from the current leaf.
        self._offset = num_read

        data = buffer.getvalue()
        buffer.close()

        return data

_block_types = {
    BTreeIndex.SIGNATURE: BTreeIndex,
    BTreeLeaf.SIGNATURE: BTreeLeaf,
    BTreeFree.SIGNATURE: BTreeFree
}
<file_sep>/starfuse/pak/pakfile.py
"""VFS tracked, BTree package backed Pakfile

Note that this implementation supports asset pakfiles (including modpak files)
that are keyed with SHA256 digests for StarBound. It will not work with anything else.
"""

import hashlib
import io
import logging

import starfuse.pak.sbon as sbon
from starfuse.pak.btreedb4 import BTreeDB4
from starfuse.fs.vfs import VFS, FileNotFoundError, IsADirError, NotADirError

log = logging.getLogger(__name__)


class KeyStore(BTreeDB4):
    """A B-tree database that uses SHA-256 hashes for key lookup."""
    def encode_key(self, key):
        return hashlib.sha256(key.encode('utf-8')).digest()


class Package(KeyStore):
    """A B-tree database representing a package of files."""
    DIGEST_KEY = '_digest'
    INDEX_KEY = '_index'

    def __init__(self, path, page_count, read_only=False):
        super(Package, self).__init__(path, page_count, read_only=read_only)
        self._index = None

    def encode_key(self, key):
        return super(Package, self).encode_key(key.lower())

    def get_digest(self):
        return self.get(Package.DIGEST_KEY)

    def get_index(self):
        if self._index:
            return self._index

        # TODO optimize this to use new regions system after refactoring BTreeDB
        stream = io.BytesIO(self.file_contents(Package.INDEX_KEY))
        if self.identifier == 'Assets1':
            self._index = sbon.read_string_list(stream)
        elif self.identifier == 'Assets2':
            self._index = sbon.read_string_digest_map(stream)

        return self._index


class Pakfile(object):
    def __init__(self, path, page_count, read_only=False):
        self.vfs = VFS()
        self.pkg = Package(path, page_count, read_only=False)

        log.debug('obtaining file list')
        file_index = self.pkg.get_index()
        log.debug('registering files with virtual filesystem')
        for filepath, lookup in file_index.iteritems():
            self.vfs.add_file(filepath, lookup, mkdirs=True)
        log.info('registered %d files with virtual filesystem', len(file_index))

    @property
    def read_only(self):
        return self.pkg.read_only

    @read_only.setter
    def read_only(self, val):
        self.pkg.read_only = val

    def entry(self, abspath):
        if abspath is None or abspath == '/':
            return (self.vfs.root, None, self.vfs.root, False)

        names = self.vfs._split_path(abspath)
        direc = self.vfs._mkdirp(names[:-1], srcpath=abspath, mkdirs=False)

        fname = names[-1]

        if fname not in direc:
            raise FileNotFoundError(abspath)

        entry = direc[fname]

        # (directory dict, filename, lookup entry, True if file, False if directory)
        return (direc, fname, entry, not isinstance(entry, dict))

    def directory_listing(self, abspath):
        (_, _, lookup, isfile) = self.entry(abspath)
        if isfile:
            raise NotADirError(abspath)
        return lookup.keys()

    def file_size(self, abspath):
        (_, _, _, isfile) = self.entry(abspath)
        if not isfile:
            raise IsADirError(abspath)
        return self.pkg.file_size(abspath)

    def file_contents(self, abspath, offset=0, size=-1):
        (_, _, _, isfile) = self.entry(abspath)
        if not isfile:
            raise IsADirError(abspath)
        return self.pkg.file_contents(abspath)[offset:offset + size]

    def readdir(self, abspath):
        (_, _, lookup, isfile) = self.entry(abspath)
        if isfile:
            raise NotADirError(abspath)

        results = []
        for name, lobj in lookup.iteritems():
            results.append((name, not isinstance(lobj, dict)))
        return results
<file_sep>/README.md
# StarFuse
Mount StarBound .pak files as FUSE filesystems.

This codebase aims to serve as a reference implementation for both reading and writing modern Starbound files.

## Installation
Just want to use StarFuse?

```console
$ pip install starfuse
```

Want to develop StarFuse?

```console
$ git clone https://github.com/qix-/starfuse
$ cd starfuse
$ virtualenv env
$ . env/bin/activate
$ pip install -r requirements.txt
```

# License
Copyright &copy; 2016 by <NAME>. Licensed under the [MIT License](LICENSE.txt) for the StarBound community.

Developers of StarBound and ChuckleFish as a whole may use this software freely both commercially and otherwise (whilst adhering to the MIT License, which doesn't ask for much). I encourage corrections from the team if they so choose.
<file_sep>/setup.py
from distutils.core import setup
setup(
    name='starfuse',
    packages=['starfuse', 'starfuse.pak', 'starfuse.pak', 'starfuse.fs'],
    version='0.3.0',
    description='Mount StarBound .pak files as FUSE filesystems',
    author='<NAME>',
    author_email='<EMAIL>',
    url='https://github.com/qix-/starfuse',
    download_url='https://github.com/qix-/starfuse/tarball/0.1.0',
    keywords=['starfuse', 'starbound', 'fuse', 'pak', 'development', 'mod', 'extension'],
    classifiers=[],
    install_requires=['fusepy'],
    entry_points={
        'console_scripts': [
            'starfuse = starfuse.main:main'
        ],
    },
)
<file_sep>/starfuse/config.py
"""StarFuse configuration/opt-parsing"""

import sys
import argparse
import logging

LOGGERFMT = '%(asctime)s [%(levelname)s] %(name)s <%(funcName)s>: %(message)s'


class Config(object):
    """StarFuse configuration class and option parser"""
    def __init__(self):
        parser = argparse.ArgumentParser(
            prog='starfuse',
            description='Mounts StarBound .pak files as FUSE filesystems')

        parser.add_argument('pakfile', type=str, help='the .pak file on which to operate')
        parser.add_argument('mount_dir', type=str, help='the directory on which to mount the PAK file')
        parser.add_argument('-v', '--verbose', help='be noisy', action='store_true')
        parser.add_argument('-w', '--write', help='allow modifications to the .pak file', action='store_true')
        parser.add_argument('--pages', type=int, help='map this number of pages at a time (default: 256)', default=256)

        self._args = parser.parse_args()

        if self._args.verbose:
            logging.basicConfig(level=logging.DEBUG, format=LOGGERFMT)
        else:
            logging.basicConfig(level=logging.INFO, format=LOGGERFMT)

    @property
    def pak_file(self):
        """Gets the .pak file path to use"""
        return self._args.pakfile

    @property
    def mount_dir(self):
        """Gets the target mount directory"""
        return self._args.mount_dir

    @property
    def page_count(self):
        """Gets the number of pages to map at once"""
        return self._args.pages

    @property
    def read_only(self):
        """Whether or not the file is read-only protected"""
        return not self._args.write

sys.modules[__name__] = Config()
<file_sep>/starfuse/fs/vfs.py
"""Virtual File System, which manages paths and their lookups"""

import os
import os.path
import logging

assert os.sep == '/', 'Starfuse cannot be used on non-unix systems'

log = logging.getLogger(__name__)


class NotADirError(Exception):
    pass


class IsADirError(Exception):
    pass


class FileNotFoundError(Exception):
    pass


class VFS(object):
    def __init__(self):
        self.root = dict()

    def add_file(self, abspath, lookup=None, mkdirs=False):
        names = self._split_path(abspath)
        direc = self._mkdirp(names[:-1], srcpath=abspath, mkdirs=mkdirs)

        filename = names[-1]

        if filename in direc:
            if isinstance(direc[filename], dict):
                raise IsADirError(abspath)
            return

        direc[filename] = lookup

    def lookup_file(self, abspath):
        names = self._split_path(abspath)
        direc = self._mkdirp(names[:-1], srcpath=abspath)

        filename = names[-1]
        if filename not in direc:
            raise FileNotFoundError(Exception)

        return direc[filename]

    def _mkdirp(self, names, srcpath=None, mkdirs=False):
        cur = self.root
        for name in names:
            if name not in cur:
                if not mkdirs:
                    raise FileNotFoundError(srcpath)
                cur[name] = dict()
            elif not isinstance(cur, dict):
                raise NotADirError(srcpath)
            cur = cur[name]
        return cur

    def _split_path(self, path):
        return path.strip(os.sep).split(os.sep)
<file_sep>/starfuse/pak/SBON.py
import collections
import struct

# Override range with xrange when running Python 2.x.
try:
    range = xrange
except:
    pass

Document = collections.namedtuple('Document', ['name', 'version', 'data'])

Tile = collections.namedtuple('Tile', [
    'foreground_material',
    'foreground_hue_shift',
    'foreground_variant',
    'foreground_sprite',
    'foreground_sprite_hue_shift',
    'background_material',
    'background_hue_shift',
    'background_variant',
    'background_sprite',
    'background_sprite_hue_shift',
    'liquid',
    'liquid_amount',
    'liquid_pressure',
    'liquid_unknown',
    'collision',
    'dungeon',
    'biome',
    'biome_2',
    'indestructible',
])


def read_bytes(stream):
    length = read_varlen_number(stream)
    return stream.read(length)


def read_document(stream, repair=False):
    name = read_string(stream)

    # Not sure what this part is.
    assert stream.read(1) == b'\x01'

    version = struct.unpack('>i', stream.read(4))[0]
    data = read_dynamic(stream, repair)

    return Document(name, version, data)


def read_document_list(stream):
    length = read_varlen_number(stream)
    return [read_document(stream) for _ in range(length)]


def read_dynamic(stream, repair=False):
    type = ord(stream.read(1))

    try:
        if type == 1:
            return None
        elif type == 2:
            format = '>d'
        elif type == 3:
            format = '?'
        elif type == 4:
            return read_varlen_number_signed(stream)
        elif type == 5:
            return read_string(stream)
        elif type == 6:
            return read_list(stream, repair)
        elif type == 7:
            return read_map(stream, repair)
        else:
            raise ValueError('Unknown dynamic type 0x%02X' % type)
    except:
        if repair:
            return None
        raise

    # Anything that passes through is assumed to have set a format to unpack.
    return struct.unpack(format, stream.read(struct.calcsize(format)))[0]


def read_fixlen_string(stream, length):
    return stream.read(length).rstrip(b'\x00').decode('utf-8')


def read_list(stream, repair=False):
    length = read_varlen_number(stream)
    return [read_dynamic(stream, repair) for _ in range(length)]


def read_map(stream, repair=False):
    length = read_varlen_number(stream)

    value = dict()
    for _ in range(length):
        key = read_string(stream)
        value[key] = read_dynamic(stream, repair)

    return value


def read_string(stream):
    return read_bytes(stream).decode('utf-8')


def read_string_list(stream):
    """Optimized structure that doesn't have a type byte for every item.

    """
    length = read_varlen_number(stream)
    return [read_string(stream) for _ in range(length)]


def read_string_digest_map(stream):
    """Special structure of string/digest pairs, used by the assets database.

    """
    length = read_varlen_number(stream)

    value = dict()
    for _ in range(length):
        path = read_string(stream)
        # Unnecessary whitespace.
        stream.seek(1, 1)
        digest = stream.read(32)
        value[path] = digest

    return value


def read_tile(world_version, stream):
    if world_version < 6:
        values = struct.unpack('>hBBhBhBBhBBHBhBB?', stream.read(23))
        values = values[:12] + (0, 0) + values[12:]
    else:
        # TODO: Figure out what the new values mean.
        values = struct.unpack('>hBBhBhBBhBBffBBhBB?', stream.read(30))
    return Tile(*values)


def read_varlen_number(stream):
    """Read while the most significant bit is set, then put the 7 least
    significant bits of all read bytes together to create a number.

    """
    value = 0
    while True:
        byte = ord(stream.read(1))
        if not byte & 0b10000000:
            return value << 7 | byte
        value = value << 7 | (byte & 0b01111111)


def read_varlen_number_signed(stream):
    value = read_varlen_number(stream)

    # Least significant bit represents the sign.
    if value & 1:
        return -(value >> 1)
    else:
        return value >> 1


def write_bytes(stream, bytes):
    write_varlen_number(stream, len(bytes))
    stream.write(bytes)


def write_varlen_number(stream, value):
    if value == 0:
        stream.write(b'\x00')
        return

    pieces = []
    while value:
        x, value = value & 0b01111111, value >> 7
        if len(pieces):
            x |= 0b10000000
        pieces.insert(0, x)
        if len(pieces) > 4096:
            raise ValueError('Number too large')

    stream.write(struct.pack('%dB' % len(pieces), *pieces))


def write_varlen_number_signed(stream, value):
    has_sign = 1 if value < 0 else 0
    write_varlen_number(stream, value << 1 | has_sign)
<file_sep>/starfuse/fs/mapped_file.py
"""A pseudo-file mapped into memory

Provides better performance for frequent reads/writes,
and makes reading/writing easier via regions (windows)
of memory. Allows memory to be accessed via array reads/
writes as well.
"""

import mmap
import logging

log = logging.getLogger(__name__)


class ReadOnlyError(Exception):
    """The mapped file is flagged as read-only"""
    def __init__(self, path):
        super(ReadOnlyError, self).__init__('mapped file is flagged as read only: %s' % path)


class RegionOverflowError(Exception):
    """Data at an offset was requested but the offset was greater than the allocated size"""
    def __init__(self, offset):
        super(RegionOverflowError, self).__init__('region overflow offset: %d (did you allocate?)' % offset)


class Region(object):
    """A virtual region of mapped memory

    This class is a 'faked' mmap() result that allows for the finer allocation of memory mappings
    beyond/below what the filesystem really allows. It is backed by true mmap()'d pages and
    uses magic methods to achieve the appearance of being an isolated region of memory."""
    __slots__ = 'parent', 'base_offset', '__size', 'cursor'

    def __init__(self, parent, base_offset, size):
        self.parent = parent
        self.base_offset = base_offset
        self.__size = size
        self.cursor = 0

    def __len__(self):
        return self.__size

    def __str__(self):
        return str(self.read(offset=0, length=len(self)))

    def __enter__(self):
        return self

    def __exit__(self, tipo, value, traceback):
        return self

    def region(self, offset=-1, size=-1):
        (offset, size) = self._sanitize_segment(offset, size)
        return self.parent.region(self.base_offset + offset, size)

    def _sanitize_segment(self, offset, length):
        if offset >= len(self):
            raise ValueError('offset falls outside region size')
        elif offset < 0:
            offset = self.cursor

        if length == 0:
            raise ValueError('length must be at least 1')
        elif length < 0:
            length = len(self) - offset

        return (offset, length)

    def read(self, length=-1, offset=-1, advance=True):
        (offset, length) = self._sanitize_segment(offset, length)
        offset += self.base_offset
        result = self.parent.read(length, offset, advance=advance)
        if advance:
            self.cursor += len(result)
        return result

    def write(self, value, length=-1, offset=-1, advance=True):
        if length < 0:
            length = len(value)
        (offset, length) = self._sanitaize_segment(offset, length)
        offset += self.base_offset
        result = self.parent.write(value, length, offset, advance=advance)
        if advance:
            self.cursor += result
        return result


class MappedFile(Region):
    """Manages mmap()-ings of a file into vmem.

    This class prevents virtual address space from growing too large by
    re-using existing maps if the requested regions have already been mapped.
    """
    def __init__(self, path, page_count, read_only=False):
        # XXX TODO NOTE remove this line when write functionality is added.
        read_only = True

        # getting 'too many files open' error? increase the constant on the next line
        # (must be an exponent of 2)
        self._page_size = page_count * mmap.PAGESIZE

        # make sure we're sane here - allocation granularity needs to divide into page size!
        assert (self._page_size % mmap.ALLOCATIONGRANULARITY) == 0, 'page size is not a multiple of allocation granularity!'

        self._file = open(path, 'r+b')
        self._pages = dict()

        self.read_only = read_only
        self._path = path

        self.cursor = 0
        super(MappedFile, self).__init__(self, base_offset=0, size=len(self))

    def __len__(self):
        self._file.seek(0, 2)
        size = self._file.tell()
        return size

    def __del__(self):
        self.close()

    def close(self):
        """Unmaps all mappings"""
        for i in self._pages:
            self._pages[i].close()
        self._file.close()

    def region(self, offset, size):
        """Requests a virtual region be 'allocated'"""
        lower_page = offset - (offset % self._page_size)
        upper_page = ((offset + size) // self._page_size) * self._page_size
        lower_page_id = lower_page // self._page_size
        upper_page_id = upper_page // self._page_size

        # make sure we're mapped
        for i in range(lower_page_id, upper_page_id + 1):
            if i not in self._pages:
                page_offset = i * self._page_size
                page_size = min(self._page_size, len(self) - page_offset)
                log.debug('mapping vfile page: id=%d offset=%d size=%d', i, page_offset, page_size)
                self._pages[i] = mmap.mmap(self._file.fileno(), offset=page_offset, length=page_size)

        # create a region
        return Region(self, base_offset=offset, size=size)

    def read(self, length=1, offset=-1, advance=True):
        """Reads data from the virtual region"""
        (offset, length) = self._sanitize_segment(offset, length)

        results = []
        length = min(length, len(self))

        abs_offset = offset
        cur_page = abs_offset // self._page_size
        abs_offset %= self._page_size

        while length > 0:
            readable = self._page_size - abs_offset
            readable = min(readable, length)

            results.append(self._pages[cur_page][abs_offset:abs_offset + readable])

            length -= readable
            abs_offset = 0
            cur_page += 1

        result = ''.join(results)
        if advance:
            self.cursor += len(result)
        return result

    def write(self, value, offset=-1, length=-1, advance=True):
        if self.read_only:
            raise ReadOnlyError(self._path)

        # TODO
        assert False, 'not implemented'
        return 0

    def __getitem__(self, offset):
        if isinstance(offset, slice):
            (start, fin, step) = offset.indices(len(self))
            result = self.read(offset=start, length=fin - start)
            if step not in [None, 1]:
                result = result[::step]
            return result

        if not isinstance(offset, int):
            raise TypeError('offset is not an integer: %s' % repr(offset))

        if offset >= len(self):
            raise RegionOverflowError(offset)

        page = offset // self._page_size
        rel_offset = offset % self._page_size
        return self._pages[page][rel_offset]

    def __setitem__(self, offset, value):
        if self.read_only:
            raise ReadOnlyError(self._path)

        if isinstance(offset, slice):
            raise ValueError('Slice assignment not supported in mapped files; assemble your data first and then write')

        if not isinstance(offset, int):
            raise TypeError('offset is not an integer: %s' % repr(offset))

        if offset >= len(self):
            raise RegionOverflowError(offset)

        page = offset // self._page_size
        rel_offset = offset % self._page_size
        self._pages[page][rel_offset] = value
<file_sep>/starfuse/fusepak.py
"""FUSE glue between the filesystem handlers and the PakVFS"""

import os
import errno
import logging
from threading import Lock
from fuse import FuseOSError, Operations, LoggingMixIn
from stat import S_IFDIR, S_IFREG
from time import time
from starfuse.pak.pakfile import Pakfile
from starfuse.fs.vfs import FileNotFoundError, IsADirError, NotADirError

log = logging.getLogger(__name__)


def fuse_op(fn):
    def handled_fn(*args, **kwargs):
        try:
            return fn(*args, **kwargs)
        except FileNotFoundError as e:
            raise FuseOSError(errno.ENOENT)
        except IsADirError as e:
            raise FuseOSError(errno.EISDIR)
        except NotADirError as e:
            raise FuseOSError(errno.ENOTDIR)
        except Exception as e:
            log.exception('EIO: %s', e.message)
            raise FuseOSError(errno.EIO)
    return handled_fn


class FusePAK(LoggingMixIn, Operations):
    """FUSE operations implementation for StarBound PAK files"""
    def __init__(self, pakfile, page_count, read_only=False):
        self.pakfile = Pakfile(pakfile, page_count, read_only=read_only)
        self._lock = Lock()

    def make_file_struct(self, size, isfile=True, ctime=time(), mtime=time(), atime=time(), read_only=False):
        stats = dict()
        # TODO replace uncommented modes with commented when write ability is added
        if isfile:
            stats['st_mode'] = S_IFREG | 0o0444
        else:
            stats['st_mode'] = S_IFDIR | 0o0555

        if not self.pakfile.read_only:
            stats['st_mode'] |= 0o0200

        stats['st_uid'] = os.getuid()
        stats['st_gid'] = os.getgid()
        stats['st_nlink'] = 1
        stats['st_ctime'] = ctime
        stats['st_mtime'] = mtime
        stats['st_atime'] = atime
        stats['st_size'] = size
        return stats

    @fuse_op
    def access(self, path, mode):
        return 0

    @fuse_op
    def flush(self, path, fn):
        return 0

    @fuse_op
    def fsync(self, path, datasync, fh):
        # we don't need to worry about fsync in StarFuse. :)
        return 0

    @fuse_op
    def getattr(self, path, fh=None):
        if path == '/':
            return self.make_file_struct(0, isfile=False)
        (_, _, _, isfile) = self.pakfile.entry(path)
        if not isfile:
            return self.make_file_struct(0, isfile=False)
        size = self.pakfile.file_size(path)
        return self.make_file_struct(size)

    @fuse_op
    def readdir(self, path, fh=None):
        return self.pakfile.directory_listing(path)

    @fuse_op
    def read(self, path, size, offset, fh=None):
        return self.pakfile.file_contents(path, offset, size)
