text:
from datetime import datetime, timezone, timedelta
import gzip
import os
import pathlib
import pickle

import pytest
import weakref

import pyarrow as pa
from pyarrow.tests.test_io import assert_file_not_found
from pyarrow.tests.util import (_filesystem_uri, ProxyHandler,
                                _configure_s3_limited_user)

from pyarrow.fs import (FileType, FileInfo, FileSelector, FileSystem,
                        LocalFileSystem, SubTreeFileSystem, _MockFileSystem,
                        FileSystemHandler, PyFileSystem, FSSpecHandler,
                        copy_files)

class DummyHandler(FileSystemHandler):
    def __init__(self, value=42):
        self._value = value

    def __eq__(self, other):
        if isinstance(other, FileSystemHandler):
            return self._value == other._value
        return NotImplemented

    def __ne__(self, other):
        if isinstance(other, FileSystemHandler):
            return self._value != other._value
        return NotImplemented

    def get_type_name(self):
        return "dummy"

    def normalize_path(self, path):
        return path

    def get_file_info(self, paths):
        info = []
        for path in paths:
            if "file" in path:
                info.append(FileInfo(path, FileType.File))
            elif "dir" in path:
                info.append(FileInfo(path, FileType.Directory))
            elif "notfound" in path:
                info.append(FileInfo(path, FileType.NotFound))
            elif "badtype" in path:
                # Will raise when converting
                info.append(object())
            else:
                raise IOError
        return info

    def get_file_info_selector(self, selector):
        if selector.base_dir != "somedir":
            if selector.allow_not_found:
                return []
            else:
                raise FileNotFoundError(selector.base_dir)
        infos = [
            FileInfo("somedir/file1", FileType.File, size=123),
            FileInfo("somedir/subdir1", FileType.Directory),
        ]
        if selector.recursive:
            infos += [
                FileInfo("somedir/subdir1/file2", FileType.File, size=456),
            ]
        return infos

    def create_dir(self, path, recursive):
        if path == "recursive":
            assert recursive is True
        elif path == "non-recursive":
            assert recursive is False
        else:
            raise IOError

    def delete_dir(self, path):
        assert path == "delete_dir"

    def delete_dir_contents(self, path, missing_dir_ok):
        if not path.strip("/"):
            raise ValueError
        assert path == "delete_dir_contents"

    def delete_root_dir_contents(self):
        pass

    def delete_file(self, path):
        assert path == "delete_file"

    def move(self, src, dest):
        assert src == "move_from"
        assert dest == "move_to"

    def copy_file(self, src, dest):
        assert src == "copy_file_from"
        assert dest == "copy_file_to"

    def open_input_stream(self, path):
        if "notfound" in path:
            raise FileNotFoundError(path)
        data = "{0}:input_stream".format(path).encode('utf8')
        return pa.BufferReader(data)

    def open_input_file(self, path):
        if "notfound" in path:
            raise FileNotFoundError(path)
        data = "{0}:input_file".format(path).encode('utf8')
        return pa.BufferReader(data)

    def open_output_stream(self, path, metadata):
        if "notfound" in path:
            raise FileNotFoundError(path)
        return pa.BufferOutputStream()

    def open_append_stream(self, path, metadata):
        if "notfound" in path:
            raise FileNotFoundError(path)
        return pa.BufferOutputStream()

@pytest.fixture
def localfs(request, tempdir):
    return dict(
        fs=LocalFileSystem(),
        pathfn=lambda p: (tempdir / p).as_posix(),
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def py_localfs(request, tempdir):
    return dict(
        fs=PyFileSystem(ProxyHandler(LocalFileSystem())),
        pathfn=lambda p: (tempdir / p).as_posix(),
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def mockfs(request):
    return dict(
        fs=_MockFileSystem(),
        pathfn=lambda p: p,
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def py_mockfs(request):
    return dict(
        fs=PyFileSystem(ProxyHandler(_MockFileSystem())),
        pathfn=lambda p: p,
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def localfs_with_mmap(request, tempdir):
    return dict(
        fs=LocalFileSystem(use_mmap=True),
        pathfn=lambda p: (tempdir / p).as_posix(),
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def subtree_localfs(request, tempdir, localfs):
    return dict(
        fs=SubTreeFileSystem(str(tempdir), localfs['fs']),
        pathfn=lambda p: p,
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def gcsfs(request, gcs_server):
    request.config.pyarrow.requires('gcs')
    from pyarrow.fs import GcsFileSystem

    host, port = gcs_server['connection']
    bucket = 'pyarrow-filesystem/'
    # Make sure the server is alive.
    assert gcs_server['process'].poll() is None

    fs = GcsFileSystem(
        endpoint_override=f'{host}:{port}',
        scheme='http',
        # Mock endpoint doesn't check credentials.
        anonymous=True,
        retry_time_limit=timedelta(seconds=45)
    )
    fs.create_dir(bucket)

    yield dict(
        fs=fs,
        pathfn=bucket.__add__,
        allow_move_dir=False,
        allow_append_to_file=False,
    )
    fs.delete_dir(bucket)

@pytest.fixture
def s3fs(request, s3_server):
    request.config.pyarrow.requires('s3')
    from pyarrow.fs import S3FileSystem

    host, port, access_key, secret_key = s3_server['connection']
    bucket = 'pyarrow-filesystem/'

    fs = S3FileSystem(
        access_key=access_key,
        secret_key=secret_key,
        endpoint_override='{}:{}'.format(host, port),
        scheme='http',
        allow_bucket_creation=True,
        allow_bucket_deletion=True
    )
    fs.create_dir(bucket)

    yield dict(
        fs=fs,
        pathfn=bucket.__add__,
        allow_move_dir=False,
        allow_append_to_file=False,
    )
    fs.delete_dir(bucket)

@pytest.fixture
def subtree_s3fs(request, s3fs):
    prefix = 'pyarrow-filesystem/prefix/'
    return dict(
        fs=SubTreeFileSystem(prefix, s3fs['fs']),
        pathfn=prefix.__add__,
        allow_move_dir=False,
        allow_append_to_file=False,
    )

_minio_limited_policy = """{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}"""

@pytest.fixture
def hdfs(request, hdfs_connection):
    request.config.pyarrow.requires('hdfs')
    if not pa.have_libhdfs():
        pytest.skip('Cannot locate libhdfs')

    from pyarrow.fs import HadoopFileSystem

    host, port, user = hdfs_connection
    fs = HadoopFileSystem(host, port=port, user=user)

    return dict(
        fs=fs,
        pathfn=lambda p: p,
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def py_fsspec_localfs(request, tempdir):
    fsspec = pytest.importorskip("fsspec")
    fs = fsspec.filesystem('file')
    return dict(
        fs=PyFileSystem(FSSpecHandler(fs)),
        pathfn=lambda p: (tempdir / p).as_posix(),
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def py_fsspec_memoryfs(request, tempdir):
    fsspec = pytest.importorskip("fsspec", minversion="0.7.5")
    if fsspec.__version__ == "0.8.5":
        # see https://issues.apache.org/jira/browse/ARROW-10934
        pytest.skip("Bug in fsspec 0.8.5 for in-memory filesystem")
    fs = fsspec.filesystem('memory')
    return dict(
        fs=PyFileSystem(FSSpecHandler(fs)),
        pathfn=lambda p: p,
        allow_move_dir=True,
        allow_append_to_file=True,
    )

@pytest.fixture
def py_fsspec_s3fs(request, s3_server):
    s3fs = pytest.importorskip("s3fs")
    host, port, access_key, secret_key = s3_server['connection']
    bucket = 'pyarrow-filesystem/'

    fs = s3fs.S3FileSystem(
        key=access_key,
        secret=secret_key,
        client_kwargs=dict(endpoint_url='http://{}:{}'.format(host, port))
    )
    fs = PyFileSystem(FSSpecHandler(fs))
    fs.create_dir(bucket)

    yield dict(
        fs=fs,
        pathfn=bucket.__add__,
        allow_move_dir=False,
        allow_append_to_file=True,
    )
    fs.delete_dir(bucket)

@pytest.fixture(params=[
    pytest.param(
        pytest.lazy_fixture('localfs'),
        id='LocalFileSystem()'
    ),
    pytest.param(
        pytest.lazy_fixture('localfs_with_mmap'),
        id='LocalFileSystem(use_mmap=True)'
    ),
    pytest.param(
        pytest.lazy_fixture('subtree_localfs'),
        id='SubTreeFileSystem(LocalFileSystem())'
    ),
    pytest.param(
        pytest.lazy_fixture('s3fs'),
        id='S3FileSystem',
        marks=pytest.mark.s3
    ),
    pytest.param(
        pytest.lazy_fixture('gcsfs'),
        id='GcsFileSystem',
        marks=pytest.mark.gcs
    ),
    pytest.param(
        pytest.lazy_fixture('hdfs'),
        id='HadoopFileSystem',
        marks=pytest.mark.hdfs
    ),
    pytest.param(
        pytest.lazy_fixture('mockfs'),
        id='_MockFileSystem()'
    ),
    pytest.param(
        pytest.lazy_fixture('py_localfs'),
        id='PyFileSystem(ProxyHandler(LocalFileSystem()))'
    ),
    pytest.param(
        pytest.lazy_fixture('py_mockfs'),
        id='PyFileSystem(ProxyHandler(_MockFileSystem()))'
    ),
    pytest.param(
        pytest.lazy_fixture('py_fsspec_localfs'),
        id='PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))'
    ),
    pytest.param(
        pytest.lazy_fixture('py_fsspec_memoryfs'),
        id='PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))'
    ),
    pytest.param(
        pytest.lazy_fixture('py_fsspec_s3fs'),
        id='PyFileSystem(FSSpecHandler(s3fs.S3FileSystem()))',
        marks=pytest.mark.s3
    ),
])
def filesystem_config(request):
    return request.param

@pytest.fixture
def fs(request, filesystem_config):
    return filesystem_config['fs']

@pytest.fixture
def pathfn(request, filesystem_config):
    return filesystem_config['pathfn']

@pytest.fixture
def allow_move_dir(request, filesystem_config):
    return filesystem_config['allow_move_dir']

@pytest.fixture
def allow_append_to_file(request, filesystem_config):
    return filesystem_config['allow_append_to_file']

def check_mtime(file_info):
    assert isinstance(file_info.mtime, datetime)
    assert isinstance(file_info.mtime_ns, int)
    assert file_info.mtime_ns >= 0
    assert file_info.mtime_ns == pytest.approx(
        file_info.mtime.timestamp() * 1e9)
    # It's an aware UTC datetime
    tzinfo = file_info.mtime.tzinfo
    assert tzinfo is not None
    assert tzinfo.utcoffset(None) == timedelta(0)

def check_mtime_absent(file_info):
    assert file_info.mtime is None
    assert file_info.mtime_ns is None

def check_mtime_or_absent(file_info):
    if file_info.mtime is None:
        check_mtime_absent(file_info)
    else:
        check_mtime(file_info)

def skip_fsspec_s3fs(fs):
    if fs.type_name == "py::fsspec+s3":
        pytest.xfail(reason="Not working with fsspec's s3fs")

@pytest.mark.s3
def test_s3fs_limited_permissions_create_bucket(s3_server):
    from pyarrow.fs import S3FileSystem
    _configure_s3_limited_user(s3_server, _minio_limited_policy)
    host, port, _, _ = s3_server['connection']

    fs = S3FileSystem(
        access_key='limited',
        secret_key='limited123',
        endpoint_override='{}:{}'.format(host, port),
        scheme='http'
    )
    fs.create_dir('existing-bucket/test')

    with pytest.raises(pa.ArrowIOError, match="Bucket 'new-bucket' not found"):
        fs.create_dir('new-bucket')

    with pytest.raises(pa.ArrowIOError, match="Would delete bucket"):
        fs.delete_dir('existing-bucket')

def test_file_info_constructor():
    dt = datetime.fromtimestamp(1568799826, timezone.utc)

    info = FileInfo("foo/bar")
    assert info.path == "foo/bar"
    assert info.base_name == "bar"
    assert info.type == FileType.Unknown
    assert info.size is None
    check_mtime_absent(info)

    info = FileInfo("foo/baz.txt", type=FileType.File, size=123,
                    mtime=1568799826.5)
    assert info.path == "foo/baz.txt"
    assert info.base_name == "baz.txt"
    assert info.type == FileType.File
    assert info.size == 123
    assert info.mtime_ns == 1568799826500000000
    check_mtime(info)

    info = FileInfo("foo", type=FileType.Directory, mtime=dt)
    assert info.path == "foo"
    assert info.base_name == "foo"
    assert info.type == FileType.Directory
    assert info.size is None
    assert info.mtime == dt
    assert info.mtime_ns == 1568799826000000000
    check_mtime(info)

def test_cannot_instantiate_base_filesystem():
    with pytest.raises(TypeError):
        FileSystem()

def test_filesystem_equals():
    fs0 = LocalFileSystem()
    fs1 = LocalFileSystem()
    fs2 = _MockFileSystem()

    assert fs0.equals(fs0)
    assert fs0.equals(fs1)
    with pytest.raises(TypeError):
        fs0.equals('string')
    assert fs0 == fs0 == fs1
    assert fs0 != 4

    assert fs2 == fs2
    assert fs2 != _MockFileSystem()

    assert SubTreeFileSystem('/base', fs0) == SubTreeFileSystem('/base', fs0)
    assert SubTreeFileSystem('/base', fs0) != SubTreeFileSystem('/base', fs2)
    assert SubTreeFileSystem('/base', fs0) != SubTreeFileSystem('/other', fs0)

def test_subtree_filesystem():
    localfs = LocalFileSystem()

    subfs = SubTreeFileSystem('/base', localfs)
    assert subfs.base_path == '/base/'
    assert subfs.base_fs == localfs
    assert repr(subfs).startswith('SubTreeFileSystem(base_path=/base/, '
                                  'base_fs=<pyarrow._fs.LocalFileSystem')

    subfs = SubTreeFileSystem('/another/base/', LocalFileSystem())
    assert subfs.base_path == '/another/base/'
    assert subfs.base_fs == localfs
    assert repr(subfs).startswith('SubTreeFileSystem(base_path=/another/base/,'
                                  ' base_fs=<pyarrow._fs.LocalFileSystem')

def test_filesystem_pickling(fs):
    if fs.type_name.split('::')[-1] == 'mock':
        pytest.xfail(reason='MockFileSystem is not serializable')

    serialized = pickle.dumps(fs)
    restored = pickle.loads(serialized)
    assert isinstance(restored, FileSystem)
    assert restored.equals(fs)

def test_filesystem_is_functional_after_pickling(fs, pathfn):
    if fs.type_name.split('::')[-1] == 'mock':
        pytest.xfail(reason='MockFileSystem is not serializable')
    skip_fsspec_s3fs(fs)

    aaa = pathfn('a/aa/aaa/')
    bb = pathfn('a/bb')
    c = pathfn('c.txt')

    fs.create_dir(aaa)
    with fs.open_output_stream(bb):
        pass  # touch
    with fs.open_output_stream(c) as fp:
        fp.write(b'test')

    restored = pickle.loads(pickle.dumps(fs))
    aaa_info, bb_info, c_info = restored.get_file_info([aaa, bb, c])
    assert aaa_info.type == FileType.Directory
    assert bb_info.type == FileType.File
    assert c_info.type == FileType.File

def test_type_name():
    fs = LocalFileSystem()
    assert fs.type_name == "local"
    fs = _MockFileSystem()
    assert fs.type_name == "mock"

def test_normalize_path(fs):
    # Trivial path names (without separators) should generally be
    # already normalized.  Just a sanity check.
    assert fs.normalize_path("foo") == "foo"

def test_non_path_like_input_raises(fs):
    class Path:
        pass

    invalid_paths = [1, 1.1, Path(), tuple(), {}, [], lambda: 1,
                     pathlib.Path()]
    for path in invalid_paths:
        with pytest.raises(TypeError):
            fs.create_dir(path)

def test_get_file_info(fs, pathfn):
    aaa = pathfn('a/aa/aaa/')
    bb = pathfn('a/bb')
    c = pathfn('c.txt')
    zzz = pathfn('zzz')

    fs.create_dir(aaa)
    with fs.open_output_stream(bb):
        pass  # touch
    with fs.open_output_stream(c) as fp:
        fp.write(b'test')

    aaa_info, bb_info, c_info, zzz_info = fs.get_file_info([aaa, bb, c, zzz])

    assert aaa_info.path == aaa
    assert 'aaa' in repr(aaa_info)
    assert aaa_info.extension == ''
    if fs.type_name == "py::fsspec+s3":
        # s3fs doesn't create empty directories
        assert aaa_info.type == FileType.NotFound
    else:
        assert aaa_info.type == FileType.Directory
        assert 'FileType.Directory' in repr(aaa_info)
    assert aaa_info.size is None
    check_mtime_or_absent(aaa_info)

    assert bb_info.path == str(bb)
    assert bb_info.base_name == 'bb'
    assert bb_info.extension == ''
    assert bb_info.type == FileType.File
    assert 'FileType.File' in repr(bb_info)
    assert bb_info.size == 0
    if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
        check_mtime(bb_info)

    assert c_info.path == str(c)
    assert c_info.base_name == 'c.txt'
    assert c_info.extension == 'txt'
    assert c_info.type == FileType.File
    assert 'FileType.File' in repr(c_info)
    assert c_info.size == 4
    if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
        check_mtime(c_info)

    assert zzz_info.path == str(zzz)
    assert zzz_info.base_name == 'zzz'
    assert zzz_info.extension == ''
    assert zzz_info.type == FileType.NotFound
    assert zzz_info.size is None
    assert zzz_info.mtime is None
    assert 'FileType.NotFound' in repr(zzz_info)
    check_mtime_absent(zzz_info)

    # with single path
    aaa_info2 = fs.get_file_info(aaa)
    assert aaa_info.path == aaa_info2.path
    assert aaa_info.type == aaa_info2.type

def test_get_file_info_with_selector(fs, pathfn):
    base_dir = pathfn('selector-dir/')
    file_a = pathfn('selector-dir/test_file_a')
    file_b = pathfn('selector-dir/test_file_b')
    dir_a = pathfn('selector-dir/test_dir_a')
    file_c = pathfn('selector-dir/test_dir_a/test_file_c')
    dir_b = pathfn('selector-dir/test_dir_b')

    try:
        fs.create_dir(base_dir)
        with fs.open_output_stream(file_a):
            pass
        with fs.open_output_stream(file_b):
            pass
        fs.create_dir(dir_a)
        with fs.open_output_stream(file_c):
            pass
        fs.create_dir(dir_b)

        # recursive selector
        selector = FileSelector(base_dir, allow_not_found=False,
                                recursive=True)
        assert selector.base_dir == base_dir

        infos = fs.get_file_info(selector)
        if fs.type_name == "py::fsspec+s3":
            # s3fs only lists directories if they are not empty, but depending
            # on the s3fs/fsspec version combo, it includes the base_dir
            # (https://github.com/dask/s3fs/issues/393)
            assert (len(infos) == 4) or (len(infos) == 5)
        else:
            assert len(infos) == 5

        for info in infos:
            if (info.path.endswith(file_a) or info.path.endswith(file_b) or
                    info.path.endswith(file_c)):
                assert info.type == FileType.File
            elif (info.path.rstrip("/").endswith(dir_a) or
                  info.path.rstrip("/").endswith(dir_b)):
                assert info.type == FileType.Directory
            elif (fs.type_name == "py::fsspec+s3" and
                  info.path.rstrip("/").endswith("selector-dir")):
                # s3fs can include base dir, see above
                assert info.type == FileType.Directory
            else:
                raise ValueError('unexpected path {}'.format(info.path))
            check_mtime_or_absent(info)

        # non-recursive selector -> not selecting the nested file_c
        selector = FileSelector(base_dir, recursive=False)

        infos = fs.get_file_info(selector)
        if fs.type_name == "py::fsspec+s3":
            # s3fs only lists directories if they are not empty
            # + for s3fs 0.5.2 all directories are dropped because of buggy
            # side-effect of previous find() call
            # (https://github.com/dask/s3fs/issues/410)
            assert (len(infos) == 3) or (len(infos) == 2)
        else:
            assert len(infos) == 4

    finally:
        fs.delete_dir(base_dir)

def test_create_dir(fs, pathfn):
    # s3fs fails deleting dir fails if it is empty
    # (https://github.com/dask/s3fs/issues/317)
    skip_fsspec_s3fs(fs)
    d = pathfn('test-directory/')

    with pytest.raises(pa.ArrowIOError):
        fs.delete_dir(d)

    fs.create_dir(d)
    fs.delete_dir(d)

    d = pathfn('deeply/nested/test-directory/')
    fs.create_dir(d, recursive=True)
    fs.delete_dir(d)

def test_delete_dir(fs, pathfn):
    skip_fsspec_s3fs(fs)

    d = pathfn('directory/')
    nd = pathfn('directory/nested/')

    fs.create_dir(nd)
    fs.delete_dir(d)
    with pytest.raises(pa.ArrowIOError):
        fs.delete_dir(nd)
    with pytest.raises(pa.ArrowIOError):
        fs.delete_dir(d)

def test_delete_dir_contents(fs, pathfn):
    skip_fsspec_s3fs(fs)

    d = pathfn('directory/')
    nd = pathfn('directory/nested/')

    fs.create_dir(nd)
    fs.delete_dir_contents(d)
    with pytest.raises(pa.ArrowIOError):
        fs.delete_dir(nd)
    fs.delete_dir_contents(nd, missing_dir_ok=True)
    with pytest.raises(pa.ArrowIOError):
        fs.delete_dir_contents(nd)
    fs.delete_dir(d)
    with pytest.raises(pa.ArrowIOError):
        fs.delete_dir(d)

def _check_root_dir_contents(config):
    fs = config['fs']
    pathfn = config['pathfn']

    d = pathfn('directory/')
    nd = pathfn('directory/nested/')

    fs.create_dir(nd)
    with pytest.raises(pa.ArrowInvalid):
        fs.delete_dir_contents("")
    with pytest.raises(pa.ArrowInvalid):
        fs.delete_dir_contents("/")
    with pytest.raises(pa.ArrowInvalid):
        fs.delete_dir_contents("//")

    fs.delete_dir_contents("", accept_root_dir=True)
    fs.delete_dir_contents("/", accept_root_dir=True)
    fs.delete_dir_contents("//", accept_root_dir=True)
    with pytest.raises(pa.ArrowIOError):
        fs.delete_dir(d)

def test_delete_root_dir_contents(mockfs, py_mockfs):
    _check_root_dir_contents(mockfs)
    _check_root_dir_contents(py_mockfs)

def test_copy_file(fs, pathfn):
    s = pathfn('test-copy-source-file')
    t = pathfn('test-copy-target-file')

    with fs.open_output_stream(s):
        pass

    fs.copy_file(s, t)
    fs.delete_file(s)
    fs.delete_file(t)

def test_move_directory(fs, pathfn, allow_move_dir):
    # move directory (doesn't work with S3)
    s = pathfn('source-dir/')
    t = pathfn('target-dir/')

    fs.create_dir(s)

    if allow_move_dir:
        fs.move(s, t)
        with pytest.raises(pa.ArrowIOError):
            fs.delete_dir(s)
        fs.delete_dir(t)
    else:
        with pytest.raises(pa.ArrowIOError):
            fs.move(s, t)

def test_move_file(fs, pathfn):
    # s3fs moving a file with recursive=True on latest 0.5 version
    # (https://github.com/dask/s3fs/issues/394)
    skip_fsspec_s3fs(fs)

    s = pathfn('test-move-source-file')
    t = pathfn('test-move-target-file')

    with fs.open_output_stream(s):
        pass

    fs.move(s, t)
    with pytest.raises(pa.ArrowIOError):
        fs.delete_file(s)
    fs.delete_file(t)

def test_delete_file(fs, pathfn):
    p = pathfn('test-delete-target-file')
    with fs.open_output_stream(p):
        pass

    fs.delete_file(p)
    with pytest.raises(pa.ArrowIOError):
        fs.delete_file(p)

    d = pathfn('test-delete-nested')
    fs.create_dir(d)
    f = pathfn('test-delete-nested/target-file')
    with fs.open_output_stream(f) as s:
        s.write(b'data')

    fs.delete_dir(d)

def identity(v):
    return v

@pytest.mark.gzip
@pytest.mark.parametrize(
    ('compression', 'buffer_size', 'compressor'),
    [
        (None, None, identity),
        (None, 64, identity),
        ('gzip', None, gzip.compress),
        ('gzip', 256, gzip.compress),
    ]
)
def test_open_input_stream(fs, pathfn, compression, buffer_size, compressor):
    p = pathfn('open-input-stream')

    data = b'some data for reading\n' * 512
    with fs.open_output_stream(p) as s:
        s.write(compressor(data))

    with fs.open_input_stream(p, compression, buffer_size) as s:
        result = s.read()

    assert result == data

def test_open_input_file(fs, pathfn):
    p = pathfn('open-input-file')

    data = b'some data' * 1024
    with fs.open_output_stream(p) as s:
        s.write(data)

    read_from = len(b'some data') * 512
    with fs.open_input_file(p) as f:
        result = f.read()
    assert result == data

    with fs.open_input_file(p) as f:
        f.seek(read_from)
        result = f.read()

    assert result == data[read_from:]

def test_open_input_stream_not_found(fs, pathfn):
    # The proper exception should be raised for this common case (ARROW-15896)
    p = pathfn('open-input-stream-not-found')
    with pytest.raises(FileNotFoundError):
        fs.open_input_stream(p)

@pytest.mark.gzip
@pytest.mark.parametrize(
    ('compression', 'buffer_size', 'decompressor'),
    [
        (None, None, identity),
        (None, 64, identity),
        ('gzip', None, gzip.decompress),
        ('gzip', 256, gzip.decompress),
    ]
)
def test_open_output_stream(fs, pathfn, compression, buffer_size,
                            decompressor):
    p = pathfn('open-output-stream')

    data = b'some data for writing' * 1024
    with fs.open_output_stream(p, compression, buffer_size) as f:
        f.write(data)

    with fs.open_input_stream(p, compression, buffer_size) as f:
        assert f.read(len(data)) == data

@pytest.mark.gzip
@pytest.mark.parametrize(
    ('compression', 'buffer_size', 'compressor', 'decompressor'),
    [
        (None, None, identity, identity),
        (None, 64, identity, identity),
        ('gzip', None, gzip.compress, gzip.decompress),
        ('gzip', 256, gzip.compress, gzip.decompress),
    ]
)
def test_open_append_stream(fs, pathfn, compression, buffer_size, compressor,
                            decompressor, allow_append_to_file):
    p = pathfn('open-append-stream')

    initial = compressor(b'already existing')
    with fs.open_output_stream(p) as s:
        s.write(initial)

    if allow_append_to_file:
        with fs.open_append_stream(p, compression=compression,
                                   buffer_size=buffer_size) as f:
            f.write(b'\nnewly added')

        with fs.open_input_stream(p) as f:
            result = f.read()

        result = decompressor(result)
        assert result == b'already existing\nnewly added'
    else:
        with pytest.raises(pa.ArrowNotImplementedError):
            fs.open_append_stream(p, compression=compression,
                                  buffer_size=buffer_size)

def test_open_output_stream_metadata(fs, pathfn):
    p = pathfn('open-output-stream-metadata')
    metadata = {'Content-Type': 'x-pyarrow/test'}

    data = b'some data'
    with fs.open_output_stream(p, metadata=metadata) as f:
        f.write(data)

    with fs.open_input_stream(p) as f:
        assert f.read() == data
        got_metadata = f.metadata()

    if fs.type_name in ['s3', 'gcs'] or 'mock' in fs.type_name:
        for k, v in metadata.items():
            assert got_metadata[k] == v.encode()
    else:
        assert got_metadata == {}

def test_localfs_options():
    # LocalFileSystem instantiation
    LocalFileSystem(use_mmap=False)

    with pytest.raises(TypeError):
        LocalFileSystem(xxx=False)

def test_localfs_errors(localfs):
    # Local filesystem errors should raise the right Python exceptions
    # (e.g. FileNotFoundError)
    fs = localfs['fs']
    with assert_file_not_found():
        fs.open_input_stream('/non/existent/file')
    with assert_file_not_found():
        fs.open_output_stream('/non/existent/file')
    with assert_file_not_found():
        fs.create_dir('/non/existent/dir', recursive=False)
    with assert_file_not_found():
        fs.delete_dir('/non/existent/dir')
    with assert_file_not_found():
        fs.delete_file('/non/existent/dir')
    with assert_file_not_found():
        fs.move('/non/existent', '/xxx')
    with assert_file_not_found():
        fs.copy_file('/non/existent', '/xxx')

def test_localfs_file_info(localfs):
    fs = localfs['fs']

    file_path = pathlib.Path(__file__)
    dir_path = file_path.parent
    [file_info, dir_info] = fs.get_file_info([file_path.as_posix(),
                                              dir_path.as_posix()])
    assert file_info.size == file_path.stat().st_size
    assert file_info.mtime_ns == file_path.stat().st_mtime_ns
    check_mtime(file_info)
    assert dir_info.mtime_ns == dir_path.stat().st_mtime_ns
    check_mtime(dir_info)

def test_mockfs_mtime_roundtrip(mockfs):
    dt = datetime.fromtimestamp(1568799826, timezone.utc)
    fs = _MockFileSystem(dt)

    with fs.open_output_stream('foo'):
        pass
    [info] = fs.get_file_info(['foo'])
    assert info.mtime == dt

@pytest.mark.gcs
def test_gcs_options():
    from pyarrow.fs import GcsFileSystem
    dt = datetime.now()
    fs = GcsFileSystem(access_token='abc',
                       target_service_account='service_account@apache',
                       credential_token_expiration=dt,
                       default_bucket_location='us-west2',
                       scheme='https', endpoint_override='localhost:8999')
    assert isinstance(fs, GcsFileSystem)
    assert fs.default_bucket_location == 'us-west2'
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs = GcsFileSystem()
    assert isinstance(fs, GcsFileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs = GcsFileSystem(anonymous=True)
    assert isinstance(fs, GcsFileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs = GcsFileSystem(default_metadata={"ACL": "authenticated-read",
                                         "Content-Type": "text/plain"})
    assert isinstance(fs, GcsFileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    with pytest.raises(ValueError):
        GcsFileSystem(access_token='access')
    with pytest.raises(ValueError):
        GcsFileSystem(anonymous=True, access_token='secret')
    with pytest.raises(ValueError):
        GcsFileSystem(anonymous=True, target_service_account='acct')
    with pytest.raises(ValueError):
        GcsFileSystem(credential_token_expiration=datetime.now())

@pytest.mark.s3
def test_s3_options():
    from pyarrow.fs import S3FileSystem

    fs = S3FileSystem(access_key='access', secret_key='secret',
                      session_token='token', region='us-east-2',
                      scheme='https', endpoint_override='localhost:8999')
    assert isinstance(fs, S3FileSystem)
    assert fs.region == 'us-east-2'
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs = S3FileSystem(role_arn='role', session_name='session',
                      external_id='id', load_frequency=100)
    assert isinstance(fs, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs2 = S3FileSystem(role_arn='role')
    assert isinstance(fs2, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs2)) == fs2
    assert fs2 != fs

    fs = S3FileSystem(anonymous=True)
    assert isinstance(fs, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs = S3FileSystem(background_writes=True)
    assert isinstance(fs, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs2 = S3FileSystem(background_writes=True,
                       default_metadata={"ACL": "authenticated-read",
                                         "Content-Type": "text/plain"})
    assert isinstance(fs2, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs2)) == fs2
    assert fs2 != fs

    fs = S3FileSystem(allow_bucket_creation=True, allow_bucket_deletion=True)
    assert isinstance(fs, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    with pytest.raises(ValueError):
        S3FileSystem(access_key='access')
    with pytest.raises(ValueError):
        S3FileSystem(secret_key='secret')
    with pytest.raises(ValueError):
        S3FileSystem(access_key='access', session_token='token')
    with pytest.raises(ValueError):
        S3FileSystem(secret_key='secret', session_token='token')
    with pytest.raises(ValueError):
        S3FileSystem(
            access_key='access', secret_key='secret', role_arn='arn'
        )
    with pytest.raises(ValueError):
        S3FileSystem(
            access_key='access', secret_key='secret', anonymous=True
        )
    with pytest.raises(ValueError):
        S3FileSystem(role_arn="arn", anonymous=True)
    with pytest.raises(ValueError):
        S3FileSystem(default_metadata=["foo", "bar"])

@pytest.mark.s3
def test_s3_proxy_options(monkeypatch):
    from pyarrow.fs import S3FileSystem

    # The following two are equivalent:
    proxy_opts_1_dict = {'scheme': 'http', 'host': 'localhost', 'port': 8999}
    proxy_opts_1_str = 'http://localhost:8999'
    # The following two are equivalent:
    proxy_opts_2_dict = {'scheme': 'https', 'host': 'localhost', 'port': 8080}
    proxy_opts_2_str = 'https://localhost:8080'

    # Check dict case for 'proxy_options'
    fs = S3FileSystem(proxy_options=proxy_opts_1_dict)
    assert isinstance(fs, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs = S3FileSystem(proxy_options=proxy_opts_2_dict)
    assert isinstance(fs, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    # Check str case for 'proxy_options'
    fs = S3FileSystem(proxy_options=proxy_opts_1_str)
    assert isinstance(fs, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    fs = S3FileSystem(proxy_options=proxy_opts_2_str)
    assert isinstance(fs, S3FileSystem)
    assert pickle.loads(pickle.dumps(fs)) == fs

    # Check that two FSs using the same proxy_options dict are equal
    fs1 = S3FileSystem(proxy_options=proxy_opts_1_dict)
    fs2 = S3FileSystem(proxy_options=proxy_opts_1_dict)
    assert fs1 == fs2
    assert pickle.loads(pickle.dumps(fs1)) == fs2
    assert pickle.loads(pickle.dumps(fs2)) == fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_2_dict)
    fs2 = S3FileSystem(proxy_options=proxy_opts_2_dict)
    assert fs1 == fs2
    assert pickle.loads(pickle.dumps(fs1)) == fs2
    assert pickle.loads(pickle.dumps(fs2)) == fs1

    # Check that two FSs using the same proxy_options str are equal
    fs1 = S3FileSystem(proxy_options=proxy_opts_1_str)
    fs2 = S3FileSystem(proxy_options=proxy_opts_1_str)
    assert fs1 == fs2
    assert pickle.loads(pickle.dumps(fs1)) == fs2
    assert pickle.loads(pickle.dumps(fs2)) == fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_2_str)
    fs2 = S3FileSystem(proxy_options=proxy_opts_2_str)
    assert fs1 == fs2
    assert pickle.loads(pickle.dumps(fs1)) == fs2
    assert pickle.loads(pickle.dumps(fs2)) == fs1

    # Check that two FSs using equivalent proxy_options
    # (one dict, one str) are equal
    fs1 = S3FileSystem(proxy_options=proxy_opts_1_dict)
    fs2 = S3FileSystem(proxy_options=proxy_opts_1_str)
    assert fs1 == fs2
    assert pickle.loads(pickle.dumps(fs1)) == fs2
    assert pickle.loads(pickle.dumps(fs2)) == fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_2_dict)
    fs2 = S3FileSystem(proxy_options=proxy_opts_2_str)
    assert fs1 == fs2
    assert pickle.loads(pickle.dumps(fs1)) == fs2
    assert pickle.loads(pickle.dumps(fs2)) == fs1

    # Check that two FSs using nonequivalent proxy_options are not equal
    fs1 = S3FileSystem(proxy_options=proxy_opts_1_dict)
    fs2 = S3FileSystem(proxy_options=proxy_opts_2_dict)
    assert fs1 != fs2
    assert pickle.loads(pickle.dumps(fs1)) != fs2
    assert pickle.loads(pickle.dumps(fs2)) != fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_1_dict)
    fs2 = S3FileSystem(proxy_options=proxy_opts_2_str)
    assert fs1 != fs2
    assert pickle.loads(pickle.dumps(fs1)) != fs2
    assert pickle.loads(pickle.dumps(fs2)) != fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_1_str)
    fs2 = S3FileSystem(proxy_options=proxy_opts_2_dict)
    assert fs1 != fs2
    assert pickle.loads(pickle.dumps(fs1)) != fs2
    assert pickle.loads(pickle.dumps(fs2)) != fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_1_str)
    fs2 = S3FileSystem(proxy_options=proxy_opts_2_str)
    assert fs1 != fs2
    assert pickle.loads(pickle.dumps(fs1)) != fs2
    assert pickle.loads(pickle.dumps(fs2)) != fs1

    # Check that two FSs (one using proxy_options and the other not)
    # are not equal
    fs1 = S3FileSystem(proxy_options=proxy_opts_1_dict)
    fs2 = S3FileSystem()
    assert fs1 != fs2
    assert pickle.loads(pickle.dumps(fs1)) != fs2
    assert pickle.loads(pickle.dumps(fs2)) != fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_1_str)
    fs2 = S3FileSystem()
    assert fs1 != fs2
    assert pickle.loads(pickle.dumps(fs1)) != fs2
    assert pickle.loads(pickle.dumps(fs2)) != fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_2_dict)
    fs2 = S3FileSystem()
    assert fs1 != fs2
    assert pickle.loads(pickle.dumps(fs1)) != fs2
    assert pickle.loads(pickle.dumps(fs2)) != fs1

    fs1 = S3FileSystem(proxy_options=proxy_opts_2_str)
    fs2 = S3FileSystem()
    assert fs1 != fs2
    assert pickle.loads(pickle.dumps(fs1)) != fs2
    assert pickle.loads(pickle.dumps(fs2)) != fs1

    # Only dict and str are supported
    with pytest.raises(TypeError):
        S3FileSystem(proxy_options=('http', 'localhost', 9090))
    # Missing scheme
    with pytest.raises(KeyError):
        S3FileSystem(proxy_options={'host': 'localhost', 'port': 9090})
    # Missing host
    with pytest.raises(KeyError):
        S3FileSystem(proxy_options={'scheme': 'https', 'port': 9090})
    # Missing port
    with pytest.raises(KeyError):
        S3FileSystem(proxy_options={'scheme': 'http', 'host': 'localhost'})
    # Invalid proxy URI (invalid scheme htttps)
    with pytest.raises(pa.ArrowInvalid):
        S3FileSystem(proxy_options='htttps://localhost:9000')
    # Invalid proxy_options dict (invalid scheme htttps)
    with pytest.raises(pa.ArrowInvalid):
        S3FileSystem(proxy_options={'scheme': 'htttp', 'host': 'localhost',
                                    'port': 8999})

@pytest.mark.hdfs
def test_hdfs_options(hdfs_connection):
    from pyarrow.fs import HadoopFileSystem
    if not pa.have_libhdfs():
        pytest.skip('Cannot locate libhdfs')

    host, port, user = hdfs_connection

    replication = 2
    buffer_size = 64*1024
    default_block_size = 128*1024**2
    uri = ('hdfs://{}:{}/?user={}&replication={}&buffer_size={}'
           '&default_block_size={}')

    hdfs1 = HadoopFileSystem(host, port, user='libhdfs',
                             replication=replication, buffer_size=buffer_size,
                             default_block_size=default_block_size)
    hdfs2 = HadoopFileSystem.from_uri(uri.format(
        host, port, 'libhdfs', replication, buffer_size, default_block_size
    ))
    hdfs3 = HadoopFileSystem.from_uri(uri.format(
        host, port, 'me', replication, buffer_size, default_block_size
    ))
    hdfs4 = HadoopFileSystem.from_uri(uri.format(
        host, port, 'me', replication + 1, buffer_size, default_block_size
    ))
    hdfs5 = HadoopFileSystem(host, port)
    hdfs6 = HadoopFileSystem.from_uri('hdfs://{}:{}'.format(host, port))
    hdfs7 = HadoopFileSystem(host, port, user='localuser')
    hdfs8 = HadoopFileSystem(host, port, user='localuser',
                             kerb_ticket="cache_path")
    hdfs9 = HadoopFileSystem(host, port, user='localuser',
                             kerb_ticket=pathlib.Path("cache_path"))
    hdfs10 = HadoopFileSystem(host, port, user='localuser',
                              kerb_ticket="cache_path2")
    hdfs11 = HadoopFileSystem(host, port, user='localuser',
                              kerb_ticket="cache_path",
                              extra_conf={'hdfs_token': 'abcd'})

    assert hdfs1 == hdfs2
    assert hdfs5 == hdfs6
    assert hdfs6 != hdfs7
    assert hdfs2 != hdfs3
    assert hdfs3 != hdfs4
    assert hdfs7 != hdfs5
    assert hdfs2 != hdfs3
    assert hdfs3 != hdfs4
    assert hdfs7 != hdfs8
    assert hdfs8 == hdfs9
    assert hdfs10 != hdfs9
    assert hdfs11 != hdfs8

    with pytest.raises(TypeError):
        HadoopFileSystem()
    with pytest.raises(TypeError):
        HadoopFileSystem.from_uri(3)

    for fs in [hdfs1, hdfs2, hdfs3, hdfs4, hdfs5, hdfs6, hdfs7, hdfs8,
               hdfs9, hdfs10, hdfs11]:
        assert pickle.loads(pickle.dumps(fs)) == fs

    host, port, user = hdfs_connection

    hdfs = HadoopFileSystem(host, port, user=user)
    assert hdfs.get_file_info(FileSelector('/'))

    hdfs = HadoopFileSystem.from_uri(
        "hdfs://{}:{}/?user={}".format(host, port, user)
    )
    assert hdfs.get_file_info(FileSelector('/'))

@pytest.mark.parametrize(('uri', 'expected_klass', 'expected_path'), [
    # leading slashes are removed intentionally, because MockFileSystem doesn't
    # have a distinction between relative and absolute paths
    ('mock:', _MockFileSystem, ''),
    ('mock:foo/bar', _MockFileSystem, 'foo/bar'),
    ('mock:/foo/bar', _MockFileSystem, 'foo/bar'),
    ('mock:///foo/bar', _MockFileSystem, 'foo/bar'),
    ('file:/', LocalFileSystem, '/'),
    ('file:///', LocalFileSystem, '/'),
    ('file:/foo/bar', LocalFileSystem, '/foo/bar'),
    ('file:///foo/bar', LocalFileSystem, '/foo/bar'),
    ('/', LocalFileSystem, '/'),
    ('/foo/bar', LocalFileSystem, '/foo/bar'),
])
def test_filesystem_from_uri(uri, expected_klass, expected_path):
    fs, path = FileSystem.from_uri(uri)
    assert isinstance(fs, expected_klass)
    assert path == expected_path

@pytest.mark.parametrize(
    'path',
    ['', '/', 'foo/bar', '/foo/bar', __file__]
)
def test_filesystem_from_path_object(path):
    p = pathlib.Path(path)
    fs, path = FileSystem.from_uri(p)
    assert isinstance(fs, LocalFileSystem)
    assert path == p.resolve().absolute().as_posix()

@pytest.mark.s3
def test_filesystem_from_uri_s3(s3_server):
    from pyarrow.fs import S3FileSystem

    host, port, access_key, secret_key = s3_server['connection']

    uri = "s3://{}:{}@mybucket/foo/bar?scheme=http&endpoint_override={}:{}"\
          "&allow_bucket_creation=True" \
          .format(access_key, secret_key, host, port)

    fs, path = FileSystem.from_uri(uri)
    assert isinstance(fs, S3FileSystem)
    assert path == "mybucket/foo/bar"

    fs.create_dir(path)
    [info] = fs.get_file_info([path])
    assert info.path == path
    assert info.type == FileType.Directory

@pytest.mark.gcs
def test_filesystem_from_uri_gcs(gcs_server):
    from pyarrow.fs import GcsFileSystem

    host, port = gcs_server['connection']

    uri = ("gs://anonymous@" +
           f"mybucket/foo/bar?scheme=http&endpoint_override={host}:{port}&" +
           "retry_limit_seconds=5")

    fs, path = FileSystem.from_uri(uri)
    assert isinstance(fs, GcsFileSystem)
    assert path == "mybucket/foo/bar"

    fs.create_dir(path)
    [info] = fs.get_file_info([path])
    assert info.path == path
    assert info.type == FileType.Directory

def test_py_filesystem():
    handler = DummyHandler()
    fs = PyFileSystem(handler)
    assert isinstance(fs, PyFileSystem)
    assert fs.type_name == "py::dummy"
    assert fs.handler is handler

    with pytest.raises(TypeError):
        PyFileSystem(None)

def test_py_filesystem_equality():
    handler1 = DummyHandler(1)
    handler2 = DummyHandler(2)
    handler3 = DummyHandler(2)
    fs1 = PyFileSystem(handler1)
    fs2 = PyFileSystem(handler1)
    fs3 = PyFileSystem(handler2)
    fs4 = PyFileSystem(handler3)

    assert fs2 is not fs1
    assert fs3 is not fs2
    assert fs4 is not fs3
    assert fs2 == fs1  # Same handler
    assert fs3 != fs2  # Unequal handlers
    assert fs4 == fs3  # Equal handlers

    assert fs1 != LocalFileSystem()
    assert fs1 != object()

def test_py_filesystem_pickling():
    handler = DummyHandler()
    fs = PyFileSystem(handler)

    serialized = pickle.dumps(fs)
    restored = pickle.loads(serialized)
    assert isinstance(restored, FileSystem)
    assert restored == fs
    assert restored.handler == handler
    assert restored.type_name == "py::dummy"

def test_py_filesystem_lifetime():
    handler = DummyHandler()
    fs = PyFileSystem(handler)
    assert isinstance(fs, PyFileSystem)
    wr = weakref.ref(handler)
    handler = None
    assert wr() is not None
    fs = None
    assert wr() is None

    # Taking the .handler attribute doesn't wreck reference counts
    handler = DummyHandler()
    fs = PyFileSystem(handler)
    wr = weakref.ref(handler)
    handler = None
    assert wr() is fs.handler
    assert wr() is not None
    fs = None
    assert wr() is None

def test_py_filesystem_get_file_info():
    handler = DummyHandler()
    fs = PyFileSystem(handler)

    [info] = fs.get_file_info(['some/dir'])
    assert info.path == 'some/dir'
    assert info.type == FileType.Directory

    [info] = fs.get_file_info(['some/file'])
    assert info.path == 'some/file'
    assert info.type == FileType.File

    [info] = fs.get_file_info(['notfound'])
    assert info.path == 'notfound'
    assert info.type == FileType.NotFound

    with pytest.raises(TypeError):
        fs.get_file_info(['badtype'])

    with pytest.raises(IOError):
        fs.get_file_info(['xxx'])

def test_py_filesystem_get_file_info_selector():
    handler = DummyHandler()
    fs = PyFileSystem(handler)

    selector = FileSelector(base_dir="somedir")
    infos = fs.get_file_info(selector)
    assert len(infos) == 2
    assert infos[0].path == "somedir/file1"
    assert infos[0].type == FileType.File
    assert infos[0].size == 123
    assert infos[1].path == "somedir/subdir1"
    assert infos[1].type == FileType.Directory
    assert infos[1].size is None

    selector = FileSelector(base_dir="somedir", recursive=True)
    infos = fs.get_file_info(selector)
    assert len(infos) == 3
    assert infos[0].path == "somedir/file1"
    assert infos[1].path == "somedir/subdir1"
    assert infos[2].path == "somedir/subdir1/file2"

    selector = FileSelector(base_dir="notfound")
    with pytest.raises(FileNotFoundError):
        fs.get_file_info(selector)

    selector = FileSelector(base_dir="notfound", allow_not_found=True)
    assert fs.get_file_info(selector) == []

def test_py_filesystem_ops():
    handler = DummyHandler()
    fs = PyFileSystem(handler)

    fs.create_dir("recursive", recursive=True)
    fs.create_dir("non-recursive", recursive=False)
    with pytest.raises(IOError):
        fs.create_dir("foobar")

    fs.delete_dir("delete_dir")
    fs.delete_dir_contents("delete_dir_contents")
    for path in ("", "/", "//"):
        with pytest.raises(ValueError):
            fs.delete_dir_contents(path)
        fs.delete_dir_contents(path, accept_root_dir=True)
    fs.delete_file("delete_file")
    fs.move("move_from", "move_to")
    fs.copy_file("copy_file_from", "copy_file_to")

def test_py_open_input_stream():
    fs = PyFileSystem(DummyHandler())

    with fs.open_input_stream("somefile") as f:
        assert f.read() == b"somefile:input_stream"
    with pytest.raises(FileNotFoundError):
        fs.open_input_stream("notfound")

def test_py_open_input_file():
    fs = PyFileSystem(DummyHandler())

    with fs.open_input_file("somefile") as f:
        assert f.read() == b"somefile:input_file"
    with pytest.raises(FileNotFoundError):
        fs.open_input_file("notfound")

def test_py_open_output_stream():
    fs = PyFileSystem(DummyHandler())

    with fs.open_output_stream("somefile") as f:
        f.write(b"data")

def test_py_open_append_stream():
    fs = PyFileSystem(DummyHandler())

    with fs.open_append_stream("somefile") as f:
        f.write(b"data")

@pytest.mark.s3
def test_s3_real_aws():
    # Exercise connection code with an AWS-backed S3 bucket.
    # This is a minimal integration check for ARROW-9261 and similar issues.
    from pyarrow.fs import S3FileSystem
    default_region = (os.environ.get('PYARROW_TEST_S3_REGION') or
                      'us-east-1')
    fs = S3FileSystem(anonymous=True)
    assert fs.region == default_region

    fs = S3FileSystem(anonymous=True, region='us-east-2')
    entries = fs.get_file_info(FileSelector(
        'voltrondata-labs-datasets/nyc-taxi'))
    assert len(entries) > 0
    key = 'voltrondata-labs-datasets/nyc-taxi/year=2019/month=6/part-0.parquet'
    with fs.open_input_stream(key) as f:
        md = f.metadata()
        assert 'Content-Type' in md
        assert md['Last-Modified'] == b'2022-07-12T23:32:00Z'
        # For some reason, the header value is quoted
        # (both with AWS and Minio)
        assert md['ETag'] == b'"4c6a76826a695c6ac61592bc30cda3df-16"'

@pytest.mark.s3
def test_s3_real_aws_region_selection():
    # Taken from a registry of open S3-hosted datasets
    # at https://github.com/awslabs/open-data-registry
    fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
    assert fs.region == 'eu-west-1'
    with fs.open_input_stream(path) as f:
        assert b"Meteo-France Atmospheric models on AWS" in f.read(50)

    # Passing an explicit region disables auto-selection
    fs, path = FileSystem.from_uri(
        's3://mf-nwp-models/README.txt?region=us-east-2')
    assert fs.region == 'us-east-2'
    # Reading from the wrong region may still work for public buckets...

    # Non-existent bucket (hopefully, otherwise need to fix this test)
    with pytest.raises(IOError, match="Bucket '.*' not found"):
        FileSystem.from_uri('s3://x-arrow-non-existent-bucket')
    fs, path = FileSystem.from_uri(
        's3://x-arrow-non-existent-bucket?region=us-east-3')
    assert fs.region == 'us-east-3'

@pytest.mark.s3
def test_resolve_s3_region():
    from pyarrow.fs import resolve_s3_region
    assert resolve_s3_region('voltrondata-labs-datasets') == 'us-east-2'
    assert resolve_s3_region('mf-nwp-models') == 'eu-west-1'

    with pytest.raises(ValueError, match="Not a valid bucket name"):
        resolve_s3_region('foo/bar')
    with pytest.raises(ValueError, match="Not a valid bucket name"):
        resolve_s3_region('s3:bucket')

@pytest.mark.s3
def test_copy_files(s3_connection, s3fs, tempdir):
    fs = s3fs["fs"]
    pathfn = s3fs["pathfn"]

    # create test file on S3 filesystem
    path = pathfn('c.txt')
    with fs.open_output_stream(path) as f:
        f.write(b'test')

    # create URI for created file
    host, port, access_key, secret_key = s3_connection
    source_uri = (
        f"s3://{access_key}:{secret_key}@{path}"
        f"?scheme=http&endpoint_override={host}:{port}"
    )
    # copy from S3 URI to local file
    local_path1 = str(tempdir / "c_copied1.txt")
    copy_files(source_uri, local_path1)

    localfs = LocalFileSystem()
    with localfs.open_input_stream(local_path1) as f:
        assert f.read() == b"test"

    # copy from S3 path+filesystem to local file
    local_path2 = str(tempdir / "c_copied2.txt")
    copy_files(path, local_path2, source_filesystem=fs)
    with localfs.open_input_stream(local_path2) as f:
        assert f.read() == b"test"

    # copy to local file with URI
    local_path3 = str(tempdir / "c_copied3.txt")
    destination_uri = _filesystem_uri(local_path3)  # file://
    copy_files(source_uri, destination_uri)

    with localfs.open_input_stream(local_path3) as f:
        assert f.read() == b"test"

    # copy to local file with path+filesystem
    local_path4 = str(tempdir / "c_copied4.txt")
    copy_files(source_uri, local_path4, destination_filesystem=localfs)

    with localfs.open_input_stream(local_path4) as f:
        assert f.read() == b"test"

    # copy with additional options
    local_path5 = str(tempdir / "c_copied5.txt")
    copy_files(source_uri, local_path5, chunk_size=1, use_threads=False)

    with localfs.open_input_stream(local_path5) as f:
        assert f.read() == b"test"

def test_copy_files_directory(tempdir):
    localfs = LocalFileSystem()

    # create source directory with 2 files
    source_dir = tempdir / "source"
    source_dir.mkdir()
    with localfs.open_output_stream(str(source_dir / "file1")) as f:
        f.write(b'test1')
    with localfs.open_output_stream(str(source_dir / "file2")) as f:
        f.write(b'test2')

    def check_copied_files(destination_dir):
        with localfs.open_input_stream(str(destination_dir / "file1")) as f:
            assert f.read() == b"test1"
        with localfs.open_input_stream(str(destination_dir / "file2")) as f:
            assert f.read() == b"test2"

    # Copy directory with local file paths
    destination_dir1 = tempdir / "destination1"
    # TODO need to create?
    destination_dir1.mkdir()
    copy_files(str(source_dir), str(destination_dir1))
    check_copied_files(destination_dir1)

    # Copy directory with path+filesystem
    destination_dir2 = tempdir / "destination2"
    destination_dir2.mkdir()
    copy_files(str(source_dir), str(destination_dir2),
               source_filesystem=localfs, destination_filesystem=localfs)
    check_copied_files(destination_dir2)

    # Copy directory with URI
    destination_dir3 = tempdir / "destination3"
    destination_dir3.mkdir()
    source_uri = _filesystem_uri(str(source_dir))  # file://
    destination_uri = _filesystem_uri(str(destination_dir3))
    copy_files(source_uri, destination_uri)
    check_copied_files(destination_dir3)

    # Copy directory with Path objects
    destination_dir4 = tempdir / "destination4"
    destination_dir4.mkdir()
    copy_files(source_dir, destination_dir4)
    check_copied_files(destination_dir4)

    # copy with additional non-default options
    destination_dir5 = tempdir / "destination5"
    destination_dir5.mkdir()
    copy_files(source_dir, destination_dir5, chunk_size=1, use_threads=False)
    check_copied_files(destination_dir5)


text:
namespace platforms {
namespace darwinn {
namespace driver {

KernelRegistersWindows::KernelRegistersWindows(
    const std::string& device_path, const std::vector<MmapRegion>& mmap_region,
    bool read_only)
    : KernelRegisters(device_path, mmap_region, read_only) {}

KernelRegistersWindows::KernelRegistersWindows(const std::string& device_path,
                                               uint64 mmap_offset,
                                               uint64 mmap_size, bool read_only)
    : KernelRegisters(device_path, mmap_offset, mmap_size, read_only) {}

KernelRegistersWindows::~KernelRegistersWindows() { UnmapAllRegions(); }

StatusOr<uint64*> KernelRegistersWindows::MapRegion(
    FileDescriptor fd, const MappedRegisterRegion& region, bool read_only) {
  gasket_address_map_ioctl ioctl = {0, region.offset, region.size, 0,
                                    0, nullptr};
  bool res;

  res = DeviceIoControl(fd, GASKET_IOCTL_MAP_HDW_VIEW, &ioctl, sizeof(ioctl),
                        &ioctl, sizeof(ioctl), NULL, NULL);
  if (!res) {
    return InternalError(StringPrintf(
        "KernelRegisters::MapRegion failed! gle=%d", GetLastError()));
  }

  return static_cast<uint64*>(ioctl.virtaddr);
}

Status KernelRegistersWindows::UnmapRegion(FileDescriptor fd,
                                           const MappedRegisterRegion& region) {
  gasket_address_map_ioctl ioctl = {0, region.offset,   region.size, 0,
                                    0, region.registers};
  bool res;

  res = DeviceIoControl(fd, GASKET_IOCTL_UNMAP_HDW_VIEW, &ioctl, sizeof(ioctl),
                        &ioctl, sizeof(ioctl), NULL, NULL);
  if (!res) {
    FailedPreconditionError(StringPrintf(
        "KernelRegisters::UnmapRegion failed! gle=%d", GetLastError()));
  }

  return OkStatus();
}

}  // namespace driver
}  // namespace darwinn
}  // namespace platforms

redpajama_set_name: RedPajamaGithub


text:
<?php

/**
 * Semver\Version class
 *
 * PHP version 5.3
 *
 * @category Semver
 * @package  Semver
 * @author   Jack Skinner <sydnerdrage@gmail.com>
 * @license  MIT http://opensource.org/licenses/MIT
 * @link     http://www.sydnerdrage.com/semver
 *
 */

namespace Semver;

/**
 * Version class
 *
 * Represents a given version
 *
 * @category Semver
 * @package  Semver
 * @author   Jack Skinner <sydnerdrage@gmail.com>
 * @license  MIT http://opensource.org/licenses/MIT
 * @link     http://www.sydnerdrage.com/semver
 */

class Version
{

}


text:
- http://phiary.me/jquery-page-link-smooth-scroll/
- jQuery animate
  - dulation
  - easing
- [demo](https://altnight.github.io/individual-sandbox/diary/20170219/1)


text:
<?php
/**
 * @file Siefa\Globales\ProveedoresBundle\Controller\TipoProveedorController.php
 * @package SiefaGlobalesProveedores
 */
namespace Siefa\Globales\ProveedoresBundle\Controller;

use Siefa\CommonsBundle\Controller\CrudController;

/**
 * Tipo Proveedor Crud Controller
 *
 * @author "Víctor Hugo Álvarez Pérez <metalvarez@gmail.com>"
 * @package SiefaGlobalesProveedores
 * @since 02/12/2013
 * @version	 0.1
 */
class TipoProveedorController extends CrudController {

}


text:
package typestats;

import java.awt.Color;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.sql.Timestamp;
import java.util.List;
import javax.swing.JOptionPane;
import javax.swing.JTextField;
import javax.swing.SwingUtilities;
import javax.swing.UIManager;
import javax.swing.event.DocumentEvent;
import javax.swing.event.DocumentListener;
import typestats.global.DataTables;
import typestats.global.ErrorHandler;
import typestats.global.Settings;

/**
 * An extended JTextField which handles the typing back methods, checking against
 * the ReadPane text, recording all typos and signaling to the ReadPane to update
 * its highlighter.
 *

 * Note: Requires a ReadPane to be linked to in order to perform methods relating
 * to typing back text.
 */
public class WriteField extends JTextField
                      implements DocumentListener, ActionListener
{
    // !!! Note of caution !!! :
    //  Internally, most timers are stored in nanoseconds (because they use
    //  System.nanoTime()), but in the database and when exported, they are in
    //  miliseconds.  When inside WriteField, assume nanoseconds unless told
    //  otherwise.

    private enum State {
        INACTIVE,
        PAUSED,
        TYPEBACK
    }

    // Links to other JContainer objects
    private ReadPane readPane;
    private TypeForm context;   //Used primarily to tell the main form you're done

    private Color enabled_color = Color.white;
    private Color error_color = Color.red;
    private Color disabled_color = UIManager.getDefaults().getColor("TextField.disabled");

    // Session tracking variables
    private char next_char;     // Next character you're supposed to type
    private long start_time;
    private long word_time;
    private long overflow_time; // Time from previous session chunks (caused by pausing)
    private javax.swing.Timer timer;
    private int characters_typed;
    private int words_typed;
    private int words_mistyped;
    private int chars_in_mistyped;
    private int chars_in_correct;
    private long time_in_mistyped;
    private long time_in_correct;
    private boolean mistyped_current_word;

    private State state = State.INACTIVE;

    public void init( ReadPane rp, TypeForm context) {
        readPane = rp;
        this.context = context;
        addActionListener(this);
        getDocument().addDocumentListener(this);
    }

    /**
     * Note: It's very important that you call this method to make sure the
     * program shuts down; otherwise the WriteField and its timer will persist
     * in a cyclical dependency that is not considered garbage.
     */
    public void deinit() {
        if( timer != null) {
            timer.stop();
            timer = null;
        }
    }

    /**
     * Either starts Typing Back with a new session or resumes the currently-
     * paused session of Typing back.
     */
    public void startTypeBack() {
        if( readPane == null || state == State.TYPEBACK)
            return;

        start_time = System.nanoTime();
        timer = new javax.swing.Timer( 50, this);
        timer.setRepeats(true);
        timer.start();

        if( state == State.PAUSED) {
            // Resume Typing back if paused
            word_time = start_time - word_time;
        }
        else {
            assert state == State.TYPEBACK;

            readPane.startTypeBack();
            String word = readPane.getHighlightedString();

            // Abort if there is no text to type back
            if( word == null || word.length() == 0) {
                context.setTypeState(TypeForm.TypeState.EDITING); //endTypeBack();
                return;
            }

            // Start with fresh stats if not.
            word_time = start_time;
            overflow_time = 0;
            next_char = getNextValidChar(word, 0);
            characters_typed = 0;

            words_typed = 0;
            words_mistyped = 0;
            chars_in_mistyped = 0;
            chars_in_correct = 0;
            time_in_mistyped = 0;
            time_in_correct = 0;

            mistyped_current_word = false;
        }

        // Enable the write pane
        setEnabled(true);
        setBackground(enabled_color);
        requestFocus();

        state = State.TYPEBACK;
    }

    /**
     * Ends the current session of typing back.
     *

     *
     * @param record Whether or not to record the currently-queued data.
     */
    public void endTypeBack( boolean record) {
        if( state == State.INACTIVE)
            return;

        readPane.endTypeBack();

        // Calculate Session WPM (different from if stopping from paused)
        long elapsed_time;
        float wpm;
        if( state == State.PAUSED) {
            elapsed_time = overflow_time;
            wpm = (characters_typed + getDocText().length()) / (elapsed_time / 12000000000.0f);
        }
        else {
            assert state == State.TYPEBACK;
            elapsed_time = System.nanoTime() - start_time + overflow_time;
            wpm = characters_typed / (elapsed_time / 12000000000.0f);
        }

        context.updateWPM( wpm);

        // Handle Database Queue
        DatabaseConnector dc = Settings.getDBConnector();
        if( record)
            recordSession( wpm, elapsed_time / 100000);
        dc.clearCollisionQueue();
        dc.clearWordQueue();

        state = State.INACTIVE;
        timer.stop();
        setEnabled(false);
        setBackground(disabled_color);
        invokeClear();
    }

    /**
     * @param wpm   session wpm
     * @param elapsed_time session length in miliseconds
     */
    private void recordSession( float wpm, long elapsed_time) {
        try {
            DatabaseConnector dc = Settings.getDBConnector();
            dc.commitWords( wpm );
            dc.commitCollisions();

            DataTables.Session session = new DataTables.Session();
            session.time = new Timestamp( System.currentTimeMillis());
            session.burst_high = 0; //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
            session.burst_low = 0; //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
            session.error_rate = words_mistyped / (double)words_typed;
            session.length = elapsed_time;
            session.wpm = wpm;
            if( time_in_correct == 0)
                session.wpm_correct = wpm;
            else
                session.wpm_correct = chars_in_correct / (time_in_correct / 12000000000.0);
            if( time_in_mistyped == 0)
                session.wpm_error = wpm;
            else
                session.wpm_error = chars_in_mistyped / (time_in_mistyped / 12000000000.0);

            dc.recordSession(session);
        } catch (Exception ex) { ErrorHandler.handleException(ex, 4);}
    }

    public void pauseTypeBack() {
        if( state != State.TYPEBACK)
            return;

        long temp_time = System.nanoTime();
        overflow_time += temp_time - start_time;
        word_time = temp_time - word_time;
        state = State.PAUSED;

        timer.stop();
        setEnabled(false);
        setBackground(disabled_color);
    }

    private char getNextValidChar( String word, int start) {
        if( word == null)
            return 0;

        List<Character> valid_chars = Settings.getAllowedChars();

        for( int i = start; i < word.length(); ++i) {
            if( valid_chars.contains(word.charAt(i)))
                return word.charAt(i);
        }

        return 0;
    }

    /**
     * Compares the currently-typed word segment against the highlighted word
     * in the read pane, returning false if the currently-typed word has a typo.
     *

     * Note: characters which are not considered typeable (not in the valid
     * character table) can either be skipped entirely or substituted for any
     * character.
     *

     * The word may end with a single whitespace character.
     *

     * WARNING: The algorithm currently flubs up when you use parts of the next
     * chunk as garbage characters without anything in-between.  Example:
     *

     * "™™™test™™" compared against "testest" will return 0, but

     * "™™™test™™" compared against "tertest" will return 2
     * @return
     * 0 if there is a conflict
     * 1 if the typed ward matches part of the highlighted word
     * 2 if they match completely
     *

     * NOTE: This function also alters next_char.  If it would return 0, next_char
     * will be unaltered.  If it would return 1, next_char will be the next valid
     * char after the partial match ends.  If it would return 2, next_char
     * will be ' '
     */
    public char compareVsHighlighted() {
        String word = readPane.getHighlightedString();
        String doc_text = getDocText();
        List<Character> valid_chars = Settings.getAllowedChars();

        if( doc_text.length() == 0) {
            next_char = getNextValidChar(word, 0);
            return 1;
        }

        // Ignore final whitespaces.
        int doc_end = doc_text.length();
        if( Character.isWhitespace(doc_text.charAt(doc_end - 1)))
        {
            doc_end = doc_text.length() - 1;
            if( doc_end == 0)
                return 0;
        }

        int word_end = word.length();
        if( word_end > 0 &&Character.isWhitespace(word.charAt(word_end - 1)))
        {
            word_end = word.length() - 1;
        }

        // When scanning through non-valid characters, the low index points
        //  to the last valid character, whereas the high index points to a
        //  non-valid character which increases as you increment through them
        int dpos = 0;
        int wstart = 0;
        int wend = 0;
        int find, garbage;
        char c;

        do {
            // Ignore garbage characters
            while(true) {
                if( wstart == word_end) {
                    // Reached the end with of the word with garbage characters.

                    if( wstart - wend < doc_end - dpos) {
                        // Too many characters at the end, conflict
                        return 0;
                    }
                    else {
                        next_char = ' ';
                        return 2;
                    }
                }

                if( valid_chars.contains(word.charAt(wstart)))
                    break;

                ++wstart;
            }
            garbage = wstart - wend;

            // Read into a chunk the rest of the word, or
            for( wend = wstart; wend < word_end; ++ wend) {
                if( !valid_chars.contains(word.charAt(wend)))
                    break;
            }

            find = _indexOfPartial( doc_text, word, wstart, wend, dpos);

            if( find == -1) {
                // Substring not found (even partially), but may be working
                //  throug garbage characters.
                if( garbage < doc_end - dpos)
                    return 0;
                else {
                    getNextValidChar(word, wstart);
                    return 1;
                }
            }
            if( find - dpos > garbage) {
                // Too much garbage
                return 0;
            }
            if( doc_end - find < wend - wstart) {
                // Partial find
                next_char = word.charAt( find + doc_end);
                return 1;
            }

            dpos = find + wend - wstart;

//            JOptionPane.showConfirmDialog(readPane, ":" + find + "," + dpos +
//                    " [" + wstart + "," + wend + "]");
            if( dpos >= doc_end) {
                // Reached the end of the typed word
                // Either partial match (if more non-garbage characters exist)
                // or full match( otherwise)
                for( ; wend < word_end; ++wend) {
                    c = word.charAt(wend);
                    if( valid_chars.contains(c)) {
                        next_char = c;
                        return 1;
                    }
                }

                next_char = ' ';
                return 2;
            }

            wstart = wend;
        }while( true);
    }

    /**
     * Acts like doc_text.indexOf( word.substring( wstart, wend), dstart)
     * Except also returns if a partial match.
     *

     * Example: if you're comparing "string" to "123456str" it would return 6.
     * But "123456str1" would return -1.
     */
    private int _indexOfPartial( String doc_text, String word,
            int wstart, int wend, int dstart)
    {
        // Automatic null string match
        if( wstart == wend)
            return dstart;

        int working_compare = 0;
        int i;

        for( i = dstart; i < doc_text.length(); ++i) {
            if( doc_text.charAt(i) == word.charAt(wstart + working_compare)) {
                ++working_compare;

                if( working_compare == (wend - wstart))
                    return i - working_compare + 1;
            }
            else
                working_compare = 0;
        }

        if( working_compare == 0)
            return -1;
        else
            return i - working_compare;
    }

    @Override
    public void actionPerformed(ActionEvent evt) {
        if( evt.getSource().equals(this)) {
            // Write Pane has been activated (generally by pressing Enter)
            if( compareVsHighlighted() == 2)
                finishWord( getDocText() + ' ');
        }
        else if( evt.getSource().equals(timer)) {
            // Recurring 50ms timer
            int typed = characters_typed + getDocText().length();
            long elapsed_time = System.nanoTime() - start_time + overflow_time;
            float wpm = typed / (elapsed_time / 12000000000.0f);
            context.updateWPM( wpm);
        }
    }

    @Override
    public void changedUpdate( DocumentEvent evt) {
        updateCompare();
    }

    @Override
    public void insertUpdate( DocumentEvent evt) {
        updateCompare();
    }

    @Override
    public void removeUpdate( DocumentEvent evt) {
        updateCompare();
    }

    /**
     * Compares the text entered into the Write Field against the highlighted
     * text in the read field, recording any mistakes made.
     */
    private void updateCompare() {
        if( !isEnabled())
            return;

        String doc_text = getDocText();

        int compare = compareVsHighlighted();

        if( compare == 0) {
            // Conflict exists
            if( getBackground() != Color.red) {
                // (Attempt to) Record the mispress in the database
                try {
                    DatabaseConnector dc = Settings.getDBConnector();

                    if( dc != null) {
                        dc.queueMistype(next_char,
                                doc_text.charAt(doc_text.length() - 1));
                    }
                } catch( Exception e) { ErrorHandler.handleException(e, 3);}
            }
            setBackground( error_color);
            mistyped_current_word = true;
        }
        else {
            // No conflict exists
            setBackground(enabled_color);

            if( compare == 2 && doc_text.length() > 0 &&
                Character.isWhitespace(doc_text.charAt(doc_text.length() - 1)))
            {
                finishWord( doc_text);
            }
        }
    }

    /**
     * Tells the readPane that you've finished typing a word.
     *

     * Note: the word is passed to this function primarily so that the enter
     * key used to finish the word is counted as a character typed (by adding
     * a space to the end).
     */
    private void finishWord( String word) {
        // Record Word into Database queue
        long temp_time = System.nanoTime();
        characters_typed += word.length();
        DatabaseConnector dc = Settings.getDBConnector();
        dc.queueWord(word, (temp_time - word_time)/1000000, mistyped_current_word);

        // Update Tracked variables
        if( mistyped_current_word) {
            words_mistyped += 1;
            chars_in_mistyped += word.length();
            time_in_mistyped += temp_time - word_time;
        }
        else {
            chars_in_correct += word.length();
            time_in_correct += temp_time - word_time;
        }
        words_typed += 1;
        word_time = temp_time;
        mistyped_current_word = false;

        // End Type-backing when at the end of the doculement
        if( !readPane.nextWord()) {
            endTypeBack(true);
            context.setTypeState(TypeForm.TypeState.EDITING);
        }
        else
            next_char = readPane.getHighlightedString().charAt(0);

        invokeClear();
    }

    /**
     * Send an asynchronous message to set the text box to ""
     * since you cannot edit text from inside text change event.
     */
    private void invokeClear() {
        SwingUtilities.invokeLater( new WriteField.RemoveText(this));
    }
    private class RemoveText implements Runnable {
        WriteField context;

        public RemoveText( WriteField context) {
            this.context = context;
        }

        @Override
        public void run() {
            context.setText("");
        }
    }

    /**
     * Gets the document text in string form.
     *

     * Note: use this as opposed to getText() because the latter
     * will have control characters in it.
     */
    public String getDocText() {
        try {
            return getDocument().getText( 0, getDocument().getLength());
        } catch( Exception e){ ErrorHandler.handleException(e, 3); }

        return "";
    }
}


text:
package org.javaplus.jmp.taglib;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import javax.activation.DataHandler;
import javax.mail.MessagingException;
import javax.mail.Part;
import javax.mail.internet.ContentType;
import javax.mail.internet.ParseException;
import javax.mail.util.ByteArrayDataSource;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.jsp.JspException;
import javax.servlet.jsp.JspTagException;
import org.javaplus.jmp.util.MessageUtil;
import org.javaplus.jmp.util.MessagingResponse;

/**
 *

 * PartTagSupport is the base class for all messaging tags
 * that create an associated instance of {@link javax.mail.Part}.

 *

 * PartTagSupport provides the support to most important
 * header fields by tag attributes, especially for Content-Type and
 * Content-Disposition. PartTagSupport also provides
 * subclasses the facilities to prepare the content from the specified
 * URL or the body content.

 *
 * @author Stephen Suen
 */
public abstract class PartTagSupport extends MessagingTagSupport
        implements PartTag {
    //
    // Tag attributes --------------------------------------------------------//
    //

    private String contentType;
    private String description;
    private String disposition;
    private String filename;
    private String url;
    private String context;
    //
    // Private used members --------------------------------------------------//
    //
    private boolean isAbsoluteUrl;
    private String charset;
    private Part part;

    /**
     * Creates new instance of tag handler
     */
    public PartTagSupport() {
        super();
    }

    /**
     * Create and return an instance of Part associated with
     * this messaging tag.
     * @return the created Part instance.
     * @see #getPart()
     */
    protected abstract Part createPart();

    public Part getPart() {
        return part;
    }

    //
    // Getters/Setters for attributes ----------------------------------------//
    //
    public String getContentType() {
        return contentType;
    }

    public void setContentType(String contentType) {
        this.contentType = normalizeAttribute(contentType);
    }

    public String getDescription() {
        return description;
    }

    public void setDescription(String description) {
        // no need to "normailize"
        this.description = description;
    }

    public String getDisposition() {
        return disposition;
    }

    public void setDisposition(String disposition) {
        this.disposition = normalizeAttribute(disposition);
    }

    public String getFileName() {
        return filename;
    }

    public void setFileName(String filename) {
        this.filename = normalizeAttribute(filename);
    }

    /**
     * Sets the URL of the resource to import.
     * @param url the URL
     */
    public void setUrl(String url) {
        this.url = normalizeAttribute(url);
    }

    /**
     * Returns the URL of the resource to import.
     * @return the URL
     */
    public String getUrl() {
        return url;
    }

    /**
     * Returns the name of the context when accessing a relative
     * URL resource that belongs to a foreign context.
     * @return the name of the context.
     */
    public String getContext() {
        return context;
    }

    /**
     * Sets the name of the context when accessing a relative
     * URL resource that belongs to a foreign context.
     * @param context the name of the context
     */
    public void setContext(String context) {
        this.context = normalizeAttribute(context);
    }

    //
    // Protected methods -----------------------------------------------------//
    //
    /**
     *

     * Set the appropriate content to the associated Part
     * instance if the content has not been set by any nested tag.
     * Subclasses must call this method to prepare the content before
     * adding their associated Part instances to
     * the enclosing entity associated with their enclosing tags.

     *

     * The overall strategy implemented by this method is:

     *

     *

     * If the content of associated Part instance is not
     * null, no content will be set by this method

     *

     * If url is specified, the resource located at the
     * URL specified by url and optional context
     * attribute will be loaded and set as the content. The Content-Type
     * determined from the resource will override the one specified by
     * the "contentType" attribute.

     *

     * Otherwise, set the text returned from the method
     * {@link #getBodyContentString()} as the content with the Content-Type
     * specified by the contentType attribute.

     *

     *
     * @throws JspException if an error occurs
     * @see #doEndTag()
     * @see #getBodyContentString()
     */
    protected void setContent() throws JspException {
        // set Content-Description if necessary
        if (description != null) {
            try {
                part.setDescription(description);
            } catch (MessagingException ex) {
                throw new JspException(MessageUtil.getMessage(
                        "parttagsupport_setdescription_failed", description),
                        ex);
            }
        }

        // set Content-Disposition if necessary
        if (disposition != null) {
            try {
                part.setDisposition(disposition);
            } catch (MessagingException ex) {
                throw new JspException(MessageUtil.getMessage(
                        "parttagsupport_setdisposition_failed", disposition),
                        ex);
            }
        }

        // check whether content has been set by any nested tag
        // NOTE:
        // an empty content could probably throw an exception,
        // ignore it
        boolean isContentSet = false;
        try {
            isContentSet = (part.getContent() != null);
        } catch (IOException ex) {
        } catch (MessagingException ex) {
        }

        if (isContentSet) {
            // content has been by nested tags (for example, MultipartTag)
        } else if (url != null) {
            setUrlContent();
        } else {
            setBodyContent();
        }

        // set the filename parameter if needed
        if (filename != null) {
            try {
                part.setFileName(filename);
            } catch (MessagingException ex) {
                throw new JspTagException(MessageUtil.getMessage(
                        "parttagsupport_setfilename_failed", filename),
                        ex);
            }
        }
    }

    /**
     * Returns the character encoding. This overrided method returns
     * the value of charset parameter specified in
     * the contentType attribute if available,
     * otherwise, delegates to the implementaion of superclass.
     *
     * @return the chracter encoding
     */
    @Override
    protected String getCharacterEncoding() {
        if (charset != null) {
            return charset;
        } else {
            return super.getCharacterEncoding();
        }
    }

    //
    // Overrided BodyTag method ----------------------------------------------//
    //
    /**
     * Performs any necessary validation on the attributes specified by
     * this implementation, and creates an empty Part
     * instance associated with this tag.
     *
     * @return the value returned from
     *          {@link MessagingTagSupport#doStartTag()}
     * @throws JspException if an error occurs
     * @see #createPart()
     */
    @Override
    public int doStartTag() throws JspException {
        int result = super.doStartTag();

        // validate the following attributes:
        // - url
        // - context
        // - contentType
        // - disposition
        if (context != null) {
            if (url == null) {
                // standalone "context" attribute is not allowed
                throw new JspTagException(MessageUtil.getMessage(
                        "parttagsupport_context_without_url"));
            } else if (!context.startsWith("/") || !url.startsWith("/")) {
                // in the case of relative URL in a foreign context,
                // values of both "context" and "url" attribute must
                // start with "/"
                throw new JspTagException(MessageUtil.getMessage(
                        "parttagsupport_bad_relative"));
            }
        }

        isAbsoluteUrl = isAbsoluteUrl(url);

        //
        // TODO: should me do like this?
        //
        // Although Content-Type can technically default to some value,
        // and even nested tags could potentially override the value
        // specified here, we enforce a Content-Type attribute
        // must be specified to avaid common mistakes that a missing
        // Content-Type leads to an ugly message to be preduced.
//        if (contentType == null) {
//            throw new JspTagException(MessageUtil.getMessage(
//                    "parttagsupport_contenttype_required"));
//        }

        // verify and "normalize" Content-Type, determine charset if
        // specified
        if (contentType != null) {
            ContentType contentTypeObj = null;
            try {
                contentTypeObj = new ContentType(contentType);
            } catch (ParseException ex) {
                throw new JspTagException(MessageUtil.getMessage(
                        "parttagsupport_invalid_contenttype", contentType),
                        ex);
            }
            charset = contentTypeObj.getParameter("charset");
            contentType = contentTypeObj.toString();
        }

        // "normalize" standard Content-Disposition value
        if (Part.ATTACHMENT.equalsIgnoreCase(disposition)) {
            disposition = Part.ATTACHMENT;
        } else if (Part.INLINE.equalsIgnoreCase(disposition)) {
            disposition = Part.INLINE;
        }

        // create an empty Part instance
        part = createPart();

        return result;
    }

    /**
     *

     * Subclasses should override this method to prepare the content of
     * the Part instance associated with the tag by calling
     * {@link #setContent()} method, and add the fully processed instance
     * to the enclosing entity aasociated with the enclosing messaing
     * tag if any.

     *

     * The default implementation of this class simply calls the superclass's
     * implementaion

     *
     * @return the value returned from superclass
     * @throws JspException if an error occurs
     * @see #setContent()
     */
    @Override
    public int doEndTag() throws JspException {
        return super.doEndTag();
    }

    @Override
    public void release() {
        this.contentType = null;
        this.description = null;
        this.disposition = null;
        this.filename = null;
        this.url = null;
        this.context = null;

        this.isAbsoluteUrl = false;
        this.charset = null;
        this.part = null;

        super.release();
    }

    //
    // Private methods -------------------------------------------------------//
    //
    private void setBodyContent() throws JspException {
        String text = getBodyContentString();
        if (text != null) {
            try {
                // use tag attribute for Content-Type header field
                // any Content-Type header added by nested tags will
                // be ignored
                if (contentType == null) {
                    logger.warning(MessageUtil.getMessage(
                            "parttagsupport_missing_contenttype"));
                    part.setText(text);
                } else {
                    part.setContent(text, contentType);
                }
            } catch (MessagingException ex) {
                throw new JspTagException(MessageUtil.getMessage(
                        "parttagsupport_settext_failed", getContentType()),
                        ex);
            }
        }
    }

    private void setUrlContent() throws JspException {
        if (url == null) {
            return;
        } else if (isAbsoluteUrl) {
            setAbsUrlContent();
        } else {
            setRelUrlContent();
        }
    }

    private void setRelUrlContent() throws JspException {
        RequestDispatcher rd = null;
        if (context == null) {
            rd = pageContext.getRequest().getRequestDispatcher(url);
        } else {
            ServletContext sc =
                    pageContext.getServletContext().getContext(context);
            if (sc == null) {
                throw new JspTagException(MessageUtil.getMessage(
                        "parttagsupport_context_not_found", context));
            }
            rd = sc.getRequestDispatcher(url);
        }

        if (rd == null) {
            throw new JspTagException(MessageUtil.getMessage(
                    "parttagsupport_url_not_found", context, url));
        }

        MessagingResponse response =
                new MessagingResponse(pageContext.getResponse());
        try {
            rd.forward(pageContext.getRequest(), response);
        } catch (ServletException ex) {
            Throwable rootCause = ex.getRootCause();
            if (rootCause == null) {
                rootCause = ex;
            }
            throw new JspTagException(MessageUtil.getMessage(
                    "parttagsupport_relurl_exception",
                    context, url, rootCause.getMessage()),
                    rootCause);
        } catch (Throwable ex) {
            throw new JspTagException(MessageUtil.getMessage(
                    "parttagsupport_relurl_exception",
                    context, url, ex.getMessage()),
                    ex);
        }

        // make sure an 2XX status (indicating success)
        int sc = response.getStatus();
        if (sc < 200 || sc > 299) {
            throw new JspTagException(MessageUtil.getMessage(
                    "parttagsupport_relurl_non_success_status",
                    context, url, sc));
        }

        // determine the Content-Type
        String type = response.getContentType();
        if (type == null) {
            // should never happen
            type = contentType;
        } else if ((contentType != null) &&
                !type.equalsIgnoreCase(contentType)) {
            //
            // TODO: should we throw an exception?
            //
            logger.warning(MessageUtil.getMessage(
                    "parttagsupport_wrong_contenttype",
                    contentType, type, url, context));
        }

        byte[] buf = response.toByteArray();
        ByteArrayDataSource ds = new ByteArrayDataSource(buf, type);
        try {
            part.setDataHandler(new DataHandler(ds));
        } catch (MessagingException ex) {
            throw new JspTagException(MessageUtil.getMessage(
                    "parttagsupport_relurl_content_failed",
                    contentType, url, context),
                    ex);
        }
    }

    private void setAbsUrlContent() throws JspException {
        URL urlObj = null;
        try {
            urlObj = new URL(url);
        } catch (MalformedURLException ex) {
            throw new JspTagException(MessageUtil.getMessage(
                    "parttagsupport_absurl_malformed", url),
                    ex);
        }

        DataHandler dh = null;
        dh = new DataHandler(urlObj);
        try {
            part.setDataHandler(dh);
        } catch (MessagingException ex) {
            throw new JspTagException(MessageUtil.getMessage(
                    "parttagsupport_absurl_content_failed", url),
                    ex);
        }

        // verify the Content-Type
        String type = dh.getContentType();
        if ((contentType != null) && !contentType.equalsIgnoreCase(type)) {
            //
            // TODO: should we throw an exception?
            //
            logger.warning(MessageUtil.getMessage(
                    "parttagsupport_wrong_contenttype",
                    contentType, type, url, context));
        }
    }

    //
    // Private helpers -------------------------------------------------------//
    //
    /**
     * The valid characters in a URL scheme part.
     */
    private static final String VALID_SCHEME_CHARS =
            "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+.-";

    /**
     *

     * Check whether the specified URL represents an absolute URL.

     *

     *  An absolute URI is defined in RFC 2396 as following:

     *

     * An absolute URI contains the name of the scheme being used (<scheme>)
     * followed by a colon (":") and then a string (the
     * <scheme-specific-part>) whose interpretation depends on the scheme.
     *

     *
     *

     * Scheme names consist of a sequence of characters beginning with a
     * lower case letter and followed by any combination of lower case
     * letters, digits, plus ("+"), period ("."), or hyphen ("-").  For
     * resiliency, programs interpreting URI should treat upper case letters
     * as equivalent to lower case in scheme names (e.g., allow "HTTP" as
     * well as "http").
     *
     *     scheme        = alpha *( alpha | digit | "+" | "-" | "." )
     *
     *

     *
     * @param url the URL to check
     * @return true if the specified URL is an absolute URL,
     *          false otherwise.
     */
    private static boolean isAbsoluteUrl(String url) {
        if (url == null)
            return false;

        int iColon = url.indexOf(':');
        if (iColon == -1)
            return false;

        for (int i = 0; i < iColon; i++) {
            if (VALID_SCHEME_CHARS.indexOf(url.charAt(i)) == -1)
                return false;
        }

        return true;
    }
}


text:
package io.jeo.cli.convert;

import com.beust.jcommander.IStringConverter;
import io.jeo.cli.Util;
import io.jeo.cli.command.DatasetSink;
import io.jeo.cli.command.GeoJSONSink;
import io.jeo.cli.command.GeobufSink;
import io.jeo.cli.command.VectorSink;
import io.jeo.cli.command.WorkspaceSink;
import io.jeo.data.Dataset;
import io.jeo.data.Driver;
import io.jeo.data.Drivers;
import io.jeo.data.Workspace;
import io.jeo.util.Pair;

import java.net.URI;

public class VectorSinkConverter implements IStringConverter<VectorSink> {
    @Override
    public VectorSink convert(String str) {
        if ("gbf".equalsIgnoreCase(str)) {
            return new GeobufSink();
        }
        if ("geojson".equalsIgnoreCase(str) || "json".equalsIgnoreCase(str)) {
            return new GeoJSONSink();
        }

        Pair<URI,String> ref = null;
        try {
            ref = Util.parseDataURI(str);
        }
        catch(IllegalArgumentException e) {
            throw new IllegalArgumentException("Unrecognized output: " + str);
        }

        Driver<?> drv = Drivers.find(ref.first);
        if (drv == null ){
            throw new IllegalArgumentException("No driver found for: " + ref.first);
        }

        if (Workspace.class.isAssignableFrom(drv.type())) {
            return new WorkspaceSink(ref);
        }
        else if (Dataset.class.isAssignableFrom(drv.type())) {
            return new DatasetSink(ref);
        }
        else {
            throw new IllegalArgumentException("Unsupported driver type: " + drv.name());
        }
    }
}