import logging
import os

from os.path import join as opj
from os.path import isdir
from os.path import exists
from os.path import realpath
from os.path import basename
from os.path import dirname

from mock import patch

from datalad.api import create
from datalad.api import install
from datalad.api import get
from datalad.consts import DATASETS_TOPURL
from datalad.utils import chpwd
from datalad.support.exceptions import InsufficientArgumentsError
from datalad.support.exceptions import InstallFailedError
from datalad.support.exceptions import IncompleteResultsError
from datalad.support.gitrepo import GitRepo
from datalad.support.gitrepo import GitCommandError
from datalad.support.annexrepo import AnnexRepo
from datalad.cmd import Runner
from datalad.tests.utils import create_tree
from datalad.tests.utils import with_tempfile
from datalad.tests.utils import assert_in
from datalad.tests.utils import with_tree
from datalad.tests.utils import with_testrepos
from datalad.tests.utils import eq_
from datalad.tests.utils import ok_
from datalad.tests.utils import assert_false
from datalad.tests.utils import ok_file_has_content
from datalad.tests.utils import assert_not_in
from datalad.tests.utils import assert_raises
from datalad.tests.utils import ok_startswith
from datalad.tests.utils import ok_clean_git
from datalad.tests.utils import serve_path_via_http
from datalad.tests.utils import swallow_logs
from datalad.tests.utils import use_cassette
from datalad.tests.utils import skip_if_no_network
from datalad.utils import _path_
from datalad.utils import rmtree

from ..dataset import Dataset
from ..install import _get_installationpath_from_url
from ..install import _get_git_url_from_source

###############
# Test helpers:
###############

def test_installationpath_from_url():
    for p in ('lastbit',
              'lastbit/',
              '/lastbit',
              'lastbit.git',
              'lastbit.git/',
              'http://example.com/lastbit',
              'http://example.com/lastbit.git',
              ):
        eq_(_get_installationpath_from_url(p), 'lastbit')

def test_get_git_url_from_source():

    # resolves datalad RIs:
    eq_(_get_git_url_from_source('///subds'), DATASETS_TOPURL + 'subds')
    assert_raises(NotImplementedError, _get_git_url_from_source,
                  '//custom/subds')

    # doesn't harm others:
    eq_(_get_git_url_from_source('http://example.com'), 'http://example.com')
    eq_(_get_git_url_from_source('/absolute/path'), '/absolute/path')
    eq_(_get_git_url_from_source('file://localhost/some'),
        'file://localhost/some')
    eq_(_get_git_url_from_source('localhost/another/path'),
        'localhost/another/path')
    eq_(_get_git_url_from_source('user@someho.st/mydir'),
        'user@someho.st/mydir')
    eq_(_get_git_url_from_source('ssh://somewhe.re/else'),
        'ssh://somewhe.re/else')
    eq_(_get_git_url_from_source('git://github.com/datalad/testrepo--basic--r1'),
        'git://github.com/datalad/testrepo--basic--r1')

@with_tree(tree={'file.txt': '123'})
@serve_path_via_http
@with_tempfile
def _test_guess_dot_git(annex, path, url, tdir):
    repo = (AnnexRepo if annex else GitRepo)(path, create=True)
    repo.add('file.txt', commit=True, git=not annex)

    # we need to prepare to be served via http, otherwise it must fail
    with swallow_logs() as cml:
        assert_raises(GitCommandError, install, path=tdir, source=url)
    ok_(not exists(tdir))

    Runner(cwd=path)(['git', 'update-server-info'])

    with swallow_logs() as cml:
        installed = install(tdir, source=url)
        assert_not_in("Failed to get annex.uuid", cml.out)
    eq_(realpath(installed.path), realpath(tdir))
    ok_(exists(tdir))
    ok_clean_git(tdir, annex=annex)

def test_guess_dot_git():
    for annex in False, True:
        yield _test_guess_dot_git, annex

######################
# Test actual Install:
######################

def test_insufficient_args():
    assert_raises(InsufficientArgumentsError, install)
    assert_raises(InsufficientArgumentsError, install, description="some")
    assert_raises(InsufficientArgumentsError, install, None)
    assert_raises(InsufficientArgumentsError, install, None, description="some")

@with_tempfile(mkdir=True)
def test_invalid_args(path):
    assert_raises(ValueError, install, 'Zoidberg', source='Zoidberg')
    # install to an invalid URL
    assert_raises(ValueError, install, 'ssh://mars:Zoidberg', source='Zoidberg')
    # install to a remote location
    assert_raises(ValueError, install, 'ssh://mars/Zoidberg', source='Zoidberg')
    # make fake dataset
    ds = create(path)
    assert_raises(ValueError, install, '/higherup.', 'Zoidberg', dataset=ds)

@skip_if_no_network
@use_cassette('test_install_crcns')
@with_tempfile(mkdir=True)
@with_tempfile(mkdir=True)
def test_install_crcns(tdir, ds_path):
    with chpwd(tdir):
        with swallow_logs(new_level=logging.INFO) as cml:
            install("all-nonrecursive", source='///')
            # since we didn't log decorations such as log level atm while
            # swallowing so lets check if exit code is returned or not
            # I will test both
            assert_not_in('ERROR', cml.out)
            # below one must not fail alone! ;)
            assert_not_in('with exit code', cml.out)

        # should not hang in infinite recursion
        with chpwd('all-nonrecursive'):
            get("crcns")
        ok_(exists(_path_("all-nonrecursive/crcns/.git/config")))
        # and we could repeat installation and get the same result
        ds1 = install(_path_("all-nonrecursive/crcns"))
        ds2 = Dataset('all-nonrecursive').install('crcns')
        ok_(ds1.is_installed())
        eq_(ds1, ds2)
        eq_(ds1.path, ds2.path)  # to make sure they are a single dataset

    # again, but into existing dataset:
    ds = create(ds_path)
    crcns = ds.install("///crcns")
    ok_(crcns.is_installed())
    eq_(crcns.path, opj(ds_path, "crcns"))
    assert_in(crcns.path, ds.get_subdatasets(absolute=True))

@skip_if_no_network
@use_cassette('test_install_crcns')
@with_tree(tree={'sub': {}})
def test_install_datasets_root(tdir):
    with chpwd(tdir):
        ds = install("///")
        ok_(ds.is_installed())
        eq_(ds.path, opj(tdir, 'datasets.datalad.org'))

        # do it a second time:
        with swallow_logs(new_level=logging.INFO) as cml:
            result = install("///")
            assert_in("was already installed from", cml.out)
            eq_(result, ds)

        # and a third time into an existing something, that is not a dataset:
        with open(opj(tdir, 'sub', 'a_file.txt'), 'w') as f:
            f.write("something")

        with swallow_logs(new_level=logging.WARNING) as cml:
            result = install("sub", source='///')
            assert_in("already exists and is not an installed dataset", cml.out)
            ok_(result is None)

@with_testrepos('.*basic.*', flavors=['local-url', 'network', 'local'])
@with_tempfile(mkdir=True)
def test_install_simple_local(src, path):
    origin = Dataset(path)

    # now install it somewhere else
    ds = install(path, source=src)
    eq_(ds.path, path)
    ok_(ds.is_installed())
    if not isinstance(origin.repo, AnnexRepo):
        # this means it is a GitRepo
        ok_(isinstance(origin.repo, GitRepo))
        # stays plain Git repo
        ok_(isinstance(ds.repo, GitRepo))
        ok_(not isinstance(ds.repo, AnnexRepo))
        ok_(GitRepo.is_valid_repo(ds.path))
        eq_(set(ds.repo.get_indexed_files()),
            {'test.dat', 'INFO.txt'})
        ok_clean_git(path, annex=False)
    else:
        # must be an annex
        ok_(isinstance(ds.repo, AnnexRepo))
        ok_(AnnexRepo.is_valid_repo(ds.path, allow_noninitialized=False))
        eq_(set(ds.repo.get_indexed_files()),
            {'test.dat', 'INFO.txt', 'test-annex.dat'})
        ok_clean_git(path, annex=True)
        # no content was installed:
        ok_(not ds.repo.file_has_content('test-annex.dat'))
        uuid_before = ds.repo.uuid

    # installing it again, shouldn't matter:
    with swallow_logs(new_level=logging.INFO) as cml:
        ds = install(path, source=src)
        cml.assert_logged(msg="{0} was already installed from".format(ds),
                          regex=False, level="INFO")
        ok_(ds.is_installed())
        if isinstance(origin.repo, AnnexRepo):
            eq_(uuid_before, ds.repo.uuid)

@with_testrepos(flavors=['local-url', 'network', 'local'])
@with_tempfile
def test_install_dataset_from_just_source(url, path):
    with chpwd(path, mkdir=True):
        ds = install(source=url)

    ok_startswith(ds.path, path)
    ok_(ds.is_installed())
    ok_(GitRepo.is_valid_repo(ds.path))
    ok_clean_git(ds.path, annex=False)
    assert_in('INFO.txt', ds.repo.get_indexed_files())

@with_testrepos(flavors=['network'])
@with_tempfile
def test_install_dataset_from_just_source_via_path(url, path):
    # for remote urls only, the source could be given to `path`
    # to allows for simplistic cmdline calls
    # Q (ben): remote urls only? Sure? => TODO

    with chpwd(path, mkdir=True):
        ds = install(url)

    ok_startswith(ds.path, path)
    ok_(ds.is_installed())
    ok_(GitRepo.is_valid_repo(ds.path))
    ok_clean_git(ds.path, annex=False)
    assert_in('INFO.txt', ds.repo.get_indexed_files())

@with_tree(tree={
    'ds': {'test.txt': 'some'},
    })
@serve_path_via_http
@with_tempfile(mkdir=True)
def test_install_dataladri(src, topurl, path):
    # make plain git repo
    ds_path = opj(src, 'ds')
    gr = GitRepo(ds_path, create=True)
    gr.add('test.txt')
    gr.commit('demo')
    Runner(cwd=gr.path)(['git', 'update-server-info'])
    # now install it somewhere else
    with patch('datalad.support.network.DATASETS_TOPURL', topurl), \
        swallow_logs():
        ds = install(path, source='///ds')
    eq_(ds.path, path)
    ok_clean_git(path, annex=False)
    ok_file_has_content(opj(path, 'test.txt'), 'some')

@with_testrepos('submodule_annex', flavors=['local', 'local-url', 'network'])
@with_tempfile(mkdir=True)
@with_tempfile(mkdir=True)
def test_install_recursive(src, path_nr, path_r):
    # first install non-recursive:
    ds = install(path_nr, source=src, recursive=False)
    ok_(ds.is_installed())
    for sub in ds.get_subdatasets(recursive=True):
        ok_(not Dataset(opj(path_nr, sub)).is_installed(),
            "Unintentionally installed: %s" % opj(path_nr, sub))
    # this also means, subdatasets to be listed as not fulfilled:
    eq_(set(ds.get_subdatasets(recursive=True, fulfilled=False)),
        {'subm 1', 'subm 2'})

    # now recursively:
    ds_list = install(path_r, source=src, recursive=True)
    # installed a dataset and two subdatasets
    eq_(len(ds_list), 3)
    eq_(sum([isinstance(i, Dataset) for i in ds_list]), 3)
    # we recurse top down during installation, so toplevel should appear at
    # first position in returned list
    eq_(ds_list[0].path, path_r)
    top_ds = ds_list[0]
    ok_(top_ds.is_installed())

    # the subdatasets are contained in returned list:
    # (Note: Until we provide proper (singleton) instances for Datasets,
    # need to check for their paths)
    assert_in(opj(top_ds.path, 'subm 1'), [i.path for i in ds_list])
    assert_in(opj(top_ds.path, 'subm 2'), [i.path for i in ds_list])

    eq_(len(top_ds.get_subdatasets(recursive=True)), 2)

    for sub in top_ds.get_subdatasets(recursive=True):
        subds = Dataset(opj(path_r, sub))
        ok_(subds.is_installed(),
            "Not installed: %s" % opj(path_r, sub))
        # no content was installed:
        ok_(not any(subds.repo.file_has_content(
            subds.repo.get_annexed_files())))
    # no unfulfilled subdatasets:
    ok_(top_ds.get_subdatasets(recursive=True, fulfilled=False) == [])

@with_testrepos('submodule_annex', flavors=['local'])
@with_tempfile(mkdir=True)
def test_install_recursive_with_data(src, path):

    # now again; with data:
    ds_list = install(path, source=src, recursive=True, get_data=True)
    # installed a dataset and two subdatasets, and two files:
    eq_(len(ds_list), 5)
    eq_(sum([isinstance(i, Dataset) for i in ds_list]), 3)
    # we recurse top down during installation, so toplevel should appear at
    # first position in returned list
    eq_(ds_list[0].path, path)
    top_ds = ds_list[0]
    ok_(top_ds.is_installed())
    if isinstance(top_ds.repo, AnnexRepo):
        ok_(all(top_ds.repo.file_has_content(top_ds.repo.get_annexed_files())))
    for sub in top_ds.get_subdatasets(recursive=True):
        subds = Dataset(opj(path, sub))
        ok_(subds.is_installed(), "Not installed: %s" % opj(path, sub))
        if isinstance(subds.repo, AnnexRepo):
            ok_(all(subds.repo.file_has_content(subds.repo.get_annexed_files())))

@with_testrepos(flavors=['local'])
# 'local-url', 'network'
# TODO: Somehow annex gets confused while initializing installed ds, whose
# .git/config show a submodule url "file:///aaa/bbb%20b/..."
# this is delivered by with_testrepos as the url to clone
@with_tempfile
def test_install_into_dataset(source, top_path):

    ds = create(top_path)
    ok_clean_git(ds.path)

    subds = ds.install("sub", source=source, save=False)
    if isinstance(subds.repo, AnnexRepo) and subds.repo.is_direct_mode():
        ok_(exists(opj(subds.path, '.git')))
    else:
        ok_(isdir(opj(subds.path, '.git')))
    ok_(subds.is_installed())
    assert_in('sub', ds.get_subdatasets())
    # sub is clean:
    ok_clean_git(subds.path, annex=False)
    # top is not:
    assert_raises(AssertionError, ok_clean_git, ds.path, annex=False)
    ds.save('addsub')
    # now it is:
    ok_clean_git(ds.path, annex=False)

    # but we could also save while installing and there should be no side-effect
    # of saving any other changes if we state to not auto-save changes
    # Create a dummy change
    create_tree(ds.path, {'dummy.txt': 'buga'})
    ok_clean_git(ds.path, untracked=['dummy.txt'])
    subds_ = ds.install("sub2", source=source, if_dirty='ignore')
    eq_(subds_.path, opj(ds.path, "sub2"))  # for paranoid yoh ;)
    ok_clean_git(ds.path, untracked=['dummy.txt'])

    # and we should achieve the same behavior if we create a dataset
    # and then decide to add it
    create(_path_(top_path, 'sub3'))
    ok_clean_git(ds.path, untracked=['dummy.txt', 'sub3/'])
    ds.add('sub3')
    ok_clean_git(ds.path, untracked=['dummy.txt'])

@skip_if_no_network
@use_cassette('test_install_crcns')
@with_tempfile
def test_failed_install_multiple(top_path):
    ds = create(top_path)

    create(_path_(top_path, 'ds1'))
    create(_path_(top_path, 'ds3'))
    ok_clean_git(ds.path, annex=False, untracked=['ds1/', 'ds3/'])

    # specify install with multiple paths and one non-existing
    with assert_raises(IncompleteResultsError) as cme:
        ds.install(['ds1', 'ds2', '///crcns', '///nonexisting', 'ds3'])

    # install doesn't add existing submodules -- add does that
    ok_clean_git(ds.path, annex=False, untracked=['ds1/', 'ds3/'])
    ds.add(['ds1', 'ds3'])
    ok_clean_git(ds.path, annex=False)
    # those which succeeded should be saved now
    eq_(ds.get_subdatasets(), ['crcns', 'ds1', 'ds3'])
    # and those which didn't -- listed
    eq_(set(cme.exception.failed), {'///nonexisting', _path_(top_path, 'ds2')})

    # but if there was only a single installation requested -- it will be
    # InstallFailedError to stay consistent with single install behavior
    # TODO: unify at some point
    with assert_raises(InstallFailedError) as cme:
        ds.install('ds2')
    with assert_raises(InstallFailedError) as cme:
        ds.install('///nonexisting')

@with_testrepos('submodule_annex', flavors=['local', 'local-url', 'network'])
@with_tempfile(mkdir=True)
def test_install_known_subdataset(src, path):

    # get the superdataset:
    ds = install(path, source=src)

    # subdataset not installed:
    subds = Dataset(opj(path, 'subm 1'))
    assert_false(subds.is_installed())
    assert_in('subm 1', ds.get_subdatasets(fulfilled=False))
    assert_not_in('subm 1', ds.get_subdatasets(fulfilled=True))
    # install it:
    ds.install('subm 1')
    ok_(subds.is_installed())
    ok_(AnnexRepo.is_valid_repo(subds.path, allow_noninitialized=False))
    # Verify that it is the correct submodule installed and not
    # new repository initiated
    eq_(set(subds.repo.get_indexed_files()),
        {'test.dat', 'INFO.txt', 'test-annex.dat'})
    assert_not_in('subm 1', ds.get_subdatasets(fulfilled=False))
    assert_in('subm 1', ds.get_subdatasets(fulfilled=True))

    # now, get the data by reinstalling with -g:
    ok_(subds.repo.file_has_content('test-annex.dat') is False)
    with chpwd(ds.path):
        result = get(path='subm 1', dataset=os.curdir)
        eq_(len(result), 1)
        eq_(result[0]['file'], opj('subm 1', 'test-annex.dat'))
        ok_(subds.repo.file_has_content('test-annex.dat') is True)
        ok_(subds.is_installed())

@with_tempfile(mkdir=True)
@with_tempfile(mkdir=True)
def test_implicit_install(src, dst):

    origin_top = create(src)
    origin_sub = origin_top.create("sub")
    origin_subsub = origin_sub.create("subsub")
    with open(opj(origin_top.path, "file1.txt"), "w") as f:
        f.write("content1")
    origin_top.add("file1.txt")
    with open(opj(origin_sub.path, "file2.txt"), "w") as f:
        f.write("content2")
    origin_sub.add("file2.txt")
    with open(opj(origin_subsub.path, "file3.txt"), "w") as f:
        f.write("content3")
    origin_subsub.add("file3.txt")
    origin_top.save(recursive=True)

    # first, install toplevel:
    ds = install(dst, source=src)
    ok_(ds.is_installed())

    sub = Dataset(opj(ds.path, "sub"))
    ok_(not sub.is_installed())
    subsub = Dataset(opj(sub.path, "subsub"))
    ok_(not subsub.is_installed())

    # fail on obscure non-existing one
    assert_raises(InstallFailedError, ds.install, source='obscure')

    # install 3rd level and therefore implicitly the 2nd:
    result = ds.install(path=opj("sub", "subsub"))
    ok_(sub.is_installed())
    ok_(subsub.is_installed())
    eq_(result, subsub)

    # fail on obscure non-existing one in subds
    assert_raises(InstallFailedError, ds.install, source=opj('sub', 'obscure'))

    # clean up:
    rmtree(dst, chmod_files=True)
    ok_(not exists(dst))

    # again first toplevel:
    ds = install(dst, source=src)
    ok_(ds.is_installed())
    sub = Dataset(opj(ds.path, "sub"))
    ok_(not sub.is_installed())
    subsub = Dataset(opj(sub.path, "subsub"))
    ok_(not subsub.is_installed())

    # now implicit but without an explicit dataset to install into
    # (deriving from CWD):
    with chpwd(dst):
        # don't ask for the file content to make return value comparison
        # simpler
        result = get(path=opj("sub", "subsub"), get_data=False)
        ok_(sub.is_installed())
        ok_(subsub.is_installed())
        eq_(result, [subsub])

@with_tempfile(mkdir=True)
def test_failed_install(dspath):
    ds = create(dspath)
    assert_raises(InstallFailedError,
                  ds.install,
                  "sub",
                  source="http://nonexistingreallyanything.somewhere/bla")

@with_testrepos('submodule_annex', flavors=['local'])
@with_tempfile(mkdir=True)
def test_install_list(path, top_path):

    # we want to be able to install several things, if these are known
    # (no 'source' allowed). Therefore first toplevel:
    ds = install(top_path, source=path, recursive=False)
    assert_not_in('annex.hardlink', ds.config)
    ok_(ds.is_installed())
    sub1 = Dataset(opj(top_path, 'subm 1'))
    sub2 = Dataset(opj(top_path, 'subm 2'))
    ok_(not sub1.is_installed())
    ok_(not sub2.is_installed())

    # fails, when `source` is passed:
    assert_raises(ValueError, ds.install,
                  path=['subm 1', 'subm 2'],
                  source='something')

    # now should work:
    result = ds.install(path=['subm 1', 'subm 2'])
    ok_(sub1.is_installed())
    ok_(sub2.is_installed())
    eq_(set([i.path for i in result]), {sub1.path, sub2.path})
    # and if we request it again via get, result should be empty
    get_result = ds.get(path=['subm 1', 'subm 2'], get_data=False)
    eq_(get_result, [])

@with_testrepos('submodule_annex', flavors=['local'])
@with_tempfile(mkdir=True)
def test_reckless(path, top_path):
    ds = install(top_path, source=path, reckless=True)
    eq_(ds.config.get('annex.hardlink', None), 'true')
    eq_(ds.repo.repo_info()['untrusted repositories'][0]['here'], True)

@with_tree(tree={'top_file.txt': 'some',
                 'sub 1': {'sub1file.txt': 'something else',
                           'subsub': {'subsubfile.txt': 'completely different',
                                      }
                           },
                 'sub 2': {'sub2file.txt': 'meaningless',
                           }
                 })
@with_tempfile(mkdir=True)
def test_install_recursive_repeat(src, path):
    subsub_src = Dataset(opj(src, 'sub 1', 'subsub')).create(force=True)
    sub1_src = Dataset(opj(src, 'sub 1')).create(force=True)
    sub2_src = Dataset(opj(src, 'sub 2')).create(force=True)
    top_src = Dataset(src).create(force=True)
    top_src.add('.', recursive=True)
    ok_clean_git(top_src.path)

    # install top level:
    top_ds = install(path, source=src)
    ok_(top_ds.is_installed() is True)
    sub1 = Dataset(opj(path, 'sub 1'))
    ok_(sub1.is_installed() is False)
    sub2 = Dataset(opj(path, 'sub 2'))
    ok_(sub2.is_installed() is False)
    subsub = Dataset(opj(path, 'sub 1', 'subsub'))
    ok_(subsub.is_installed() is False)

    # install again, now with data and recursive, but recursion_limit 1:
    result = get(os.curdir, dataset=path, recursive=True, recursion_limit=1)
    # top-level dataset was not reobtained
    assert_not_in(top_ds, result)
    assert_in(sub1, result)
    assert_in(sub2, result)
    assert_not_in(subsub, result)
    ok_(top_ds.repo.file_has_content('top_file.txt') is True)
    ok_(sub1.repo.file_has_content('sub1file.txt') is True)
    ok_(sub2.repo.file_has_content('sub2file.txt') is True)

    # install sub1 again, recursively and with data
    top_ds.install('sub 1', recursive=True, get_data=True)
    ok_(subsub.is_installed())
    ok_(subsub.repo.file_has_content('subsubfile.txt'))

@with_testrepos('submodule_annex', flavors=['local'])
@with_tempfile(mkdir=True)
@with_tempfile
def test_install_skip_list_arguments(src, path, path_outside):
    ds = install(path, source=src)
    ok_(ds.is_installed())

    # install a list with valid and invalid items:
    with swallow_logs(new_level=logging.WARNING) as cml:
        with assert_raises(IncompleteResultsError) as cme:
            ds.install(
                path=['subm 1', 'not_existing', path_outside, 'subm 2'],
                get_data=False)
        result = cme.exception.results
        for skipped in [opj(ds.path, 'not_existing'), path_outside]:
            cml.assert_logged(msg="ignored non-existing paths: {}\n".format(
                              [opj(ds.path, 'not_existing'), path_outside]),
                              regex=False, level='WARNING')
            pass
        ok_(isinstance(result, list))
        eq_(len(result), 2)
        for sub in [Dataset(opj(path, 'subm 1')), Dataset(opj(path, 'subm 2'))]:
            assert_in(sub, result)
            ok_(sub.is_installed())

    # return of get is always a list, even if just one thing was gotten
    # in this case 'subm1' was already obtained above, so this will get this
    # content of the subdataset
    with assert_raises(IncompleteResultsError) as cme:
        ds.install(path=['subm 1', 'not_existing'])
    with assert_raises(IncompleteResultsError) as cme:
        ds.get(path=['subm 1', 'not_existing'])
    result = cme.exception.results
    eq_(len(result), 1)
    eq_(result[0]['file'], 'subm 1/test-annex.dat')

@with_testrepos('submodule_annex', flavors=['local'])
@with_tempfile(mkdir=True)
def test_install_skip_failed_recursive(src, path):

    # install top level:
    ds = install(path, source=src)
    sub1 = Dataset(opj(path, 'subm 1'))
    sub2 = Dataset(opj(path, 'subm 2'))
    # sabotage recursive installation of 'subm 1' by polluting the target:
    with open(opj(path, 'subm 1', 'blocking.txt'), "w") as f:
        f.write("sdfdsf")

    with swallow_logs(new_level=logging.WARNING) as cml:
        result = ds.get(os.curdir, recursive=True)
        # toplevel dataset was in the house already
        assert_not_in(ds, result)
        assert_in(sub2, result)
        assert_not_in(sub1, result)
        cml.assert_logged(
            msg="Target {} already exists and is not an installed dataset. Skipped.".format(sub1.path),
            regex=False, level='WARNING')

@with_tree(tree={'top_file.txt': 'some',
                 'sub 1': {'sub1file.txt': 'something else',
                           'subsub': {'subsubfile.txt': 'completely different',
                                      }
                           },
                 'sub 2': {'sub2file.txt': 'meaningless',
                           }
                 })
@with_tempfile(mkdir=True)
def test_install_noautoget_data(src, path):
    subsub_src = Dataset(opj(src, 'sub 1', 'subsub')).create(force=True)
    sub1_src = Dataset(opj(src, 'sub 1')).create(force=True)
    sub2_src = Dataset(opj(src, 'sub 2')).create(force=True)
    top_src = Dataset(src).create(force=True)
    top_src.add('.', recursive=True)

    # install top level:
    cdss = install(path, source=src, recursive=True)
    # there should only be datasets in the list of installed items,
    # and none of those should have any data for their annexed files yet
    for ds in cdss:
        assert_false(any(ds.repo.file_has_content(ds.repo.get_annexed_files())))

@with_tempfile
@with_tempfile
def test_install_source_relpath(src, dest):
    ds1 = create(src)
    src_ = basename(src)
    with chpwd(dirname(src)):
        ds2 = install(dest, source=src_)


"""
    Custom Http handler
    Author: Pascal Noisette

"""

from http import HTTPStatus
from http.server import SimpleHTTPRequestHandler
from tinytag import TinyTag

class ImageHandler(SimpleHTTPRequestHandler):

    def do_GET(self):
        """Serve image"""
        path = self.translate_path(self.path)

        if not path.endswith(".mp3.png"):
            return super().do_GET()

        try:
            file = path[:-len(".png")]
            tag = TinyTag.get(file, image=True)
            image = tag.get_image()
            assert image is not None
        except (OSError, AssertionError):
            self.send_error(HTTPStatus.NOT_FOUND, "File not found")
            return None

        self.send_response(HTTPStatus.OK)
        self.send_header("Content-type", "image/png")
        self.send_header("Content-Length", str(len(image)))
        self.end_headers()
        self.wfile.write(image)


import smtplib
from django.shortcuts import render, redirect
from django.contrib import messages
from .forms import MovingForm, DeliveryForm
from services.send_application_by_email import send_email
from captcha_config import SITE_KEY
from services.captcha_is_valid import check_captcha

def main(request):
    return render(request, 'main.html')

def moving(request):
    form = MovingForm()

    if request.method == 'POST':
        form = MovingForm(request.POST)
        if form.is_valid():
            data = form.cleaned_data
            captcha_is_valid = check_captcha(request)
            if captcha_is_valid:
                try:
                    send_email(data, 'Moving')
                    messages.success(request, "Success")
                    return redirect('moving')
                except smtplib.SMTPException:
                    messages.error(request, "Server error occurred, we didn't get your application")
            else:
                messages.error(request, "Invalid Captcha, try again, please")
        else:
            messages.error(request, "Error")

    return render(request, 'moving.html', {'form': form, 'captcha_key': SITE_KEY})

def delivery(request):
    form = DeliveryForm()

    if request.method == 'POST':
        form = DeliveryForm(request.POST)
        if form.is_valid():
            data = form.cleaned_data
            captcha_is_valid = check_captcha(request)
            if captcha_is_valid:
                try:
                    send_email(data, 'Delivery')
                    messages.success(request, "Success")
                    return redirect('delivery')
                except smtplib.SMTPException:
                    messages.error(request, "Server error occurred, we didn't get your application")
            else:
                messages.error(request, "Invalid Captcha, try again, please")
        else:
            messages.error(request, "Error")

    return render(request, 'delivery.html', {'form': form, 'captcha_key': SITE_KEY})

def terms_and_conditions(request):
    return render(request, 'terms_and_conditions.html')


import socket, threading, pickle,player,json
import game

class Server():
    #game1 = game.Game("deneme6.json")
    def __init__(self):
        LOCALHOST = "127.0.0.1"
        PORT = 8080
        server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server.bind((LOCALHOST, PORT))
        #super(Server, self).__init__("deneme6.json")
        game1 = game.Game("deneme6.json")
        print("Server started")
        print("Waiting for client request..")
        #print("{0}".format(self.gameType))
        self.started = 0
        while True:
            server.listen(5)
            clientsock, clientAddress = server.accept()
            newthread = self.ClientThread(clientAddress, clientsock, game1)
            newthread.start()

    class ClientThread(threading.Thread):
        def __init__(self, clientAddress, clientsocket, serv):
            self.serv = serv    # yanlis bir method gibi. gamein fonksiyonlarını burada kullanabilmek icin yapilmis. bunun yerine server classı yapmazsın. game objesini initte verirsin.
            self.clientsocket = clientsocket
            self.clientAddress = clientAddress
            self.csocket = clientsocket
            threading.Thread.__init__(self)
            print ("New connection added: ", self.clientAddress)

        def run(self):
            print ("Connection from : ", self.clientAddress)
            print(self.serv.gameTypeValue)
            self.csocket.sendall(bytes("hi", 'UTF-8'))
            msg = ''
            data =b""

            while True:
                data = self.csocket.recv(1024)
                #msg = data
                msg = data.decode()
                #print(msg)
                if data == b"":
                    print("data bos")
                    break
                elif msg[0] == '{':
                    #print(msg)
                    msg1 = eval(msg)
                    print(msg1["request"])
                    if msg1["request"] == "join":
                        print(msg)
                        p1 = player.Player(msg1["pId"], msg1["name"])
                        self.serv.join(p1)
                        self.csocket.sendall(bytes("joined", 'UTF-8'))

                    elif msg1["request"] == "ready":

                        readyPlayer =list(filter(lambda x: x.id == msg1["pId"], self.serv.players))[0]
                        if readyPlayer:
                            self.serv.ready(readyPlayer)
                        self.csocket.sendall(bytes("ready", 'UTF-8'))
                        break

                    elif msg1["request"] == "notify":
                        self.csocket.sendall(bytes(str(self.serv.state()), 'UTF-8'))
                    # elif msg1["request"] == "choice":
                    #     if msg1["value"] == "True":
                    #         choicePlayer = list(filter(lambda x: x.id == msg1["pId"], self.serv.players))[0]
                    #         self.serv.pick(choicePlayer, True)
                    #     else:
                    #         choicePlayer = list(filter(lambda x: x.id == msg1["pId"], self.serv.players))[0]
                    #         self.serv.pick(choicePlayer, False)
                print(msg)
            self.csocket.close()
            print(self.serv.players)

            #print ("from client", msg)

            #print ("Client at ", self.clientAddress , " disconnected...")

if __name__ == '__main__':
    Server()

ceng445 Software Development with Scripting Languages/phase2/server.py


import os
import sys
from logging.handlers import SysLogHandler, RotatingFileHandler
import logging

# Logging Levels
# https://docs.python.org/3/library/logging.html#logging-levels
# CRITICAL  50
# ERROR 40
# WARNING   30
# INFO  20
# DEBUG 10
# NOTSET    0

loggers = {}

def set_up_logging(name):
    global loggers

    if loggers.get(name):
        return loggers.get(name)
    else:
        if not 'logs' in os.getcwd():
            log_path = os.getcwd() + '/logs/falcon_pedidos.log'
        else:
            log_path = os.getcwd() + '/falcon_pedidos.log'

        with open(log_path, 'a+'):
            pass

        logger = logging.getLogger(name)
        logger.setLevel(logging.INFO)
        handler = RotatingFileHandler(log_path,maxBytes=5242880,backupCount=5)
        #handler = logging.StreamHandler()
        handler.setLevel(logging.INFO)
        formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(process)d]:[%(thread)d] [%(levelname)s]: %(message)s]')
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        loggers[name] = logger

        return logger


#!/usr/bin/env python
import argparse
import glob
import itertools
import logging
import multiprocessing as mp
import os
import sys
from gzip import open as gzopen
from typing import Dict, List, Optional, Tuple

import pandas as pd
from Bio.SeqIO.QualityIO import FastqGeneralIterator

config: Dict[str, int] = {
    "LEFT_ADAPTER": 3,
    "RIGHT_ADAPTER": 2,
    "POOL_BARCODE": 4,
    "CELL_BARCODE": 7,
    "UMI": 8,
    "DEFAULT_SAMPLE_NAME": "GGTTTACT",  # arbitrary sequence
}

def check_folder(folder: str, mandatory: Optional[bool] = False):
    """Checks if folder exists, otherwise create it.

    Args:
        folder (str): Output folder
    """
    if not os.path.exists(folder):
        if mandatory:
            logging.error(f"Folder {folder} not found!")
            sys.exit(-1)
        else:
            logging.warning(f"Missing folder {folder}, creating ...")
            os.mkdir(folder)

def trim_cdna(r1: List[str]) -> Tuple[str, str]:
    """Trim cdna by removing adapters and Pool barcode.

    Args:
        r1 (List[str]): full cdna sequence

    Returns:
        Tuple[str, str]: trimmed cdna and quality
    """
    global config

    # [0]: header, [1]: seq, [2]: quality
    _, seq, qa = r1
    cdna: str = seq[(config["LEFT_ADAPTER"] + config["POOL_BARCODE"]) : -config["RIGHT_ADAPTER"]]
    cdna_quality: str = qa[(config["LEFT_ADAPTER"] + config["POOL_BARCODE"]) : -config["RIGHT_ADAPTER"]]

    return cdna, cdna_quality

def create_r2(r1: FastqGeneralIterator, r2: FastqGeneralIterator) -> Tuple[str, str]:
    """Create R2 consisting of
        - pool barcode (4bp)
        - cell barcode (7bp)
        - UMI          (8bp)

    Args:
        r1 ([FastqGeneralIterator]): R1 read
        r2 ([FastqGeneralIterator]): R2 read

    Returns:
        Tuple[str, str]: New barcode sequence and quality
    """
    global config

    _, r1_seq, r1_qa = r1
    _, r2_seq, r2_qa = r2

    # get pool barcode
    pb_seq: str = r1_seq[config["LEFT_ADAPTER"] : config["LEFT_ADAPTER"] + config["POOL_BARCODE"]]
    pb_qa: str = r1_qa[config["LEFT_ADAPTER"] : config["LEFT_ADAPTER"] + config["POOL_BARCODE"]]

    barcode_len: int = config["POOL_BARCODE"] + config["CELL_BARCODE"] + config["UMI"]
    barcode_seq: str = f"{pb_seq}{r2_seq}"[:barcode_len]
    barcode_qa: str = f"{pb_qa}{r2_qa}"[:barcode_len]

    return barcode_seq, barcode_qa

def convert_to_10x(params: Tuple[str, str, str, str], chunk_size: int = 100) -> None:
    """Main function processing the reads.

    Args:
        params (Tuple[str, str, str, str]): R1, R2, input, output folder
    """
    fastq_r1, fastq_r2, input_folder, output_folder = params

    with gzopen(fastq_r1, "rt") as fq_r1, gzopen(fastq_r2, "rt") as fq_r2, gzopen(
        f"{output_folder}/{os.path.basename(fastq_r1)}", "wt"
    ) as fastq_r1_out, gzopen(f"{output_folder}/{os.path.basename(fastq_r2)}", "wt") as fastq_r2_out:
        fastq_r1 = FastqGeneralIterator(fq_r1)
        fastq_r2 = FastqGeneralIterator(fq_r2)

        counter: int = 0
        fastq_r1_content, fastq_r2_content = "", ""
        for r1, r2 in zip(fastq_r1, fastq_r2):
            # [0]: header, [1]: seq, [2]: quality

            cdna, cdna_quality = trim_cdna(r1)
            barcode, barcode_quality = create_r2(r1, r2)

            fastq_r1_content += f'@{r1[0]}:{config["DEFAULT_SAMPLE_NAME"]}\n{barcode}\n+\n{barcode_quality}\n'
            fastq_r2_content += f'@{r2[0]}:{config["DEFAULT_SAMPLE_NAME"]}\n{cdna}\n+\n{cdna_quality}\n'

            if counter % chunk_size == 0:
                fastq_r1_out.write(fastq_r1_content)
                fastq_r2_out.write(fastq_r2_content)
                counter, fastq_r1_content, fastq_r2_content = 0, "", ""

        # write the rest of the file
        if counter != 0:
            fastq_r1_out.write(fastq_r1_content)
            fastq_r2_out.write(fastq_r2_content)

def convert(fastqs_folder: str, output_folder: str, threads: int):
    """Main function for converting MARS-seq raw files to 10X format.

    Args:
        input_folder (str): Input folder
        output_folder (str): Output folder
    """

    check_folder(fastqs_folder, mandatory=True)
    check_folder(output_folder)

    r1_files = sorted(glob.glob(f"{fastqs_folder}/*R1*.fastq.gz"))
    r2_files = sorted(glob.glob(f"{fastqs_folder}/*R2*.fastq.gz"))

    if len(r1_files) != len(r2_files):
        print(f"Something is off, please check you have the same amount of paired-end files!")
        sys.exit(-1)

    data = zip(
        r1_files,
        r2_files,
        itertools.repeat(fastqs_folder, len(r1_files)),
        itertools.repeat(output_folder, len(r1_files)),
    )
    with mp.Pool(threads) as pool:
        _ = pool.map(convert_to_10x, data)

def whitelist(batch: str, amp_batches: str, well_cells: str):
    """Generate whitelist for StarSolo.
    The file should contain cell tags.

    Args:
        batch (str): batch name
        amp_batches (str): Amplification batches [xls]
        well_cells (str): Well cells [xlsx]
    """
    batches = pd.read_excel(amp_batches)
    wells = pd.read_excel(well_cells)

    # preprocess and clean-up
    batches = batches[batches.Seq_batch_ID == batch].astype("category")
    wells = wells[wells.Amp_batch_ID.isin(batches.Amp_batch_ID)]
    wells = wells[["Amp_batch_ID", "Cell_barcode"]]
    wells["Amp_batch_ID"] = wells["Amp_batch_ID"].str.strip().astype("category")
    wells["Cell_barcode"] = wells["Cell_barcode"].str.strip().astype("category")

    # merge and concat barcodes
    wells["Amp_batch_ID"] = wells["Amp_batch_ID"].cat.rename_categories(batches.Pool_barcode.unique())
    wells["whitelist"] = wells[["Amp_batch_ID", "Cell_barcode"]].agg("".join, axis=1)

    # save
    wells["whitelist"].to_csv(f"whitelist.txt", header=None, index=None)

if __name__ == "__main__":
    logging.getLogger().setLevel(logging.INFO)
    formatter = logging.Formatter("%(asctime)s  %(message)s", "%d-%m-%Y %H:%M:%S")

    arg_parser = argparse.ArgumentParser(description="Plugin for converting and running velocity on MARS-seq v2.")

    arg_parser.add_argument("--version", "-v", action="version", version=f"velocity 0.1")
    command_parser = arg_parser.add_subparsers(dest="command")

    # convert
    convert_parser = command_parser.add_parser("convert", help="Convert reads to 10X format")
    convert_parser.add_argument("--input", type=str, help="Input folder with fastq files", required=True)
    convert_parser.add_argument("--output", type=str, help="Output folder", required=True)
    convert_parser.add_argument("--threads", type=int, help="Number of threads", required=True, default=4)

    # whitelist
    whitelist_parser = command_parser.add_parser("whitelist", help="Create whitelist for StarSolo.")
    whitelist_parser.add_argument("--batch", type=str, help="Batch name", required=True)
    whitelist_parser.add_argument("--amp_batches", type=str, help="Amplification batches [xls]", required=True)
    whitelist_parser.add_argument("--well_cells", type=str, help="Well cells [xls]", required=True)

    args = arg_parser.parse_args()

    if args.command == "convert":
        convert(args.input, args.output, args.threads)
    elif args.command == "whitelist":
        whitelist(args.batch, args.amp_batches, args.well_cells)
    else:
        logging.error(f"Command {args.command} not recognized!")


import re

def scrape_synopsis(soup):
    scraped_synopsis = soup.find(itemprop='description').get_text()
    formatted_synopsis = scraped_synopsis.strip()
    for match in re.finditer(r'[\[<(](.*?)[\]<)]', formatted_synopsis):
        if 'Source' in match.group(0):
            formatted_synopsis = formatted_synopsis.replace(match.group(0), '')
        if 'MAL' in match.group(0):
            formatted_synopsis = formatted_synopsis.replace(match.group(0), '')
    return formatted_synopsis

def scrape_main_name(soup):
    main_name = soup.select("h1.h1")[0].text
    formatted_main_name = main_name.strip()
    return formatted_main_name

def scrape_english_name(soup):
    english_div = soup.find_all(text=re.compile(r'^English.*'))
    if len(english_div) <= 0:
        return
    english_name = english_div[0]
    formatted_english_name = (" ".join(english_name.string.parent.parent.text.strip().split(" ")[1:])
                              if len(english_name) > 0 else '')
    return formatted_english_name

def scrape_japanese_name(soup):
    japanese_div = soup.find_all(text=re.compile(r'^Japanese.*'))
    if len(japanese_div) <= 0:
        return
    japanese_name = japanese_div[0]
    formatted_japanese_name = (" ".join(japanese_name.string.parent.parent.text.strip().split(" ")[1:])
                               if len(japanese_name) > 0 else '')
    return formatted_japanese_name

def scrape_synonyms(soup):
    synonyms_div = soup.find_all(text=re.compile(r'^Synonyms.*'))
    if len(synonyms_div) <= 0:
        return
    synonyms = synonyms_div[0]
    formatted_synonyms = (" ".join(synonyms.string.parent.parent.text.strip().split(" ")[1:])
                          if len(synonyms) > 0 else '').split(', ')
    return formatted_synonyms

def scrape_anime_type(soup):
    anime_type = soup.select("div > a")[15].text
    return anime_type

def scrape_episodes(soup):
    episodes = soup.select("div.spaceit")[0].text
    formatted_episodes = episodes.split()[1]
    return formatted_episodes

def scrape_status(soup):
    status = soup.find_all(text=re.compile(r'\b(?:%s)\b' % '|'.join(['Currently Airing',
                                                                     'Finished Airing',
                                                                     'Not yet aired'])))[0]
    formatted_status = status.strip()
    return formatted_status

def scrape_airdate(soup):
    airdate = soup.select("div.spaceit")[1].text
    formatted_airdate = " ".join(airdate.strip().split(" ")[2:])
    return formatted_airdate

def scrape_source(soup):
    source = (soup.select("div.spaceit")[3].text.split()
              if ("Source:" in soup.select("div.spaceit")[3].text)
              else soup.select("div.spaceit")[4].text.split())
    formatted_source = " ".join(source).split(" ")[1]
    return formatted_source

def scrape_genres(soup):
    genres = soup.select("div")[soup.select("div").index(soup.select("div.spaceit")[3]
                                                         if ("Source:" in soup.select("div.spaceit")[3].text)
                                                         else soup.select("div.spaceit")[4]) + 1]
    formatted_genres = genres.text.strip().split("\n")[1]
    formatted_genres = [g.strip() for g in formatted_genres.split(',')]
    return formatted_genres

def scrape_duration(soup):
    duration = (soup.select("div.spaceit")[4].text
                if ("Duration:" in soup.select("div.spaceit")[4].text)
                else soup.select("div.spaceit")[5].text)
    formatted_duration = " ".join(duration.strip().split(" ")[2:])
    return formatted_duration

def scrape_rating(soup):
    rating = soup.select("div.fl-l.score")[0].text
    formatted_rating = rating.strip()
    return formatted_rating


import random

# define some responses
responses = {
    "hello": ["Hi there!", "Hello!", "Greetings!"],
    "what is your name": ["my name is mrs eric bot"],
    "where were you born": ["i was born in a computer ", "still figuring the date i was created"],
    "are you married": ["i am a bot and dont need to be married", "my married status remains unknown to me only"],
    "how are you": ["I'm doing well, thank you.", "I'm good, thanks for asking."],
    "bye": ["Goodbye!", "See you later!", "Farewell!"],
}

# define a function to generate a response

def generate_response(user_input):
    # convert user input to lowercase
    user_input = user_input.lower()
    # check if input matches a response key
    for key in responses.keys():
        if user_input in key:
            return random.choice(responses[key])
    # if no key matches, return a default response
    return "I'm sorry, I didn't understand what you said."

# main loop
while True:
    # get user input
    user_input = input("You: ")
    # generate and print response
    bot_response = generate_response(user_input)
    print("Bot:", bot_response)


# Exercício 1 Dado um array com os seguintes elementos ["zebra", "macaco",
# "elefante", "arara", "javali"] , após duas iterações utilizando bubble sort
# , como estaria este array?

from Cronometro import Cronometro

def bubble_sort(array):
    # variável utilizado na iteração
    # para marcar se houve ou não trocas naquela iteração
    # Quando falsa, indica que o array está ordenado
    has_swapped = True

    # armazena o número de iterações para evitar
    # a iteração sobre índices já ordenados
    num_of_iterations = 0

    # Enquanto ainda não está ordenado (ocorreram trocas na iteração)
    while has_swapped:
        has_swapped = False

        # percorra o array até o ultimo índice não ordenado
        for i in range(len(array) - num_of_iterations - 1):
            # caso a posição corrente seja maior que a posterior
            if array[i] > array[i + 1]:
                # realiza a troca entre as posições
                array[i], array[i + 1] = array[i + 1], array[i]
                # marca que tivemos trocas nesta iteração
                has_swapped = True
        num_of_iterations += 1

    return array

def quicksort(array, low, high):
    # caso base: se já atingiu a menor porção (1)
    if len(array) == 1:
        # retorne o array
        return array

    # os índices irão se cruzar quando o array estiver ordenado
    if low < high:
        # índice da partição é o índice onde o array foi divido
        # e é determinado a partir do pivô.
        # Este índice já está ordenado
        partition_index = partition(array, low, high)

        # Ordena os elementos presentes antes da partição (menores que o pivô)
        # e depois (maiores que o pivô)
        quicksort(array, low, partition_index - 1)
        quicksort(array, partition_index + 1, high)

# função auxiliar responsável pela partição do array
# escolhendo um pivô e fazendo movimentações dos sub arrays gerados
def partition(array, low, high):
    # índice do menor elemento
    i = low - 1
    # o pivô será escolhido
    # através do índice que divide o array
    pivot = array[high]  # pivot

    # itera sobre os elementos
    for j in range(low, high):

        # se o elemento corrente é menor ou igual ao pivô
        if array[j] <= pivot:

            # incrementa o índice do menor elemento
            i = i + 1
            array[i], array[j] = array[j], array[i]
    array[i + 1], array[high] = array[high], array[i + 1]

    return i + 1

with Cronometro("algoritmo"):
    print(quicksort(["zebra", "macaco", "elefante", "arara", "javali"], 0, 4))

bloco_36 augoritmo e estrutua de dados/dia_2 recursividade e estrategias para solucao de problemas/ex1.py


# --- Day 10: Syntax Scoring ---
#
# https://adventofcode.com/2021/day/10

import requests

openers = {'(', '{', '<', '['}
closers = {')', '}', '>', ']'}

o2c = {'(': ')',
       '{': '}',
       '<': '>',
       '[': ']'}
c2o = {')': '(',
       '}': '{',
       '>': '<',
       ']': '['}

def syntax_error_score_v1(syntax_lines):
    score = 0
    for line in syntax_lines:
        score = score + __get_corrupted_score(__get_corrupted_bracket(line))
    return score

def syntax_error_score_v2(syntax_lines):
    scores = list()
    for line in syntax_lines:
        if __get_corrupted_bracket(line) is None:
            scores.append(__get_autocomplete_score(__complete_line(line)))
    scores.sort()
    return scores[int(len(scores) / 2)]

def __complete_line(line):
    stack = []
    for bracket in line:
        if c2o.get(peek_stack(stack)) == o2c.get(bracket):
            stack.pop()
        else:
            stack.append(bracket)
    completion_chunk = ""
    while stack:
        completion_chunk = completion_chunk + o2c.get(peek_stack(stack))
        stack.pop()
    return completion_chunk

def __get_autocomplete_score(completion_chunk):
    points = {')': 1, '}': 3, '>': 4, ']': 2}
    score = 0
    for bracket in completion_chunk:
        score = score * 5 + points.get(bracket)

    return score

def __get_corrupted_score(bracket):
    points = {')': 3, '}': 1197, '>': 25137, ']': 57}
    if bracket:
        return points[bracket]
    else:
        return 0

def __get_corrupted_bracket(line):
    stack = []
    for bracket in line:
        if bracket in openers:
            stack.append(bracket)
        else:
            if peek_stack(stack) == c2o.get(bracket):
                stack.pop()
            else:
                return bracket
    return None

def peek_stack(stack):
    if stack:
        return stack[-1]
    else:
        return None

if __name__ == '__main__':
    m_headers = {
        'cookie': '<TOKEN HERE>'}
    url_input = requests.get("https://adventofcode.com/2021/day/10/input", headers=m_headers).text.splitlines()
    file_input = open('syntax.txt', 'r').read().splitlines()
    print("Part: 1 = " + str(syntax_error_score_v1(url_input)))
    print("Part: 2 = " + str(syntax_error_score_v2(url_input)))


from mtcnn import MTCNN
import cv2

def main():
    img = cv2.imread(".\\images\\war.jpg")
    #img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    detector = MTCNN()
    bound_boxes = detector.detect_faces(img)
    for box in bound_boxes:
        img = bound_box_draw(box, img)
    cv2.imshow("Image", img)
    cv2.waitKey(0)

def bound_box_draw(bb, img):
    #face
    red = (0, 0, 255)
    green = (0, 255, 0)
    blue = (255, 0, 0)
    x, y, w, h = bb['box']

    #dynamic thickness
    thickness = int(w/30)
    if(thickness<=0):
        thickness = 1

    img = cv2.rectangle(img, (x, y+h), (x+w, y), red, thickness=thickness)
    img = cv2.putText(img, "face", (x,y), cv2.FONT_HERSHEY_PLAIN, thickness, green, thickness=thickness)
    #other face parts
    attrs = bb['keypoints']
    img = cv2.circle(img, attrs['left_eye'], thickness, blue, thickness=-1)
    img = cv2.circle(img, attrs['right_eye'], thickness, blue, thickness=-1)
    img = cv2.circle(img, attrs['nose'], thickness, blue, thickness=-1)
    img = cv2.circle(img, attrs['mouth_left'], thickness, blue, thickness=-1)
    img = cv2.circle(img, attrs['mouth_right'], thickness, blue, thickness=-1)
    return img

if __name__ == "__main__":
    main()


import time

import requests
from flask import current_app as app
from flask import request
from flask_restplus import Namespace, Resource
from flask_jwt_extended import get_jwt_identity
from flask_jwt_extended.exceptions import *
from flask_jwt_extended import jwt_required
from jwt.exceptions import *
from commons.jwt_helpers import access_required

from core.classroom_services import search_class_lists

api = Namespace('classroom_class', description='Namespace for classroom_class service')

_http_headers = {'Content-Type': 'application/json'}

_es_index_classroom_classes = 'cfs_classroom_classes'
_es_type = '_doc'
_es_size = 100

@api.errorhandler(NoAuthorizationError)
def handle_auth_error(e):
    return {'message': str(e)}, 401

@api.errorhandler(CSRFError)
def handle_auth_error(e):
    return {'message': str(e)}, 401

@api.errorhandler(ExpiredSignatureError)
def handle_expired_error(e):
    return {'message': 'Token has expired'}, 401

@api.errorhandler(InvalidHeaderError)
def handle_invalid_header_error(e):
    return {'message': str(e)}, 422

@api.errorhandler(InvalidTokenError)
def handle_invalid_token_error(e):
    return {'message': str(e)}, 422

@api.errorhandler(JWTDecodeError)
def handle_jwt_decode_error(e):
    return {'message': str(e)}, 422

@api.errorhandler(WrongTokenError)
def handle_wrong_token_error(e):
    return {'message': str(e)}, 422

@api.errorhandler(RevokedTokenError)
def handle_revoked_token_error(e):
    return {'message': 'Token has been revoked'}, 401

@api.errorhandler(FreshTokenRequired)
def handle_fresh_token_required(e):
    return {'message': 'Fresh token required'}, 401

@api.errorhandler(UserLoadError)
def handler_user_load_error(e):
    identity = get_jwt_identity().get('id')
    return {'message': "Error loading the user {}".format(identity)}, 401

@api.errorhandler(UserClaimsVerificationError)
def handle_failed_user_claims_verification(e):
    return {'message': 'User claims verification failed'}, 400

@api.route('/<string:classroom_class_id>')
class ClassroomClassByID(Resource):

    @api.doc('get classroom_class details by id')
    def get(self, classroom_class_id):
        app.logger.info('Get classroom_class_details method called')
        rs = requests.session()
        search_url = 'http://{}/{}/{}/{}'.format(app.config['ES_HOST'], _es_index_classroom_classes, _es_type, classroom_class_id)
        response = rs.get(url=search_url, headers=_http_headers).json()
        print(response)
        if 'found' in response:
            if response['found']:
                data = response['_source']
                data['id'] = response['_id']
                app.logger.info('Get classroom_class_details method completed')
                return data, 200
            app.logger.warning('classroom_class not found')
            return {'found': response['found']}, 404
        app.logger.error('Elasticsearch down, response: ' + str(response))
        return response, 500

    @access_required(access="ALL")
    @api.doc('update classroom_class by id')
    def put(self, classroom_class_id):
        app.logger.info('Update classroom_class_details method called')
        rs = requests.session()
        post_data = request.get_json()

        search_url = 'http://{}/{}/{}/{}'.format(app.config['ES_HOST'], _es_index_classroom_classes, _es_type, classroom_class_id)
        response = rs.get(url=search_url, headers=_http_headers).json()
        print(response)
        if 'found' in response:
            if response['found']:
                data = response['_source']
                for key, value in post_data.items():
                    data[key] = value
                data['updated_at'] = int(time.time())
                response = rs.put(url=search_url, json=data, headers=_http_headers).json()
                if 'result' in response:
                    app.logger.info('Update classroom_class_details method completed')
                    return response['result'], 200
                else:
                    app.logger.error('Elasticsearch down, response: ' + str(response))
                    return response, 500
            app.logger.warning('classroom_class not found')
            return {'message': 'not found'}, 404
        app.logger.error('Elasticsearch down, response: ' + str(response))
        return response, 500

    @access_required(access="ALL")
    @api.doc('delete classroom_class by id')
    def delete(self, classroom_class_id):
        app.logger.info('Delete classroom_class_details method called')
        rs = requests.session()
        search_url = 'http://{}/{}/{}/{}'.format(app.config['ES_HOST'], _es_index_classroom_classes, _es_type, classroom_class_id)
        response = rs.delete(url=search_url, headers=_http_headers).json()
        print(response)
        if 'result' in response:
            if response['result'] == 'deleted':
                app.logger.info('Delete classroom_class_details method completed')
                return response['result'], 200
            else:
                return response['result'], 400
        app.logger.error('Elasticsearch down, response: ' + str(response))
        return response, 500

@api.route('/')
class CreateClassroomClass(Resource):

    @access_required(access="ALL")
    @api.doc('create classroom_class')
    def post(self):
        app.logger.info('Create classroom_class method called')
        rs = requests.session()
        data = request.get_json()
        print('data: ', data)

        data['created_at'] = int(time.time())
        data['updated_at'] = int(time.time())

        post_url = 'http://{}/{}/{}'.format(app.config['ES_HOST'], _es_index_classroom_classes, _es_type)
        response = rs.post(url=post_url, json=data, headers=_http_headers).json()
        print(response)

        if 'result' in response and response['result'] == 'created':
            app.logger.info('Create classroom_class method completed')
            return response['_id'], 201
        app.logger.error('Elasticsearch down, response: ' + str(response))
        return response, 500

@api.route('/search', defaults={'page': 0})
@api.route('/search/<int:page>')
class SearchClassroomClass(Resource):

    @api.doc('search classroom_class based on post parameters')
    def post(self, page=0):
        try:
            app.logger.info('Classroom class search method called')
            param = request.get_json()
            class_list = search_class_lists(param, page, _es_size)
            return {
                'class_list': class_list
            }
        except Exception as e:
            return {'message': str(e)}, 500


import sys
str = sys.stdin.readlines()
str = [s.strip('\n') for s in str]

target = str[0]
ct = int(str[1])
data = []
for i in range(2, ct+2):
    data.append(str[i])

data = target.join(data)
print(data)


from account import Account

class BalanceSheet:
    """This is the class for a balance sheet used in financial accounting"""

    def __init__(self, company_name, date):
        """This is the constructor for the balance sheet class"""
        self.company_name = company_name
        self.date = date
        self.current_assets_list = []
        self.fixed_assets_list = []
        self.current_liabilities_list = []
        self.long_term_liabilities_list = []
        self.equities_list = []
        self.current_assets = ['Cash', 'Accounts Receivable', 'Inventory', 'Prepaid Expenses','Prepaid Liabilities', 'Marketable Securities', 'Other Short Term Investments', 'Cash Equivalent']
        self.fixed_assets = ['Land', 'Buildings', 'Machinery', 'Furniture', 'Vehicles', 'Accumulated Depreciation', 'Goodwill', 'Investment']
        self.current_liabilities = ['Accounts Payable', 'Notes Payable', 'Accrued Expense', 'Taxes Payable', 'Credit Card Debt', 'Accrued Expenses']
        self.long_term_liabilities = ['Long Term Debt', 'Bank Debt', 'Other Long Term Notes Payable']
        self.equities = ['Preferred Stock', 'Common Stock', 'Retained Earnings']

    def __repr__(self):
        """returns a string representation of the balance sheet"""
        s = self.company_name + '\n'
        s += 'Balance Sheet' + '\n'
        s += "As of: " + self.date + '\n'
        s += '-----------------------------------------------------------' + '\n'
        s += 'ASSETS' + '\n'
        s += 'Current Assets:' + '\n'
        for x in range(len(self.current_assets_list)):
            s += self.current_assets_list[x].name + '   ' + str(self.current_assets_list[x].amount) + "\n"
        s += '\n' + 'The total current assets balance is: ' + str(self.current_asset_balance()) + "\n" + '\n'
        s += 'Fixed Assets: ' + '\n'
        for x in range(len(self.fixed_assets_list)):
            s += self.fixed_assets_list[x].name + '   ' + str(self.fixed_assets_list[x].amount) + "\n"
        s += 'The total fixed assets balance is: ' + str(self.fixed_asset_balance()) + "\n" + '\n'
        s += 'The total asset balance is: ' + str(self.fixed_asset_balance() + self.current_asset_balance()) + '\n'
        s += '-----------------------------------------------------------' + '\n'
        s += 'LIABILITIES' + '\n'
        s += 'Current Liabilities:' + '\n'
        for x in range(len(self.current_liabilities_list)):
            s += self.current_liabilities_list[x].name + '   ' + str(self.current_liabilities_list[x].amount) + "\n"
        s += '\n' + 'The total current liabilities balance is: ' + str(self.current_liabilities_balance()) + "\n" + '\n'
        s += 'Long Term Liabilities: ' + '\n'
        for x in range(len(self.long_term_liabilities_list)):
            s += self.long_term_liabilities_list[x].name + '   ' + str(self.long_term_liabilities_list[x].amount) + "\n"
        s += 'The total long term liabilities balance is: ' + str(self.long_term_liabilities_balance()) + "\n" + '\n'
        s += 'The total liabilities balance is: ' + str(self.current_liabilities_balance() + self.long_term_liabilities_balance()) + '\n'
        s += '-----------------------------------------------------------' + '\n'
        s += "OWNER'S EQUITIY" +'\n'
        s += 'Equities:' + '\n'
        for x in range(len(self.equities_list)):
            s += self.equities_list[x].name + '   ' + str(self.equities_list[x].amount) + "\n"
        s += '\n' + 'The total equities balance is: ' + str(self.equities_balance()) + "\n" + '\n'
        s += 'The total equities and liabilities is: ' + str(self.current_liabilities_balance() + self.long_term_liabilities_balance() + self.equities_balance())
        s += '\n'
        if (self.fixed_asset_balance() + self.current_asset_balance()) == self.current_liabilities_balance() + self.long_term_liabilities_balance() + self.equities_balance():
            s += 'The total assets and total liabilities and shareholders equities balance'
        else:
            s += 'The total assets and total liabilities and shareholders equities do not balance'
        return s

    def current_asset_balance(self):
        """returns the current asset balance"""
        total = 0
        if self.current_assets_list == []:
            return total
        else:
            for x in range(len(self.current_assets_list)):
                total += self.current_assets_list[x].amount
            return total

    def fixed_asset_balance(self):
        """returns the fixed asset balance"""
        total = 0
        if self.fixed_assets_list == []:
            return total
        else:
            for x in range(len(self.fixed_assets_list)):
                total += self.fixed_assets_list[x].amount
            return total

    def current_liabilities_balance(self):
        """returns the current liabilities balance"""
        total = 0
        if self.current_liabilities_list == []:
            return total
        else:
            for x in range(len(self.current_liabilities_list)):
                total += self.current_liabilities_list[x].amount
            return total

    def long_term_liabilities_balance(self):
        """returns the long term liabilities balance"""
        total = 0
        if self.long_term_liabilities_list == []:
            return total
        else:
            for x in range(len(self.long_term_liabilities_list)):
                total += self.long_term_liabilities_list[x].amount
            return total

    def equities_balance(self):
        """returns the equities balance"""
        total = 0
        if self.equities_list == []:
            return total
        else:
            for x in range(len(self.equities_list)):
                total += self.equities_list[x].amount
            return total

    def categorize(self, name):
        """returns 1 for current asset, 2 for fixed asset, 3 for current liability, 4 for long term liability, 5 for equity"""
        if name in self.current_assets:
            return 1
        elif name in self.fixed_assets:
            return 2
        elif name in self.current_liabilities:
            return 3
        elif name in self.long_term_liabilities:
            return 4
        elif name in self.equities:
            return 5
        return -1

    def add_to(self, amount, name):
        """adds to the assets, liabilities, equities section of a balance sheet"""
        category = self.categorize(name)
        if category == -1:
            print('This account does not exist')
        elif isinstance(amount, int) != True:
            print('The amount is invalid')
        elif self.is_in_balance_sheet(name) == True:
            if category == 1:
                for x in range(len(self.current_assets_list)):
                    if name == self.current_assets_list[x].name:
                        if (self.current_assets_list[x].amount + amount) > 0:
                            self.current_assets_list[x].amount += amount
                        else:
                            print('Cannot have a negative value for an account!')
            if category == 2:
                for x in range(len(self.fixed_assets_list)):
                    if name == self.fixed_assets_list[x].name:
                        if (self.fixed_assets_list[x].amount + amount) > 0:
                            self.fixed_assets_list[x].amount += amount
                        else:
                            print('Cannot have a negative value for an account!')
            if category == 3:
                for x in range(len(self.current_liabilities_list)):
                    if name == self.current_liabilities_list[x].name:
                        if (self.current_liabilities_list[x].amount + amount) > 0:
                            self.current_liabilities_list[x].amount += amount
                        else:
                            print('Cannot have a negative value for an account!')
            if category == 4:
                for x in range(len(self.long_term_liabilities_list)):
                    if name == self.long_term_liabilities_list[x].name:
                        if (self.long_term_liabilities_list[x].amount + amount) > 0:
                            self.long_term_liabilities_list[x].amount += amount
                        else:
                            print('Cannot have a negative value for an account!')
            if category == 5:
               for x in range(len(self.equities_list)):
                   if name == self.equities_list[x].name:
                       if (self.equities_list[x].amount + amount) > 0:
                           self.equities_list[x].amount += amount
                       else:
                           print('Cannot have a negative value for an account!')
        else:
            print('name is not in balance sheet')
            if amount < 0:
                print('You cannot have a negative value for an account!')
            else:
                account = Account(amount, name, category)
                if account.category == 1:
                    self.current_assets_list += [account]
                elif account.category == 2:
                    self.fixed_assets_list += [account]
                elif account.category == 3:
                    self.current_liabilities_list += [account]
                elif account.category == 4:
                    self.long_term_liabilities_list += [account]
                elif account.category == 5:
                    self.equities_list += [account]

    def is_in_balance_sheet(self, name):
        """helper function for add_to to check to see if account is already in balance sheet"""
        category = self.categorize(name)
        if category == 1:
            for x in range(len(self.current_assets_list)):
                if name == self.current_assets_list[x].name:
                    return True
        if category == 2:
            for x in range(len(self.fixed_assets_list)):
                if name == self.fixed_assets_list[x].name:
                    return True
        if category == 3:
            for x in range(len(self.current_liabilities_list)):
                if name == self.current_liabilities_list[x].name:
                    return True
        if category == 4:
            for x in range(len(self.long_term_liabilities_list)):
                if name == self.long_term_liabilities_list[x].name:
                    return True
        if category == 5:
           for x in range(len(self.equities_list)):
               if name == self.equities_list[x].name:
                   return True
        return False

    def remove(self, name):
        """removes from the account"""
        category = self.categorize(name)
        if category == -1:
            print('No such account exists')
        elif isinstance(name,str) == False:
            print('The name is invalid')
        elif category == 1:
            for x in range(len(self.current_assets_list)):
                if name == self.current_assets_list[x].name:
                    self.current_assets_list.remove(self.current_assets_list[x])
        elif category == 2:
            for x in range(len(self.fixed_assets_list)):
                if name == self.fixed_assets_list[x].name:
                    self.fixed_assets_list.remove(self.fixed_assets_list[x])
        elif category == 3:
            for x in range(len(self.current_liabilities_list)):
                if name == self.current_liabilities_list[x].name:
                    self.current_liabilities_list.remove(self.current_liabilities_list[x])
        elif category == 4:
            for x in range(len(self.long_term_liabilities_list)):
                if name == self.long_term_liabilities_list[x].name:
                    self.long_term_liabilities_list.remove(self.long_term_liabilities_list[x])
        elif category == 5:
           for x in range(len(self.equities_list)):
               if name == self.equities_list[x].name:
                   self.equities_list.remove(self.equities_list[x])

    def print_account_options(self):
        """prints the account options"""
        print('These are the options to choose from: ')
        print('Assets')
        print('The options for current assets are: ')
        for x in range(len(self.current_assets)):
            print(self.current_assets[x])
        print()
        print('The options for fixed assets are: ')
        for x in range(len(self.fixed_assets)):
            print(self.fixed_assets[x])
        print()
        print('Liabilities')
        print('The options for current liabilities are: ')
        for x in range(len(self.current_liabilities)):
            print(self.current_liabilities[x])
        print()
        print('The options for long term liabilities are: ')
        for x in range(len(self.long_term_liabilities)):
            print(self.long_term_liabilities[x])
        print()
        print('Equities')
        print('The options for equities are: ')
        for x in range(len(self.equities)):
            print(self.equities[x])


import threading
import time
import uuid
from typing import Any, Callable

from danswer.configs.constants import DocumentSource
from danswer.connectors.factory import instantiate_connector
from danswer.connectors.interfaces import LoadConnector, PollConnector, EventConnector
from danswer.connectors.models import InputType, Document
from tinyfish.imp.models import ConnectorTaskTfCallbackType, ConnectorTaskTfCredentialsCallback, \
    ConnectorTaskTfDocumentCallback, ConnectorTaskTfFinishedCallback

def _merge_split_docs(doc: Document) -> str:
    s = ''
    for section in doc.sections:
        s += section.text
    return s

class TinyFishSyncThread(threading.Thread):
    def __init__(self,
                 source: str,
                 input_type: str,
                 credentials: dict[str, Any],
                 config: dict[str, Any],
                 last_run_time: float,
                 event_callback: Callable[[ConnectorTaskTfCallbackType, Any], None]):
        super().__init__()

        self.source = source
        self.input_type = input_type
        self.credentials = credentials
        self.config = config
        self.last_run_time = last_run_time
        self.event_callback = event_callback
        self.run_time = time.time()
        self.sync_id = str(uuid.uuid4())
        self.cancelled = False

    def stop(self):
        self.cancelled = True

    def run(self):
        err = None
        try:
            self.work()
        except Exception as e:
            print(e)
            err = str(e)

        if self.cancelled:
            err = "Cancelled"

        self.on_sync_done(err)

    def work(self):
        connector, new_credentials = instantiate_connector(
            source=DocumentSource(self.source),
            input_type=InputType(self.input_type),
            connector_specific_config=self.config,
            credentials=self.credentials)
        new_credentials = connector.load_credentials(new_credentials or self.credentials)
        if new_credentials is not None and new_credentials != self.credentials:
            self.on_credential_changed(new_credentials)

        if self.input_type == InputType.LOAD_STATE:
            assert isinstance(connector, LoadConnector)
            generator = connector.load_from_state()
        elif self.input_type == InputType.POLL:
            assert isinstance(connector, PollConnector)
            generator = connector.poll_source(0, 0)
        elif self.input_type == InputType.EVENT:
            assert isinstance(connector, EventConnector)
            generator = connector.handle_event(0)
        else:
            raise ValueError(f"Invalid input_type={self.input_type}")

        for batch_docs in generator:
            if self.cancelled:
                break

            for doc in batch_docs:
                if self.cancelled:
                    break
                self.on_doc_fetched(doc)

            if self.cancelled:
                break

    def on_credential_changed(self, new_credentials: dict[str, Any]):
        event_type = ConnectorTaskTfCallbackType.CREDENTIAL_CHANGED
        data = ConnectorTaskTfCredentialsCallback(
            sync_id=self.sync_id,
            credentials=new_credentials,
        )
        self.event_callback(event_type, data)

    def on_doc_fetched(self, doc: Document):
        event_type = ConnectorTaskTfCallbackType.DOCUMENT
        data = ConnectorTaskTfDocumentCallback(
            sync_id=self.sync_id,
            meta=doc.metadata,
            data=_merge_split_docs(doc),
        )
        self.event_callback(event_type, data)

    def on_sync_done(self, error: str | None):
        event_type = ConnectorTaskTfCallbackType.FINISHED
        data = ConnectorTaskTfFinishedCallback(
            sync_id=self.sync_id,
            success=error is None,
            error=error,
            run_time=self.run_time,
        )

        print("on_sync_done", data)
        self.event_callback(event_type, data)


import argparse
import json
import os

full_path_to_file = None
path_to_preserved_file = None
label_titles = ['Heading', 'Education', 'Experience']

def parse_cli_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('tex_file')
    args = parser.parse_args()

    if args.tex_file:
        return args.tex_file

def get_all_lines_from_file():
    all_lines = list()
    with open(full_path_to_file, 'r') as tex_file:
        for line in tex_file:
            all_lines.append(line)
    return all_lines

def get_label_lines(label):
    label = '\label{{{}}}'.format(label)
    found_label = False
    label_lines = dict()

    with open(full_path_to_file, 'r') as tex_file:
        line_counter = 0
        for line in tex_file:
            if '\label' in line:
                if line.split()[0] == label:
                    found_label = True
                else:
                    found_label = False
            if found_label:
                label_lines[line_counter] = line
            line_counter += 1

    return label_lines

def get_proper_values(label_title):
    full_path_to_proper_values_file = '{}\{}'.format(dir_path, 'proper_values.json')
    with open(full_path_to_proper_values_file, 'r') as proper_values_file:
        full_proper_values = json.load(proper_values_file)
    proper_values = full_proper_values[label_title]

    proper_value_dictionary = dict()
    for feature_name, feature_value in proper_values.items():
        proper_value_dictionary[feature_name] = feature_value

    return proper_value_dictionary

def search_and_replace_improper_values(label_lines, proper_values):
    for line_number, line_content in label_lines.items():
        for phony_value, real_value in proper_values.items():
            if phony_value in line_content:
                line_content = line_content.replace(phony_value, real_value)
                label_lines[line_number] = line_content

def log_back_proper_values(all_lines, label_lines):
    for line_number in range(len(all_lines)):
        if label_lines.get(line_number):
            all_lines[line_number] = label_lines[line_number]

def write_values_to_file(lines, path):
    with open(path, 'w') as tex_file:
        for i in range(len(lines)):
            line = lines[i]
            tex_file.write(line)

if __name__ == "__main__":
    file_name = parse_cli_args()

    if file_name:
        dir_path = os.path.dirname(os.path.realpath(__file__))
        full_path_to_file = '{}\{}'.format(dir_path, file_name)
        path_to_preserved_file = '{}\{}'.format(dir_path, file_name.replace('.', '-old.'))
        all_lines = get_all_lines_from_file()
        preserved_lines = all_lines.copy()

        for label_title in label_titles:
            # Get line number of values.
            heading_label = '\label{{{}}}'.format(label_title)
            heading_lines = get_label_lines(heading_label)

            # Get a dictionary of 'improper values' mapped to their proper values, as defined by the user.
            proper_values = get_proper_values(label_title)

            if proper_values:
                # We have a dictionary of lines mapping the label to values to be replaced. Complete the replacements.
                search_and_replace_improper_values(heading_lines, proper_values)

                # Log back the real values to the list representation of the .tex document.
                log_back_proper_values(all_lines, heading_lines)

        # Write the proper values back to the tex file and the old values to another tex file.
        if all_lines:
            write_values_to_file(all_lines, full_path_to_file)
            write_values_to_file(preserved_lines, path_to_preserved_file)


import time

import numpy as np
from sklearn.metrics import (
    roc_curve,
    auc,
    average_precision_score,
    precision_recall_curve
)
from sklearn.manifold import TSNE
import plotly.express as px
import torch
import pandas as pd

from utils.generic import get_device
from training.miners import RandomTripletMiner
from datasets.TripletDataset import TripletDataset
from datasets.SimpleDataset import SimpleDataset

def generate_metrics(model, data_set, segmented, results_path, balanced):
    if segmented:
        distances, clf_labels = generate_posteriors_segments(model, data_set)
        df = generate_ROC(distances, clf_labels, results_path)
        df_prc = generate_PRC(distances, clf_labels, results_path)
        return df, 0, 0, 0, 0
    else:
        distances, clf_labels, embeddings, song_labels, cover_names = generate_posteriors_full(model, data_set,
                                                                                balanced)
        df = generate_ROC(distances, clf_labels, results_path)
        df_prc = generate_PRC(distances, clf_labels, results_path)
        map, mrr, mr1, pr10 = ranking_metrics(model, data_set)
        generate_tsne(embeddings, song_labels, cover_names)
        return df, map.item(), mrr, mr1, pr10

def generate_embeddings_metrics(model, data_set, results_path):
    class TempClass:
        def __init__(self, frames, labels, song_names):
            self.frames = frames.unsqueeze(1)
            self.labels = labels
            self.song_names = song_names

    embeddings, song_labels, cover_names = model.calculate_embeddings(data_set)

    distances, clf_labels, embeddings, song_labels, cover_names = generate_posteriors_full(model, TempClass(embeddings,
                                                                                song_labels,
                                                                                cover_names))
    df = generate_ROC(distances, clf_labels, results_path)
    ap = average_precision(distances, clf_labels)
    df_prc = generate_PRC(distances, clf_labels, results_path)
    generate_tsne(embeddings, song_labels, cover_names)
    return df, ap, pd.DataFrame(), None, None

def generate_posteriors_segments(
        model, data_set: TripletDataset):
    model.eval()
    device = get_device()
    model.type(torch.FloatTensor).to(device)
    distances = []
    clf_labels = []
    miner = RandomTripletMiner
    inference_time = 0
    data_len = 0
    with torch.no_grad():
        for i in range(len(data_set)):
            # N X 1 X num_feats X num_samples, N
            (data, labels) = data_set[i]
            data_len += len(data)
            data = data.type(torch.FloatTensor).to(device)

            st = time.time() * 1000  # ms
            embeddings = model(data)
            et = time.time() * 1000  # ms
            inference_time += (et - st)

            a, p, n = miner(embeddings, labels)

            pos_dist = torch.norm(embeddings[a] - embeddings[p], dim=1)
            neg_dist = torch.norm(embeddings[a] - embeddings[n], dim=1)

            distances.append(pos_dist)
            clf_labels.extend([1] * pos_dist.size()[0])

            distances.append(neg_dist)
            clf_labels.extend([0] * neg_dist.size()[0])

    print(f"Average inference time: {inference_time / data_len}")

    distances, clf_labels = torch.cat(distances).cpu().numpy(), np.array(clf_labels)
    return distances, clf_labels

def generate_posteriors_full(model, data_set: SimpleDataset, balanced=False):
    model.eval()
    device = get_device()
    model.type(torch.FloatTensor).to(device)
    miner = RandomTripletMiner
    embeddings = []
    song_labels = []
    cover_names = []
    distances = []
    clf_labels = []
    with torch.no_grad():
        for frames, song_label, cover_name in zip(data_set.frames, data_set.labels, data_set.song_names):
            x = frames.to(device)
            embeddings.append(model(x))
            song_labels.append(song_label)
            cover_names.append(cover_name)

        embeddings = torch.cat(embeddings, dim=0)
        if not balanced:
            distance_matrix = torch.cdist(embeddings, embeddings, p=2)
            song_labels = torch.tensor(song_labels)

            for id, (d, lab) in enumerate(zip(distance_matrix, song_labels)):
                ids = torch.argwhere(song_labels == lab)
                ids = ids.flatten()
                ids = ids[ids != id]
                pos_dist = d[ids]
                distances.append(pos_dist)
                clf_labels.extend([1] * pos_dist.size()[0])

                ids = torch.argwhere(song_labels != lab)
                ids = ids.flatten()
                neg_dist = d[ids]
                distances.append(neg_dist)
                clf_labels.extend([0] * neg_dist.size()[0])
        else:
            for i in range(128):
                a, p, n = miner(embeddings, torch.tensor(data_set.labels))

                pos_dist = torch.norm(embeddings[a] - embeddings[p], dim=1)
                neg_dist = torch.norm(embeddings[a] - embeddings[n], dim=1)

                distances.append(pos_dist)
                clf_labels.extend([1] * pos_dist.size()[0])

                distances.append(neg_dist)
                clf_labels.extend([0] * neg_dist.size()[0])

        distances, clf_labels = torch.cat(distances).cpu().numpy(), np.array(clf_labels)
        return distances, clf_labels, embeddings, song_labels, cover_names

def generate_ROC(distances: np.ndarray, clf_labels: np.ndarray, results_path: str):
    permute_ids = np.random.permutation(len(distances))
    sample_distances = distances[permute_ids][:100000]
    sample_clf_labels = clf_labels[permute_ids][:100000]

    fpr, tpr, thresholds = roc_curve(sample_clf_labels, 2 - sample_distances)
    df = pd.DataFrame({"tpr": tpr, "fpr": fpr, "thr": thresholds})
    roc_auc = auc(fpr, tpr)

    fig = px.area(
        data_frame=df,
        x="fpr",
        y="tpr",
        title=f"ROC Curve (AUC={roc_auc:.4f})",
        labels=dict(x="False Positive Rate", y="True Positive Rate"),
        hover_data=["thr"],
        width=700,
        height=500,
    )
    fig.add_shape(type="line", line=dict(dash="dash"), x0=0, x1=1, y0=0, y1=1)

    fig.update_yaxes(scaleanchor="x", scaleratio=1)
    fig.update_xaxes(constrain="domain")

    if results_path:
        df.to_csv(results_path + "/thresholds.csv")
        fig.write_image(results_path + "/roc.png")

    fig.show()
    return df

def calc_MRR(distance_matrix: torch.Tensor, labels: torch.Tensor):
    # Size N x N
    rr = []

    for d, lab in zip(distance_matrix, labels):
        sorted_ids_by_dist = d.argsort()
        sorted_labels_by_dist = labels[sorted_ids_by_dist]

        # MRR
        rank = (sorted_labels_by_dist[1:] == lab).nonzero()[0].item() + 1
        rr.append(1 / rank)

    mrr = np.mean(rr)
    return mrr

def calc_MAP(array2d, version):
    MAP, top10, rank1 = 0, 0, 0

    for d, lab in zip(array2d, version):
        per_top10, per_rank1, per_MAP = 0, 0, 0
        version_cnt = 0.

        sorted_ids_by_dist = d.argsort()
        sorted_labels_by_dist = version[sorted_ids_by_dist][1:]

        ids = torch.argwhere(sorted_labels_by_dist == lab)
        for id in ids:
            version_cnt += 1
            per_MAP += version_cnt / (id + 1)
            if per_rank1 == 0:
                per_rank1 = id + 1
            if id < 10:
                per_top10 += 1

        per_MAP /= 1 if version_cnt < 0.0001 else version_cnt
        MAP += per_MAP
        top10 += per_top10
        rank1 += per_rank1

    return MAP / float(len(array2d)), top10 / float(len(array2d)) / 10, rank1 / float(len(array2d))

def ranking_metrics(model, data_set):
    model.eval()
    device = get_device()
    model.type(torch.FloatTensor).to(device)
    with torch.no_grad():
        embeddings = []
        labels = torch.tensor(data_set.labels).to(device)
        for frames, label in zip(data_set.frames, data_set.labels):
            x = frames.to(device)
            embeddings.append(model(x))

        embeddings = torch.cat(embeddings, dim=0)
        distance_matrix = torch.cdist(embeddings, embeddings, p=2)

        map, prk, mr1 = calc_MAP(distance_matrix, labels)
        mrr = calc_MRR(distance_matrix, labels)

        return map, mrr, mr1, prk

def generate_PRC(distances: np.ndarray, clf_labels: np.ndarray, results_path: str):
    permute_ids = np.random.permutation(len(distances))
    sample_distances = distances[permute_ids][:100000]
    sample_clf_labels = clf_labels[permute_ids][:100000]

    # Use the precision_recall_curve function to get the precision, recall, and thresholds arrays
    precision, recall, thresholds = precision_recall_curve(sample_clf_labels, 2 - sample_distances)
    df = pd.DataFrame({"precision": precision[:-1], "recall": recall[:-1], "thr": thresholds})
    # Compute the average precision score
    ap = average_precision_score(sample_clf_labels, 2 - sample_distances)

    # Create a Plotly area plot using the precision, recall, and thresholds arrays
    fig = px.line(
        data_frame=df,
        x="recall",
        y="precision",
        title=f"Precision-Recall Curve (AP={ap:.4f})",
        labels=dict(x="Recall", y="Precision"),
        hover_data=["thr"],
        width=700,
        height=500,
    )

    # Update the y-axis to have the same scale as the x-axis
    fig.update_yaxes(scaleanchor="x", scaleratio=1)
    # Constrain the x-axis to the [0, 1] domain
    fig.update_xaxes(constrain="domain")

    # Save the dataframe and the plot figure as a CSV file and a PNG image, respectively, if a results path is provided
    if results_path:
        fig.write_image(results_path + "/prc.png")

    # Display the plot
    fig.show()
    return df

def generate_tsne(embeddings: torch.Tensor, song_labels: list, cover_names: list) -> None:
    X = embeddings.cpu().numpy()
    X_embed = TSNE(n_components=2, learning_rate='auto',
                   init='random', perplexity=3).fit_transform(X)

    y = np.array(song_labels)
    fig = px.scatter(x=X_embed[:, 0], y=X_embed[:, 1], color=y, hover_name=cover_names)
    fig.show()


import json
import os
import tqdm

lengths_data_path = "/home/alanes/wimbd/lengths_data/"

for filename in os.listdir(lengths_data_path):
    toks_data = dict()
    chars_data = dict()
    dataset = filename.split(".")[0]

    print(f'analyzing {dataset}')

    with open(os.path.join(lengths_data_path, filename)) as infile:
        for line in tqdm.tqdm(infile.readlines()):
            data = json.loads(line)
            key = data['key']
            count = int(data['count'])

            if key == 'text is None':
                continue

            count_type, value = key.split(' ')
            length = int(value)
            if count_type.lower() == 'tokens':
                chars_data[length] = count
            elif count_type.lower() == 'chars':
                toks_data[length] = count

    count_sum = sum(toks_data.values())
    dist = {length: count / count_sum for length, count in toks_data.items()}
    print(f'Found {count_sum} total token datapoints; values sum to {sum(dist.values())}')
    with open(os.path.join("/home/alanes/wimbd/lengths_tok_summary/", f'toks_{dataset}.json'), 'w') as ofile:
        json.dump(dist, ofile)

    count_sum = sum(chars_data.values())
    dist = {length: count / count_sum for length, count in chars_data.items()}
    print(f'Found {count_sum} total char datapoints; values sum to {sum(dist.values())}')
    with open(os.path.join("/home/alanes/wimbd/lengths_char_summary/", f'chars_{dataset}.json'), 'w') as ofile:
        json.dump(dist, ofile)


"""
deepdataspace.utils.classes
"""

import copy
from threading import RLock

class SingletonMeta(type):
    """
    Any class whose metaclass is SingletonMeta will be a singleton class.
    """

    __instances = dict()
    __lock = RLock()

    @staticmethod
    def __copy__(self):
        name = self.__class__.__qualname__
        print(f"An instance of singleton class[{name}] already exists, returning it on __copy__")
        return self

    @staticmethod
    def __deepcopy__(self, *args, **kwargs):
        name = self.__class__.__qualname__
        print(f"An instance of singleton class[{name}] already exists, returning it on __deepcopy__")
        return self

    def __new__(mcs, name, bases, attrs):
        for base in bases:
            if isinstance(base, SingletonMeta):
                name = base.__qualname__
                raise RuntimeError(f"Singleton class[{name}] cannot be inherited by any class.")

        attrs["__copy__"] = SingletonMeta.__copy__
        attrs["__deepcopy__"] = SingletonMeta.__deepcopy__

        return super(SingletonMeta, mcs).__new__(mcs, name, bases, attrs)

    def __call__(cls, *args, **kwargs):
        name = cls.__qualname__
        instance = SingletonMeta.__instances.get(name, None)
        if instance is None:
            with SingletonMeta.__lock:
                instance = SingletonMeta.__instances.setdefault(name,
                                                                super(SingletonMeta, cls).__call__(*args, **kwargs))
        else:
            print(f"An instance of singleton class[{name}] already exists, returning it on instance initiation")
        return instance

def test():
    class A(metaclass=SingletonMeta):
        def __init__(self, k, v):
            self.k = k
            self.v = v

    a = A(1, 2)
    b = A(3, 4)
    c = a
    d = copy.copy(a)
    e = copy.deepcopy(a)
    assert len({id(a), id(b), id(c), id(d), id(e)}) == 1

    try:
        class B(A, metaclass=SingletonMeta):
            def __init__(self, k, v):
                self.k = k
                self.v = v
                super(B, self).__init__(k, v)
    except Exception as e:
        err = e
    else:
        err = None
    assert isinstance(err, RuntimeError)

    class C(metaclass=SingletonMeta):
        def __init__(self, k, v):
            self.k = k
            self.v = v

    a = C(1, 2)
    b = C(3, 4)
    c = a
    d = copy.copy(a)
    e = copy.deepcopy(a)
    assert len({id(a), id(b), id(c), id(d), id(e)}) == 1

if __name__ == "__main__":
    test()


# -*- coding: utf-8 -*-
"""Module defines a fixture for :class:`~ms_graph_exporter.ms_graph.api.MsGraph`."""
from os import environ

import pytest

from ms_graph_exporter.ms_graph.api import MsGraph

@pytest.fixture(scope="session")
def graph_real(logger):
    """Provide MsGraph instance with access to actual MS Graph API."""
    try:
        _graph = MsGraph(
            client_id=environ.get("GRAPH_CLIENT_ID", "undefined"),
            client_secret=environ.get("GRAPH_CLIENT_SECRET", "undefined"),
            tenant=environ.get("GRAPH_TENANT", "undefined"),
        )

    except Exception:
        logger.exception("[MsGraph]: Fixture not properly instantiated")
        yield None

    else:
        logger.info("%s: Provide 'graph_real' fixture", _graph)
        yield _graph

    finally:
        logger.info("%s: Destroy 'graph_real' fixture", _graph)


# https://github.com/RsaCtfTool/RsaCtfTool
# ./RsaCtfTool.py --createpub -n n -e e > key.pub
# ./RsaCtfTool.py --publickey key.pub --private --dump

import gmpy2

def small_m_large_n(n, e, c):
    gmpy2.get_context().precision = gmpy2.bit_length(c)
    m = int(gmpy2.root(c, e))
    assert pow(m, e, n) == c
    return m

if __name__ == '__main__':
    from Crypto.Util.number import getPrime, getRandomNBitInteger

    _BITS = 256

    _n = getPrime(_BITS) * getPrime(_BITS)
    _m = getRandomNBitInteger(8)
    _e = 3
    _c = pow(_m, _e, _n)
    assert _m == small_m_large_n(_n, _e, _c)


"""Assignment: Use the Titanic data set which is provided.
Please write your code after the comments, and upload the completed .py file along with the excel files.
"""
# survived:	Survival	0 = No, 1 = Yes
# pclass:	Ticket class	1 = 1st, 2 = 2nd, 3 = 3rd
# sex:	Sex
# Age:	Age in years
# Siblings/Spouses Aboard': siblings / spouses aboard the Titanic
# Parents/Children Aboard:	# of parents / children aboard the Titanic
# fare:	Passenger fare

'''Part 1'''

import pandas as pd

#Import the data and relevant libraries
df = pd.read_csv("Titanic-2.csv")

##Is there any missing values in the data? If yes, which columns have missing data?
df.info() # THERE IS NO MISSING INFO (there are 887 non-null values in each column)

#sort data based on decreasing "age" and assign it to a variable called age_sorted
age_sorted = df.sort_values(by="Age", ascending=False)

#get all rows for which the "Survived" value is 1, and assign to a variable called survivors.
survivors = df.loc[df["Survived"] == 1]

#get all rows for which the "Survived" value is 0, and assign to a variable called nonsurvivors.
nonsurvivors = df.loc[df["Survived"] == 0]

#get all rows for which "Sex" is male, "Age" is more than 30, and "Fare" is more than 15
df.loc[(df["Sex"] == "male") & (df["Age"] > 30) & (df["Fare"] > 15)]

#How many males were in the first Pclass?
df.loc[(df["Sex"] == "male") & (df["Pclass"] == 1)].info() # 122

#Get the name of the person(s) who paid the highest fare?
df.sort_values(by="Fare", ascending=False).head(1)["Name"]
# Mr. Thomas Drake Martinez Cardeza

#The person who paid the highest fare was in which Pclass?
df.sort_values(by="Fare", ascending=False).head(1)["Pclass"]
# 1

##how many females survived?
df[(df["Sex"] == "female") & (df["Survived"] == 1)].info()
# 233

##How many males survived?
df[(df["Sex"] == "male") & (df["Survived"] == 1)].info()
# 109

#How many females less than 35 years of age survived?
df[(df["Sex"] == "female") & (df["Survived"] == 1) & (df["Age"] < 35)].info()
# 158

#How many males in Pclass 1 paid more than 15$?
df[(df["Sex"] == "male") & (df["Pclass"] == 1) & (df["Fare"] > 15)].info()
# 116

#Get rows 3-9 (including both 3 and 9)
df.loc[3:9]

#get last 10 rows
df.tail(10)

#Passengers in which Pclass (1,2,3) paid the highest mean fare?
df.groupby("Pclass").mean()["Fare"]
# Pclass 1

#Compare the mean fare price of people who survived and who did not.
df.groupby("Survived").mean()["Fare"]
# People who survived had a mean fare price of 48.395 as opposed to
# people who did not survive who only had a mean of 22.209 (26.186 difference)

#create a new column which is sum of 'Siblings/Spouses Aboard' and 'Parents/Children Aboard'. Name the column appropriately.
df["People Abroad"] = df["Siblings/Spouses Aboard"] + df["Parents/Children Aboard"]

#save the file as an Excel file with name updated_file
df.to_excel("updated_file.xlsx");

'''Part 2'''

#Write a program to print the cube of numbers from 2 to 100 (2 and 100 included)
def cube():
    for i in range(2,101):
        print(i**3)
cube()

#Write a program to print all odd numbers from 2 to 100
def oddNumbers():
    for i in range(2,101):
        if i%2 == 1:
            print(i)
oddNumbers()

#What is a library in python? Name 6 python libraries.
# A library in python is a module of commands or functionality that you can import
# to use in your python program.

# 6 python libraries are pandas, numpy, matplotlib, tensorflow, pytorch, and
# Beautiful Soup