from datetime import datetime
import itertools
import os

from trac.api import ISystemInfoProvider
from trac.cache import cached
from trac.config import BoolOption, IntOption, ListOption, PathOption, Option
from trac.core import Component, TracError, implements
from trac.util.datefmt import FixedOffset, to_timestamp, format_datetime
from trac.util.html import Markup, tag
from trac.util.text import exception_to_unicode, shorten_line, to_unicode
from trac.util.translation import _
from trac.versioncontrol.api import Changeset, Node, Repository, \
                                    IRepositoryConnector, InvalidRepository,\
                                    NoSuchChangeset, NoSuchNode, \
                                    RepositoryManager, IRepositoryProvider
from trac.versioncontrol.cache import CACHE_YOUNGEST_REV, CachedRepository, \
                                      CachedChangeset
from trac.versioncontrol.web_ui import IPropertyRenderer
from trac.web.chrome import Chrome
from trac.wiki import IWikiSyntaxProvider

from tracopt.versioncontrol.git import PyGIT

class GitCachedRepository(CachedRepository):
    """Git-specific cached repository."""

    def display_rev(self, rev):
        return self.short_rev(rev)

    def short_rev(self, path):
        return self.repos.short_rev(path)

    def normalize_rev(self, rev):
        if not rev:
            return self.get_youngest_rev()
        normrev = self.repos.git.verifyrev(rev)
        if normrev is None:
            raise NoSuchChangeset(rev)
        return normrev

    def get_youngest_rev(self):
        # return None if repository is empty
        return CachedRepository.get_youngest_rev(self) or None

    def child_revs(self, rev):
        return self.repos.child_revs(rev)

    def get_changesets(self, start, stop):
        for key, csets in itertools.groupby(
                CachedRepository.get_changesets(self, start, stop),
                key=lambda cset: cset.date):
            csets = list(csets)
            if len(csets) == 1:
                yield csets[0]
                continue
            rev_csets = {cset.rev: cset for cset in csets}
            while rev_csets:
                revs = [rev for rev in rev_csets
                            if not any(r in rev_csets
                                       for r in self.repos.child_revs(rev))]
                for rev in sorted(revs):
                    yield rev_csets.pop(rev)

    def get_changeset(self, rev):
        return GitCachedChangeset(self, self.normalize_rev(rev), self.env)

    def sync(self, feedback=None, clean=False):
        if clean:
            self.remove_cache()

        metadata = self.metadata
        self.save_metadata(metadata)
        meta_youngest = metadata.get(CACHE_YOUNGEST_REV, '')
        repos = self.repos

        def is_synced(rev):
            for count, in self.env.db_query("""
                    SELECT COUNT(*) FROM revision WHERE repos=%s AND rev=%s
                    """, (self.id, rev)):
                return count > 0
            return False

        def needs_sync():
            max_holders = 999
            revs = sorted(set(rev for refname, rev in repos.git.get_refs()))
            step = max_holders - 1
            for idx in range(0, len(revs), step):
                revs_ = revs[idx:idx + step]
                holders = ','.join(('%s',) * len(revs_))
                args = [self.id]
                args.extend(revs_)
                query = """SELECT COUNT(*) FROM revision
                           WHERE repos=%%s AND rev IN (%s)""" % holders
                for count, in self.env.db_query(query, args):
                    if count < len(revs_):
                        return True
            return False

        def traverse(rev, seen):
            revs = []
            merge_revs = []
            while True:
                if rev in seen:
                    break
                seen.add(rev)
                if is_synced(rev):
                    break
                revs.append(rev)
                parent_revs = repos.parent_revs(rev)
                if not parent_revs:  # root commit?
                    break
                rev = parent_revs[0]
                if len(parent_revs) > 1:
                    merge_revs.append((len(revs), parent_revs[1:]))
            for idx, parent_revs in reversed(merge_revs):
                for rev in parent_revs:
                    revs[idx:idx] = traverse(rev, seen)
            return revs

        def sync_revs():
            updated = False
            seen = set()

            for rev in repos.git.all_revs():
                if repos.child_revs(rev):
                    continue
                revs = traverse(rev, seen)  # topology ordered
                while revs:
                    # sync revision from older revision to newer revision
                    rev = revs.pop()
                    self.log.info("Trying to sync revision [%s]", rev)
                    cset = repos.get_changeset(rev)
                    try:
                        self.insert_changeset(rev, cset)
                        updated = True
                    except self.env.db_exc.IntegrityError as e:
                        self.log.info('Revision %s already cached: %r', rev, e)
                        continue
                    if feedback:
                        feedback(rev)

            return updated

        with self.env.db_query:
            while True:
                repos.sync()
                if needs_sync() and sync_revs():
                    continue  # sync again
                repos_youngest = repos.youngest_rev or ''
                if meta_youngest != repos_youngest:
                    with self.env.db_transaction as db:
                        db("""
                            UPDATE repository SET value=%s
                            WHERE id=%s AND name=%s
                            """, (repos_youngest, self.id, CACHE_YOUNGEST_REV))
                        del self.metadata
                return

class GitCachedChangeset(CachedChangeset):
    """Git-specific cached changeset."""

    def get_branches(self):
        _rev = self.rev

        return [(k, v == _rev) for k, v in
                 self.repos.repos.git.get_branch_contains(_rev, resolve=True)]

    def get_tags(self):
        return self.repos.repos.git.get_tags(self.rev)

def _last_iterable(iterable):
    """helper for detecting last iteration in for-loop"""
    i = iter(iterable)
    try:
        v = next(i)
    except StopIteration:
        return
    for nextv in i:
        yield False, v
        v = nextv
    yield True, v

def intersperse(sep, iterable):
    """The 'intersperse' generator takes an element and an iterable and
    intersperses that element between the elements of the iterable.

    inspired by Haskell's ``Data.List.intersperse``
    """

    for i, item in enumerate(iterable):
        if i: yield sep
        yield item

# helper
def _parse_user_time(s):
    """Parse author or committer attribute lines and return
    corresponding ``(user, timestamp)`` pair.
    """

    user, time, tz_str = s.rsplit(None, 2)
    tz = FixedOffset((int(tz_str) * 6) // 10, tz_str)
    time = datetime.fromtimestamp(float(time), tz)
    return user, time

_file_type_mask = 0o170000

def _is_dir(mode):
    if mode is None:
        return False
    return (mode & _file_type_mask) in (0o040000, 0o160000)

def _is_submodule(mode):
    if mode is None:
        return False
    return (mode & _file_type_mask) == 0o160000

class GitConnector(Component):

    implements(IRepositoryConnector, ISystemInfoProvider, IWikiSyntaxProvider)

    def __init__(self):
        self._version = None

        try:
            self._version = PyGIT.Storage.git_version(git_bin=self.git_bin)
        except PyGIT.GitError as e:
            self.log.error("GitError: %s", e)

        if self._version:
            self.log.info("detected GIT version %s", self._version['v_str'])
            if not self._version['v_compatible']:
                self.log.error("GIT version %s installed not compatible"
                               "(need >= %s)", self._version['v_str'],
                               self._version['v_min_str'])

    # ISystemInfoProvider methods

    def get_system_info(self):
        if self._version:
            yield 'GIT', self._version['v_str']

    # IWikiSyntaxProvider methods

    def _format_sha_link(self, formatter, sha, label):
        # FIXME: this function needs serious rethinking...

        reponame = ''

        context = formatter.context
        while context:
            if context.resource.realm in ('source', 'changeset'):
                reponame = context.resource.parent.id
                break
            context = context.parent

        try:
            repos = RepositoryManager(self.env).get_repository(reponame)

            if not repos:
                raise Exception("Repository '%s' not found" % reponame)

            sha = repos.normalize_rev(sha) # in case it was abbreviated
            changeset = repos.get_changeset(sha)
            return tag.a(label, class_='changeset',
                         title=shorten_line(changeset.message),
                         href=formatter.href.changeset(sha, repos.reponame))
        except Exception as e:
            return tag.a(label, class_='missing changeset',
                         title=to_unicode(e), rel='nofollow')

    def get_wiki_syntax(self):
        yield (r'(?:\b|!)r?[0-9a-fA-F]{%d,40}\b' % self.wiki_shortrev_len,
               lambda fmt, sha, match:
                    self._format_sha_link(fmt, sha.startswith('r')
                                          and sha[1:] or sha, sha))

    def get_link_resolvers(self):
        yield ('sha', lambda fmt, _, sha, label, match=None:
                        self._format_sha_link(fmt, sha, label))

    # IRepositoryConnector methods

    persistent_cache = BoolOption('git', 'persistent_cache', 'false',
        """Enable persistent caching of commit tree.""")

    cached_repository = BoolOption('git', 'cached_repository', 'false',
        """Wrap `GitRepository` in `CachedRepository`.""")

    shortrev_len = IntOption('git', 'shortrev_len', 7,
        """The length at which a sha1 is abbreviated (must be >= 4
        and <= 40).
        """)

    wiki_shortrev_len = IntOption('git', 'wikishortrev_len', 40,
        """The minimum length at which a hex-string in wiki content
        is formatted as a changeset TracLink (must be >= 4 and <= 40).
        """)

    trac_user_rlookup = BoolOption('git', 'trac_user_rlookup', 'false',
        """Enable reverse mapping of git email addresses to trac user ids.
        Performance will be reduced if there are many users and the
        `cached_repository` option is `disabled`.

        A repository resync is required after changing the value of this
        option.
        """)

    use_committer_id = BoolOption('git', 'use_committer_id', 'true',
        """Use git-committer id instead of git-author id for the
        changeset ''Author'' field.
        """)

    use_committer_time = BoolOption('git', 'use_committer_time', 'true',
        """Use git-committer timestamp instead of git-author timestamp
        for the changeset ''Timestamp'' field.
        """)

    git_fs_encoding = Option('git', 'git_fs_encoding', 'utf-8',
        """Define charset encoding of paths within git repositories.""")

    git_bin = Option('git', 'git_bin', 'git',
        """Path to the git executable.""")

    def get_supported_types(self):
        yield ('git', 8)

    def get_repository(self, type, dir, params):
        """GitRepository factory method"""
        assert type == 'git'

        if not (4 <= self.shortrev_len <= 40):
            raise TracError(_("%(option)s must be in the range [4..40]",
                              option="[git] shortrev_len"))

        if not (4 <= self.wiki_shortrev_len <= 40):
            raise TracError(_("%(option)s must be in the range [4..40]",
                              option="[git] wikishortrev_len"))

        if not self._version:
            raise TracError(_("GIT backend not available"))
        elif not self._version['v_compatible']:
            raise TracError(_("GIT version %(hasver)s installed not "
                              "compatible (need >= %(needsver)s)",
                              hasver=self._version['v_str'],
                              needsver=self._version['v_min_str']))

        if self.trac_user_rlookup:
            def rlookup_uid(email):
                """Reverse map 'real name <user@domain.tld>' addresses to trac
                user ids.

                :return: `None` if lookup failed
                """

                try:
                    _, email = email.rsplit('<', 1)
                    email, _ = email.split('>', 1)
                    email = email.lower()
                except Exception:
                    return None

                for _uid, _name, _email in self.env.get_known_users():
                    try:
                        if email == _email.lower():
                            return _uid
                    except Exception:
                        continue

        else:
            def rlookup_uid(_):
                return None

        repos = GitRepository(self.env, dir, params, self.log,
                              persistent_cache=self.persistent_cache,
                              git_bin=self.git_bin,
                              git_fs_encoding=self.git_fs_encoding,
                              shortrev_len=self.shortrev_len,
                              rlookup_uid=rlookup_uid,
                              use_committer_id=self.use_committer_id,
                              use_committer_time=self.use_committer_time,
                              )

        if self.cached_repository:
            repos = GitCachedRepository(self.env, repos, self.log)
            self.log.debug("enabled CachedRepository for '%s'", dir)
        else:
            self.log.debug("disabled CachedRepository for '%s'", dir)

        return repos

class CsetPropertyRenderer(Component):

    implements(IPropertyRenderer)

    # relied upon by GitChangeset
    def match_property(self, name, mode):
        # default renderer has priority 1
        return (name in ('Parents',
                         'Children',
                         'Branches',
                         'git-committer',
                         'git-author',
                         ) and mode == 'revprop') and 4 or 0

    def render_property(self, name, mode, context, props):

        def sha_link(sha, label=None):
            # sha is assumed to be a non-abbreviated 40-chars sha id
            try:
                reponame = context.resource.parent.id
                repos = RepositoryManager(self.env).get_repository(reponame)
                cset = repos.get_changeset(sha)
                if label is None:
                    label = repos.display_rev(sha)

                return tag.a(label, class_='changeset',
                             title=shorten_line(cset.message),
                             href=context.href.changeset(sha, repos.reponame))

            except Exception as e:
                return tag.a(sha, class_='missing changeset',
                             title=to_unicode(e), rel='nofollow')

        if name == 'Branches':
            branches = props[name]

            # simple non-merge commit
            return tag(*intersperse(', ', (sha_link(rev, label)
                                           for label, rev in branches)))

        elif name in ('Parents', 'Children'):
            revs = props[name] # list of commit ids

            if name == 'Parents' and len(revs) > 1:
                # we got a merge...
                current_sha = context.resource.id
                reponame = context.resource.parent.id

                parent_links = intersperse(', ',
                                           ((sha_link(rev),
                      ' (',
                      tag.a(_("diff"),
                            title=_("Diff against this parent (show the "
                                    "changes merged from the other parents)"),
                            href=context.href.changeset(current_sha, reponame,
                                                        old=rev)),
                      ')')
                     for rev in revs))

                return tag(list(parent_links),
                           tag.br(),
                           tag.span(Markup(_("Note: this is a merge"
                                             " changeset, the "
                                             "changes displayed below "
                                             "correspond to the merge "
                                             "itself.")),
                                    class_='hint'),
                           tag.br(),
                           tag.span(Markup(_("Use the (diff) "
                                             "links above to see all the "
                                             "changes relative to each "
                                             "parent.")),
                                    class_='hint'))

            # simple non-merge commit
            return tag(*intersperse(', ', map(sha_link, revs)))

        elif name in ('git-committer', 'git-author'):
            user_, time_ = props[name]
            _str = "%s (%s)" % (
                Chrome(self.env).format_author(context.req, user_),
                format_datetime(time_, tzinfo=context.req.tz))
            return str(_str)

        raise TracError(_("Internal error"))

class GitRepository(Repository):
    """Git repository"""

    def __init__(self, env, path, params, log,
                 persistent_cache=False,
                 git_bin='git',
                 git_fs_encoding='utf-8',
                 shortrev_len=7,
                 rlookup_uid=lambda _: None,
                 use_committer_id=False,
                 use_committer_time=False,
                 ):

        self.env = env
        self.logger = log
        self.gitrepo = path
        self.params = params
        self.persistent_cache = persistent_cache
        self.shortrev_len = max(4, min(shortrev_len, 40))
        self.rlookup_uid = rlookup_uid
        self.use_committer_time = use_committer_time
        self.use_committer_id = use_committer_id

        try:
            factory = PyGIT.StorageFactory(path, log, not persistent_cache,
                                           git_bin=git_bin,
                                           git_fs_encoding=git_fs_encoding)
            self._git = factory.getInstance()
        except PyGIT.GitError as e:
            log.error(exception_to_unicode(e))
            raise InvalidRepository(
                _('"%(name)s" is not readable or not a Git repository.',
                  name=params.get('name') or '(default)'))

        Repository.__init__(self, 'git:' + path, self.params, log)
        self._cached_git_id = str(self.id)

    def close(self):
        self._git = None

    @property
    def git(self):
        if self.persistent_cache:
            return self._cached_git
        else:
            return self._git

    @cached('_cached_git_id')
    def _cached_git(self):
        self._git.invalidate_rev_cache()
        return self._git

    def get_youngest_rev(self):
        return self.git.youngest_rev()

    def get_path_history(self, path, rev=None, limit=None):
        raise TracError(_("Unsupported \"Show only adds and deletes\""))

    def get_oldest_rev(self):
        return self.git.oldest_rev()

    def normalize_path(self, path):
        return path and path.strip('/') or '/'

    def normalize_rev(self, rev):
        if not rev:
            return self.get_youngest_rev()
        normrev = self.git.verifyrev(rev)
        if normrev is None:
            raise NoSuchChangeset(rev)
        return normrev

    def display_rev(self, rev):
        return self.short_rev(rev)

    def short_rev(self, rev):
        return self.git.shortrev(self.normalize_rev(rev),
                                 min_len=self.shortrev_len)

    def get_node(self, path, rev=None):
        return self._get_node(path, rev)

    def _get_node(self, path, rev, ls_tree_info=None, historian=None):
        return GitNode(self, path, rev, self.log, ls_tree_info, historian)

    def get_quickjump_entries(self, rev):
        for bname, bsha in self.git.get_branches():
            yield 'branches', bname, '/', bsha
        for t in self.git.get_tags():
            yield 'tags', t, '/', t

    def get_path_url(self, path, rev):
        return self.params.get('url')

    def get_changesets(self, start, stop):
        for rev in self.git.history_timerange(to_timestamp(start),
                                              to_timestamp(stop)):
            yield self.get_changeset(rev)

    def get_changeset(self, rev):
        """GitChangeset factory method"""
        return GitChangeset(self, rev)

    def get_changeset_uid(self, rev):
        return self.normalize_rev(rev)

    def get_changes(self, old_path, old_rev, new_path, new_rev,
                    ignore_ancestry=0):
        # TODO: handle renames/copies, ignore_ancestry
        old_path = self.normalize_path(old_path)
        new_path = self.normalize_path(new_path)
        if old_path != new_path:
            raise TracError(_("Not supported in git_fs"))

        old_rev = self.normalize_rev(old_rev)
        new_rev = self.normalize_rev(new_rev)
        if old_rev == new_rev:
            return

        def get_tree(rev):
            results = self.git.ls_tree(rev, target_path, recursive=True)
            return {result[4]: result for result in results}

        target_path = old_path.strip('/')
        old_tree = get_tree(old_rev)
        new_tree = get_tree(new_rev)

        with self.git.get_historian(old_rev, target_path) as old_historian:
            with self.git.get_historian(new_rev, target_path) as new_historian:
                for chg in self.git.diff_tree(old_rev, new_rev, target_path):
                    mode1, mode2, obj1, obj2, action, path, path2 = chg
                    kind = Node.DIRECTORY \
                           if _is_dir(mode2) or _is_dir(mode1) \
                           else Node.FILE

                    change = GitChangeset.action_map[action]
                    old_node = self._get_node(path, old_rev,
                                              old_tree.get(path, False),
                                              old_historian) \
                               if change != Changeset.ADD else None
                    new_node = self._get_node(path, new_rev,
                                              new_tree.get(path, False),
                                              new_historian) \
                               if change != Changeset.DELETE else None

                    yield old_node, new_node, kind, change

    def next_rev(self, rev, path=''):
        return self.git.hist_next_revision(rev)

    def previous_rev(self, rev, path=''):
        return self.git.hist_prev_revision(rev)

    def parent_revs(self, rev):
        return self.git.parents(rev)

    def child_revs(self, rev):
        return self.git.children(rev)

    def rev_older_than(self, rev1, rev2):
        return self.git.rev_is_anchestor_of(self.normalize_rev(rev1),
                                            self.normalize_rev(rev2))

    # def clear(self, youngest_rev=None):
    #     self.youngest = None
    #     if youngest_rev is not None:
    #         self.youngest = self.normalize_rev(youngest_rev)
    #     self.oldest = None

    def clear(self, youngest_rev=None):
        self.sync()

    def sync(self, rev_callback=None, clean=None):
        if rev_callback:
            revs = set(self.git.all_revs())

        if self.persistent_cache:
            del self._cached_git  # invalidate persistent cache
        if not self.git.sync():
            return None # nothing expected to change

        if rev_callback:
            revs = set(self.git.all_revs()) - revs
            for rev in revs:
                rev_callback(rev)

class GitNode(Node):

    def __init__(self, repos, path, rev, log, ls_tree_info=None,
                 historian=None):
        self.log = log
        self.repos = repos
        self.fs_sha = None # points to either tree or blobs
        self.fs_perm = None
        self.fs_type = None
        self.fs_size = None
        if rev:
            rev = repos.normalize_rev(to_unicode(rev))
        else:
            rev = repos.youngest_rev
        created_rev = rev

        kind = Node.DIRECTORY
        p = path.strip('/')
        if p:  # ie. not the root-tree
            if not rev:
                raise NoSuchNode(path, rev)
            if ls_tree_info is None:
                ls_tree_info = repos.git.ls_tree(rev, p)
                if ls_tree_info:
                    ls_tree_info = ls_tree_info[0]
            if not ls_tree_info:
                raise NoSuchNode(path, rev)
            self.fs_perm, self.fs_type, self.fs_sha, self.fs_size, fname = \
                ls_tree_info

            # fix-up to the last commit-rev that touched this node
            created_rev = repos.git.last_change(rev, p, historian)

            if self.fs_type == 'tree':
                kind = Node.DIRECTORY
            elif self.fs_type == 'blob':
                kind = Node.FILE
            elif _is_submodule(self.fs_perm):
                # FIXME: this is a workaround for missing git submodule
                #        support in the plugin
                kind = Node.DIRECTORY
            else:
                self.log.warning('Got unexpected object %r', ls_tree_info)
                raise TracError(_("Internal error (got unexpected object "
                                  "kind '%(kind)s')", kind=self.fs_type))

        self.created_path = path
        self.created_rev = created_rev

        Node.__init__(self, repos, path, rev, kind)

    def __git_path(self):
        """return path as expected by PyGIT"""
        p = self.path.strip('/')
        if self.isfile:
            assert p
            return p
        if self.isdir:
            return p and (p + '/')

        raise TracError(_("Internal error"))

    def get_content(self):
        if not self.isfile:
            return None
        return self.repos.git.get_file(self.fs_sha)

    def get_properties(self):
        if self.fs_perm is None:
            return {}
        props = {'mode': '%06o' % self.fs_perm}
        if _is_submodule(self.fs_perm):
            props['commit'] = self.fs_sha
        return props

    def get_annotations(self):
        if not self.isfile:
            return

        return [rev for rev, lineno in
                self.repos.git.blame(self.rev,self.__git_path())]

    def get_entries(self):
        if not self.rev:  # if empty repository
            return
        if not self.isdir:
            return
        if _is_submodule(self.fs_perm):
            return

        with self.repos.git.get_historian(self.rev,
                                          self.path.strip('/')) as historian:
            for ent in self.repos.git.ls_tree(self.rev, self.__git_path()):
                yield GitNode(self.repos, ent[-1], self.rev, self.log, ent,
                              historian)

    def get_content_type(self):
        if self.isdir:
            return None

        return ''

    def get_content_length(self):
        if not self.isfile:
            return None

        if self.fs_size is None:
            self.fs_size = self.repos.git.get_obj_size(self.fs_sha)

        return self.fs_size

    def get_history(self, limit=None):
        if not self.rev:  # if empty repository
            return
        # TODO: find a way to follow renames/copies
        for is_last, rev in _last_iterable(self.repos.git.history(self.rev,
                                                self.__git_path(), limit)):
            yield (self.path, rev, Changeset.EDIT if not is_last else
                                   Changeset.ADD)

    def get_last_modified(self):
        if not self.isfile:
            return None

        try:
            msg, props = self.repos.git.read_commit(self.rev)
            user, ts = _parse_user_time(props['committer'][0])
        except:
            self.log.error("internal error (could not get timestamp from "
                           "commit '%s')", self.rev)
            return None

        return ts

class GitChangeset(Changeset):
    """A Git changeset in the Git repository.

    Corresponds to a Git commit blob.
    """

    action_map = { # see also git-diff-tree(1) --diff-filter
        'A': Changeset.ADD,
        'M': Changeset.EDIT, # modified
        'T': Changeset.EDIT, # file type (mode) change
        'D': Changeset.DELETE,
        'R': Changeset.MOVE, # renamed
        'C': Changeset.COPY
        } # TODO: U, X, B

    def __init__(self, repos, sha):
        if sha is None:
            raise NoSuchChangeset(sha)

        try:
            msg, props = repos.git.read_commit(sha)
        except PyGIT.GitErrorSha:
            raise NoSuchChangeset(sha)

        self.props = props

        assert 'children' not in props
        _children = list(repos.git.children(sha))
        if _children:
            props['children'] = _children

        committer, author = self._get_committer_and_author()
        # use 1st author/committer as changeset owner/timestamp
        c_user = a_user = c_time = a_time = None
        if committer:
            c_user, c_time = _parse_user_time(committer)
        if author:
            a_user, a_time = _parse_user_time(author)

        if repos.use_committer_time:
            time = c_time or a_time
        else:
            time = a_time or c_time

        if repos.use_committer_id:
            user = c_user or a_user
        else:
            user = a_user or c_user

        # try to resolve email address to trac uid
        user = repos.rlookup_uid(user) or user

        Changeset.__init__(self, repos, rev=sha, message=msg, author=user,
                           date=time)

    def _get_committer_and_author(self):
        committer = author = None
        if 'committer' in self.props:
            committer = self.props['committer'][0]
        if 'author' in self.props:
            author = self.props['author'][0]
        return committer, author

    def get_properties(self):
        properties = {}

        if 'parent' in self.props:
            properties['Parents'] = self.props['parent']

        if 'children' in self.props:
            properties['Children'] = self.props['children']

        committer, author = self._get_committer_and_author()
        if author != committer:
            properties['git-committer'] = _parse_user_time(committer)
            properties['git-author'] = _parse_user_time(author)

        branches = list(self.repos.git.get_branch_contains(self.rev,
                                                           resolve=True))
        if branches:
            properties['Branches'] = branches

        return properties

    def get_changes(self):
        # Returns the differences against the first parent
        parent = self.props.get('parent')
        parent = parent[0] if parent else None

        changes = self.repos.git.get_changes(parent, self.rev)
        for mode1, mode2, obj1, obj2, action, path1, path2 in changes:
            path = path2 or path1
            p_path, p_rev = path1, parent

            kind = Node.DIRECTORY \
                   if _is_dir(mode2) or _is_dir(mode1) else \
                   Node.FILE

            action = GitChangeset.action_map[action]

            if action == Changeset.ADD:
                p_path = p_rev = None

            yield path, kind, action, p_path, p_rev

    def get_branches(self):
        _rev = self.rev

        return [(k, v == _rev)
                for k, v in self.repos.git.get_branch_contains(_rev,
                                                               resolve=True)]

    def get_tags(self):
        return self.repos.git.get_tags(self.rev)

class GitwebProjectsRepositoryProvider(Component):

    implements(IRepositoryProvider)

    projects_list = PathOption('gitweb-repositories', 'projects_list', doc=
        """Path to a gitweb-formatted projects.list""")

    projects_base = PathOption('gitweb-repositories', 'projects_base', doc=
        """Path to the base of your git projects""")

    projects_url = Option('gitweb-repositories', 'projects_url', doc=
        """Template for project URLs. `%s` will be replaced with the repo
        name""")

    sync_per_request = ListOption('gitweb-repositories',
        'sync_per_request', '', doc="""Repositories to sync on every request
        (not recommended).""")

    def get_repositories(self):
        """Retrieve repositories specified in a `projects_list` file."""
        if not self.projects_list:
            return

        if not os.path.exists(self.projects_list):
            self.log.warning("The [git] projects_list file was not found at "
                             "'%s'", self.projects_list)
            return

        with open(self.projects_list, 'r', encoding='utf-8') as fp:
            for line in fp:
                entries = line.strip().split()
                if entries:
                    name = entries[0]
                    reponame = name.rstrip('.git')
                    info = {
                        'dir': os.path.join(self.projects_base, name),
                        'sync_per_request': reponame in self.sync_per_request,
                        'type': 'git',
                    }
                    description_path = \
                        os.path.join(info['dir'], 'description')
                    if os.path.exists(description_path):
                        with open(description_path, 'r',
                                  encoding='utf-8') as fd:
                            info['description'] = fd.read().strip()
                    if self.projects_url:
                        info['url'] = self.projects_url % reponame
                    yield reponame, info


from db.models.movies import Movie
from schemas.movies import MovieCreate,FilterMovie
from .movie_genres import create_movie_genres,delete_movie_genres
from sqlalchemy.orm import Session
from db.models.genres import Genre
from db.models.reviews import Review
from sqlalchemy import or_,and_

# create movie
def create_new_movie(movie: MovieCreate,db: Session):
    # obtengo el dict; le separo los ids de los géneros y luego inserto apropiadamente
    data = movie.dict()
    genres = data.pop('genres')

    # agrego la peli
    movie_object = Movie(**data)
    db.add(movie_object)
    db.flush()

    # los géneros
    create_movie_genres(movie_object.id,genres,db)

    # commit
    db.commit()
    db.refresh(movie_object)
    return movie_object

# retrieve movie by id
def retreive_movie(id: int, db: Session):
    return db.query(Movie).filter(Movie.id == id).first()

#http://127.0.0.1:8000/movies/?date__lte=218-07-03&date__lte=2018-07-03
#http://127.0.0.1:8000/movies/?date__gte=2022-09-21&date__lte=2022-09-21#

def list_movies(db: Session,f:FilterMovie):

    filters = [Movie.id>0]

    if f.title:
        filters.append(Movie.title==f.title)

    if f.title__contains:
        filters.append(Movie.title.contains(f.title__contains))

    if f.genres:
        generos = db.query(Genre.id).filter(Genre.id.in_(f.genres))
        filters.append(Genre.id.in_(generos))

    # rango de fechas
    if f.date__gte:
        filters.append( Movie.date >=  f.date__gte)
    if f.date__lte:
        filters.append( Movie.date <=  f.date__lte)

    # junto todo
    filters = and_(*filters)

    query = db.query(Movie).join(Movie.genres).filter(filters).limit(f.limit).offset(f.offset-1).all()
     # le agrego count, page, limit
    count = db.query(Movie).filter(filters).count() # join en la cuenta??

    #
    data = [
        {
            'id':d.id,
            'title': d.title,
            'date': d.date,
            'description': d.description,
            'image_link':d.image_link,
            'genres': [{'description':t.description,'id':t.id} for t in d.genres]
        }
        for d in query
    ]

    # respuesta
    response = {
        'data': data,
        'count': count,
        'limit':f.limit,
        'offset':f.offset
    }
    return response

# update by id
def update_movie_by_id(id: int, movie: MovieCreate, db: Session):

    existing_movie = db.query(Movie).filter(Movie.id == id)
    if not existing_movie.first():
        return 0

    # actualizo la peli y sus géneros
    data = movie.dict(exclude_unset=True) # ipte
    genres = data.pop('genres')

    # géneros: borro los viejos y agrego los nuevos
    delete_movie_genres(id,db)
    create_movie_genres(id,genres,db)

    # la peli
    existing_movie.update(data)
    #existing_movie.update(movie.__dict__)
    db.commit()
    return 1

def delete_movie_by_id(id: int, db: Session):
    existing_movie = db.query(Movie).filter(Movie.id == id)
    if not existing_movie.first():
        return 0
    # borro sus géneros
    delete_movie_genres(movie_id=id,db=db)
    # borro la peli
    existing_movie.delete(synchronize_session=False) # ver esto
    db.commit()
    return 1

#movie_object = Movie(**movie.dict()); antes, sin tener que agregar en la otra tabla
#db.add(movie_object)


import ipaddress
from typing import Any

class IPAddress:
    def __init__(self, ip: str, mask: int):
        self.ip = ip
        self.mask = mask

    def __int__(self) -> int:
        int_ip = int(ipaddress.ip_address(self.ip))
        return int_ip

    def _int_to_str(self, ip_as_int: int) -> str:
        return str(ipaddress.ip_address(ip_as_int))

    def __str__(self) -> str:
        return f"{self.ip}/{self.mask}"

    def __repr__(self) -> str:
        return f"IPAddress('{self.ip}', {self.mask})"

    def __add__(self, other: int) -> IPAddress:
        if type(other) != int:
            raise TypeError(
                f"'+' not supported between instances of 'IPAddress'"
                f" and '{type(other).__name__}'"
            )
        new_ip = self._int_to_str(int(self) + other)
        return IPAddress(new_ip, self.mask)

    def __eq__(self, second_ip: object) -> bool:
        if type(second_ip) != IPAddress:
            raise TypeError(
                f"'==' not supported between instances of 'IPAddress'"
                f" and '{type(second_ip).__name__}'"
            )
        return (int(self), self.mask) == (int(second_ip), second_ip.mask)

    def __lt__(self, second_ip: IPAddress) -> bool:
        if type(second_ip) != IPAddress:
            raise TypeError(
                f"'<' not supported between instances of 'IPAddress'"
                f" and '{type(second_ip).__name__}'"
            )
        return (int(self), self.mask) < (int(second_ip), second_ip.mask)

if __name__ == "__main__":
    ip1 = IPAddress("10.1.1.1", 25)
    ip2 = IPAddress("10.2.2.2", 25)
    ip3 = IPAddress("10.1.1.1", 25)
    ip4 = IPAddress("10.10.1.1", 25)
    ip5 = IPAddress("10.1.1.1", 29)


import typing
from collections import defaultdict

class Node:
    nodes = {}

    def __init__(self, value: int, left=None, right=None):
        self.value = value
        self.left = left
        self.right = right

__nodes = {}

def node(value: int) -> Node:
    return __nodes.setdefault(value, Node(value))

def print_nodes(node: Node):
    if node.left:
        print_nodes(node.left)
    print(node.value)
    if node.right:
        print_nodes(node.right)

def print_nodes_2(node: Node, lines: typing.List[str], level: int, pos: int):
    if len(lines) <= level:
        lines.append("")
    if node.left:
        print_nodes_2(node.left, lines, level + 1, pos + 1)
    else:
        lines[level] += " "
    if node.right:
        print_nodes_2(node.right, lines, level + 1, pos + 2)
    lines[level] = add(lines[level], node.value, pos)

def add(s: str, value: int, pos: int) -> str:
    s += " " * ((pos * 2) - len(s)) + str(value)
    return s

def main():
    head_node = node(3)
    node(3).left = node(1)
    node(1).right = node(2)
    node(3).right = node(5)
    node(5).right = node(6)
    print_nodes(head_node)
    print("")
    arr = []
    print_nodes_2(head_node, arr, 0, 0)
    for s in arr:
        print(s)

if __name__ == "__main__":
    main()


from typing import Optional

import importlib
import os
import sys
from pathlib import Path

from deepcave.config import Config

def parse_config(filename: Optional[str] = None) -> Config:
    """
    Parses the config given the filename. Both relative and absolute paths are possible.

    Parameters
    ----------
    filename : Optional[str], optional
        Location of the config. Must be a python file.
        By default None (default configuration will be used).

    Note
    ----
    The python file must contain a class named ``Config`` and inherit ``deepcave.config.Config``.

    Returns
    -------
    Config
        Either the default config (if no filename is given) or the config parsed from the given
        filename.

    Raises
    ------
    RuntimeError
        If config class could not be loaded.
    """
    config = Config()
    if filename is not None and filename != "default":
        try:
            p = Path(filename)

            # Absolute path
            if filename.startswith("/") or filename.startswith("~"):
                path = p.parent
                script_dir = path.stem
                module_name = p.stem
            else:
                path = Path(os.getcwd()) / p.parent

            script_dir = path.stem  # That's the path without the script name
            module_name = p.stem  # That's the script name without the extension

            # Now we add to sys path
            sys.path.append(str(path))

            module = importlib.import_module(f"{script_dir}.{module_name}")
            config = module.Config()

        except Exception:
            raise RuntimeError(f"Could not load class Config from {p}.")

    return config


import os
import random
import numpy as np
import scipy.io as sio
from PIL import Image
from glob import glob
from natsort import natsorted

import torch
import torchvision
from torchvision import transforms
import torchvision.transforms.functional as F
from torch.utils.data import Dataset, DataLoader

from .utils import *

import ipdb

# This class is for SIDD_Medium train on sRGB patches
from natsort import natsorted
from glob import glob
class SIDD_sRGB_Train_DataLoader(Dataset):
    def __init__(self, path, length, patch_size, patched_input=False):
        super(SIDD_sRGB_Train_DataLoader, self).__init__()
        assert length % 160 == 0, 'You should specify a proper length.'

        self.len            = length
        self.patch_size     = patch_size
        self.patched_input  = patched_input
        self.noisy_imgs, self.clean_imgs = [], []
        self.transform      = torchvision.transforms.ToTensor()
        self.return_name    = False

        if not self.patched_input:
            # get file names from original dataset
            imgs = natsorted(glob(os.path.join(path, '*', '*.PNG')))
            for img in imgs:
                img_name = os.path.split(img)[-1]
                if 'GT' in img_name:
                    self.clean_imgs.append(img)
                if 'NOISY' in img_name:
                    self.noisy_imgs.append(img)
        else:
            # get file names from cropped dataset
            self.noisy_imgs = natsorted(glob(os.path.join(path, 'input', '*.png')))
            self.clean_imgs = natsorted(glob(os.path.join(path, 'target', '*.png')))

    def __len__(self):
        return self.len if self.len else len(self.noisy_imgs)

    def __getitem__(self, index):
        index = index % len(self.noisy_imgs) # in case out of range.
        aug   = random.randint(0, 7)

        # read image
        noisy_img   = np.array(Image.open(self.noisy_imgs[index]))
        clean_img   = np.array(Image.open(self.clean_imgs[index]))

        # random crop
        if self.patch_size:
            noisy_patch, clean_patch = crop_patches(noisy_img, clean_img,
                                                    self.patch_size)

        # random augmentation
        noisy_patch = Data_Augmentation(self.transform(noisy_patch), aug)
        clean_patch = Data_Augmentation(self.transform(clean_patch), aug)

        filename = os.path.splitext(os.path.split(self.clean_imgs[index])[-1])[0]
        if self.return_name:
            return noisy_patch, clean_patch, filename
        else:
            return noisy_patch, clean_patch

# this class is for SIDD val on sRGB patches.
# could download patched val data from https://drive.google.com/drive/folders/1S44fHXaVxAYW3KLNxK41NYCnyX9S79su
# If the full dataset is found, this class could be extended to support full graph/any patched size input.
class SIDD_sRGB_Val_DataLoader(Dataset):
    def __init__(self, path, patch_size=None):
        super(SIDD_sRGB_Val_DataLoader, self).__init__()
        self.noisy_imgs = natsorted(glob(os.path.join(path, 'input', '*.png')))
        self.clean_imgs = natsorted(glob(os.path.join(path, 'target', '*.png')))
        self.length     = len(self.noisy_imgs)
        self.patch_size = patch_size
        self.transform  = torchvision.transforms.ToTensor()
        self.return_name= False

    def __len__(self):
        return self.length

    def __getitem__(self, index):
        index %= self.length

        noisy_patch = np.array(Image.open(self.noisy_imgs[index]))
        clean_patch = np.array(Image.open(self.clean_imgs[index]))

        noisy_patch, clean_patch = self.transform(noisy_patch), \
                                   self.transform(clean_patch)

        if self.patch_size:
            noisy_patch = F.center_crop(noisy_patch, (self.patch_size, self.patch_size))
            clean_patch = F.center_crop(clean_patch, (self.patch_size, self.patch_size))

        filename = os.path.splitext(os.path.split(self.clean_imgs[index])[-1])[0]

        if self.return_name:
            return noisy_patch, clean_patch, filename
        else:
            return noisy_patch, clean_patch

# this class is for SIDD test on sRGB patches stored in .mat format
class SIDD_sRGB_mat_Test_DataLoader(Dataset):
    """
    Args:
        path - the dir to '*.mat' file
    """
    def __init__(self, path):
        super(SIDD_sRGB_mat_Test_DataLoader, self).__init__()
        self.len = 1280
        noisy_patch  = sio.loadmat(os.path.join(path, 'ValidationNoisyBlocksSrgb.mat'))
        target_patch = sio.loadmat(os.path.join(path, 'ValidationGtBlocksSrgb.mat'))
        noisy_array  = np.float32(np.array(noisy_patch['ValidationNoisyBlocksSrgb']))
        assert noisy_array.shape == (40, 32, 256, 256, 3)
        noisy_array  = np.resize(noisy_array / 255., (self.len, 256, 256, 3))
        target_array = np.float32(np.array(target_patch['ValidationGtBlocksSrgb']))
        target_array = np.resize(target_array / 255., (self.len, 256, 256, 3))

        self.transform    = torchvision.transforms.ToTensor()
        self.noise_patch  = noisy_array
        self.target_patch = target_array

    def __getitem__(self, index):
        input_patch = self.noise_patch[index]
        target_patch = self.target_patch[index]

        return self.transform(input_patch), self.transform(target_patch)

    def __len__(self):
        return self.len


# PYTHON_ARGCOMPLETE_OK

import argcomplete
import argparse

from . import io
from . import cfg
from . import utils

@utils.logExceptionsAndRaise
def main():
  # show startup message
  io.startup()

  # options used by multiple subparsers
  def addVerboseArg(parser):
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='print more detailed output')
  def addTexFileArg(parser):
    parser.add_argument('tex_file', default='', nargs='?',
                        help='toplevel tex file of the document')

  # setup argparser and global arguments
  p = argparse.ArgumentParser()
  addVerboseArg(p)
  p.add_argument('--version', action='store_true',
                 help='print installed paperman version')

  sub = p.add_subparsers(metavar='subcommands', dest='command')

  # img command
  s = sub.add_parser('img', help='image helper: check if unused or missing '
                                 'images exist or import missing images')
  addTexFileArg(s)
  addVerboseArg(s)
  s.add_argument('-i', '--import', action='store_true',
                 help='try importing missing images from image search path')
  #s.add_argument('-c', '--clean', action='store_true',
  #               help='remove unused images')

  # bib command
  s = sub.add_parser('bib', help='bibliography helper: check if unused or '
                                 'missing bib entries exist or import '
                                 'missing entries')
  addTexFileArg(s)
  addVerboseArg(s)
  s.add_argument('-p', '--print', action='store_true',
                 help='try to find missing citations in existing bibliographies '
                      'and show them in the shell, conflicts with --import, '
                      '--rewrite, and --sort')
  s.add_argument('-i', '--import', action='store_true',
                 help='try importing missing citations from existing '
                      'bibliographies')
  #s.add_argument('-c', '--clean', action='store_true',
  #               help='remove unused citations from bibliography, implies '
  #                    '--rewrite')
  s.add_argument('-r', '--rewrite', action='store_true',
                 help='rewrite and repair entire bibliography')
  s.add_argument('-s', '--sort', nargs='?', default=False,
                 help='sort bibliography, sort order can be one of "date", '
                      '"author" or "key", default is "key", implies --rewrite')

  # input command
  s = sub.add_parser('input',
                     help=r'input helper: check if all \input{} files exist '
                          r'or import files')
  addVerboseArg(s)
  s.add_argument('-i', '--import', action='store_true',
                 help=r'try importing missing \input{} files from input '
                      r'search path')

  # call img, bib and input command with --import flag set
  s = sub.add_parser('import-all',
                     help='shortcut to run img, bib and input subcommands '
                          'with --import option enabled')
  addVerboseArg(s)

  # sort authors command
  s = sub.add_parser('sort-authors',
                     help='alphabetic sorting and formatting of author lists '
                          'for scientific publications and presentations')
  addVerboseArg(s)
  s.add_argument('authors', nargs='+',
                 help='comma separated list of authors seperated. Use -s '
                      'option for a different separator')
  s.add_argument('-s', '--separator', default=',',
                 help='separation character in author list')
  s.add_argument('-f', '--full-name', action='store_true',
                 help='output full author names.')
  s.add_argument('--keep-first', action='store_true',
                 help='keep position of first author in list.')
  s.add_argument('--keep-last', action='store_true',
                 help='keep position of last author in list.')
  s.add_argument('-k', '--keep-first-and-last', action='store_true',
                 help='keep position of first and last author in list.')
  s.add_argument('-q', '--quiet', action='store_true',
                 help='just output the sorted author list, nothing else.')
  s.add_argument('--no-and', action='store_true',
                 help='use the separator instead of "and" before the final '
                      'name and ignore ands in input.')

  # collect bib/pdf pairs from specified collect directories
  s = sub.add_parser('collect',
                     help='rename and move bib-pdf file pairs from specified '
                          'folders to library')
  addVerboseArg(s)
  s.add_argument('paths', nargs='*',
                 help='specify paths to import bib-pdf file pairs from, '
                      'defaults to paths given in config')
  s.add_argument('-w', '--watch', action='store_true',
                 help='never exit and keep watching the collect directories')
  s.add_argument('-e', '--err-to-file', action='store_true',
                 help='write errors not only to stdout, but also to a txt file '
                      'located in the same position as the bib-pdf file pair '
                      'that caused the error')

  # library subcommand
  s = sub.add_parser('lib',
                     help='search library, check library health, detect '
                          'corrupt bib files, duplicates and possibly '
                          'broken pdfs')
  addVerboseArg(s)
  s.add_argument('-f', '--find', nargs='+',
                 help='find library entries that contain given words in '
                      'bibtex fields')
  s.add_argument('-F', '--find-fulltext', nargs='+',
                 help='find library entries that contain given words in '
                      'full manuscript text (requires pdf2txt program to '
                      'to be installed on your $PATH)')
  s.add_argument('-k', '--key', action='store_true',
                 help='print citation key instead of path')
  s.add_argument('-l', '--long', action='store_true',
                 help='print full bibtex entries instead of path only')
  #s.add_argument('-L', '--extra-long', action='store_true',
  #               help='print full pdf2txt result instead of path only')

  # sync subcommand
  s = sub.add_parser('sync',
                     help='sync library pdfs to a tablet or other device')
  addVerboseArg(s)
  s.add_argument('-p', '--sync-path',
                 help='path to the device root that is to be synced.')

  # diff subcommand
  s = sub.add_parser('diff',
                     help='build pdfs that show changes between file versions')
  addVerboseArg(s)
  s.add_argument('-t' , '--old-is-tag', action='store_true',
                 help='do not treat "old" argument as filename, '
                      'but as git tag name (requires git to be installed '
                      'in your $PATH, requires current directory to be '
                      'git repository)')
  s.add_argument('-T' , '--new-is-tag', action='store_true',
                 help='do not treat "new" argument as filename, '
                      'but as git tag name (requires git to be installed '
                      'in your $PATH, requires current directory to be '
                      'git repository)')
  s.add_argument('-o', '--outfile', nargs='?', default='diff.pdf',
                 help='name of the pdf file to be built')
  s.add_argument('-c', '--clean', action='store_true',
                 help='remove all build files except for output pdf')
  s.add_argument('old', help='filename of old version')
  s.add_argument('new', help='filename of new version')
  s.add_argument('filename', nargs='?',
                 help='only used if both --old-is-tag and '
                      '--new-is-tag are set, specifies filename')

  # make-diff subcommand
  s = sub.add_parser('journal',
                     help='find full journal names and abbreviations')
  addVerboseArg(s)
  s.add_argument('-f', '--full', action='store_true',
                 help='print full names of found journals')
  s.add_argument('-a', '--abbreviated', action='store_true',
                 help='print abbreviated names of found journals ')
  s.add_argument('--list-suspicious', action='store_true',
                 help='print all journals in database that might by errornous, '
                      'i.e., abbreviations that do not end with a dot')
  s.add_argument('name', nargs='*',
                 help='journal name, abbreviation or partial name')

  # lint subcommand
  s = sub.add_parser('lint',
                     help='search latex project for potential errors')
  addVerboseArg(s)
  addTexFileArg(s)

  # clean subcommand
  s = sub.add_parser('clean',
                     help='recurse into subfolders of current directory and '
                          'remove latex build files')
  addVerboseArg(s)

  # config subcommand
  s = sub.add_parser('config',
                     help='print location of config file and exit')
  s.add_argument('-o', '--open', action='store_true',
                 help='open config file in vim')
  addVerboseArg(s)

  # enable autocompletion and parse args
  argcomplete.autocomplete(p)
  args = p.parse_args()

  # apply verbosity setting
  io.isVerbose = args.verbose

  # if version is requested, print version and exit
  if args.version:
    io.info(f'paperman version {io.__version__}')
    return

  # select submodule for subcommands
  cmd, cmds = None, None
  if not args.command:
    io.err('subcommand is required')
    return

  elif args.command == 'img':
    from .subcommands import img as cmd

  elif args.command == 'bib':
    from .subcommands import bib as cmd

  elif args.command == 'input':
    from .subcommands import inp as cmd

  elif args.command == 'import-all':
    from .subcommands import inp as cmd1
    from .subcommands import img as cmd2
    from .subcommands import bib as cmd3
    setattr(args, 'import', True)
    args.clean = False
    args.rewrite = False
    args.sort = False
    args.print = False
    cmds = [cmd1, cmd2, cmd3]

  elif args.command == 'sort-authors':
    from .subcommands import sort_authors as cmd

  elif args.command == 'collect':
    from .subcommands import collect as cmd

  elif args.command == 'lib':
    from .subcommands import lib as cmd

  elif args.command == 'sync':
    from .subcommands import sync as cmd

  elif args.command == 'diff':
    from .subcommands import diff as cmd

  elif args.command == 'journal':
    from .subcommands import journal as cmd

  elif args.command == 'lint':
    from .subcommands import lint as cmd

  elif args.command == 'clean':
    from .subcommands import clean as cmd

  elif args.command == 'config':
    from .subcommands import config as cmd

  else:
    io.dbg(f'args={args}')
    raise ValueError(f'args.command has unexpected value')

  # raise error if required config keys are missing
  cfg.testIfRequiredExist()

  # run subcommand module
  cmdStr = str(cmd or '\n'+'\n'.join([str(c) for c in cmds]))
  io.verb(f'selected subcommand(s): {cmdStr}')
  try:
    if cmd:
      cmds = [cmd]
    if cmds:
      for cmd in cmds:
        cmd.main(args)
  except KeyboardInterrupt:
    raise
  except RuntimeError as e:
    io.err(str(e))
  except Exception as e:
    if cfg.get('debug'):
      raise
    io.err(str(e))

if __name__ == '__main__':
  main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 20 14:51:43 2020

@author: andreas
"""

import os
import time

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from Clustering_CAML import Clustering_CAML
from finder import Finder
from scipy.spatial.distance import euclidean
from sklearn.cluster import DBSCAN, OPTICS

# Clustering class

class Clustering:
    def __init__(self, G, basefolder):

        """

        Clusters points generated by `Geometry`

        Parameters
        ----------
        G: Geometry
            The `Geometry` that generates the point to be clustered.
        basefolder: str
            The folder in which the data are saved.
        """
        print(basefolder)
        self.Geometry = G
        self.basefolder = basefolder
        self.min_overlap_per_ref = 0.3

    def fit(self, algo, params=None):
        """

        Parameters
        ----------
        algo: list
            Possible values to  are:
            * 'OPTICS'
            * 'CAML'
            * 'FINDER_1D_loop'
            * 'FINDER_1d'
            * 'FINDER_loop'
            * 'FINDER'
            * 'dbscan'
            *
        params: dict
            The parameters needed for the algorithm.

        Returns
        -------
        results_:
            the labels of the clustering.

        """

        if params is None:
            params = []

        self.algo = algo
        XC = self.Geometry.XC

        result_ = []

        t_start = time.time()

        if algo == "OPTICS":
            params_ = params["OPTICS"]
            threshold = params_["min_samples"]
            xi = params_["xi"]
            eps_max = params_["max_eps"]

            clustering_optics = OPTICS(
                min_samples=threshold,
                xi=xi,
                max_eps=eps_max,
                metric=euclidean,
                cluster_method="xi",
            ).fit(XC)
            # min_cluster_size=.05
            labels = clustering_optics.labels_

        if "CAML" in algo:
            labels = Clustering_CAML(algo, XC, datafolder=self.basefolder)

        elif algo == "FINDER_1D_loop":
            FD = Finder(algo="DbscanLoop", one_two_d="oneD")
            labels = FD.fit(XC)
            result_ = FD.selected_parameters

        elif algo == "FINDER_1D":
            FD = Finder(algo="dbscan", one_two_d="oneD")
            labels = FD.fit(XC)
            result_ = FD.selected_parameters

        elif algo == "FINDER_full_loop":
            FD = Finder(algo="DbscanLoop")
            labels = FD.fit(XC)
            result_ = FD.selected_parameters

        elif algo == "FINDER_full":
            FD = Finder(algo="dbscan")
            labels = FD.fit(XC)
            result_ = FD.selected_parameters

        elif algo == "FINDER_loop":
            FD = Finder(
                algo="DbscanLoop", similarity_score_computation="threshold"
            )
            labels = FD.fit(XC)
            result_ = FD.selected_parameters

            phase_space = FD.phasespace
            phase_space.to_pickle(self.basefolder + "phasespace.pkl")

        elif algo == "FINDER":
            FD = Finder(
                algo="dbscan", similarity_score_computation="threshold"
            )
            labels = FD.fit(XC)
            result_ = FD.selected_parameters

            phase_space = FD.phasespace
            phase_space.to_pickle(self.basefolder + "phasespace.pkl")

        elif algo == "dbscan":
            params_ = params["dbscan"]
            eps = params_["eps"]
            min_samples = params_["min_samples"]
            DB = DBSCAN(eps=eps, min_samples=min_samples).fit(XC)
            labels = DB.labels_

        # clustering_optics = OPTICS(min_samples=threshold, xi=sigma,max_eps=0.3,metric=euclidean,cluster_method='xi').fit(XC) # min_cluster_size=.05

        self.computationTime = time.time() - t_start
        self.labels = labels
        return result_

    def evaluate(self):
        """
        Evaluate the clustering results.
        """

        self.__labelEvaluation()
        self.__clusterEvaluation()
        self.__locEvaluation()
        self.__subClustersPerRefCluster()

    def __labelEvaluation(self):

        """
        Evaluate the labeling performed by the algorithm, comparing it with the ground truth provided
        by the `Geometry` object.

        Returns
        -------
        clusters_correct: np.array

        """

        labels = self.labels
        labels_groundTruth = self.Geometry.labels_groundtruth

        no_clusters = np.max(labels) + 1
        no_clusters_gT = np.max(labels_groundTruth) + 1

        min_overlap_per_ref = self.min_overlap_per_ref
        clusters_correct = -np.ones((no_clusters,), dtype=np.int)

        clusters = [np.where(labels == i)[0] for i in np.arange(no_clusters)]
        clusters_ref = [
            np.where(labels_groundTruth == i)[0]
            for i in np.arange(no_clusters_gT)
        ]

        for i, cl in enumerate(clusters):

            coverage = 0
            idx_chosen = -1

            for ii, cl_ref in enumerate(clusters_ref):
                cl_and_clRef = np.intersect1d(cl, cl_ref)
                clusters_correct[i] = -1
                if len(cl_and_clRef) > min_overlap_per_ref * len(cl_ref):
                    if (len(cl_and_clRef) / len(cl)) > coverage:
                        idx_chosen = ii
                        coverage = len(cl_and_clRef) / len(cl)

            clusters_ref = [
                clusters_ref[i]
                for i in np.arange(len(clusters_ref))
                if (i != idx_chosen)
            ]
            clusters_correct[i] = idx_chosen

        self.clusters_correct = clusters_correct
        return clusters_correct

    def __clusterEvaluation(self):
        """
        Evaluate the performance of the clustering algorithm.

        This is done considering `Clustering.clusters_correct' computed by 'Clustering.__labelEvaluation`.

        Returns
        -------

        cluster_evaluation: dict
            * A dictionary containing:
            * true_positives
            * false_positives
            * false_negatives
            * n_groundtruth
            * false_negative_clusters
            * false_positive_clusters
            * true_positive_clusters
        """

        clusters_correct = self.clusters_correct

        n_groundtruth = np.max((self.Geometry.labels_groundtruth)) + 1
        n_identified = np.max((self.labels)) + 1

        true_positives = np.sum(clusters_correct != -1)
        false_positives = np.sum(clusters_correct == -1)
        false_negatives = n_groundtruth - true_positives

        all_groundtruth = np.arange(n_groundtruth)
        all_identified = np.arange(n_identified)

        false_negative_clusters = np.asarray(
            [a for a in all_groundtruth if (not (a in list(clusters_correct)))]
        )
        false_positive_clusters = np.asarray(
            [i for i in all_identified if (clusters_correct[i] == -1)]
        )
        true_positive_clusters = np.asarray(
            [i for i in all_identified if (clusters_correct[i] != -1)]
        )

        cluster_evaluation = {
            "true_positives": true_positives,
            "false_positives": false_positives,
            "false_negatives": false_negatives,
            "n_groundtruth": n_groundtruth,
            "false_negative_clusters": false_negative_clusters,
            "false_positive_clusters": false_positive_clusters,
            "true_positive_clusters": true_positive_clusters,
        }
        # COMPUTE
        self.cluster_evaluation = cluster_evaluation
        return cluster_evaluation

    def __locEvaluation(self):

        """
        An evaluation performed point by point (instead of comparing clusters).

        Returns
        -------
        dict
            A dict with
            * `true_positives`
            * `false_positives`
            * `false_negatives`
        """

        labels = self.labels
        labels_groundTruth = self.Geometry.labels_groundtruth

        false_positives = np.sum((labels != -1) * (labels_groundTruth == -1))
        false_negatives = np.sum((labels == -1) * (labels_groundTruth != -1))
        true_positives = np.sum((labels != -1) * (labels_groundTruth != -1))

        loc_evaluation = {
            "true_positives": true_positives,
            "false_positives": false_positives,
            "false_negatives": false_negatives,
        }

        self.loc_evaluation = loc_evaluation
        return loc_evaluation

    def __subClustersPerRefCluster(self):

        """

        Returns
        -------
        number_of_subclusters : ndarray
            Entry `i` is how many subclusters reference cluster `i` is divided into
        """
        # Return an array. Entry i is how many subclusters reference cluster
        # i is devided into

        labels_groundTruth = self.Geometry.labels_groundtruth
        labels = self.labels

        n_cl = np.max(labels_groundTruth)

        number_of_subclusters = np.zeros((n_cl + 1,))

        for idx_ref in np.unique(labels_groundTruth):
            if idx_ref == -1:
                continue
            # Get the points
            mark = (labels_groundTruth == idx_ref) * (labels != -1)

            number_of_subclusters[idx_ref] = len(np.unique(labels[mark]))
        self.number_of_subclusters = number_of_subclusters
        return number_of_subclusters

    def plotScatter(self, filename):

        """

        Parameters
        ----------
        filename: str
            Name (and address) of the file to be saved.

        """

        labels = self.labels
        XC = self.Geometry.XC

        # Get correctly detected:
        correct_detected = np.zeros_like(labels)
        for i, cl_ in enumerate(self.clusters_correct):
            if cl_ != -1:  # if not correctly detected
                correct_detected[labels == i] = 1

        fig, ax = plt.subplots()
        mark = labels == -1
        sns.scatterplot(
            x=XC[mark, 0], y=XC[mark, 1], color="grey", s=1, alpha=0.2
        )
        mark = labels >= 0
        sns.scatterplot(
            x=XC[mark, 0],
            y=XC[mark, 1],
            hue=labels[mark],
            palette="bright",
            s=1,
            style=-1 * correct_detected[mark],
            legend=False,
        )
        ax.set_aspect("equal")

        x_0 = 0
        y_0 = np.min(XC[:, 1]) - 50
        ax.plot([x_0, x_0 + 100], [y_0, y_0], "k")
        ax.annotate(
            "$100nm$", (x_0 + 50, y_0 + 10), fontsize="large", ha="center"
        )
        ax.set_aspect(1)
        ax.set_xticks([])
        ax.set_yticks([])
        ax.axis("off")

        plt.savefig(filename)

        # plt.show()


import pandas as pd
import matplotlib.pyplot as plt
import numpy as NP
import sys
Benchmark = input("Which benchmark? ")
sys.stdout=open(Benchmark+"_python_output.txt", "w")

data = pd.read_csv(Benchmark+'_outputlog.txt', skiprows=3)
lengde = len(data)
PC = []
Inst = []
for i in range(int(lengde)):
    testline = data.iloc[i,0]
    if type(testline) is str:
        x = testline.split("] ")
        # print(x)
        for k in x:
            if "inst=[" in k:
                Inst.append(k)
            if "pc=[" in k:
                PC.append(k)

print("Total elements of PC: ",len(PC))
print("Total elements of instructions: ",len(Inst))

df = pd.DataFrame(list(zip(PC,Inst)), columns=['PC', 'instruction'])
values = pd.DataFrame(df['PC'].value_counts()[:10])
fig = plt.figure(figsize=(10, 5))
ax = fig.add_axes([0,0,1,1])
X = values.index
y = [(el[0]/len(PC))*100 for el in values.values]
ax.bar(X,y)
plt.title("PC")
plt.xlabel("PC")
plt.ylabel("%")
plt.ylim([0,30])
plt.xticks(rotation=-20)
plt.savefig(Benchmark+'_pc_plot.png', bbox_inches='tight')

print(df['PC'].value_counts()[:10])
print("% of each address:")
for i in range(10):
    print(i, "=", X[i], "=", y[i],"%")

values = pd.DataFrame(df['instruction'].value_counts()[:10])
fig = plt.figure(figsize=(10, 5))
ax = fig.add_axes([0,0,1,1])
X = values.index
y = [(el[0]/len(Inst))*100 for el in values.values]
ax.bar(X,y)
plt.title("Instruction")
plt.xlabel("Instruction")
plt.ylabel("%")
plt.ylim([0,30])
plt.xticks(rotation=-20)
plt.savefig(Benchmark+'_instruction_plot.png', bbox_inches='tight')

print(df['instruction'].value_counts()[:10])
print("% of each address:")
for i in range(10):
    print(i, "=", X[i], "=", y[i], "%")

sys.stdout.close()


#!/usr/bin/env python3
"""Returning the location of the ISS in latitude/longitude"""
import requests

NASA= "http://api.open-notify.org/iss-now.json"
def main():
    resp= requests.get(NASA).json()

    print("Current location of the ISS: ")
    print("Lon: ", resp["iss_position"]["longitude"])
    print("Lat: ", resp["iss_position"]["latitude"])

if __name__ == "__main__":
    main()


# coding=utf-8
import numpy as np
import tensorflow as tf

def sparse_tuple_from(sequences, dtype=np.int32):
    indices = []
    values = []

    for n, seq in enumerate(sequences):
        indices.extend(zip([n] * len(seq), range(len(seq))))
        values.extend(seq)

    indices = np.asarray(indices, dtype=np.int64)
    values = np.asarray(values, dtype=dtype)
    shape = np.asarray([len(sequences), indices.max(0)[1] + 1], dtype=np.int64)

    # return tf.SparseTensor(indices=indices, values=values, shape=shape)
    return indices, values, shape

def sparsetensor_show(st):
    b = tf.sparse_tensor_to_dense(st, default_value=-1)
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        res1, res2 = sess.run([st, b])
        print("res1:{}".format(res1))
        print("res2:{}".format(res2))

def get_sparsetensor(indices_, values_, dense_shape_):
    st = tf.SparseTensor(indices_, values_, dense_shape_)
    return st

if __name__ == '__main__':
    s = [[10, 27, 32, 14, 50, 6], [1, 2, 3, 4, 5, 9], [11, 22, 33, 44, 55, 666, 77, 88, 99, 400, 1111]]
    indices_t, values_t, shape_t = sparse_tuple_from(s)
    print(indices_t, values_t, shape_t)
    print([1] * 10)
    zipped = zip([0] * 10, [500, 2, 3, 4, 100])
    print(list(zipped))
    s_t = get_sparsetensor(indices_t, values_t, shape_t)
    sparsetensor_show(s_t)
    print(s)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Amna Khalid
October 31, 2018.
Project5.py
This program displays cryptography.
"""
def readfile (filename):
    '''
    This function opens a file, reads it and then converts it into
    a single string variable.
    PARAMETERS:
        filename - name of the file the function is going to open
    RETURN VALUE:
        data - the string that it returns from the file
    '''
    file = open(filename,'r')
    data = file.read()
    file.close()

    return data

def decode(cipher, key):
    '''
    This function deciphers a file.
    PARAMETERS:
        cipher - the file that needs to be deciphered
        key - number of spaces the letters have to move
    RETURN VALUE:
        s - the decoded text

    key = 17
    source = Frankenstein by Mary Shelly - Letter 1
    '''
    s = ''
    for x in range(0,len(cipher)):
        c = cipher[x]
        if c.isalpha():
            t = ord(cipher[x]) - 65
            l = ((t - key) % 26) + 65
            c = chr(l)
        s = s + c

    return s

def encode (plain, key):
    '''
    This function turns a file into a cipher.
    PARAMETERS:
        plain - the file that needs to be ciphered
        key - number of spaces the letters have to move
    RETURN VALUE:
        s - encoded version of the text

    Source = Dr. Jekyll and Mr. Hyde by Robert Louis Stevenson
    '''
    s = ''
    for x in range(0,len(plain)):
        c = plain[x]
        if c.isalpha():
            t = ord(plain[x]) - 65
            l = ((t + key) % 26) + 65
            c = chr(l)

        s = s + c
    return s

def main():
    '''
    Calls other functions and tries different keys for the decode
    function.
    '''
    f1 = readfile('cipher.txt')
    for key in range(0,26):
        d = decode(f1,key)
        print(key,':','\n', d, '\n')
    f2 = readfile('plain.txt')
    e = encode(f2, 11)
    print(e)

main()


class SimpleTreeNode:

    def __init__(self, val, parent):
        self.NodeValue = val  # значение в узле
        self.Parent = parent  # родитель или None для корня
        self.Children = []  # список дочерних узлов

class SimpleTree:

    def __init__(self, root):
        self.Root = root # корень, может быть None

    def AddChild(self, ParentNode, NewChild):
        ParentNode.Children.append(NewChild)
        NewChild.Parent = ParentNode

    def DeleteNode(self, NodeToDelete):
        NodeToDelete.Parent.Children.remove(NodeToDelete)

    def GetAllNodes(self):
        # ваш код выдачи всех узлов дерева в определённом порядке
        if self.Root is None:
            return
        nodes = []
        nodes.append(self.Root)
        all_nodes = []
        while len(nodes) > 0:
            node = nodes.pop()
            all_nodes.append(node)
            if len(node.Children) != 0:
                for child in node.Children:
                    nodes.append(child)

        return all_nodes

    def FindNodesByValue(self, val):
        if self.Root is None:
            return
        nodes = []
        val_nodes = []
        nodes.append(self.Root)
        while len(nodes) > 0:
            node = nodes.pop()
            if node.NodeValue == val:
                val_nodes.append(node)
            if len(node.Children) != 0:
                for child in node.Children:
                    nodes.append(child)

        return val_nodes

    def MoveNode(self, OriginalNode, NewParent):
        NewParent.Children.append(OriginalNode)
        OriginalNode.Parent.Children.remove(OriginalNode)

    def Count(self):
        def calculate_count(node):
            if node is None:
                return 0
            count = 1
            for child in node.Children:
                count += calculate_count(child)
            return count

        return calculate_count(self.Root)

    def LeafCount(self):
        # количество листьев в дереве
        if self.Root is None:
            return 0
        count_leaves = 0
        nodes = []
        nodes.append(self.Root)
        while len(nodes) > 0:
            node = nodes.pop()
            if len(node.Children) != 0:
                for child in node.Children:
                    nodes.append(child)
            else:
                count_leaves += 1

        return count_leaves

    def create_matrix_from_tree(self):
        size = self.Count()
        nodes = self.GetAllNodes()
        tree_matrix = [[] for _ in range(size + 2)]
        for i in range(len(nodes)):
            #if len(nodes[i].Children) != 0:
            for child in nodes[i].Children:
                child_ind = nodes.index(child)
                tree_matrix[i].append(child.NodeValue)
                tree_matrix[child_ind].append(nodes[i].NodeValue)
        return tree_matrix

    def EvenTrees(self):
        nodes = self.GetAllNodes()
        total_num = self.Count()
        tree_matrix = self.create_matrix_from_tree()
        def find_link_to_break(tree, visited, res, node):
            num, temp = 0, 0
            visited[node] = 1

            for i in range(len(tree[node])):
                if visited[tree[node][i]] == 0:
                    temp = find_link_to_break(tree, visited, res, tree[node][i])
                    if temp % 2  == 0:
                        num += temp
                    else:
                        res.append(node)
                        #res.append(i)
            return  num + 1

        visited = [0] * (total_num + 2)
        nodes_to_break_link = []
        find_link_to_break(tree_matrix, visited, nodes_to_break_link, 0)
        result = [node for node in nodes_to_break_link][:2]
        fin = []
        for i in result:
            for node in nodes:
                if node.NodeValue == 1 or node.NodeValue == i:
                    fin.append(node)

        return fin


import winsound

def main():
    freqs = {"la": 220, "si": 247, "do": 261, "re": 293, "mi": 329, "fa": 349, "sol": 392}
    notes = "sol,250-mi,250-mi,500-fa,250-re,250-re,500-do,250-re,250-mi,250-fa,250-sol,250-sol,250-sol,500"

    notes_splitted = notes.split('-')
    itr = iter(notes_splitted)
    while True:
        try:
            note = next(itr).split(',')
            winsound.Beep(freqs[note[0]], int(note[1]))
        except StopIteration:
            break

if __name__ == "__main__":
    main()


import math
import logging

import numpy as np
from scipy import integrate, interpolate, constants

from hyperspy.misc.eels.base_gos import GOSBase

_logger = logging.getLogger(__name__)

XU = [
    .82, .52, .52, .42, .30, .29, .22, .30, .22, .16, .12, .13, .13, .14, .16,
    .18, .19, .22, .14, .11, .12, .12, .12, .10, .10, .10]
# IE3=[73,99,135,164,200,245,294,347,402,455,513,575,641,710,
# 779,855,931,1021,1115,1217,1323,1436,1550,1675]

# IE1=[118,149,189,229,270,320,377,438,500,564,628,695,769,846,
# 926,1008,1096,1194,1142,1248,1359,1476,1596,1727]

R = constants.value("Rydberg constant times hc in eV")

class HydrogenicGOS(GOSBase):

    """Computes the K and L GOS using R. Egerton's  routines.

    Parameters
    ----------
    element_subshell : str
        For example, 'Ti_L3' for the GOS of the titanium L3 subshell

    Methods
    -------
    parametrize_GOS()
        Parametrize the GOS to speed up the calculation.
    get_qaxis_and_gos(ienergy, qmin, qmax)
        Given the energy axis index and qmin and qmax values returns
        the qaxis and gos between qmin and qmax using linear
        interpolation to include qmin and qmax in the range.

    Attributes
    ----------
    energy_axis : array
        The tabulated energy axis
    qaxis : array
        The tabulated qaxis
    energy_onset: float
        The energy onset for the given element subshell as obtained
        from iternal tables.

    Notes
    -----
    The Hydrogeninc GOS are calculated using R. Egerton's SIGMAK3 and
    SIGMAL3 routines that has been translated from Matlab to Python by
    I. Iyengar. See http://www.tem-eels.ca/ for the original code.

    """
    _name = 'hydrogenic'

    def __init__(self, element_subshell):
        """
        Parameters
        ----------

        element_subshell : str
            For example, 'Ti_L3' for the GOS of the titanium L3 subshell

        """
        # Check if the Peter Rez's Hartree Slater GOS distributed by
        # Gatan are available. Otherwise exit

        self.element, self.subshell = element_subshell.split('_')
        self.read_elements()
        self.energy_shift = 0

        if self.subshell[:1] == 'K':
            self.gosfunc = self.gosfuncK
            self.rel_energy_axis = self.get_parametrized_energy_axis(
                50, 3, 50)
        elif self.subshell[:1] == 'L':
            self.gosfunc = self.gosfuncL
            self.onset_energy_L3 = self.element_dict['Atomic_properties'][
                'Binding_energies']['L3']['onset_energy (eV)']
            self.onset_energy_L1 = self.element_dict['Atomic_properties'][
                'Binding_energies']['L1']['onset_energy (eV)']
            self.onset_energy = self.onset_energy_L3
            relative_axis = self.get_parametrized_energy_axis(
                50, 3, 50)
            dL3L2 = self.onset_energy_L1 - self.onset_energy_L3
            self.rel_energy_axis = np.hstack((
                relative_axis[:relative_axis.searchsorted(dL3L2)],
                relative_axis + dL3L2))
        else:
            raise ValueError(
                'The Hydrogenic GOS currently can only'
                'compute K or L shells. Try using Hartree-Slater GOS')

        self.energy_axis = self.rel_energy_axis + self.onset_energy

        info_str = (
            "\nHydrogenic GOS\n" +
            ("\tElement: %s " % self.element) +
            ("\tSubshell: %s " % self.subshell) +
            ("\tOnset Energy = %s " % self.onset_energy))
        _logger.info(info_str)

    def integrateq(self, onset_energy, angle, E0):
        energy_shift = onset_energy - self.onset_energy
        self.energy_shift = energy_shift
        gamma = 1 + E0 / 511.06
        T = 511060 * (1 - 1 / gamma ** 2) / 2
        qint = np.zeros((self.energy_axis.shape[0]))
        for i, E in enumerate(self.energy_axis + energy_shift):
            qa0sqmin = (E ** 2) / (4 * R * T) + (E ** 3) / (
                8 * gamma ** 3 * R * T ** 2)
            p02 = T / (R * (1 - 2 * T / 511060))
            pp2 = p02 - E / R * (gamma - E / 1022120)
            qa0sqmax = qa0sqmin + 4 * np.sqrt(p02 * pp2) * \
                (math.sin(angle / 2)) ** 2

            # dsbyde IS THE ENERGY-DIFFERENTIAL X-SECN (barn/eV/atom)
            qint[i] = 3.5166e8 * (R / T) * (R / E) * (
                integrate.quad(
                    lambda x: self.gosfunc(E, np.exp(x)),
                    math.log(qa0sqmin), math.log(qa0sqmax))[0])
        self.qint = qint
        return interpolate.interp1d(self.energy_axis + energy_shift, qint)

    def gosfuncK(self, E, qa02):
        # gosfunc calculates (=DF/DE) which IS PER EV AND PER ATOM
        z = self.Z
        r = 13.606
        zs = 1.0
        rnk = 1
        if z != 1:
            zs = z - 0.5
            rnk = 2

        q = qa02 / zs ** 2
        kh2 = E / (r * zs ** 2) - 1
        akh = np.sqrt(abs(kh2))
        if akh < 0.01:
            akh = 0.01
        if kh2 >= 0.0:
            d = 1 - np.e ** (-2 * np.pi / akh)
            bp = np.arctan(2 * akh / (q - kh2 + 1))
            if bp < 0:
                bp = bp + np.pi
            c = np.e ** ((-2 / akh) * bp)
        else:
            d = 1
            y = -1 / akh * np.log((q + 1 - kh2 + 2 * akh) / (
                q + 1 - kh2 - 2 * akh))
            c = np.e ** y
        a = ((q - kh2 + 1) ** 2 + 4 * kh2) ** 3
        return 128 * rnk * E / (
            r * zs ** 4) * c / d * (q + kh2 / 3 + 1 / 3) / (a * r)

    def gosfuncL(self, E, qa02):
        # gosfunc calculates (=DF/DE) which IS PER EV AND PER ATOM
        # Note: quad function only works with qa02 due to IF statements in
        # function

        z = self.Z
        r = 13.606
        zs = z - 0.35 * (8 - 1) - 1.7
        iz = z - 11
        if iz >= len(XU):
            # Egerton does not tabulate the correction for Z>36.
            # This produces XSs that are within 10% of Hartree-Slater XSs
            # for these elements.
            u = .1
        else:
            # Egerton's correction to the Hydrogenic XS
            u = XU[int(iz)]
        el3 = self.onset_energy_L3 + self.energy_shift
        el1 = self.onset_energy_L1 + self.energy_shift

        q = qa02 / zs ** 2
        kh2 = E / (r * zs ** 2) - 0.25
        akh = np.sqrt(abs(kh2))
        if kh2 >= 0.0:
            d = 1 - np.exp(-2 * np.pi / akh)
            bp = np.arctan(akh / (q - kh2 + 0.25))
            if bp < 0:
                bp = bp + np.pi
            c = np.exp((-2 / akh) * bp)
        else:
            d = 1
            y = -1 / akh * \
                np.log((q + 0.25 - kh2 + akh) / (q + 0.25 - kh2 - akh))
            c = np.exp(y)

        if E - el1 <= 0:
            g = 2.25 * q ** 4 - (0.75 + 3 * kh2) * q ** 3 + (
                0.59375 - 0.75 * kh2 - 0.5 * kh2 ** 2) * q * q + (
                0.11146 + 0.85417 * kh2 + 1.8833 * kh2 * kh2 + kh2 ** 3) * \
                q + 0.0035807 + kh2 / 21.333 + kh2 * kh2 / 4.5714 + kh2 ** 3 \
                / 2.4 + kh2 ** 4 / 4

            a = ((q - kh2 + 0.25) ** 2 + kh2) ** 5
        else:
            g = q ** 3 - (5 / 3 * kh2 + 11 / 12) * q ** 2 + (
                kh2 * kh2 / 3 + 1.5 * kh2 + 65 / 48) * q + kh2 ** 3 / 3 + \
                0.75 * kh2 * kh2 + 23 / 48 * kh2 + 5 / 64
            a = ((q - kh2 + 0.25) ** 2 + kh2) ** 4
        rf = ((E + 0.1 - el3) / 1.8 / z / z) ** u
        # The following commented lines are to give a more accurate GOS
        # for edges presenting white lines. However, this is not relevant
        # for quantification by curve fitting.
        # if abs(iz - 11) <= 5 and E - el3 <= 20:
        #     rf = 1
        return rf * 32 * g * c / a / d * E / r / r / zs ** 4


import time
from datetime import datetime, timedelta
from urllib.parse import urlparse
from typing import Callable, Dict
import pandas as pd
import requests
from bs4 import BeautifulSoup
from lib.helper import configs, setup_logger

logger = setup_logger(' SCRAPER:PY ', 'lib/log.xml', level="INFO")
header = configs.header
timeout = configs.timeout

def get_request(url, proxies=None):
    try:
        response = requests.get(url, headers=header, timeout=timeout, proxies=proxies)  # TODO Proxy ayarları eklenecek
        response.raise_for_status()
        return response

    except requests.exceptions.HTTPError as errh:
        logger.error("HTTP Hatası: %s. Proxy: %s", errh, proxies)
        raise errh
    except requests.exceptions.ConnectionError as errc:
        logger.error("Bağlantı Hatası: %s.  Proxy: %s", errc, proxies)
        raise errc
    except requests.exceptions.Timeout as errt:
        logger.error("Zaman Aşımı Hatası: %s.  Proxy: %s", errt, proxies)
        raise errt
    except requests.exceptions.RequestException as err:
        logger.error("Bir Hata Oluştu: %s.  Proxy: %s", err, proxies)
        raise err

def amazon(url):
    response = get_request(url)
    try:

        runtime = datetime.now()
        soup = BeautifulSoup(response.text, 'html.parser')
        title = soup.select_one('#productTitle').getText().strip()
        try:
            price = soup.select_one(
                '#corePriceDisplay_desktop_feature_div > div.a-section.a-spacing-none.aok-align-center > span > span:nth-child(2)').getText()
        except:
            price = soup.find(id="price").getText()

        ean = soup.find(id="ASIN").get("value")
        stock = False if soup.find('input', value="Add to Basket") == None else True

        results = {"Product ID": "", "Runtime DateTime": runtime, "Product Title": title, "Price": price,
                   "EAN": ean,
                   "Stock Availability": stock}

        return results
    except (AttributeError, TypeError, ValueError) as err:
        logger.error("Amazon BeautifulSoup Hatası: %s. URL: %s", err, url)
        raise
    except Exception as e:
        raise e

def argos(url):
    response = get_request(url)
    try:

        soup = BeautifulSoup(response.text, 'html.parser')
        runtime = datetime.now()
        title = soup.select_one(
            '#content > main > div:nth-child(2) > div.xs-block > div:nth-child(1) > section.xs-12--none.md-7.xl-8.pdp-core.bolt-v2 > div.Namestyles__ProductName-sc-269llv-0.kEQsqD.bolt-v2 > h1 > span:nth-child(1)'
        ).getText().strip()

        price = soup.select_one(
            '#content > main > div:nth-child(2) > div.xs-block > div:nth-child(1) > section.xs-12--none.md-5--none.xl-4--none.pdp-right > section > ul > li > h2'
        ).getText()
        ean = soup.select_one(
            '#content > main > div:nth-child(2) > div.xs-block > div:nth-child(1) > section.xs-12--none.md-7.xl-8.pdp-core.bolt-v2 > div.Namestyles__ProductName-sc-269llv-0.kEQsqD.bolt-v2 > h1 > span.Namestyles__CatNumber-sc-269llv-2.lbKeLk'
        ).getText()

        stock = False if soup.find('button', {"data-test": "add-to-trolley-button-button"}) == None else True

        results = {"Product ID": "", "Runtime DateTime": runtime, "Product Title": title, "Price": price,
                   "EAN": ean,
                   "Stock Availability": stock}

        return results
    except (AttributeError, TypeError, ValueError) as err:
        logger.error("Argos BeautifulSoup Hatası: %s. URL: %s", err, url)
        raise
    except Exception:
        pass

def bestwaywholesale(url):
    response = get_request(url)
    try:

        soup = BeautifulSoup(response.text, 'html.parser')
        runtime = datetime.now()
        title = soup.select_one(
            '#shop-products > div.productpagedetail-inner > div.right > h1'
        ).getText().strip()
        table = soup.find('table', {'class': 'prodtable'})
        price = ""
        ean = ""
        for tr in table.find_all('tr'):
            th = tr.find('th').text
            td = tr.find('td').text
            if ("RSP" in th):
                price = td
            if ("EAN" in th):
                ean = td

        stock = False if soup.find('span', {"class": "must"}) == None else True
        results = {"Product ID": "", "Runtime DateTime": runtime, "Product Title": title, "Price": price,
                   "EAN": ean, "Stock Availability": stock}

        return results
    except (AttributeError, TypeError, ValueError) as err:
        logger.error("Bestwaywholesale BeautifulSoup Hatası: %s. URL: %s", err, url)
        raise
    except Exception:
        raise

def read_data():
    # Son bir saat içindeki okunmamış dataları DataFrame olarak döndürür.
    try:
        df = pd.read_csv(configs.input_file)
    except Exception as e:
        logger.error("İnput file okuma hatası ERR: {}".format(e))
        raise e
    else:
        df.dropna(subset=['Products'], inplace=True)
        try:
            df['LastRead'] = pd.to_datetime(df['LastRead'])  # TODO Dt atama işlemi gereksiz olabilir bak!
            df['LastScrape'] = pd.to_datetime(df['LastScrape'])
        except Exception as e:
            logger.error("İnput file sütun isimleri hatalı ERR: {}".format(e))
            raise e
        return df

scrapers: Dict[str, Callable[[str], dict]] = {
    "www.amazon.co.uk": amazon,
    "www.argos.co.uk": argos,
    "www.bestwaywholesale.co.uk": bestwaywholesale

}

def run_scraper():
    try:
        df = read_data()
        last_read = datetime.now() - timedelta(minutes=configs.run_period)
        selected_product_df = df[df['LastRead'] < last_read][:configs.max_product]
        selected_product_df['LastRead'] = datetime.now()

        scraped_product_list = []

        if selected_product_df is not None:
            for index, row in selected_product_df.iterrows():

                for url in row['Products'].split(';'):  # TODO Loglama ve hata handle
                    try:
                        scraper = scrapers.get(urlparse(url).netloc)
                        print(url)

                        result = scraper(url)
                        if result is not None:
                            dict_data = result
                            dict_data['Product ID'] = str(index)
                            dict_data['Runtime DateTime'] = str(datetime.now())
                            scraped_product_list.append(dict_data)

                            time.sleep(configs.frequency)
                    except TypeError:
                        logger.error("Scraper Bulunamadı : {} ".format(url))
                    except Exception as e:
                        # logger.error(e)
                        pass
                selected_product_df.at[
                    index, 'LastScrape'] = datetime.now()  # TODO işlem başarıyla tamamlanınca değişmeli
            df.update(selected_product_df)
            df.to_csv(configs.input_file, header=True, mode='w', index=False)

            output_data = pd.DataFrame(scraped_product_list,
                                       columns=["Product ID", "Runtime DateTime", "Product Title", "Price",
                                                "EAN", "Stock Availability"])
            output_data.to_csv(configs.output_file, header=False, mode='a', index=False)

    except Exception as e:
        logger.error(e)
        # raise e

if __name__ == "__main__":
    while True:
        run_scraper()
        print("Scraper çalışıyor : ", datetime.now())
        time.sleep(5)


#
# @lc app=leetcode.cn id=154 lang=python3
#
# [154] 寻找旋转排序数组中的最小值 II
#
from typing import List
import unittest
# @lc code=start

class Solution:
    def findMin(self, nums: List[int]) -> int:
        left, right = 0, len(nums)-1
        while left < right:
            mid = (left + right) // 2
            if nums[mid] > nums[right]:
                left = mid + 1
            elif nums[mid] < nums[right]:
                right = mid
            else:  # nums[mid] == nums[right]
                right -= 1

        return nums[left]

# @lc code=end

class Test(unittest.TestCase):
    def test(self):
        def f(arr, ans):
            self.assertEqual(Solution().findMin(arr), ans)
        f([3, 4, 5, 1, 2], 1)
        f([2, 2, 2, 0, 1], 0)
        f([1], 1)
        f([1, 1], 1)
        f([1, 1, 1], 1)
        f([1, 1, 1, 1], 1)
        f([1, 1, 2, 1, 1], 1)
        f([1, 1, 0, 1, 1], 0)
        f([1, 2, 3, 4, 5], 1)
        f([0, 0, 0, -1, 0], -1)
        f([3, 1], 1)
        f([5, 1, 1, 1, 1, 1, 1], 1)

if __name__ == "__main__":
    unittest.main()

154.寻找旋转排序数组中的最小值-ii.py


import serial.tools.list_ports as port_list
import serial
from time import sleep

def idn_scan(dump_yml=False, baud_rates=(9600,)):
    """
    Sends "*IDN?" command to all COM ports where hardware is present.
    Commands are sent blindly with EOL (\r\n) termination and at baud_rates specified.
    Parameters
    ----------
    dump_yml - enables (True) or disables (False) dumping of .yaml file containing the mapping of
                IDN string to COM and BAUD
    baud_rates - tuple of baud rates at which the IDN query sequence will be repeated

    Returns - idn_to_com dictionary mapping IDN string (key) to a dictionary of (PORT) and (BAUD)
    This dictionary is also output to a .yaml file if dump_yaml argument is set to true.
    -------

    """
    TIMEOUT = 0.5  # If a large number of serial devices are present you might wish to shorten the timeout time to
    # speed up the query rounds however this may be at the cost of some devices not having enough time to respond

    idn_to_com = {}

    for baud in baud_rates:
        print(f'\n<NOTE>: *IDN? requests are sent blindly as:\n ASCII, EOL terminated,\n baudrate={str(baud)}, '
              f'bytesize=EIGHTBITS, \n parity=PARITY_NONE, stopbits=STOPBITS_ONE \n Device may not respond with IDN '
              f'if:\n  -device uses other baudrate or encoding, \n  -device is non SCPI compliant.\n\n')

        for port, desc, hwid in port_list.comports():

            try:
                ser = serial.Serial(port, timeout=TIMEOUT, baudrate=baud, writeTimeout=0)
                sleep(0.5)
                ser.write(b'*IDN?\r\n')
                idn = ser.readline()
                print(f'PORT: {port}\n IDN: {idn}\n DESC: {desc} \n HWID: {hwid}\n')
                ser.close()
                if idn != b'':
                    idn_to_com.update({idn.strip(): {'PORT': port, 'BAUD': baud}})
            except serial.SerialException as exc:
                print(f'PORT: {port}\n IDN: ?\n DESC: {exc} \n HWID: ?\n')

    if dump_yml:
        import yaml
        with open('idn_to_com.yml', 'w') as f:
            f.write(yaml.dump(idn_to_com, default_flow_style=False))

    return idn_to_com

if __name__ == "__main__":
    import sys

    if len(sys.argv) == 1:
        idn_scan()
    elif len(sys.argv) == 2:
        idn_scan(dump_yml=(sys.argv[1] == 'dump_yml=True'))
    else:
        idn_scan(dump_yml=(sys.argv[1] == 'dump_yml=True'), baud_rates=tuple(sys.argv[2:]))