from typing import Dict, Iterable, Optional

from crowdin_api.api_resources.abstract.resources import BaseResource
from crowdin_api.api_resources.enums import ExportProjectTranslationFormat
from crowdin_api.api_resources.translations.enums import (
    CharTransformation,
    PreTranslationApplyMethod,
    PreTranslationAutoApproveOption,
)

class TranslationsResource(BaseResource):
    """
    Resource for Translations.

    Translators can work with entirely untranslated project or you can pre-translate the files to
    ease the translations process.

    Use API to pre-translate files via Machine Translation (MT) or Translation Memory (TM), upload
    your existing translations, and download translations correspondingly. Pre-translate and build
    are asynchronous operations and shall be completed with sequence of API methods.

    Link to documentation:
    https://developer.crowdin.com/api/v2/#tag/Translations
    """

    def get_builds_path(self, projectId: int, buildId: Optional[int] = None):
        if buildId:
            return f"projects/{projectId}/translations/builds/{buildId}"

        return f"projects/{projectId}/translations/builds"

    def pre_translation_status(
        self, preTranslationId: str, projectId: Optional[int] = None
    ):
        """
        Pre-Translation Status.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#tag/Translations/paths/~1projects~1{projectId}~1pre-translations~1{preTranslationId}/get
        """

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="get",
            path=f"projects/{projectId}/pre-translations/{preTranslationId}",
        )

    def apply_pre_translation(
        self,
        languageIds: Iterable[str],
        fileIds: Iterable[int],
        projectId: Optional[int] = None,
        method: Optional[PreTranslationApplyMethod] = None,
        engineId: Optional[int] = None,
        autoApproveOption: Optional[PreTranslationAutoApproveOption] = None,
        duplicateTranslations: Optional[bool] = None,
        translateUntranslatedOnly: Optional[bool] = None,
        translateWithPerfectMatchOnly: Optional[bool] = None,
        labelIds: Optional[Iterable[int]] = None,
        excludeLabelIds: Optional[Iterable[int]] = None,
    ):
        """
        Apply Pre-Translation.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.pre-translations.post
        """
        if labelIds is None:
            labelIds = []

        if excludeLabelIds is None:
            excludeLabelIds = []

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="post",
            path=f"projects/{projectId}/pre-translations",
            request_data={
                "languageIds": languageIds,
                "fileIds": fileIds,
                "method": method,
                "engineId": engineId,
                "autoApproveOption": autoApproveOption,
                "duplicateTranslations": duplicateTranslations,
                "translateUntranslatedOnly": translateUntranslatedOnly,
                "translateWithPerfectMatchOnly": translateWithPerfectMatchOnly,
                "labelIds": labelIds,
                "excludeLabelIds": excludeLabelIds,
            },
        )

    def build_project_directory_translation(
        self,
        directoryId: int,
        projectId: Optional[int] = None,
        targetLanguageIds: Optional[Iterable[str]] = None,
        skipUntranslatedStrings: Optional[bool] = None,
        skipUntranslatedFiles: Optional[bool] = None,
        exportApprovedOnly: Optional[bool] = None,
    ):
        """
        Build Project Directory Translation.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.directories.post
        """

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="post",
            path=f"{self.get_builds_path(projectId=projectId)}/directories/{directoryId}",
            request_data={
                "targetLanguageIds": targetLanguageIds,
                "skipUntranslatedStrings": skipUntranslatedStrings,
                "skipUntranslatedFiles": skipUntranslatedFiles,
                "exportApprovedOnly": exportApprovedOnly,
            },
        )

    def build_project_file_translation(
        self,
        fileId: int,
        targetLanguageId: str,
        projectId: Optional[int] = None,
        skipUntranslatedStrings: Optional[bool] = None,
        skipUntranslatedFiles: Optional[bool] = None,
        exportApprovedOnly: Optional[bool] = None,
        eTag: Optional[str] = None,
    ):
        """
        Build Project File Translation.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.files.post
        """

        if eTag is not None:
            headers = {"If-None-Match": eTag}
        else:
            headers = None

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="post",
            headers=headers,
            path=f"{self.get_builds_path(projectId=projectId)}/files/{fileId}",
            request_data={
                "targetLanguageId": targetLanguageId,
                "skipUntranslatedStrings": skipUntranslatedStrings,
                "skipUntranslatedFiles": skipUntranslatedFiles,
                "exportApprovedOnly": exportApprovedOnly,
            },
        )

    def list_project_builds(
        self,
        projectId: Optional[int] = None,
        branchId: Optional[int] = None,
        page: Optional[int] = None,
        offset: Optional[int] = None,
        limit: Optional[int] = None,
    ):
        """
        List Project Builds.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.getMany
        """

        projectId = projectId or self.get_project_id()
        params = {"branchId": branchId}
        params.update(self.get_page_params(page=page, offset=offset, limit=limit))

        return self._get_entire_data(
            method="get",
            path=self.get_builds_path(projectId=projectId),
            params=params,
        )

    def build_project_translation(
        self, request_data: Dict, projectId: Optional[int] = None
    ):
        """
        Build Project Translation.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.post
        """

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="post",
            path=self.get_builds_path(projectId=projectId),
            request_data=request_data,
        )

    def build_crowdin_project_translation(
        self,
        projectId: Optional[int] = None,
        branchId: Optional[int] = None,
        targetLanguageIds: Optional[Iterable[str]] = None,
        skipUntranslatedStrings: Optional[bool] = None,
        skipUntranslatedFiles: Optional[bool] = None,
        exportApprovedOnly: Optional[bool] = None,
        exportWithMinApprovalsCount: Optional[int] = None,
    ):
        """
        Build Project Translation(Crowdin Translation Create Project Build Form).

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.post
        """

        projectId = projectId or self.get_project_id()

        return self.build_project_translation(
            projectId=projectId,
            request_data={
                "branchId": branchId,
                "targetLanguageIds": targetLanguageIds,
                "skipUntranslatedStrings": skipUntranslatedStrings,
                "skipUntranslatedFiles": skipUntranslatedFiles,
                "exportApprovedOnly": exportApprovedOnly,
                "exportWithMinApprovalsCount": exportWithMinApprovalsCount,
            },
        )

    def build_pseudo_project_translation(
        self,
        pseudo: bool,
        projectId: Optional[int] = None,
        prefix: Optional[str] = None,
        suffix: Optional[str] = None,
        lengthTransformation: Optional[int] = None,
        charTransformation: Optional[CharTransformation] = None,
    ):
        """
        Build Project Translation(Translation Create Project Pseudo Build Form).

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.post
        """

        projectId = projectId or self.get_project_id()

        return self.build_project_translation(
            projectId=projectId,
            request_data={
                "pseudo": pseudo,
                "prefix": prefix,
                "suffix": suffix,
                "lengthTransformation": lengthTransformation,
                "charTransformation": charTransformation,
            },
        )

    def upload_translation(
        self,
        languageId: str,
        storageId: int,
        fileId: int,
        projectId: Optional[int] = None,
        importEqSuggestions: Optional[bool] = None,
        autoApproveImported: Optional[bool] = None,
        translateHidden: Optional[bool] = None,
    ):
        """
        Upload Translations.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.postOnLanguage
        """

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="post",
            path=f"projects/{projectId}/translations/{languageId}",
            request_data={
                "storageId": storageId,
                "fileId": fileId,
                "importEqSuggestions": importEqSuggestions,
                "autoApproveImported": autoApproveImported,
                "translateHidden": translateHidden,
            },
        )

    def download_project_translations(
        self, buildId: int, projectId: Optional[int] = None
    ):
        """
        Download Project Translations.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.download.download
        """

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="get",
            path=f"{self.get_builds_path(projectId=projectId, buildId=buildId)}/download",
        )

    def check_project_build_status(self, buildId: int, projectId: Optional[int] = None):
        """
        Check Project Build Status.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.get
        """

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="get",
            path=self.get_builds_path(projectId=projectId, buildId=buildId),
        )

    def cancel_build(self, buildId: int, projectId: Optional[int] = None):
        """
        Cancel Build.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.builds.delete
        """

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="delete",
            path=self.get_builds_path(projectId=projectId, buildId=buildId),
        )

    def export_project_translation(
        self,
        targetLanguageId: str,
        projectId: Optional[int] = None,
        format: Optional[ExportProjectTranslationFormat] = None,
        labelIds: Optional[Iterable[int]] = None,
        branchIds: Optional[Iterable[int]] = None,
        directoryIds: Optional[Iterable[int]] = None,
        fileIds: Optional[Iterable[int]] = None,
        skipUntranslatedStrings: Optional[bool] = None,
        skipUntranslatedFiles: Optional[bool] = None,
        exportApprovedOnly: Optional[bool] = None,
    ):
        """
        Export Project Translation.

        Link to documentation:
        https://developer.crowdin.com/api/v2/#operation/api.projects.translations.exports.post
        """

        projectId = projectId or self.get_project_id()

        return self.requester.request(
            method="post",
            path=f"projects/{projectId}/translations/exports",
            request_data={
                "targetLanguageId": targetLanguageId,
                "format": format,
                "labelIds": labelIds,
                "branchIds": branchIds,
                "directoryIds": directoryIds,
                "fileIds": fileIds,
                "skipUntranslatedStrings": skipUntranslatedStrings,
                "skipUntranslatedFiles": skipUntranslatedFiles,
                "exportApprovedOnly": exportApprovedOnly,
            },
        )


import subprocess
from datetime import datetime
import csv
import time

cmd_prefix = 'sudo java -d64 -Xms15g -Xmx15g -XX:MaxDirectMemorySize=50G -cp .:lib/*:bin/'
server_ip = '192.5.87.104'
bandwidth = '10000'
rtt = '32'
tcp_buf = '40'
max_channels = '60'
alg_interval = '10'
output_log = 'output/results_LuigiMin_diffetentActiveCores_Sept5.csv'

#datasets = ['20000 5000 128','0 0 128','0 5000 0','20000 0 0']
#datasets = ['20000 0 0','0 5000 0','0 0 128','20000 5000 128']
datasets = ['20000 0 0']
numActiveCoresSet = ['24','12','6','4','2']
#algorithms = ['ismailMinEnergy', 'ismailMaxThroughput', 'luigiMinEnergy', 'luigiEEMT']
#algorithms = ['luigiMinEnergy','ismailMinEnergy', 'luigiEEMT', 'ismailMaxThroughput']
algorithms = ['luigiMinEnergy']

for i in range(3):
    print('\nStarting iteration ' + str(i))
    for dataset in datasets:
        for numActiveCores in numActiveCoresSet:
            for algorithm in algorithms:
                if algorithm == 'ismailMinEnergy' or algorithm == 'ismailMaxThroughput':
                    tcp_buf = '40'
                cmd = cmd_prefix + ' ' + \
                    'algorithms.OptimalDataTransfer' + ' ' + \
                    algorithm + ' ' + \
                    dataset + ' ' + \
                    server_ip + ' ' + \
                    str(80) + ' ' + \
                    bandwidth + ' ' + \
                    rtt + ' ' + \
                    tcp_buf + ' ' + \
                    max_channels + ' ' + \
                    alg_interval + ' ' + \
                    output_log + ' ' + \
                    numActiveCores

            print('Start transfer of ' + dataset + ' with algorithm = ' + algorithm + ' iteration' + str(i) + ' maxChannels = ' + max_channels)
            subprocess.run(cmd, shell=True)
            print('Finished transfer of ' + dataset + ' with algorithm = ' + algorithm + ' iteration' + str(i) + ' maxChannels = ' + max_channels)
            time.sleep(10)


import pathlib
import requests
from pathlib import Path

def getLinkAndToken():
    path = Path(pathlib.Path.cwd(),'mispibot','src','main','resources','application.properties')
    print(str(path))
    paramlist = []
    with open(path) as file:
        for line in file:
            if 'bot.path' in line or 'bot.token' in line:
                paramlist.append(str(line.strip('\n').split('=')[1].strip()))

    print(paramlist)
    return paramlist

def regTelegramWebHook(linkFromNgrok,token):
    link = "https://api.telegram.org/bot"+token+"/setwebhook?url="+linkFromNgrok+"/bots/index.php"
    print(link)
    response = requests.get(link)
    valueslist = []
    for item in response.json().values():
        valueslist.append(item)

    if valueslist[0]:
        print("удачненько зарегались 👍")
    elif not valueslist[0]:
        print(response.text)
        print("не зарегано 👎")
    else:
        print(response.text)
        print('непонял ошибку☹☹')

def main():

    linkAndToken = getLinkAndToken()
    link = 0
    token = 0
    for item in linkAndToken:
        if 'https' in item:
            link = item
        else:
            token = item

    regTelegramWebHook(link,token)

if __name__ == "__main__":
    main()


from ast import literal_eval
from distutils.core import setup

def get_version(source='src/vcfarray.py'):
    with open(source) as f:
        for line in f:
            if line.startswith('VERSION'):
                return literal_eval(line.partition('=')[2].lstrip())
    raise ValueError("VERSION not found")

setup(
    name='vcfarray',
    version=get_version(),
    author='Alistair Miles',
    author_email='alimanfoo@googlemail.com',
    package_dir={'': 'src'},
    py_modules=['vcfarray'],
    url='https://github.com/alimanfoo/pyvcfarray',
    license='MIT License',
    description='Please note, development of this package has moved to https://github.com/alimanfoo/vcfnp',
    classifiers=['Intended Audience :: Developers',
                 'License :: OSI Approved :: MIT License',
                 'Programming Language :: Python',
                 'Topic :: Software Development :: Libraries :: Python Modules'
                 ]
)


import re
from orangecontrib.bio.ontology import OBOOntology
import itertools
import logging
from pprint import pprint

# The IDs from the ontology file
TECH_REP = "EFO:0002090"
BIOL_REP = "EFO:0002091"
HOUR_TERM = "UO:0000032"
MINUTE_TERM = "UO:0000031"
DAY_TERM = "UO:0000033"

def get_rex_value(regex, str):
    """
    Wrapper for regex parsing functions.

    Returns None if parsing fails.
    """
    try:
        return regex.search(str).group()
    except AttributeError:
        return None

def get_obo_id(str):
    """
    Get OBO id from column header.

    Throw exception if parsing fails.

    >>> get_obo_id("tpm of 293SLAM rinderpest infection, 00hr, biol_rep1.CNhs14406.13541-145H4")
    'FF:13541-145H4'
    """
    OBO_ID_REGEX = re.compile(r'CNhs\d+.(\w+)-(\w+)')
    return "FF:" + "-".join(OBO_ID_REGEX.search(str).groups())

def get_lib_id(str):
    """
    Get library id from column header

    Throw exception if parsing failes.

    >>> get_lib_id("tpm of 293SLAM rinderpest infection, 00hr, biol_rep1.CNhs14406.13541-145H4")
    'CNhs14406'
    """
    LIB_ID_REGEX = re.compile(r'CNhs(\d+)')
    return LIB_ID_REGEX.search(str).group()

def get_time(str):
    """
    Parse time from column header

    >>> get_time("tpm of 293SLAM rinderpestday01 infectionday01, 00hr, biol_rep1.CNhs14406.13541-145H4")
    Traceback (most recent call last):
    ...
    AssertionError: multiple times per unit day not allowed: tpm of 293SLAM rinderpestday01 infectionday01, 00hr, biol_rep1.CNhs14406.13541-145H4
    >>> get_time("tpm of 293SLAM rinderpestd infectionday01, 00hr, biol_rep1.CNhs14406.13541-145H4")
    'day01;00hr'
    >>> get_time("tpm of 293SLAM rinderpestd infectionday12, biol_rep1.CNhs14406.13541-145H4")
    'day12'
    >>> get_time("tpm of 293SLAM rinderpestdayd01 infectionddady01, 00hxr, biol_rep1.CNhs14406.13541-145H4")

    >>> get_time("tpm of 293SLAM rinderpestd infectiond, 01hr40min, biol_rep1.CNhs14406.13541-145H4")
    '01hr40min'
    """
    time = {"hr": None, "min": None, "day": None}
    TIME_REGEX = re.compile(r'(\d+)(hr|min)|day(\d+)')
    INT_REGEX = re.compile(r'(\d+)')
    for match_obj in TIME_REGEX.finditer(str):
        time_str = match_obj.group()
        for unit in time:
            if unit in time_str:
                assert time[unit] is None, 'multiple times per unit {} not allowed: {}'.format(unit, str)
                time[unit] = INT_REGEX.search(time_str).group()

    return time_dict_to_str(time)

def time_dict_to_str(time_dict):
    """
    Format a dictionary `time = {"hr": j, "min": k, "day": l}` to a string e.g. 02hr40min.

    >>> time_dict_to_str({"hr": None, "min": None, "day": None})

    >>> time_dict_to_str({"hr": None, "min": None, "day": 1})
    'day01'
    >>> time_dict_to_str({"hr": 12, "min": None, "day": "01"})
    'day01;12hr'
    >>> time_dict_to_str({"hr": 12, "min": 15, "day": None})
    '12hr15min'
    """
    out = []
    if time_dict["day"] is not None:
        out.append("day{:02d}".format(int(time_dict["day"])))
        if time_dict["hr"] is not None or time_dict['min'] is not None:
            out.append(";")  # we need a separator in that case
    if time_dict['hr'] is not None:
        out.append("{:02d}hr".format(int(time_dict['hr'])))
    if time_dict['min'] is not None:
        out.append("{:02d}min".format(int(time_dict['min'])))
    out = "".join(out)
    if out == "":
        return None
    else:
        return out

def get_donor(str):
    """
    Parse donor from column header.

    >>> get_donor("tpm of 293SLAM rinderpestdayd01 infection, 00hr, donor1, biol_rep1.CNhs14406.13541-145H4")
    'donor1'
    >>> get_donor("tpm of 293SLAM rinderpestdayd01 infection, 00hr, dondor1, biol_rep1.CNhs14406.13541-145H4")

    """
    DONOR_REGEX = re.compile(r'donor(\d+)')
    return get_rex_value(DONOR_REGEX, str)

def get_biol_replicate(str):
    """
    Parse biological replicate from column header.
    Both rep1 and biol_rep1 are considered as a biological replicate.

    >>> get_biol_replicate("tpm of 293SLAM rinderpestdayd01 infection, 00hr, donor1, biol_rep1.CNhs14406.13541-145H4")
    'biol_rep1'
    >>> get_biol_replicate("tpm of 293SLAM rinderpestdayd01 infection, 00hr, donor1, rep1.CNhs14406.13541-145H4")
    'rep1'
    >>> get_biol_replicate("tpm of 293SLAM rinderpestdayd01 infection, 00hr, donor1, CNhs14406.13541-145H4")

    >>> get_biol_replicate("tpm of 293SLAM rinderpestdayd01 infection, 00hr, donor1, some_rep1.CNhs14406.13541-145H4")

    """
    REPLICATE_REGEX = re.compile(r'([ ]+|biol_)rep(\d+)')
    rep = get_rex_value(REPLICATE_REGEX, str)
    if rep is not None:
        rep = rep.strip()
    return rep

def get_tech_replicate(str):
    """
    Parse replicate from column header

    >>> get_tech_replicate("tpm of 293SLAM rinderpestdayd01 infection, 00hr, donor1, tech_rep1.CNhs14406.13541-145H4")
    'tech_rep1'
    >>> get_tech_replicate("tpm of 293SLAM rinderpestdayd01 infection, 00hr, donor1, rep1.CNhs14406.13541-145H4")

    """
    REPLICATE_REGEX = re.compile(r'tech_rep(\d+)')
    return get_rex_value(REPLICATE_REGEX, str)

def contains_term(term, tag_name, tag_value):
    """Check if a set of tags (from a certain term) contains a certain annotation (e.g. is_a: 24-hour-sample)"""
    return any(a == tag_name and b == tag_value for a, b in term.tags())

def get_time_terms(obo):
    hour_terms = [term.id for term in obo.child_terms(HOUR_TERM)]
    minute_terms = [term.id for term in obo.child_terms(MINUTE_TERM)]
    day_terms = [term.id for term in obo.child_terms(DAY_TERM)]
    return hour_terms, minute_terms, day_terms

def is_time_term(obo, term_id):
    """
    Return true if the term_id is a time-ontology

    >>> obo = OBOOntology()
    >>> obo.load(open("test/testdata/ff-phase2-140729.obo"))
    >>> is_time_term(obo, "FF:0000357")
    True
    >>> is_time_term(obo, "FF:0350357")
    False
    """
    return True if term_id in itertools.chain(*get_time_terms(obo)) else False

def ontology_to_time(obo, obo_id):
    """
    Convert an ontology id to a readable time string.

    >>> obo = OBOOntology()
    >>> obo.load(open("test/testdata/ff-phase2-140729.obo"))
    >>> ontology_to_time(obo, "FF:0000357")
    '00hr'

    """
    hour_terms, minute_terms, day_terms = get_time_terms(obo)
    NO_REGEX = re.compile(r'(\d+)')
    term = obo.term(obo_id)
    number = NO_REGEX.search(term.name).group()
    time_dict = {"hr": None, "min": None, "day": None}
    if obo_id in hour_terms:
        time_dict['hr'] = number
    elif obo_id in minute_terms:
        time_dict['min'] = number
    elif obo_id in day_terms:
        time_dict['day'] = number
    return time_dict_to_str(time_dict)

def get_sample_type_from_ontology(obo, obo_term):
    """
    Get the sample_type from the ontology.

    There are three sample types defined in the ontology. The sample belongs to
    a certain type if it is a descendant of the respective type definition object.

    There are certain cases which are not unique. Thus a list of the associated types is returned.

    Args:
        obo: OBOOntology Object
        obo_term: OBOObject (term)

    Returns:
        List containing None/primary cell/tissue/cell line

    >>> obo = OBOOntology()
    >>> obo.load(open("test/testdata/ff-phase2-140729.obo"))
    >>> obo_term = obo.term("FF:10100-102D1")  # tpm of uterus, adult, pool1.CNhs1167...
    >>> get_sample_type_from_ontology(obo, obo_term)
    ['tissue']
    >>> obo_term = obo.term("FF:11931-125I5") # not unique
    >>> sorted(get_sample_type_from_ontology(obo, obo_term))
    ['primary cell', 'tissue']
    """
    mapping = {
        "FF:0000002": "primary cell",
        "FF:0000004": "tissue",
        "FF:0000003": "cell line",
    }
    super_terms = [term.id for term in obo.super_terms(obo_term.id)]
    sample_type = []  # cases like FF:11931-125I5 which are not unique
    for key, val in mapping.items():
        if key in super_terms:
            sample_type.append(val)
    return sample_type

def process_sample_name(sample_info):
    """
    Extract sample information from the sample name.

    >>> pprint(process_sample_name("tpm of 293SLAM rinderpest infection, 00hr, biol_rep1.CNhs14406.13541-145H4"))
    {'biol_rep': 'biol_rep1',
     'donor': None,
     'lib_id': 'CNhs14406',
     'name_orig': 'tpm of 293SLAM rinderpest infection, 00hr, '
                  'biol_rep1.CNhs14406.13541-145H4',
     'obo_id': 'FF:13541-145H4',
     'tech_rep': None,
     'time': '00hr'}

    """
    info_n = dict()
    info_n["obo_id"] = get_obo_id(sample_info)
    info_n["lib_id"] = get_lib_id(sample_info)
    info_n["name_orig"] = sample_info
    logging.info("{}: Parsing information from sample name".format(info_n["lib_id"]))

    # we use the 'original' name here, as some information e.g. tech_rep is not available
    # in the name retrieved from the ontology.
    info_n["donor"] = get_donor(sample_info)  # donor is some sort of biological replicate
    info_n["time"] = get_time(sample_info)
    info_n["tech_rep"] = get_tech_replicate(sample_info)
    info_n["biol_rep"] = get_biol_replicate(sample_info)

    assert not (bool(info_n["biol_rep"]) and bool(info_n["donor"])), \
        "as donor is a form of biological replicate they should not co-occur in a name: {}".format(sample_info)

    return info_n

def process_sample_ontology(obo, sample_info):
    """
     loops through all is_a relationships in tags and look if it contains any relevant annotation.
     Here, we extract the time and whether the sample is a biological or technical replicate.

     Args:
         obo: OBOOntology Object
         obo_term: OBOObject (term)

     Returns:
         time_o: term_id of a TIME ontology or None if not specified.
         tech_rep_o: True, if the sample is a technical replicate, else None
         biol_rep_o, True, if the sample is a biological replicate, else None

    >>> obo = OBOOntology()
    >>> obo.load(open("test/testdata/ff-phase2-140729.obo"))
    >>> sample_info = "tpm of occipital lobe, fetal, donor1.CNhs11784.10073-102A1"
    >>> pprint(process_sample_ontology(obo, sample_info))
    {'biol_rep': True,
     'lib_id': 'CNhs11784',
     'name': 'occipital lobe, fetal, donor1',
     'obo_id': 'FF:10073-102A1',
     'sample_type': ['tissue'],
     'tech_rep': None,
     'time': None}
    """
    info_o = {
        "tech_rep": None,
        "biol_rep": None,
        "time": None,
        "obo_id": get_obo_id(sample_info),
        "lib_id": get_lib_id(sample_info),
    }
    logging.info("{}: Searching Ontology".format(info_o["lib_id"]))
    obo_term = obo.term(info_o["obo_id"])
    info_o["name"] = obo_term.name
    tags = obo_term.tags()
    for tag, tag_value, _, _ in tags:
        if tag == "is_a":
            if is_time_term(obo, tag_value):
                assert info_o["time"] is None, 'multiple matches: {0}'.format(tags)
                info_o["time"] = ontology_to_time(obo, tag_value)
            if tag_value == TECH_REP:
                assert info_o["tech_rep"] is None, 'multiple matches: {0}'.format(tags)
                info_o["tech_rep"] = True
            if tag_value == BIOL_REP:
                assert info_o["biol_rep"] is None, 'multiple matches: {0}'.format(tags)
                info_o["biol_rep"] = True
    info_o["sample_type"] = get_sample_type_from_ontology(obo, obo_term)
    return info_o

def merge_sample_info(info_n, info_o, info_si, annot_notes=[]):
    """
    Put the information from name, ontology and supplementary information together.
    Check for consistency and take note, if information is lacking in the ontology.

    Args:
        info_n: dict, info from name, retrieve through process_sample_name
        info_o: dict, info from ontology, retrieve trough process_sample_ontology
        info_si: dict, info from supplement, get from data frame
        annot_notes: list passed by reference that will be filled with a note if an annotation is
            missing in the ontology

    Returns:
        annot, dict with the merged annotations

    >>> obo = OBOOntology()
    >>> obo.load(open("test/testdata/ff-phase2-140729.obo"))
    >>> sample_info = "tpm of 293SLAM rinderpest infection, 00hr, biol_rep1.CNhs14406.13541-145H4"
    >>> info_n = process_sample_name(sample_info)
    >>> info_o = process_sample_ontology(obo, sample_info)
    >>> info_si = {"obo_id": "FF:13541-145H4", "lib_id": 'CNhs14406', "sample_type": "tissue"}
    >>> annotation_notes = []
    >>> pprint(merge_sample_info(info_n, info_o, info_si, annotation_notes))
    {'biol_rep': True,
     'donor': None,
     'lib_id': 'CNhs14406',
     'name': '293SLAM rinderpest infection, 00hr, biol_rep1',
     'name_orig': 'tpm of 293SLAM rinderpest infection, 00hr, '
                  'biol_rep1.CNhs14406.13541-145H4',
     'obo_id': 'FF:13541-145H4',
     'sample_type': 'tissue',
     'tech_rep': False,
     'time': '00hr'}
    >>> pprint(annotation_notes)
    [{'field_name': 'biol_rep',
      'lib_id': 'CNhs14406',
      'new_value': 'biol_rep1',
      'obo_id': 'FF:13541-145H4'},
     {'field_name': 'sample_type',
      'lib_id': 'CNhs14406',
      'new_value': 'tissue',
      'obo_id': 'FF:13541-145H4'}]

    apparently, for CNhs14406, the ontology contains no information, that is is a biological replicate.

    """
    assert info_n["obo_id"] == info_o["obo_id"] == info_si["obo_id"] is not None, "obo_ids not matching"
    assert info_n["lib_id"] == info_o["lib_id"] == info_si["lib_id"] is not None, "lib_ids not matching"
    logging.info("{}: Merging Information".format(info_n["lib_id"]))

    def add_annotation_note(field_name, new_value):
        """add note, that an annotation is missing in the ontology"""
        logging.debug("{}: missing in ontology. ".format(field_name))
        annot_notes.append({
            "lib_id": info_n["lib_id"],
            "obo_id": info_n["obo_id"],
            "field_name": field_name,
            "new_value": new_value
        })

    def simply_merge(field_name):
        """Merge the three dicts, assert that the same if not None or key does not exist"""
        dicts = [info_n, info_o, info_si]
        vals = [d.get(field_name, None) for d in dicts]
        true_vals = [v for v in vals if v is not None]
        assert len(set(true_vals)) <= 1, "value mismatch"
        val = true_vals[0] if len(true_vals) > 0 else None
        if vals[1] is None and len(true_vals) > 1:
            add_annotation_note(field_name, val)
        return val

    def merge_biol_rep():
        if bool(info_n["biol_rep"]) or bool(info_n["donor"]):
            if not bool(info_o["biol_rep"]):
                add_annotation_note("biol_rep", info_n["biol_rep"] if info_n["biol_rep"] else info_n["donor"])
        return bool(info_o["biol_rep"]) or bool(info_n["biol_rep"]) or bool(info_n["donor"])

    def merge_tech_rep():
        if bool(info_n["tech_rep"]):
            if not bool(info_o["tech_rep"]):
                add_annotation_note("tech_rep", info_n["tech_rep"])
        return bool(info_o["tech_rep"]) or bool(info_n["tech_rep"])

    def merge_sample_type():
        # here, we tolerate inconsistencies and prefer the ontology if available
        if len(info_o["sample_type"]) > 0:
            if info_si["sample_type"] is not None and (info_si["sample_type"] not in info_o["sample_type"]):
                logging.warning("sample_type: inconsistent. Ontology: {}, SI: {}".format(
                    info_o["sample_type"], info_si["sample_type"]))
            if len(info_o["sample_type"]) == 1:  # there are cases with multiple sample types in the ontology
                return info_o["sample_type"][0]
            else:
                logging.warning("sample_type: multiple sample types in ontology: {}".format(info_o["sample_type"]))
                return "multiple"
        else:
            add_annotation_note("sample_type", info_si["sample_type"])
            return info_si["sample_type"]

    annot = {
        "lib_id": info_n["lib_id"], # no merge required due to assertion
        "obo_id": info_n["obo_id"], # no merge required due to assertion
        "biol_rep": merge_biol_rep(),
        "tech_rep": merge_tech_rep(),
        "donor": simply_merge("donor"),
        "time": simply_merge("time"),
        "name": simply_merge("name"),
        "name_orig": simply_merge("name_orig"),
        "sample_type": merge_sample_type()
    }

    return annot


#!/usr/bin/env python3

# Dependencies
# ============

# Standard
# --------
import argparse
import os
import sys
import re
import random
import string
import codecs
from datetime import date

# Non-standard
# ------------
# See http://tinydb.readthedocs.io/
from tinydb import TinyDB, Query
from tinydb.storages import Storage, touch

# See https://www.dulwich.io/
from dulwich.repo import Repo
from dulwich.errors import NotGitRepository
import dulwich.porcelain as git

# See https://passlib.readthedocs.io/
from passlib.apps import custom_app_context as pwd_context

# Initializing
# ============

# Calculate defaults
# ------------------

default_users_file = os.path.join(os.getcwd(), 'db.json')

mscwg_email = 'mscwg@rda-groups.org'

db_format = {
    'indent': 1,
    'ensure_ascii': False
}

# Command-line arguments
# ----------------------
# We reuse the following help text a lot:
user_argument = {'help': 'openID/OAuth ID code of the user'}
apiuser_argument = {'help': 'user ID of the API user'}

# Here is the actual parser:
parser = argparse.ArgumentParser(
    prog='python3 -m rdamsc_userctl',
    description='Metadata Standards Catalog User Control Tool.'
                ' Registers new API users, and blocks/unblocks both API and'
                ' regular users.')
parser.add_argument(
    '-u',
    '--user-db', help='location of user database file (instance/users/db.json)',
    action='store',
    default=default_users_file,
    dest='userdb')
subparsers = parser.add_subparsers(
    title='subcommands',
    help='perform database operation')
parser_blockuser = subparsers.add_parser(
    'block-user',
    help='block regular user of the Catalog')
parser_blockuser.add_argument(
    'userid',
    **user_argument)
parser_blockapiuser = subparsers.add_parser(
    'block-api-user',
    help='block user of the restricted Catalog API')
parser_blockapiuser.add_argument(
    'userid',
    **apiuser_argument)
parser_unblockuser = subparsers.add_parser(
    'unblock-user',
    help='unblock regular user of the Catalog')
parser_unblockuser.add_argument(
    'userid',
    **user_argument)
parser_unblockapiuser = subparsers.add_parser(
    'unblock-api-user',
    help='unblock user of the restricted Catalog API')
parser_unblockapiuser.add_argument(
    'userid',
    **apiuser_argument)
parser_addapiuser = subparsers.add_parser(
    'add-api-user',
    help='add API user to user database')
parser_addapiuser.add_argument(
    'name',
    help='name of the API user')
parser_addapiuser.add_argument(
    'userid',
    **apiuser_argument)
parser_addapiuser.add_argument(
    'email',
    help='contact email address for the API user')

# Operations
# ==========
# Block user
# ----------
def dbBlock(args, api=False, toggle=True):
    """Blocks or unblocks user. If `api` is False, operates on regular Web
    user; if True, operates on API user. If `toggle` is True, blocks the user;
    if False, unblocks the user.
    """
    # Retrieve user record
    db = TinyDB((args.userdb), **db_format)
    User = Query()

    if api:
        table = db.table('api_users')
    else:
        table = db

    if toggle:
        verb = ('Block', 'Blocking', 'blocked')
    else:
        verb = ('Unblock', 'Unblocking', 'unblocked')

    user_list = table.search(User.userid == args.userid)
    status = len(user_list)
    if status < 1:
        print(f"User {args.userid} not found. Exiting.")
        sys.exit(1)
    elif status > 1:
        print('ID not unique. Is there a problem in the database?')
        sys.exit(2)

    print(f"{verb[1]} user {args.userid}...")
    user = user_list[0]

    # Update user record
    table.update({'blocked': toggle}, doc_ids=[user.doc_id])

    # Add file to Git index
    try:
        repo = Repo(os.path.dirname(args.userdb))
    except NotGitRepository:
        repo = Repo.init(os.path.dirname(args.userdb))

    git.add(repo=repo, paths=[args.userdb])

    # Prepare commit information
    committer = f"MSCWG <{mscwg_email}>".encode('utf8')
    author = committer
    message = (f"{verb[0]} user {args.userid}\n\nChanged by userctl"
               .encode('utf8'))

    # Execute commit
    git.commit(repo, message=message, author=author, committer=committer)
    print(f"User successfully {verb[2]}.")

def dbBlockApi(args):
    dbBlock(args, api=True)

def dbUnblock(args):
    dbBlock(args, toggle=False)

def dbUnblockApi(args):
    dbBlock(args, api=True, toggle=False)

parser_blockuser.set_defaults(func=dbBlock)
parser_blockapiuser.set_defaults(func=dbBlockApi)
parser_unblockuser.set_defaults(func=dbUnblock)
parser_unblockapiuser.set_defaults(func=dbUnblockApi)

# Add API user
# ------------
def dbAdd(args):
    db = TinyDB((args.userdb), **db_format)
    table = db.table('api_users')
    name = args.name
    userid = args.userid
    email = args.email

    # Validate user input...
    errors = list()
    badchars = set()
    # Check username:
    for char in userid:
        if char not in (string.ascii_letters + string.digits + '-_.'):
            badchars.add(char)
    if badchars:
        errors.append("These characters are not allowed in the user ID: "
                        f'''"{''.join(badchars)}"''')
    # Check email:
    if not re.match('[^@\\s]+@[^@\\s\\.]+\\.[^@\\s]+', email):
        errors.append('That email address does not look quite right.')
    # Were there errors?
    if errors:
        print('\n'.join(errors))
        sys.exit(1)

    # Generate pseudo-random string
    try:
        rng = random.SystemRandom()
    except NotImplementedError:
        rng = random
    password = ''.join(rng.choice(string.ascii_letters + string.digits)
                       for _ in range(12))

    # Update user record
    table.insert({
        'name': name,
        'email': email,
        'userid': userid,
        'password_hash': pwd_context.hash(password)})

    # Add file to Git index
    try:
        repo = Repo(os.path.dirname(args.userdb))
    except NotGitRepository:
        repo = Repo.init(os.path.dirname(args.userdb))
    git.add(repo=repo, paths=[args.userdb])

    # Prepare commit information
    committer = f"MSCWG <{mscwg_email}>".encode('utf8')
    author = committer
    message = (f"Add API user {name}\n\nChanged by userctl"
               .encode('utf8'))

    # Execute commit
    git.commit(repo, message=message, author=author, committer=committer)
    print(f"User successfully added with password {password}")

parser_addapiuser.set_defaults(func=dbAdd)

# Processing
# ==========
def main():
    args = parser.parse_args()
    args.func(args)

if __name__ == '__main__':
    main()


# Definition for a binary tree node.
# class TreeNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution(object):
    def helper(self, node, depth):
        if node == None:
            if depth > self.max: self.max = depth
            return

        self.helper(node.right, depth + 1)
        self.helper(node.left, depth + 1)

    def maxDepth(self, root):
        """
        :type root: TreeNode
        :rtype: int
        """
        self.max = 0
        self.helper(root, 0)
        return self.max


import inspect
import math
import os

import networkx as nx
import osmnx as ox
import grinpy as gp

def map_to_graph_from_point(lst, key=None, plot=False, dist=None):
    x = lst[0]
    y = lst[1]
    if dist is not None:
        dist = dist
    else:
        dist = lst[2]

    ox.config(use_cache=True, log_console=False)

    if key is not None:
        filepath = "graph_" + key + "_" + str(dist)
    else:
        filepath = "graph_" + str(x) + "_" + str(y) + "_" + str(dist)

    if os.path.exists("./map/" + filepath):

        G = ox.load_graphml("./map/" + filepath)

    else:

        G = ox.graph_from_point((x, y), dist=dist, network_type="drive", retain_all=False)
        ox.save_graphml(G, filepath="./map/" + filepath)

    if plot == True:
        plotting_graph(G)

    return G

def map_to_graph_from_bbox(lst, key=None, plot=False):
    dist = lst[3]
    ox.config(use_cache=True, log_console=False)

    if key is not None:
        filepath = "graph_" + key + "_" + str(dist)
    else:
        filepath = "graph_" + str(lst[0]) + "_" + str(lst[1]) + "_" + str(lst[2]) + "_" + str(dist)

    if os.path.exists("./map/" + filepath):

        G = ox.load_graphml("./map/" + filepath)
    else:

        G = ox.graph_from_bbox(lst[0], lst[1], lst[2], lst[3], network_type="drive", retain_all=False)

        ox.save_graphml(G, filepath="./map/" + filepath)

    if plot == True:
        plotting_graph(G)
    return G

def plotting_graph(G):
    fig, ax = ox.plot_graph(G, bgcolor="k", node_color="#FFFFFF", edge_color="#333333", node_size=20, edge_linewidth=2)

def get_adjacency_matrix(G):
    G_un = G.to_undirected()
    a = nx.adjacency_matrix(G_un)
    print(a.todense())

def get_diameter(G):
    G_un = G.to_undirected()
    diam = nx.diameter(G_un)
    print(diam)
    return diam

def get_radius(G):
    G_un = G.to_undirected()
    rad = nx.radius(G_un)
    print(rad)
    return rad

def get_average_shortest_path_length(G):
    G_un = G.to_undirected()
    avg = nx.average_shortest_path_length(G_un)
    print(avg)
    return avg

def get_independence_number(G):
    G_un = G.to_undirected()
    ind = len(nx.maximal_independent_set(G_un))
    return ind

def get_average_degree(G):
    G_un = G.to_undirected()
    deg = nx.average_degree_connectivity(G_un)
    print(deg)
    return deg

def get_irregularity(G):
    G_un = G.to_undirected()
    irr = gp.irregularity(G_un)
    print(irr)
    return irr

def get_degree_sequence(G):
    G_un = G.to_undirected()
    deg = sorted([val for (node, val) in G_un.degree()])
    return deg

def get_surface(G):
    min_x = +1000000000
    max_x = -1000000000
    min_y = +1000000000
    max_y = -1000000000
    for e in list(G.nodes(data=True)):

        if e[1]["x"] < min_x:
            min_x = e[1]["x"]
        if e[1]["x"] > max_x:
            max_x = e[1]["x"]
        if e[1]["y"] < min_y:
            min_y = e[1]["y"]
        if e[1]["y"] > max_y:
            max_y = e[1]["y"]

    latitude = max_y - min_y
    longitude = max_x - min_x

    # latitude in km = 111 * latitude
    latitude_km = 111 * latitude

    # longitude in km = 111 * cos(latitude) * longitude
    longitude_km = 111 * math.cos(latitude) * longitude

    surface = latitude_km * longitude_km

    return surface

def find_angle(Ax, Ay, Bx, By, Cx, Cy):

    AB = math.sqrt(math.pow(Bx - Ax, 2) + math.pow(By - Ay, 2))

    BC = math.sqrt(math.pow(Bx - Cx, 2) + math.pow(By - Cy, 2))

    AC = math.sqrt(math.pow(Cx - Ax, 2) + math.pow(Cy - Ay, 2))

    if (BC * BC + AB * AB - AC * AC) / (2 * BC * AB) > 1 :
        return math.cos(1)
    if (BC * BC + AB * AB - AC * AC) / (2 * BC * AB) < -1 :
        return math.cos(-1)
    return math.acos((BC * BC + AB * AB - AC * AC) / (2 * BC * AB));


####write a pytho program to print dictionary in a table format
dict1 = {'NAME': ['John', 'Tim', 'Steve'],
         'AGE': [21, 20, 21],
         'COURSE': ['Data Structures', 'Machine Learning', 'OOPS with Java']}

for each_row in zip(*([i] + (j)
                      for i, j in dict1.items())):
    print(*each_row," ")

dict1 = {1:["John", 21, 'CS'],
          2: ["Tim", 20, 'EE'],
          3: ["Steve", 21, 'Civil'],
         }

# Print the names of the columns.
print("{:<10} {:<10} {:<10}".format('NAME', 'AGE', 'COURSE'))

# print each data item.
for key, value in dict1.items():
    name, age, course = value
    print("{:<10} {:<10} {:<10}".format(name, age, course))


import os
import sys
from copy import deepcopy
from typing import Optional, List, Tuple, Union

import torch
from torch.nn import Module, Softmax, ModuleList
from transformers import BartForConditionalGeneration, LogitsProcessorList, BartTokenizerFast

DEVICE=torch.device("cuda")

COPY_LAYERS = {
    "encoder": [0, 6, 11],
    "decoder": [0]
}

class BartModel(Module):
    def __init__(self):
        super().__init__()
        self.model = BartForConditionalGeneration.from_pretrained("facebook/bart-large")

    def get_n_samples(self,
                      input_ids: torch.Tensor,
                      attention_mask: torch.Tensor = None,
                      n: int = 3,
                      max_len: int = 100):
        sequences = []
        for seq_id in range(input_ids.shape[0]):
            input_ids_sentence_repeated = torch.cat([input_ids[seq_id][None, :] for _ in range(n)], dim=0)
            attention_mask_sentence_repeated = (
                    None if attention_mask is None
                    else torch.cat([attention_mask[seq_id][None, :] for _ in range(n)], dim=0)
            )
            gen_outputs = self.model.generate(inputs=input_ids_sentence_repeated,
                                              attention_mask=attention_mask_sentence_repeated,
                                              use_cache=True,
                                              decoder_start_token_id=self.model.config.decoder_start_token_id,
                                              max_length=max_len,
                                              num_beams=1,
                                              do_sample=False)
            print("GEN OUTPUTS", gen_outputs)
            sequences.append(gen_outputs)
        return sequences
    def forward(self,
                input_ids: torch.LongTensor = None,
                attention_mask: Optional[torch.Tensor] = None,
                decoder_input_ids: Optional[torch.LongTensor] = None,
                decoder_attention_mask: Optional[torch.LongTensor] = None,
                head_mask: Optional[torch.Tensor] = None,
                decoder_head_mask: Optional[torch.Tensor] = None,
                cross_attn_head_mask: Optional[torch.Tensor] = None,
                encoder_outputs: Optional[List[torch.FloatTensor]] = None,
                past_key_values: Optional[List[torch.FloatTensor]] = None,
                inputs_embeds: Optional[torch.FloatTensor] = None,
                decoder_inputs_embeds: Optional[torch.FloatTensor] = None,
                labels: Optional[torch.LongTensor] = None,
                use_cache: Optional[bool] = None,
                output_attentions: Optional[bool] = None,
                output_hidden_states: Optional[bool] = None,
                return_dict: Optional[bool] = None,):
        return self.model(input_ids,
                          attention_mask=attention_mask,
                          decoder_input_ids=decoder_input_ids,
                          decoder_attention_mask=decoder_attention_mask,
                          head_mask=head_mask,
                          decoder_head_mask=decoder_head_mask,
                          cross_attn_head_mask=cross_attn_head_mask,
                          encoder_outputs=encoder_outputs,
                          past_key_values=past_key_values,
                          inputs_embeds=inputs_embeds,
                          decoder_inputs_embeds=decoder_inputs_embeds,
                          labels=labels,
                          use_cache=use_cache,
                          output_attentions=output_attentions,
                          output_hidden_states=output_hidden_states,
                          return_dict=return_dict)

    def get_n_samples_2(self,
                      input_ids: torch.Tensor,
                      attention_mask: torch.Tensor=None,
                      n: int=3,
                      max_len:int=100):
        """
        for every input sentence, generate n output samples

        @param input_ids: (N, S)
        @param attention_mask: (N, S)

        @returns sequences: list(torch.Tensor(n, s)): List of size N, containing tensors
                            with n samples and sequence length of s

        """
        encoder = self.model.get_encoder()
        decoder = self.model.get_decoder()

        sequences = []
        logits_processor = LogitsProcessorList()

        for seq_id in range(input_ids.shape[0]):

            input_ids_sentence_repeated = torch.cat([input_ids[seq_id][None, :] for _ in range(n)], dim=0)
            attention_mask_sentence_repeated = None if attention_mask is None else torch.cat([attention_mask[seq_id][None, :] for _ in range(n)], dim=0)

            # print("INPUT SHAPE:", input_ids_sentence_repeated.shape)
            # print("ATTN SHAPE: ", attention_mask_sentence_repeated.shape)
            enc_outputs = encoder(input_ids_sentence_repeated,
                                  attention_mask=attention_mask_sentence_repeated,
                                  output_attentions=True,
                                  output_hidden_states=True)
            enc_hidden_state, all_hidden_states, enc_attentions = enc_outputs.last_hidden_state, enc_outputs.hidden_states, enc_outputs.attentions

            past_key_values = None
            dec_input_ids = torch.ones((input_ids_sentence_repeated.shape[0], 1), dtype=torch.long, device=DEVICE) * self.model.config.decoder_start_token_id
            sequence = None

            for i in range(max_len):
                # print("ITERATION:", i)
                # print("DECODER INPUT IDS", dec_input_ids)

                # print("DEC_INPUT_IDS",  dec_input_ids)
                dec_outputs = decoder(dec_input_ids,
                                      encoder_hidden_states=enc_hidden_state,
                                      encoder_attention_mask=attention_mask_sentence_repeated,
                                      past_key_values=past_key_values,
                                      output_hidden_states=True,
                                      use_cache=True)

                last_hidden_state = dec_outputs.last_hidden_state
                past_key_values = dec_outputs.past_key_values

                lm_logits = self.model.lm_head(last_hidden_state)
                lm_logits = lm_logits + self.model.final_logits_bias.to(lm_logits.device)
                lm_logits[:, -1, self.model.config.bos_token_id] -= 100 if sequence is not None else 0

                # greedy
                # last_val = torch.argmax(Softmax(dim=-1)(lm_logits[:, -1:, :]), dim=-1)
                # print(last_val.shape)
                # dec_input_ids = last_val

                # Sampled
                last_val = self.compute_random_sample(lm_logits[:, -1:, :])
                # print(last_val.shape)
                dec_input_ids = last_val

                sequence = last_val if sequence is None else torch.cat([sequence, last_val], dim=-1)
                if self.all_eos(sequence):
                    break

            sequences.append(sequence)

        return sequences

    def compute_random_sample(self, logits):
        """
        @params logits: (N, S, C)
        @returns (N, S)
        """

        probs = Softmax(dim=-1)(logits)
        cdf = torch.cumsum(probs, dim=-1)
        cdf[:, :, -1] += 0.1 # Final bin of cdf is 1.1, to account for floating point imprecision
        thresh = torch.rand((logits.shape[0], logits.shape[1]), device=DEVICE)[:, :, None]
        thresh_mask = cdf < thresh
        sample_output = torch.sum(thresh_mask, dim=-1)

        if False:
            print(probs)
            print(cdf)
            print(thresh)
            print(thresh_mask)
            print(torch.sum(thresh_mask, dim=-1))
        return sample_output

    def all_eos(self, sequences):
        """
        Checks if all sequences in the batch has EOS token
        @param sequence: (N, S)
        """
        return (torch.sum(torch.sum(sequences == self.model.config.eos_token_id, dim=-1) == 0) == 0)

    def load_pretrained(self, path: str):

        # self.model.load_state_dict(torch.load(os.path.join(path, "pytorch_model.bin")))
        self.model = BartForConditionalGeneration.from_pretrained(path).to(DEVICE)

class BartStudentModel(BartModel):
    def __init__(self, teacher_model:BartModel=None, config:dict=None):
        super().__init__()

        if teacher_model is not None:
            assert config is not None, "If teacher model is defined, so should the config argument"

        self.config = config

        self.model = self.copy_from_teacher(teacher_model, config)

    def copy_from_teacher(self, teacher_model:BartModel=None, config:dict=None):

        if teacher_model is None:
            teacher_model = BartForConditionalGeneration.from_pretrained("facebook/bart-large")
            config = {"decoder":[0], "encoder":[0, 6, 11]}

        teacher_encoder = teacher_model.model.get_encoder()
        teacher_decoder = teacher_model.model.get_decoder()

        student_encoder = deepcopy(teacher_encoder)
        student_encoder.layers = ModuleList([teacher_encoder.layers[i] for i in config["encoder"]])
        student_decoder = deepcopy(teacher_decoder)
        student_decoder.layers = ModuleList([teacher_decoder.layers[i] for i in config["decoder"]])

        student_model = BartForConditionalGeneration.from_pretrained("facebook/bart-base")
        student_model.model.decoder = student_decoder
        student_model.model.encoder = student_encoder

        return student_model

if __name__ == "__main__":

    teacher_model = BartModel()
    student_model = BartStudentModel(teacher_model, config=COPY_LAYERS)

    print(teacher_model)
    print("\n\n")
    print(student_model)

    tok = BartTokenizerFast.from_pretrained("facebook/bart-large")
    print(tok.pad_token_id)
    print(tok.bos_token_id)
    print(tok.eos_token_id)

    tok = BartTokenizerFast.from_pretrained("facebook/bart-base")
    print(tok.pad_token_id)
    print(tok.bos_token_id)
    print(tok.eos_token_id)


from django.contrib import admin
from django.urls import path
from firstapp import views
from django.views.generic import TemplateView
from django.conf.urls import include, url

admin.autodiscover()

urlpatterns = [
    path('',views.great, name='great'),
    url(r'^admin/backups/', include('dbbackup_ui.urls')),
    url(r'^admin/', admin.site.urls),
    path('zoo/', views.index, name='home'),
    path('animal/', views.index_animal, name='animal'),
    path('area/', views.index_habitat_area, name='area'),
    path('user/', views.index_user, name='user'),
    path('ration/', views.index_feeding_ration, name='ration'),
    path('bird/', views.index_bird, name='bird'),
    path('reptile/', views.index_reptile, name='reptile'),
    #пути для животных
    path('animal/add/', views.view_animal.add_animal, name='add_animal'),
    path("animal/del/", views.view_animal.del_animal, name="del_animal"),
    path("animal/up/", views.view_animal.update_animal, name="update_animal"),
    #пути для зон обитания
    path('area/adde/', views.view_habitat_area.add_habitat_area, name='add_habitat_area'),
    path("area/dele/", views.view_habitat_area.del_habitat_area, name="del_habitat_area"),
    path("area/upe/", views.view_habitat_area.update_habitat_area, name="update_habitat_area"),
    #пути для пользователей
    path('user/addc/', views.view_user.add_user, name='add_author'),
    path("user/delc/", views.view_user.del_user, name="del_data"),
    path("user/upc/", views.view_user.update_user, name="update_data"),
    #пути для рационов
    path('ration/addo/', views.view_feeding_ration.add_feeding_ration, name='add_ration'),
    path("ration/delo/", views.view_feeding_ration.del_feeding_ration, name="del_ration"),
    path("ration/upo/", views.view_feeding_ration.update_feeding_ration, name="update_ration"),
    #пути для птиц
    path('bird/addb/', views.view_bird.add_bird, name='add_bird'),
    path("bird/delb/", views.view_bird.del_bird, name="del_bird"),
    path("bird/upb/", views.view_bird.update_bird, name="update_bird"),
    #пути для рептилий
    path('reptile/addr/', views.view_reptile.add_reptile, name='add_reptile'),
    path("reptile/delr/", views.view_reptile.del_reptile, name="del_reptile"),
    path("reptile/upr/", views.view_reptile.update_reptile, name="update_reptile"),
]
urlpatterns += [
    path('accounts/', include('django.contrib.auth.urls')),
]


from libs.comnetpr.Graph import *
from libs.comnetpr.Graph import *
from libs.comnetpr.Algorithms import *
from mpl_toolkits.mplot3d import Axes3D
import pandas as pd
import matplotlib.pyplot as plt

graph = CreateGraph()
graph.readCSV('graph1.csv')
#graph.readCSV('internal.csv')

df = pd.DataFrame(graph.getNodeEdge()[1], columns=['Source', 'Destination', 'Cost'])

fig = plt.figure()
ax = fig.add_subplot(111, projection = '3d')
ax.title.set_text('Source-Destination-Cost 3D')

x = df['Source']
y = df['Destination']
z = df['Cost']

ax.scatter(x, y, z)
ax.set_xlabel("Source")
ax.set_ylabel("Destination")
ax.set_zlabel("Cost")

plt.show()


from requests import get
from yt_dlp import YoutubeDL
import discord
from discord.ext import commands
import logging
import asyncio

# bot setup
print(discord.__version__)
description = 'I have no enemies, merely topologies of ignorance!'
intents = discord.Intents.default()
intents.members = True
bot = commands.Bot(command_prefix='$', description=description, intents=intents, help_command=None)

# Error logging class, unused right now.
class ErrorLogging(object):
    def debug(self, msg):
        pass
    def warning(self, msg):
        pass
    def error(self, msg):
        print(msg)

# Queries youtube with passed search term or link and downloads an audio file in opus format.
def download_video(search):
    ydl_opts = {
        'format': 'bestaudio/best',
        'postprocessors': [{
            'key': 'FFmpegExtractAudio',
            'preferredcodec': 'opus',
        }],
        'outtmpl': 'output.%(ext)s',
        'verbose': False,
        'logger': ErrorLogging(),
    }
    with YoutubeDL(ydl_opts) as ydl:
        try:
            get(search)
        except:
            video = ydl.extract_info(f"ytsearch:{search}", download=False)['entries'][0]['webpage_url']
        else:
            video = search
        ydl.download([video])

# commands
@bot.event
async def on_ready():
    print("Behold! The great and powerful Elminster!")
    print(bot.user.name + '@%s' % bot.user.id)

@bot.command()
async def help(ctx):
    await ctx.send('Begin a request with \'$\' and I will dazzle you with my abilities!')

@bot.command()
async def play(ctx, *, search):
    download_video(search)
    source = discord.PCMVolumeTransformer(discord.FFmpegPCMAudio("output.opus"), volume=0.1)
    if ctx.voice_client is None:
        channel = ctx.message.author.voice.channel
        await channel.connect()
    ctx.voice_client.play(source)
    while ctx.voice_client.is_playing():
        await asyncio.sleep(.1)
    await ctx.voice_client.disconnect()

@bot.command()
async def developers(ctx):
    await ctx.send("DEVELOPERS DEVELOPERS DEVELOPERS DEVELOPERS")
    source = discord.PCMVolumeTransformer(discord.FFmpegPCMAudio("developers.opus"), volume=0.1)
    if ctx.voice_client is None:
        channel = ctx.message.author.voice.channel
        await channel.connect()
    ctx.voice_client.play(source)

@bot.command()
async def dc(ctx):
    if ctx.voice_client is not None:
        if ctx.voice_client:
            await ctx.guild.voice_client.disconnect()
            await ctx.send('Elminster shall take his leave!')
        else:
            await ctx.send('Silence knave, I\'m not in a voice channel.')

# discord.py logs
logger = logging.getLogger('discord')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

# open
with open('./secret', 'r') as f:
    token = f.readline()
bot.run(token)


class SingleTonDecorator:

    def __init__(self, clss):
        self.clss = clss
        self.instance = None

    def __call__(self, *args, **kwargs):
        print(self.instance)
        if self.instance == None:
            self.instance = self.clss(*args, **kwargs)
        return self.instance

# Normal single Ton
class President:

    def __init__(self, name):
        self.name = name

    def __new__(cls, *args, **kwargs):
        if not hasattr(cls, "_instance"):
            cls._instance = super().__new__(cls)
        return cls._instance

o = President("Obama")

p = President("Trump")

q = President("Biden")

print(o, p, q)

print(o.name)
print(p.name)
print(q.name)

# SignleTon using decorator

@SingleTonDecorator
class President:

    def __init__(self, name):
        self.name = name

o = President("Obama")

p = President("Trump")

q = President("Biden")

print(o.name)
print(p.name)
print(q.name)


from setuptools import setup
import sys, os

version = '0.2'

long_description = (
    open('README.txt').read()
    + '\n' +
    open('CHANGES.txt').read())

setup(name='typeinfo',
      version=version,
      description='A library to add optional type information to python objects.',
      long_description=long_description,
      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
      keywords='',
      author='TypeInfo Developers',
      author_email='b.leskes@gmail.com',
      url='',
      license='apache',
      py_modules=['typeinfo'],
      package_dir={ "": "src"},
      zip_safe=False,
      install_requires=[
      ])


# coding: utf-8
"""
-------------------------------------------------
   File Name:      main
   Author :        Tiancheng Song
   E-mail :        tiancheng.song@mailbox.tu-dresden.de
   Date:           2022-01-15 12:10 AM
-------------------------------------------------
Description :

    Face Depth Measurement

"""
import cv2
import cvzone
from cvzone.FaceMeshModule import FaceMeshDetector

cap = cv2.VideoCapture('http://192.168.178.48:4747/video')
detector = FaceMeshDetector(maxFaces=1)

while True:
    success, img = cap.read()
    img = cv2.rotate(img, cv2.ROTATE_90_COUNTERCLOCKWISE)
    img, faces = detector.findFaceMesh(img, draw=False)

    if faces:
        face = faces[0]
        pointLeft = face[145]
        pointRight = face[374]
        # Drawing
        # cv2.line(img, pointLeft, pointRight, (0, 200, 0), 3)
        # cv2.circle(img, pointLeft, 5, (255, 0, 255), cv2.FILLED)
        # cv2.circle(img, pointRight, 5, (255, 0, 255), cv2.FILLED)
        w, _ = detector.findDistance(pointLeft, pointRight)
        W = 6.3

        # # Finding the Focal Length
        # d = 50
        # f = (w*d)/W
        # print(f)

        # Finding distance
        f = 840
        d = (W * f) / w
        print(d)

        cvzone.putTextRect(img, f'Depth: {int(d)}cm', (face[10][0] - 100, face[10][1] - 50), scale=2)

    cv2.imshow("Image", img)
    key = cv2.waitKey(1) & 0xFF
    if key == 27:
        break

cap.release()
cv2.destroyAllWindows()

print("********************************************************")
print("***                   By Song T.C.                   ***")
print("***       tiancheng.song@mailbox.tu-dresden.de       ***")
print("********************************************************")

Module/Extern/OpenCV/BasicProject/44 Face Distance Measurement/FaceDepthMeasurement.py


"""
Module for koinex upstream
"""
import requests
from django.utils import timezone as dj_timezone

class KoinexDriver:
    """Driver class for koinex upstream"""

    def __init__(self):
        self.__latest = {}
        self.__url = 'https://koinex.in/api/ticker'
        self.__time_of_update = None

    def get_data(self):
        """Returns latest data"""
        return self.__latest

    def get_time_of_update(self):
        """Provides last updated data"""
        return self.__time_of_update

    def get_latest_updates(self):
        """Refreshes the data from upstream and updates last updated time"""
        try:
            response = requests.get(self.__url)
            if response.ok:
                self.__latest = response.json()
                self.__time_of_update = dj_timezone.now()
        except:
            pass
        return self.get_data(), self.get_time_of_update()

    def get_coin_details(self, coin):
        """Provides value and stats for a particular coin"""
        try:
            return dict(
                value=self.__latest['prices']['inr'][coin],
                stats=self.__latest['stats']['inr'][coin]
            )
        except KeyError:
            raise Exception("No Data Found")

koinex = KoinexDriver()


# https://www.codewars.com/kata/55e67e44bf97fa66900000a0
class Node(object):
    def __init__(self, data=None):
        self.data = data
        self.next = None

def sorted_intersect(first, second):
    if first is None or second is None:
        return None

    head = current = None
    while first is not None and second is not None:
        if first.data > second.data:
            first, second = second, first

        while first is not None and first.data < second.data:
            first = first.next
        if first is None:
            break

        if first.data == second.data:
            if current is None:
                head = current = first
            else:
                current.next = first
                current = current.next
            while first is not None and first.data == current.data:
                first = first.next
            while second is not None and second.data == current.data:
                second = second.next

    if current is not None:
        current.next = None
    return head

katas/Python/6 kyu/Linked Lists  Sorted Intersect 55e67e44bf97fa66900000a0.py


import doctest
import os
import re

from data_warehouse_utils.helpers import clean_filename, replace_diacritics

def is_complete_message(s):
    """
    Given a WhatsApp chat exported string, returns if string is a complete message

    :param s: string
    :return: bool

    >>> is_complete_message("[2021-05-05, 11:44:26] A: message content")
    True

    >>> is_complete_message("Message content")
    False
    """
    pattern = r"^\[\d{4}-\d{2}-\d{2},\W\d{2}:\d{2}:\d{2}]"
    results = re.search(pattern, s)
    return True if results else False

def count_messages(filepath):
    """
    Given a WhatsApp chat txt export, returns the number of messages exchanged.

    :param filepath: string
    :return: int
    """
    total_messages = 0
    pattern = r"\[\d{4}-\d{2}-\d{2},\s\d{2}:\d{2}:\d{2}\]"

    with open(filepath) as f:
        for line in f:
            matches = re.findall(pattern, line)
            total_messages += len(matches)
    return total_messages

def parse_message(s):
    """
    Given a WhatsApp chat string, returns the message as a dict.

    :param s: string
    :return: dict

    >>> parse_message("[2021-01-01, 00:00:00] Author: Message")
    {'timestamp': '2021-01-01 00:00:00', 'author': 'Author', 'content': 'Message'}
    """
    pattern = r"\[(\d{4}-\d{2}-\d{2}),(\W\d{2}:\d{2}:\d{2})]\W(.*):\W(.*)"
    # Use regex to extract the timestamp, author, and content from the input string
    match = re.match(pattern, s)
    timestamp = match.group(1) + match.group(2)
    author = match.group(3)
    content = match.group(4)
    # Create a dictionary with the extracted data
    return {"timestamp": timestamp, "author": author, "content": content}

def pre_process_whatsapp_filename(gcs_raw_filepath):
    """
    Given a raw filepath in GCS for a WhatsApp chat, return the pre_processed filename

    :param gcs_raw_filepath: string
    :return: string

    >>> pre_process_whatsapp_filename("whatsapp/chats/WhatsApp Chat - Gonçalo Miranda/_chat.txt")
    'whatsapp/chats/goncalo_miranda/chat.json'

    >>> pre_process_whatsapp_filename("whatsapp/chats/WhatsApp Chat - António Bênção/_chat.txt")
    'whatsapp/chats/antonio_bencao/chat.json'

    >>> pre_process_whatsapp_filename("whatsapp/chats/WhatsApp Chat - 🐮/_chat.txt")
    'whatsapp/chats/🐮/chat.json'
    """
    pattern = r"whatsapp\/chats\/WhatsApp\WChat\W-\W(.*)\/_(chat)\.txt"
    matches = re.match(pattern, gcs_raw_filepath)
    entity = matches.group(1)
    filename = matches.group(2)

    entity = replace_diacritics(entity)
    entity = clean_filename(entity.lower())
    return os.path.join("whatsapp", "chats", entity, filename + ".json")

if __name__ == "__main__":
    doctest.testmod()


class Lesson:

    lesson_name = None
    _tests_marks = []
    _marks = []

    def __init__(self, lesson_name, test_marks:list, marks:list):
        self.lesson_name = lesson_name
        self._tests_marks = test_marks
        self._marks = marks

    def add_marks(self, value):
        if value in range(2,6):
            self._marks.append(value)
        return self._marks

    def add_tests_marks(self, value):
        if value in range(0,101):
            self._tests_marks.append(value)
        return self._tests_marks

    def get_marks(self):
        return self._marks

    def get_tests_marks(self):
        return self._tests_marks

    def lesson_print(self):
        print(self.lesson_name)
        print(self._marks)
        print(self._tests_marks)

if __name__ == '__main__':
    math = Lesson("math")
    math.add_marks(99)
    for i in range(0,10):
        math.add_tests_marks(i)
        math.add_marks(i)
    math.lesson_print()


import json

from flask import render_template

from models.Business import Business
from models.Form1099NECCreateRequest import Form1099NECCreateRequest
from models.Form1099NecList import Form1099NecList
from models.NECFormData import NECFormData
from models.Recipient import Recipient
from models.ReturnData import ReturnData
from models.ReturnHeader import ReturnHeader
from models.ScheduleFiling import ScheduleFiling
from models.States import States
from models.SubmissionManifest import SubmissionManifest
from models.USAddress import USAddress

def save_form_nec(input_request_json):
    businessId = ''
    if 'business_list' in input_request_json:
        businessId = input_request_json['business_list'][0]

    rName = ''
    if 'rName' in input_request_json:
        rName = input_request_json['rName'][0]

    rTIN = ''
    if 'rTIN' in input_request_json:
        rTIN = input_request_json['rTIN'][0]

    amount = ''
    if 'amount' in input_request_json:
        amount = input_request_json['amount'][0]

    recipientId = None

    if 'recipientsDropDown' in input_request_json:
        recipientId = input_request_json['recipientsDropDown'][0]

    requestModel = Form1099NECCreateRequest()

    returnHeader = ReturnHeader()

    business = Business()
    business.set_BusinessId(businessId)
    returnHeader.set_Business(business.__dict__)
    requestModel.set_ReturnHeader(returnHeader.__dict__)

    submissionManifest = SubmissionManifest()
    submissionManifest.set_TaxYear(2020)
    submissionManifest.set_IsFederalFiling(2020)
    submissionManifest.set_IsStateFiling(True)
    submissionManifest.set_IsPostal(True)
    submissionManifest.set_IsOnlineAccess(True)
    submissionManifest.set_IsTinMatching(True)
    submissionManifest.set_IsScheduleFiling(True)
    scheduleFiling = ScheduleFiling()
    scheduleFiling.set_EfileDate("04/21/2021")
    submissionManifest.set_ScheduleFiling(scheduleFiling.__dict__)
    requestModel.set_SubmissionManifest(submissionManifest.__dict__)
    returnDataList = []
    returnData = ReturnData()
    # returnData.set_RecordId(null)
    returnData.set_SequenceId("1")
    # set Recipient data
    recipientModel = Recipient()
    if recipientId != '-1':
        recipientModel.set_RecipientId(recipientId)
    else:
        recipientModel.set_RecipientId('')

    recipientModel.set_TINType("EIN")
    recipientModel.set_TIN(rTIN)
    recipientModel.set_FirstPayeeNm(rName)
    recipientModel.set_SecondPayeeNm("")
    recipientModel.set_IsForeign(False)
    usAddress = USAddress()
    usAddress.set_Address1("1751 Kinsey Rd")
    usAddress.set_Address2("Main St")
    usAddress.set_City("Dothan")
    usAddress.set_State("AL")
    usAddress.set_ZipCd("36303")
    recipientModel.set_USAddress(usAddress.__dict__)
    # recipientModel.set_ForeignAddress(null)
    recipientModel.set_Email("sharmila.k@dotnetethics.com")
    recipientModel.set_Fax("1234567890")
    recipientModel.set_Phone("1234567890")
    returnData.set_Recipient(recipientModel.__dict__)
    # set NEC data
    necFormDataModel = NECFormData()
    necFormDataModel.set_B1NEC(amount)
    necFormDataModel.set_B4FedTaxWH(54.12)
    necFormDataModel.set_IsFATCA(True)
    necFormDataModel.set_Is2ndTINnot(True)
    necFormDataModel.set_AccountNum("20123130000009000001")
    statesList = []
    stateModel = States()
    stateModel.set_StateCd("PA")
    stateModel.set_StateWH(15)
    stateModel.set_StateIdNum("99999999")
    stateModel.set_StateIncome(16)
    statesList.append(stateModel.__dict__)  # State 1
    stateModel = States()
    stateModel.set_StateCd("AZ")
    stateModel.set_StateWH(17)
    stateModel.set_StateIdNum("99-999999")
    stateModel.set_StateIncome(18)
    statesList.append(stateModel.__dict__)  # State 2
    necFormDataModel.set_States(statesList)

    returnData.set_NECFormData(necFormDataModel.__dict__)
    returnData.set_MISCFormData(None)
    returnDataList.append(returnData.__dict__)

    requestModel.set_ReturnData(returnDataList)

    return requestModel

# Validate 1099 NEC Save response
def nec_save_response_validation(response):
    if response['StatusCode'] == 200:

        return render_template('success.html',
                               response='StatusMessage=' + response['StatusMessage'] + '<br>SubmissionId =' + response[
                                   'SubmissionId'], ErrorMessage=' Form 1099-NEC Created Successfully',
                               ButtonText="Form 1099-NEC", FormType="NEC")

    elif 'Form1099Records' in response and response['Form1099Records'] is not None and 'ErrorRecords' in response[
        'Form1099Records'] and response['Form1099Records']['ErrorRecords'][0] is not None and 'Errors' in \
            response['Form1099Records']['ErrorRecords'][0] and response['Form1099Records']['ErrorRecords'][0][
        'Errors'] is not None:

        errorRecords = []

        for errorList in response['Form1099Records']['ErrorRecords']:

            if 'Errors' in errorList and errorList['Errors'] is not None:

                for err in errorList['Errors']:
                    errorRecords.append(err)

        return render_template('error_list.html', errorList=errorRecords,
                               status=str(response['StatusCode']) + " - " + str(response['StatusName']) + " - " + str(
                                   response['StatusMessage']))
    else:

        return render_template('success.html', response='StatusMessage=' + str(response['StatusCode']),
                               ErrorMessage='Message=' + json.dumps(response))

# Validate 1099 NEC List response
def form_nec_list_response_validation(response):
    form1099NecList = []
    if response is not None:
        if 'Form1099Records' in response:
            if response['Form1099Records'] is not None:
                for records in response['Form1099Records']:
                    recipientData = Form1099NecList()
                    if 'RecipientNm' in records['Recipient']:
                        recipientData.set_RecipientNm(
                            records['Recipient']['RecipientNm'])
                    elif 'RecipientName' in records['Recipient']:
                        recipientData.set_RecipientNm(
                            records['Recipient']['RecipientName'])

                    recipientData.set_TIN(records['Recipient']['TIN'])
                    recipientData.set_RecipientId(
                        records['Recipient']['RecordId'])
                    recipientData.set_SubmissionId(records['SubmissionId'])
                    recipientData.set_BusinessNm(records['BusinessNm'])
                    recipientData.set_Status(records['Recipient']['Status'])
                    form1099NecList.append(recipientData.__dict__)

    return json.dumps(form1099NecList)

# Validate 1099 NEC Transmit response
def form_1099_nec_transmit_response_validation(response):
    if response is not None:

        if response['StatusCode'] == 200:

            return render_template('success.html',
                                   response='Status Timestamp=' + response['Form1099Records']['SuccessRecords'][0][
                                       'StatusTs'],
                                   ErrorMessage='Status= ' + response['Form1099Records']['SuccessRecords'][0]['Status'],
                                   ButtonText="Form 1099-NEC", FormType="NEC")

        elif 'Errors' in response and response['Errors'] is not None:

            return render_template('error_list.html', errorList=response['Errors'],
                                   status=str(response['StatusCode']) + " - " + str(
                                       response['StatusName']) + " - " + str(response['StatusMessage']))

        else:
            return render_template('success.html', response='StatusMessage=' + str(response['StatusCode']),
                                   ErrorMessage='Message=' + json.dumps(response))

# Validate 1099 NEC Get Pdf response
def form_1099_nec_get_pdf_response_validation(response):
    if 'Form1099NecRecords' in response and response['Form1099NecRecords'] is not None:
        if 'Message' in response['Form1099NecRecords'][0]:
            return render_template('pdf_response.html', errorList=response['Form1099NecRecords'],
                                   FormType="Form 1099-NEC")
        else:
            return "OK"
    elif 'Errors' in response and response['Errors'] is not None:
        return render_template('pdf_response.html', errorList=response['Errors'])

    return "OK"


# -*- coding: utf-8 -*-
"""
Created on Fri Jan  5 10:42:16 2018

E-mail: xieydd@gmail.com

@author: xieydd

@description:使用SVM进行人脸识别
"""
import numpy,time
from sklearn import datasets
from sklearn import svm
from sklearn import decomposition
from sklearn import manifold
from sklearn.cross_validation import train_test_split as tts
from sklearn import svm,neural_network
from sklearn.metrics import classification_report,precision_score,recall_score,f1_score
from sklearn import pipeline
from sklearn.preprocessing import StandardScaler,MinMaxScaler
from sklearn.grid_search import GridSearchCV
from matplotlib import pyplot
import matplotlib.colors as colors

def get_data():
    face_data=datasets.fetch_olivetti_faces()
    #face_data=datasets.load_iris()
    data=face_data.data
    target=face_data.target
    return data,target

def pca(x,n):
    pca_learner=decomposition.PCA(n_components=n)
    x=pca_learner.fit_transform(x)
    return x

def pca_svm(pca_n=10,svm_C=1):
    t1=time.time()
    data,target=get_data()
    #scale_learner=StandardScaler()
    #data=scale_learner.fit_transform(data)
    x_train,x_test,y_train,y_test=tts(data,target,random_state=33)
    pca_learner=decomposition.PCA(n_components=pca_n)
    x_train=pca_learner.fit_transform(x_train)
    svm_learner=svm.SVC(C=svm_C)
    svm_learner.fit(x_train,y_train)
    x_test_pre=pca_learner.transform(x_test)
    y_test_pre=svm_learner.predict(x_test_pre)
    # report=classification_report(y_test,y_test_pre)
    # print 'The Main Explanied: ',numpy.sum(pca_learner.explained_variance_ratio_)
    # print report
    # print x_test_pre.shape,y_test_pre.shape,y_test.shape
    ac=svm_learner.score(x_test_pre,y_test)
    p=precision_score(y_test,y_test_pre,average='weighted')
    r=recall_score(y_test,y_test_pre,average='weighted')
    f1=2.0/(1.0/p+1.0/r)
    t=time.time()-t1
    return ac,p,r,f1,t

def pca_svm_time_score_compare():
    ac_score=[]
    p_score=[]
    r_score=[]
    f1_score=[]
    tt=[]
    stand=MinMaxScaler((20,30))
    steps=numpy.arange(10,410,10)
    for n in steps:
        ac,p,r,f1,t=pca_svm(pca_n=n)
        p_score.append(p)
        f1_score.append(f1)
        r_score.append(r)
        ac_score.append(ac)
        tt.append(t)
    p_score_stand=stand.fit_transform(numpy.array(p_score).reshape((-1,1)))
    r_score_stand=stand.fit_transform(numpy.array(r_score).reshape((-1,1)))
    f1_score_stand=stand.fit_transform(numpy.array(f1_score).reshape((-1,1)))
    ac_score_stand=stand.fit_transform(numpy.array(ac_score).reshape((-1,1)))
    figure=pyplot.figure()

    pyplot.subplot(2,1,1)
    pyplot.scatter(steps,f1_score,label='f1-score',color='red',s=p_score_stand,alpha=0.7)
    pyplot.scatter(steps,r_score,label='recall-score',color='blue',s=r_score_stand,alpha=0.7)
    pyplot.scatter(steps,p_score,label='precision-score',color='yellow',s=f1_score_stand,alpha=0.7)
    pyplot.scatter(steps,ac_score,label='accuracy-score',color='purple',s=ac_score_stand,alpha=0.7)
    pyplot.xlabel('n-components')
    pyplot.ylabel('score')
    pyplot.legend()
    pyplot.title('The Score Of SVM After PCA To N_components')
    pyplot.subplot(2,1,2)
    pyplot.plot(steps,tt,label='cost-time',color='black',marker='o')
    # for i in range(len(tt)):
        # pyplot.text(steps[i],ac_score[i],str(round(tt[i],1))+'s',fontdict=dict(size=10,weight='normal'))
        # pyplot.plot([steps[i],steps[i]],[0,ac_score[i]],'--b')
    pyplot.legend()
    pyplot.xlabel('n-components')
    pyplot.ylabel('time')
    pyplot.show()

# pca before svm fitting is better
def pca_svm_pipeline():
    #svm_C=numpy.linspace(0.5,10,10)
    svm_C=[1]
    pca_n_components=numpy.arange(5,200,10)
    data,target=get_data()
    x_train,x_test,y_train,y_test=tts(data,target,random_state=33)
    #scale_learner=StandardScaler()
    pca_learner=decomposition.PCA()
    svm_learner=svm.SVC()
    pipe=pipeline.Pipeline([('pca',pca_learner),('svm',svm_learner)])
    gscv=GridSearchCV(pipe,
                      {'pca__n_components':pca_n_components,'svm__C':svm_C},n_jobs=-1)
    gscv.fit(x_train,y_train)
    y_test_pre=gscv.predict(x_test)
    report=classification_report(y_test,y_test_pre)
    print(gscv.best_params_ )
    print(report)
    target_pre=gscv.predict(data)
    n1,n2=data.shape
    figure=pyplot.figure()
    L=numpy.zeros((40,))
    xx=numpy.linspace(0,1,64)+13
    yy=numpy.linspace(1,0,64)+13
    xx,yy=numpy.meshgrid(xx,yy)
    for i in range(n1):
        k=target_pre[i]
        g=L[k]
        L[k]+=1
        xx1=xx-k
        yy1=yy-g
        pyplot.contourf(xx1,yy1,data[i].reshape((64,64)),cmap='gray')
        if target[i]!=target_pre[i]:
            pyplot.scatter(numpy.mean(xx1),numpy.mean(yy1),marker='x',c='red',s=40)
    pyplot.axis('off')
    pyplot.grid('off')
    pyplot.title('PCA & SVM Recongnize Faces')
    pyplot.show()

if __name__=='__main__':
    pca_svm_pipeline()  #Grid Search and show the results
    pca_svm_time_score_compare()  #Direct Search


"""
    This module contains functions relevant to Animating solutions
"""

import matplotlib.pyplot as plt
from matplotlib import animation, rc

def animate_heatmap(weight_trajectory, weight_bounds=3):
    """ Calls the solution_trajectory function, then creates an animated line plot of the solutions

        training_run -- an object containing a 'model', and a 'weight_trajectory'
        model -- and ODE
        weight_trajectory -- a series of weight matrices that represent progress along the way from the initial weights to final weights
        inital_conds -- the initial values of each of the variables in the model
    """
    fig, ax = plt.subplots(figsize = (6,6))
    fig.subplots_adjust(left=0, bottom=0.1, right=1, top=.9, wspace=None, hspace=None)
    # ax.set_title('Weight Trajectory')
    # ax.set_xlabel('Connections In')
    # ax.set_ylabel('Connections Out')
    ax = plt.imshow(weight_trajectory[0], cmap="coolwarm", vmin = -weight_bounds, vmax = weight_bounds)

    class ani:
        n=0
    def update(frame):
        ax.set_array(weight_trajectory[ani.n-2])
        ani.n = ani.n+1
        return ax,

    ani = animation.FuncAnimation(fig, update, interval=200, blit=True, frames= len(weight_trajectory)-1 )
    plt.close()
    return ani


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2018/7/30 16:53
# Project:
# @Author: ZQJ
# @Email : zihe@yscredit.com
from pyspider.libs.base_handler import *
from lxml import etree
import re

class Handler(BaseHandler):
    crawl_config = {
        'itag': 'v6.5',
        'time_out': 4000,
        # 'proxy': 'H14LXDJ6O07CAFDP:150D24D434AC09EE@proxy.abuyun.com:9010',
    }

    retry_delay = {
        0: 60,
        1: 60 * 5,
        2: 60 * 10,
        3: 60 * 15,
        4: 60 * 20,
        5: 60 * 25,
    }

    schema_def = [{"name": "id", "type": "string"},
                  {"name": "_id_", "type": "string"},
                  {"name": "ann_type", "type": "string"},
                  {"name": "announcer", "type": "string"},
                  {"name": "case_no", "type": "string"},
                  {"name": "ann_content", "type": "string"},
                  {"name": "ann_date", "type": "string"},
                  {"name": "content_url", "type": "string"},
                  {"name": "ann_html", "type": "string"},
                  {"name": "pdf_url", "type": "string"},
                  {"name": "source", "type": "string"},
                  {"name": "defendant", "type": "string"}
                  ]

    @every(minutes=24 * 60)
    @config(age=60 * 60)
    def on_start(self):
        url = 'http://zqzyssfw.gdzqfy.gov.cn/zqzy/front/ggxxList?ah=&fjm=JE0&gglx=sd&xszt=splc&pagenumber=2&dir=up&cs_token_flag=1&cs_token=21a2b8d642f845738e119bf21e63da63'
        self.crawl(url, callback=self.get_page)

    @config(age=60 * 60)
    def get_page(self, response):
        html = response.etree
        last_page = html.xpath('//div[@class="left"]//text()')
        all_page = int(re.findall(r'共(\d+)页', ''.join(last_page))[0]) + 1
        for i in range(1, 51):  # all_page
            url = 'http://zqzyssfw.gdzqfy.gov.cn/zqzy/front/ggxxList?ah=&fjm=JE0&gglx=sd&xszt=splc&pagenumber=' + str(
                i) + '&dir=up&cs_token_flag=1&cs_token=21a2b8d642f845738e119bf21e63da63'
            self.crawl(url, callback=self.index_page)

    @config(age=60 * 60)
    def index_page(self, response):
        try:
            html = response.etree
            table = html.xpath('//div[@class="doclist"]/table/tr')
            key = table[0].xpath('.//text()')
            key = list(map(lambda x: re.sub(r'\s', '', x), key))
            key = list(filter(lambda x: len(x) > 0, key))

            for each in table[1:]:
                case = each.xpath('.//text()')
                case = list(map(lambda x: re.sub(r'\s', '', x), case))
                case = list(filter(lambda x: len(x) > 0, case))
                print(case)

                ann_content = re.sub(r'[\{\}\']', '', str(dict(zip(key[1:], case[1:]))))
                case_on = case[1]
                ann_date = case[3] + 'T00:00:00+08:00'
                ann_html = etree.tounicode(each)
                defendant = case[5]

                yield {
                    'ann_type': '送达公告',
                    'announcer': '肇庆市中级人民法院',
                    'defendant': str(defendant),
                    'case_no': str(case_on),
                    'ann_content': str(ann_content),
                    'ann_date': str(ann_date),
                    'content_url': str(response.url),
                    'ann_html': str(ann_html),
                    'pdf_url': '',
                    'source': '肇庆市中级人民法院网上诉讼平台',
                    'id': '',
                    '_id_': ''
                }
        except Exception as e:
            print(e)