from pyteomics import mzid # @UnresolvedImport
import zipfile, os, gzip

class MZIdentReader():
    def __init__(self, filename, **kwargs):

        if filename.endswith('.gz') or filename.endswith('.zip'):
            self.filename = self.extract_file(filename)
        else:
            self.filename = filename
        self.source = self.create_parser()
        self.index_dict = {}
        self.index_dict_num = 0

    def extract_file(self, archive):
        if archive.endswith('gz'):
            in_f = gzip.open(archive, 'rb')
            archive = archive.replace(".gz", "")
            out_f = open(archive, 'wb')
            out_f.write(in_f.read())
            in_f.close()
            out_f.close()
            return archive

        elif archive.endswith('zip'):
            unzip_path = os.path.dirname(archive)
            zip_ref = zipfile.ZipFile(archive, 'r')
            zip_ref.extractall(unzip_path)
            zip_ref.close()

            return os.path.splitext(archive)

    def create_parser(self):
        return mzid.read(self.filename)

    def get_next(self):
        return next(self.source)

    def reset(self):
        self.source.reset()

    def set_index_dict(self, index_dict):
        self.index_dict = index_dict
        self.index_dict_num = len(index_dict)

    def match_identification_with_peaklist(self, peaklist, index_dict=None):
        if index_dict is not None:
            self.index_dict = index_dict

        if len(self.index_dict) == 0:
            print("Index dictionary is empty")
            return

        self.reset()

        found, notfound = 0, 0
        for identity in self.source:
            check, title, scanID = self.find_spectral_match(identity)
            if check:
                identification = self.get_spectrum_information(identity)
                peaklist[scanID]['identification'] = identification
                found += 1
            else:
                notfound += 1

        msg = "Found {}/{} | Not found {}/{}. There are {} unassigned spectra".format(found, found+notfound,
                                                                                notfound, found+notfound,
                                                                                len(peaklist)-found)
        print(msg)
        return peaklist

    def find_spectral_match(self, identification):
        """
        Check whether PSM match the peak list
        -----
        identification (dict): mzIdent dataset
        spectral_dict (dict): key - value dictionary of ORIGAMI scan names and peaklist file titles
        """

        if identification.get("spectrum title", "") in self.index_dict:
            return True, identification['spectrum title'], self.index_dict[identification['spectrum title']]
        elif identification.get("name", "") in self.index_dict:
            return True, identification['name'], self.index_dict[identification['name']]
        else:
            return False, None, None

    def get_spectrum_information(self, spectrum_in):
        spectrum_out = {}
        for spec in xrange(len(spectrum_in.get('SpectrumIdentificationItem', []))):
            peptide_sequence = spectrum_in['SpectrumIdentificationItem'][spec]['PeptideSequence']
            experimental_mz = spectrum_in['SpectrumIdentificationItem'][spec]['experimentalMassToCharge']
            calculated_mz = spectrum_in['SpectrumIdentificationItem'][spec]['calculatedMassToCharge']
            charge = spectrum_in['SpectrumIdentificationItem'][spec].get('chargeState', 0)

            # get dictionaries
            scores = self.get_scores(spectrum_in['SpectrumIdentificationItem'][spec])
            peptide_info = self.get_peptide_information(spectrum_in['SpectrumIdentificationItem'][spec].get('PeptideEvidenceRef', []))
            modification_info = self.get_modifications(spectrum_in['SpectrumIdentificationItem'][spec].get('Modification', []))
            spectrum_out[spec] = {'peptide_seq':peptide_sequence,
                                  'experimental_mz':experimental_mz,
                                  'calculated_mz':calculated_mz,
                                  'charge':charge,
                                  'scores':scores,
                                  'peptide_info':peptide_info,
                                  'modification_info':modification_info}

        return spectrum_out

    def get_peptide_information(self, peptide_in):
        """
        Retrieve peptide information from mzIdent file
        """
        peptide_out = {}
        for pep in xrange(len(peptide_in)):
            peptide_sequence = peptide_in[pep].get('Seq', "")
            protein_description = peptide_in[pep].get('protein description', "")
            accession = peptide_in[pep].get('accession', "")
            start = peptide_in[pep].get('start', "")
            end = peptide_in[pep].get('end', "")
            pre = peptide_in[pep].get('pre', "")
            post = peptide_in[pep].get('post', "")

            peptide_out[pep] = {'peptide_seq':peptide_sequence,
                                'protein_description':protein_description,
                                'accession':accession,
                                'start':start,
                                'end':end,
                                'peptide_prev_aa':pre,
                                'peptide_next_aa':post}

        return peptide_out

    def get_modifications(self, modifications_in):
        """
        Retrieve modification information from mzIdent file
        """
        modifications_out = {}
        for mod in xrange(len(modifications_in)):
            modification_location = modifications_in[mod].get('location', "")
            modification_mass_delta = modifications_in[mod].get('monoisotopicMassDelta', "")
            modification_name = modifications_in[mod].get('name', "")
            modification_residues = modifications_in[mod].get('residues', "")

            modifications_out[mod] = {'location':modification_location,
                                      'mass_delta':modification_mass_delta,
                                      'name':modification_name,
                                      'residues':modification_residues}

        return modifications_out

    def get_scores(self, spectrum_in):
        """
        Retrieve a dictionary with scores
        """
        scaffold_pep_prob = spectrum_in.get('Scaffold:Peptide Probability', "")
        identity_e_score = spectrum_in.get('IdentityE Score', "")
        mascot_score = spectrum_in.get('Mascot:score', "")
        mascot_threshold = spectrum_in.get('Mascot:identity threshold', "")

        scores = {'scaffold_peptide_probability':scaffold_pep_prob,
                  'identity_e_score':identity_e_score,
                  'mascot_score':mascot_score,
                  'mascot_threshold':mascot_threshold}

        return scores


import logging
import os
import tempfile

import boto3
import pandas as pd
from airflow import DAG
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.python import PythonOperator
from airflow.operators.subdag_operator import SubDagOperator
from airflow.providers.amazon.aws.hooks.s3 import S3Hook
from airflow.providers.postgres.operators.postgres import PostgresOperator
from airflow.utils.helpers import chain
from botocore import UNSIGNED
from botocore.client import Config
from custom_postgreshook import CustomPostgresHook

POSTGRES_CONN_ID = "postgres_default"
pg_hook = CustomPostgresHook(postgres_conn_id=POSTGRES_CONN_ID)
logger = logging.getLogger("Historical refresh")

def create_historical_refresh_dag(
    # fmt: off
    dag_id, schedule, symbol, granularity,
    bucket, prefix, symbol_id, default_args
    # fmt: on
):
    """
    Function returns a dag instance which loads data from
    Binance's S3 bucket to load historical data quickly.
    S3 Bucket containing monthly/daily data can be used to load
    data containing different intervals.

    Args:
        schedule: schedule at which DAG should run.
        symbol (str): Stock/Crypto symbol which will be used to pull data.
        symbol_id (int): Symbol Id of the symbol which was shared.
        granularity (str): Time period for which candles are created.
                Ex: 1m, 5m etc.
        bucket (str): Binance's S3 bucket from which data is procured.
        prefix (str): Prefix to S3 folder within bucket.
    Returns:
        instance of Airflow DAG.
    """

    def get_list_from_s3(*args, **context):
        """
        Get list of data files from S3 bucket
        """
        updated_prefix = prefix.format(
            frequency="monthly", symbol=symbol, granularity=granularity
        )

        filepaths = []
        bucket_obj = S3Hook("aws_default").get_bucket(bucket)

        for obj in bucket_obj.objects.filter(Prefix=updated_prefix):
            path, filename = os.path.split(obj.key)
            if filename.endswith("zip"):
                filepaths.append(obj.key)

        context["ti"].xcom_push(key="file_list", value=filepaths)
        logging.info("Found {:d} files".format(len(filepaths)))

    def get_historical_data(*args, **context):

        filepaths = context["ti"].xcom_pull(
            task_ids="get_list_from_s3_{:s}".format(symbol), key="file_list"
        )
        for key in filepaths:
            logging.info("Downloading file: {:s}".format(key))
            # do not sign s3 requests
            config_no_sign = Config(signature_version=UNSIGNED)
            client = boto3.client("s3", config=config_no_sign)
            obj = client.get_object(Bucket=bucket, Key=key)
            file = obj.get("Body").read()
            temp_file = tempfile.NamedTemporaryFile().name
            with open(temp_file, "wb") as file_handle:
                file_handle.write(file)

            df = pd.read_csv(
                temp_file,
                compression="zip",
                header=None,
                names=[
                    "open_time",
                    "open",
                    "high",
                    "low",
                    "close",
                    "volume",
                    "close_time",
                    "quote_av",
                    "trades",
                    "tb_base_av",
                    "tb_quote_av",
                    "ignore",
                ],
                dtype={"open_time": int, "close_time": int},
            )
            df = df.drop("ignore", axis=1)
            df.loc[:, "symbol_id"] = symbol_id

            pg_hook.write_pandas_df(
                df.to_json(),
                name="tickerdata_{:s}".format(granularity),
                if_exists="append",
                index=False,
            )

    dag = DAG(dag_id, schedule_interval=schedule, default_args=default_args)

    with dag:
        t1 = PythonOperator(
            task_id="get_list_from_s3_{:s}".format(symbol),
            python_callable=get_list_from_s3,
            provide_context=True,
        )
        t2 = PythonOperator(
            task_id="get_historical_data_{:s}".format(symbol),
            python_callable=get_historical_data,
            provide_context=True,
        )

        SQL = """UPDATE symbols
            SET symbol_last_updated_at=timezone('utc', NOW())
            WHERE symbol_id = {:d}""".format(
            symbol_id
        )
        t3 = PostgresOperator(
            task_id="Set_{:s}_last_updated_at".format(symbol),
            postgres_conn_id=POSTGRES_CONN_ID,
            sql=SQL,
        )
        t1 >> t2 >> t3

    return dag

def get_symbols_needing_historical():
    """
    Function runs a SQL statement to identify
    symbols which do not have any historical data.
    Idea is to backfill large years of data using historical feeds
    and use incremental data refresh to fill data spanning a few days.
    """
    N = 1  # number of symbols to update per dag run
    symbols_with_no_historic_data = """
        SELECT x.symbol_id, x.symbol_name
        FROM symbols x
        LEFT JOIN (SELECT symbol_id, max(close_time) last_close_time
          FROM tickerdata_1m
          GROUP BY 1) y ON x.symbol_id = y.symbol_id
        WHERE last_close_time IS NULL
    """
    # For Testing purposes:
    # symbols_with_no_historic_data = """
    #     SELECT 3649 as symbol_id, 'EOSUSDT' as symbol_name
    # """
    symbol_list = pg_hook.get_pandas_df(symbols_with_no_historic_data).head(N)
    logger.info("Getting historical data for {:d} symbols".format(N))
    return symbol_list

def create_subdag_operator(
    # fmt: off
    parent_dag, schedule, symbol, granularity,
    bucket, prefix, symbol_id, default_args
    # fmt: on
):
    """

    :meta private:
    Function creates dag which will be consumed by SubDagOperator.

    Args:
        schedule: schedule at which DAG should run.
        symbol (str): Stock/Crypto symbol which will be used to pull data.
        symbol_id (int): Symbol Id of the symbol which was shared.
        granularity (str): Time period for which candles are created.
                Ex: 1m, 5m etc.
        bucket (str): Binance's S3 bucket from which data is procured.
        prefix (str): Prefix to S3 folder within bucket.

    Returns:
        instance of Subdag operator.
    """
    subdag_id_suffix = "historical_data_refresh_{}".format(str(symbol))
    parent_dag_id = parent_dag.dag_id.split(".")[0]
    subdag_id = "{}.{}".format(parent_dag_id, subdag_id_suffix)
    logger.info("Creating dag for {:s}".format(subdag_id_suffix))
    subdag = create_historical_refresh_dag(
        subdag_id,
        schedule,
        symbol,
        granularity,
        bucket,
        prefix,
        symbol_id,
        default_args,
    )
    # fmt: off
    sd_op = SubDagOperator(task_id=subdag_id_suffix,
                           dag=parent_dag,
                           subdag=subdag)
    # fmt: on
    return sd_op

def create_subdag_operators(parent_dag, symbol_list):
    """
    :meta private:

    Private function to create subdag operators based on
    given parent dag and list of symbols.
    """
    subdags = []
    granularity = "1m"
    bucket = "data.binance.vision"
    prefix = "data/spot/{frequency}/klines/{symbol}/{granularity}/"
    schedule = "@once"
    subdags = [
        create_subdag_operator(
            parent_dag,
            schedule,
            row.loc["symbol_name"],
            granularity,
            bucket,
            prefix,
            row.loc["symbol_id"],
            default_args,
        )
        for index, row in symbol_list.iterrows()
    ]
    # chain subdag-operators together
    chain(*subdags)
    return subdags

default_args = {
    "owner": "airflow",
    "start_date": "2020-05-09",
    "catchup": False,  # prevents backfill jobs
}

parent_dag = DAG(
    dag_id="historical_refresh_dag_v1",
    default_args=default_args,
    schedule_interval=None,
)

symbol_list = get_symbols_needing_historical()
subdag_ops = create_subdag_operators(parent_dag, symbol_list)

# Dummy operations added to see quick progress on Airflow UI
dummy_op_start = DummyOperator(task_id="dummy_op_start", dag=parent_dag)
dummy_op_start >> subdag_ops[0]

dummy_op_end = DummyOperator(task_id="dummy_op_end", dag=parent_dag)
subdag_ops[-1] >> dummy_op_end


#!/usr/bin/env python

import requests
import os

GITHUB_URL = "https://api.github.com/orgs/{}/repos"

def main():
    github_token = os.getenv("GITHUB_TOKEN")
    github_org = os.getenv("GITHUB_ORGANIZATION")
    s3_bucket_name = os.getenv("BACKUP_BUCKET")
    repo_blacklist = os.getenv("REPO_BLACKLIST").split("|")

    res = requests.get(url=GITHUB_URL.format(github_org), auth=requests.auth.HTTPBasicAuth(github_token, "x-oauth-basic"))
    repo_list = [x["name"] for x in res.json()]
    print(f"Retrieved {len(repo_list)} for {github_org} organization: {', '.join(repo_list)}")

    repo_list_clean = [r for r in repo_list if r.lower() not in [b.lower() for b in repo_blacklist]]
    print(f"Removed blacklisted repos, {len(repo_list_clean)} left for {github_org} organization: {', '.join(repo_list_clean)}")

    with open("repo_list.txt", "w") as f:
        for repo in repo_list_clean:
            f.write(f"{repo}\n")

    print("Wrote repo list to repo_list.txt")

if __name__ == "__main__":
    main()


import face_recognition
import os, sys
import cv2
import numpy as np
import math

# Helper
def face_confidence(face_distance, face_match_threshold=0.6):
    range = (1.0 - face_match_threshold)
    linear_val = (1.0 - face_distance) / (range * 2.0)

    if face_distance > face_match_threshold:
        return str(round(linear_val * 100, 2)) + '%'
    else:
        value = (linear_val + ((1.0 - linear_val) * math.pow((linear_val - 0.5) * 2, 0.2))) * 100
        return str(round(value, 2)) + '%'

class FaceRecognition:
    face_locations = []
    face_encodings = []
    face_names = []
    known_face_encodings = []
    known_face_names = []
    process_current_frame = True
    students_present =[]

    def __init__(self):
        self.encode_faces()

    def encode_faces(self):
        for image in os.listdir('faces'):
            face_image = face_recognition.load_image_file(f"faces/{image}")
            face_encoding = face_recognition.face_encodings(face_image)[0]

            self.known_face_encodings.append(face_encoding)
            self.known_face_names.append(image)

    def run_recognition(self):
        my_list = []
        video_capture = cv2.VideoCapture(0)

        if not video_capture.isOpened():
            sys.exit('Video source not found...')

        while True:
            ret, frame = video_capture.read()

            if self.process_current_frame:
                small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
                    #frame small akaumpo error but yyyyy??
                rgb_small_frame = small_frame[:, :, ::-1]

                self.face_locations = face_recognition.face_locations(frame)
                self.face_encodings = face_recognition.face_encodings(frame, self.face_locations)

                self.face_names = []
                for face_encoding in self.face_encodings:
                    matches = face_recognition.compare_faces(self.known_face_encodings, face_encoding)
                    name = "Unknown"
                    confidence = '???'

                    face_distances = face_recognition.face_distance(self.known_face_encodings, face_encoding)

                    best_match_index = np.argmin(face_distances)
                    if matches[best_match_index]:
                        name = self.known_face_names[best_match_index]
                        confidence = face_confidence(face_distances[best_match_index])
                        print(f'{name} ({confidence})')
                        if (confidence) > '85':
                            my_list.append(name)
                            print(name)
                            if name not in self.students_present:
                                self.students_present.append(name)

                    self.face_names.append(f'{name} ({confidence})')

            cv2.imshow('Face Recognition', frame)

            if cv2.waitKey(1) == ord('q'):
                video_capture.release()
                cv2.destroyAllWindows()

if __name__ == '__main__':
     fr = FaceRecognition()
     fr.run_recognition()


import socket
import dnslib

UDP_IP = "0.0.0.0"
UDP_PORT = 53

sock = socket.socket(socket.AF_INET, # Internet
                     socket.SOCK_DGRAM) # UDP
sock.bind((UDP_IP, UDP_PORT))

def isint(INTEGER):
    try:
        int(INTEGER)
        return True
    except:
        return False

while True:
    try:
        data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
        myreq = dnslib.DNSRecord.parse(data)
        print("------START------")
        print("RECEIVED MESSAGE --> ", myreq)
        print("^^^ FROM ADDRESS --> ", addr)
        for line in str(myreq).split("\n"):
            if "id:" in line:
                request = line.split(' ')[7].replace(';', '')
                print("REQUEST_ID: ", request)
                break

        returnaddr = []
        inreq = ""
        for line in str(myreq).split("\n"):
            if ".simple.io" in line:
                inreq = line.split(' ')[0].replace(';', '')
                for line in inreq.split('.'):
                    if isint(line):
                        returnaddr.append(str(line)+".")

        if len(returnaddr) > 4:
            returnaddr = returnaddr[:4]

        addrstring = ''.join(e for e in returnaddr)
        print("RETURNING IP ", addrstring[:len(addrstring)-1])
        d = dnslib.DNSRecord(dnslib.DNSHeader(id=int(request),qr=1,aa=1,ra=1), q = dnslib.DNSQuestion(inreq), a = dnslib.RR(inreq,rdata=dnslib.A(addrstring[:len(addrstring)-1])))

        sock.sendto(d.pack(), addr)
        print("======END=======")
        print()
    except Exception as e:
        print()
        print("FAILURE IN PARSER: ", e)
        print()


import random

dic_v1 = {}

for i in range(1,11):
  dic_v1[i] = i * 2

print(dic_v1)

dic_v2 = {i: i * 2 for i in range(1,11)}
print(dic_v2)

contries_mercosur = ['ar','br','pr','uy']
population = {}
for i in contries_mercosur:
  population[i] = random.randint(3,220)
print(population)

pop_v2 = {i: random.randint(3,220) for i in contries_mercosur}
print(pop_v2)

names = ['pedro','marcelo','cacho']
ages = [21,54,35]

print(list(zip(names,ages)))

new_dic = {names: ages for (names,ages) in zip(names,ages)}
print(new_dic)


import data_loader
from tqdm import tqdm
import json
import argparse
import models
import os
import paddle
class Predictor(object):
    def __init__(self, model,config):
        self.model = model
        self.config = config
    def load(self, path=None):
        if path:
            self.model.set_state_dict(paddle.load(path))

        else:
            self.model.set_state_dict(paddle.load(
                os.path.join(self.config.save_path, self.config.dataset, self.model.model_name + ".pt")
            ))

    def sequence_tag2tag(self, text, label):
        # 统计序列标注中的实体及其类型
        # 存放实体相关信息，以字典结构保存，其中包括entity、type以及index
        item = dict()
        # 保存当前正在读取的实体，实体结束后会存入item["entity"]中
        _entity = str()
        # ner中存放当前语料包含的所有实体
        ner = list()
        index = list()
        # 遍历序列标注形式的标签，如果当前标签中包含“B-”则表明“上一个实体已经读取完毕，现在开始要开始读取一个新的实体”
        # 如果当前标签中包含“I-”，说明正在读取的实体还未结束，将当前标签所对应的字添加进_entity中，继续遍历
        # 循环结束后，如果item中不为空，说明存在有未保存的实体，将相关实体信息添加到字典中，最后添加到数据集中。
        for i, (t, l) in enumerate(zip(text, label)):
            if "B-" in l:
                if item:
                    item["entity"] = _entity
                    item["index"] = index
                    ner.append(item)
                    _entity = str()
                    item = dict()
                    index = list()
                item["type"] = l.split("-")[1]
                _entity = t
                index.append(i)
            if "I-" in l and item is not None:
                _entity += t
                index.append(i)
        if item:
            item["entity"] = _entity
            item["index"] = index
            ner.append(item)
            _entity = str()
            item = dict()
            index = list()
        return ner
    def predcit(self,data_loader,origin_data):
        result = list()
        batch = 0
        with paddle.no_grad():
            for data_batch in tqdm(data_loader):
                sentence_batch = origin_data[batch : batch + args.batch_size]
                data_batch = [data for data in data_batch]
                bert_inputs, sent_length = data_batch
                outputs = self.model(bert_inputs)
                for sentence, pred_label, bert_input in zip(sentence_batch,outputs,bert_inputs):
                    sentence = sentence["sentence"]
                    _valid_index = bert_input.not_equal(paddle.to_tensor(0, dtype="int64")).astype("int64")
                    pred_label = paddle.argmax(pred_label, -1)[_valid_index].cpu().numpy()
                    pred_label = [args.vocab.id_to_label(str(i)) for i in pred_label]
                    result.append({"sentence":sentence,"label":self.sequence_tag2tag(sentence,pred_label)})
                batch += args.batch_size
        with open(os.path.join(args.save_path,args.task,"model_predicted.json"), "w", encoding="utf-8") as f:
            json.dump(result, f, indent=2, ensure_ascii=False)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--task', type=str, default='参照组')
    parser.add_argument('--save_path', type=str, default='./outputs')
    parser.add_argument('--predict_path', type=str, default='./outputs')
    parser.add_argument('--bert_name', type=str, default=r"bert-base-chinese")
    parser.add_argument('--batch_size', type=int, default=4)
    parser.add_argument('--bert_hid_size', type=int, default=768)
    parser.add_argument('--conv_hid_size', type=int, default=96)
    parser.add_argument('--dilation', type=list, default=[1,2,3,4])
    parser.add_argument('--device', type=str, default="cuda")
    args = parser.parse_args(args=[])

    dataset, ori_data = data_loader.load_real_bert(args)
    data_loader = paddle.io.DataLoader(dataset=dataset
                                       ,collate_fn=data_loader.pred_collate_fn
                                       ,batch_size=args.batch_size
                                    )
    model = models.bertCNN(args)
    predictor = Predictor(model,args)
    predictor.load(os.path.join("./outputs",args.task,model.model_name + ".pt"))
    predictor.predcit(data_loader,ori_data)


import pandas as pd
import numpy as np

def fill_missing(values):
    one_day = 60*24
    for row in range(values.shape[0]):
        for col in range(values.shape[1]):
            if(np.isnan(values[row, col])):
                values[row, col] = values[row-one_day, col]

#load the dataset
dataset = pd.read_csv('data_1.txt', sep=';', header=0, low_memory=False, infer_datetime_format=True, parse_dates={'datetime':[0,1]}, index_col=['datetime'])

#missing value as NaN
dataset.replace('?', np.nan, inplace=True)
#convert all to float type
dataset = dataset.astype('float32')

#populate missing values
fill_missing(dataset.values)

#add a col for remainder meter
values = dataset.values
dataset['sub_metering_4'] = (values[:,0]*1000/60) - (values[:,4]+values[:,5]+values[:,6])

#save the new dataset to csv format
dataset.to_csv('data_2.csv')


#!/usr/bin/env python3
import os
import re

INPUT_REGEX_PATTERN = re.compile(
    r"target area: "
    r"x=(?P<min_x>\d+)\.\.(?P<max_x>\d+), "
    r"y=(?P<min_y>-?\d+)..(?P<max_y>-?\d+)"
)

def parse_input(f) -> tuple[int, int, int, int]:
    m = INPUT_REGEX_PATTERN.match(f.readline().strip())
    if m is None:
        raise ValueError("Expected input match")
    return (
        int(m.group("min_x")),
        int(m.group("max_x")),
        int(m.group("min_y")),
        int(m.group("max_y")),
    )

def max_trajectory_height(y_0: int) -> int:
    if y_0 <= 0:
        return 0
    # Based on the gravity rules the max trajectory height is 1 + 2 + ... + y_0
    # i.e. the y_0th triangle number.
    return y_0 * (y_0 + 1) // 2

def find_initial_velocity_values(
    target_area: tuple[int, int, int, int]
) -> list[tuple[int, int]]:
    (min_x, max_x, min_y, max_y) = target_area

    initial_velocity_values: list[tuple[int, int]] = []

    # We can use some simple bounds to narrow the search space to:
    #
    #   1 <= x_0 <= max_x, assuming min_x > 0
    #
    # - Initial x velocity must be positive to get to a positive
    #   target area width.
    # - Initial x velocity can be no larger than the max target area
    #   width otherwise we overshoot in a single step.
    #
    #   min_y <= y_0 < abs(min_y), assuming min_y < 0
    #
    # - Initial y velocity cannot be lower than the target area floor
    #   otherwise we drop below it in a single step.
    # - Initial y velocity can be no larger than the absolute value of
    #   the target area floor as our y velocity when we cross the x
    #   will be (1 + y_0) so otherwise we would drop below it in the
    #   following step.
    for x_0 in range(1, max_x + 1):
        for y_0 in range(min_y, abs(min_y)):
            x, y = 0, 0
            dx, dy = x_0, y_0
            while x <= max_x and y >= min_y:
                if min_x <= x <= max_x and min_y <= y <= max_y:
                    initial_velocity_values.append((x_0, y_0))
                    break
                x += dx
                y += dy
                dx = 0 if dx == 0 else dx - 1
                dy -= 1

    return initial_velocity_values

if __name__ == "__main__":
    with open(os.path.join(os.path.dirname(__file__), "input17.txt")) as f:
        target_area = parse_input(f)

    initial_velocity_values = find_initial_velocity_values(target_area)

    print("Part One:")
    print(max(max_trajectory_height(y_0) for _, y_0 in initial_velocity_values))

    print("Part Two:")
    print(len(initial_velocity_values))


"""
https://leetcode.com/problems/longest-palindromic-substring/
Author: Will Cray
Date: 3/17/2022
Time: O(N^2)
Space: O(1)
"""

class Solution:

    # input: string, english letters and digits
    # input: s is at least of length one

    # output: longest palindromic substring

    # ex. - "bab" -> "bab"
    # ex. - "babcac" -> "bab"
    # ex. - "abc" -> "a"
    # ex. - "a" -> "a"
    # ex - "aaaaaa" -> "aaaaaa"
    # ex - "aaabaab" -> "baab"

    def longestPalindrome(self, s: str) -> str:

        # solution - brute force
        # try all substrings
        # function to check for palindrome, maintain max string
        # time: O(N^3)
        # space: O(N)

        # DP
        # time: O(N^2)
        # space: O(N^2)

        # expand from middle
        # time: O(N^2)
        # space: O(1)

        # init longest substring
        best_l, best_r = 0, 0

        # iterate over string with window of size 1 and of size 2
        for i in range(len(s)):

            # call function that returns longest palidrome with that being the start
            # single char pivot
            # r is inclusive
            s_l, s_r = self.getLongest(s, i, i)
            if s_r - s_l > best_r - best_l:
                best_l, best_r = s_l, s_r

            # double char pivot
            # ensure not at end of list and two characters match
            if i + 1 < len(s) and s[i] == s[i + 1]:
                d_l, d_r = self.getLongest(s, i, i + 1)

                if d_r - d_l > best_r - best_l:
                    best_l, best_r = d_l, d_r

        # return longest substring
        return s[best_l:best_r + 1]

    # returning tuple of longest valid palindrome from starting point
    # r is inclusive
    def getLongest(self, s: str, l: int, r: int) -> Tuple[int, int]:

        # if moving outward one step is in bounds and still valid, move outward
        while l - 1 >= 0 and r + 1 < len(s) and s[l - 1] == s[r + 1]:

            l -= 1
            r += 1

        return (l, r)


#!/usr/bin/env python3
import glob, sys, os.path, re, gzip
from datetime import datetime, date, timedelta
import time
from math import inf, floor

root_dir = sys.argv[1]
stats = {
    "server": {
        "last": None,
        "total": timedelta(),
        "idle": timedelta(),
        "idlestart": None
    },
    "players": {}
}

def _stripext(fname: str) -> str:
    fname = os.path.basename(fname)
    while fname.endswith(".gz") or fname.endswith(".log"):
        fname = fname[:fname.rindex(".")]

    return fname

def _sortkey(fname: str) -> list:
    fname = _stripext(fname)
    if fname == "latest":
        return [inf]
    return [int(i) for i in fname.split("-")]

for logfile in sorted(glob.iglob("*.log*", root_dir=root_dir), key=_sortkey):
    logfile = os.path.join(root_dir, logfile)
    print(f"  opening {logfile}", file=sys.stderr)

    _fname = _stripext(logfile)
    if _fname == "latest": # assume latest.log has correct timestamp
        filedate = date.fromtimestamp(os.path.getctime(logfile))
        print(f"  assuming latest.log has correct ctime {filedate}", file=sys.stderr)
    else:
        filedate = datetime.strptime(_fname[:_fname.rindex("-")], "%Y-%m-%d")

    if logfile.endswith(".gz"):
        print("  -> is gzip...", file=sys.stderr)
        f = gzip.open(logfile, "rt")
    else:
        f = open(logfile, "rt")

    with f:
        for line in f:
            line = line.strip()
            if not re.match(r"^\[\d\d:\d\d:\d\d\] \[", line): continue
            msgtime = datetime.combine(filedate, datetime.strptime(line[:line.index(" ")], "[%H:%M:%S]").time())
            line = line[line.index(" ")+1:]

            if line.startswith("[Server thread/INFO]: "):
                line = line[22:]

                # start
                if line.startswith("Starting minecraft server"):
                    stats["server"]["idlestart"] = stats["server"]["last"] = msgtime
                    print(f"! {msgtime} : start", file=sys.stderr)
                    print("* idle: started", file=sys.stderr)

                # stop
                elif line == "Closing Server":
                    stats["server"]["total"] += msgtime - stats["server"]["last"]
                    stats["server"]["idle"] += msgtime - stats["server"]["idlestart"]
                    stats["server"]["last"] = None
                    print(f"! {msgtime} : stop -> running count is {stats['server']['total']}", file=sys.stderr)
                    print(f"* dump idle: server stop -> running count is {stats['server']['idle']}", file=sys.stderr)

                # joined
                elif re.match(r"^[a-zA-Z0-9_]* joined the game$", line):
                    player = line[:line.index(" ")]
                    print(f"! {msgtime} : joined:  '{player: <16}'", file=sys.stderr)
                    if player not in stats["players"]: stats["players"][player] = {
                        "last": None, "playtime": timedelta(),
                        "deaths": 0, "commands": 0,
                        "messages": 0, "advancements": 0
                    }
                    stats["players"][player]["last"] = msgtime
                    stats["players"][player]["online"] = True
                    if len(list(filter(lambda p: p["online"], stats["players"].values()))) < 2:
                        stats["server"]["idle"] += msgtime - stats["server"]["idlestart"]
                        stats["server"]["idlestart"] = None
                        print(f"* no longer idle: first player joined -> running count is {stats['server']['idle']}", file=sys.stderr)

                # left
                elif re.match(r"^[a-zA-Z0-9_]* lost connection: .*$", line):
                    player = line[:line.index(" ")]
                    reason = line[line.index(" "):][18:]
                    stats["players"][player]["playtime"] += msgtime - stats["players"][player]["last"]
                    stats["players"][player]["online"] = False
                    print(f"! {msgtime} : left:    '{player: <16}', '{reason}' -> running count is {stats['players'][player]['playtime']}", file=sys.stderr)
                    if not any([p["online"] for p in stats["players"].values()]):
                        stats["server"]["idlestart"] = msgtime
                        print("* idle: no online players", file=sys.stderr)

                # death
                elif re.match(r"^[a-zA-Z0-9_]* (was|drowned|experienced|blew|hit|fell|went|walked|burned|discovered|froze|starved|suffocated|didn't|withered|died).*$", line):
                    player = line[:line.index(" ")]
                    reason = line[line.index(" ")+1:]
                    print(f"! {msgtime} : death:   '{player: <16}', '{reason}'", file=sys.stderr)
                    stats["players"][player]["deaths"] += 1

                # command
                elif re.match(r"^[a-zA-Z0-9_]* issued server command: .*$", line):
                    player = line[:line.index(" ")]
                    command = line[line.index(": ")+2:]
                    print(f"! {msgtime} : command: '{player: <16}', '{command}'", file=sys.stderr)
                    stats["players"][player]["commands"] += 1

                elif re.match(r"^[a-zA-Z0-9_]* has made the advancement \[.*\]$", line):
                    player = line[:line.index(" ")]
                    advname = line[line.index(" [")+2:-1]
                    print(f"! {msgtime} : adv:     '{player: <16}', '{advname}'", file=sys.stderr)
                    stats["players"][player]["advancements"] += 1

            # chatmsg
            elif re.match(r"^\[Async Chat Thread - #\d+/INFO\]:( \[Not Secure\]){0,1} <", line):
                line = line[line.index("]: ")+3:]
                if line.startswith("[Not Secure] "):
                    line = line[line.index("] <")+3:]
                else:
                    line = line[1:]
                player = line[:line.index("> ")]
                content = line[line.index("> ")+2:]
                print(f"! {msgtime} : chatmsg: '{player: <16}', '{content}'", file=sys.stderr)
                stats["players"][player]["messages"] += 1

stats["server"]["running"] = stats["server"]["last"] is not None
stats["server"]["isidle"] = stats["server"]["idlestart"] is not None
now = datetime.fromtimestamp(floor(time.time()))
if stats["server"]["running"]:
    stats["server"]["total"] += now - stats["server"]["last"]
if stats["server"]["isidle"]:
    stats["server"]["idle"] += now - stats["server"]["idlestart"]

for p in stats["players"]:
    if stats["players"][p]["online"]:
        stats["players"][p]["playtime"] += now - stats["players"][p]["last"]

#print(stats)
if "-fjson" in sys.argv:
    import json
    newstats = {
        "server": {
            "total_uptime": stats["server"]["total"].seconds,
            "idle_time": stats["server"]["idle"].seconds,
            "is_idle": stats["server"]["isidle"],
            "running": stats["server"]["running"]
        },
        "players": {}
    }
    for p in stats["players"]:
        newstats["players"][p] = {
            "playtime": stats["players"][p]["playtime"].seconds,
            "is_online": stats["players"][p]["online"],
            "deaths": stats["players"][p]["deaths"],
            "messages": stats["players"][p]["messages"],
            "commands": stats["players"][p]["commands"],
            "advancements": stats["players"][p]["advancements"]
        }
    json.dump(newstats, sys.stdout)
    sys.stdout.write("\n")
else:
    print(f"""Server:
  Total uptime: {stats['server']['total']}{' + running' if stats['server']['running'] else ''}
  Idle: {stats['server']['idle']}{' + idling' if stats['server']['isidle'] else ''}
Players:""")
    for p, s in sorted(stats["players"].items(), key=lambda p: p[1]["playtime"], reverse=True):
        print(f"""  {p: <16}
    total playtime: {stats['players'][p]['playtime']}{' + online' if stats['players'][p]['online'] else ''}
    deaths:{stats['players'][p]['deaths']: >7}, messages:{stats['players'][p]['messages']: >8}
    commands:{stats['players'][p]['commands']: >5}, advancements:{stats['players'][p]['advancements']: >4}""")


import random
import numpy as np
import torch.utils.data as data
import utils.utils_image as util
import os
from utils import utils_blindsr as blindsr

import json
from collections import Counter

def check_list(*l):
    count = Counter(l)
    p_count = count['Positive']
    s_count = count['Similar']
    n_count = count['Negative']
    return p_count, s_count, n_count

def find_max_repetition(*l):
    count = Counter(l)
    most_counterNum = count.most_common(1)
    most_element = most_counterNum[0][0]
    most_num = most_counterNum[0][1]
    return most_element, most_num

class DatasetBlindSROursP(data.Dataset):
    def __init__(self, opt):
        super(DatasetBlindSROursP, self).__init__()
        self.opt = opt
        self.n_channels = opt['n_channels'] if opt['n_channels'] else 3
        self.sf = opt['scale'] if opt['scale'] else 4
        self.shuffle_prob = opt['shuffle_prob'] if opt['shuffle_prob'] else 0.1
        self.use_sharp = opt['use_sharp'] if opt['use_sharp'] else False
        self.degradation_type = opt['degradation_type'] if opt['degradation_type'] else 'bsrgan'
        self.lq_patchsize = self.opt['lq_patchsize'] if self.opt['lq_patchsize'] else 64
        self.patch_size = self.opt['H_size'] if self.opt['H_size'] else self.lq_patchsize * self.sf

        self.all_gt_folder = self.opt['dataroot_all_gt']  # 0 - 20000 sets of images folder. The folder has five sub folders, i.e. original, 01, 02, 03, 04, 05, each sub folder contains different GTs.
        self.all_json_folder = self.opt['dataroot_all_json']

        # Filter out the sets which don't have any effective P labels at all.
        self.no_effective_set = self.filter_no_effective_set(self.all_json_folder)

        self.all_json = [json_file for json_file in os.listdir(os.path.join(self.all_json_folder, 'A'))]
        # Get the final effective sets, the four images of which has at least have one effective P label
        self.effective_json = [json_file for json_file in self.all_json if json_file not in self.no_effective_set]

        self.effective_img = []
        for json_file in self.effective_json:
            img_name = os.path.splitext(json_file)[0]
            self.effective_img.append(img_name)

        # self.paths_H = util.get_image_paths(opt['dataroot_H'])
        # print(len(self.paths_H))

        #        for n, v in enumerate(self.paths_H):
        #            if 'face' in v:
        #                del self.paths_H[n]
        #        time.sleep(1)
        # assert self.paths_H, 'Error: H path is empty.'

    def __getitem__(self, index):

        L_path = None

        json_file = self.effective_json[index]
        img_name = self.effective_img[index]

        img_original_path = os.path.join(self.all_gt_folder, "original", f"{img_name}.png")
        img_01_path = os.path.join(self.all_gt_folder, "01", f"{img_name}_01.png")
        img_02_path = os.path.join(self.all_gt_folder, "02", f"{img_name}_02.png")
        img_03_path = os.path.join(self.all_gt_folder, "03", f"{img_name}_03.png")
        img_04_path = os.path.join(self.all_gt_folder, "04", f"{img_name}_04.png")

        img_list_path = [img_original_path, img_01_path, img_02_path, img_03_path, img_04_path]

        self.Ajson_path = os.path.join(self.all_json_folder, 'A', json_file)
        self.Bjson_path = os.path.join(self.all_json_folder, 'B', json_file)
        self.Cjson_path = os.path.join(self.all_json_folder, 'C', json_file)

        with open(self.Ajson_path, mode='r', encoding='utf-8') as fA:
            jsonA = json.load(fA)
        with open(self.Bjson_path, mode='r', encoding='utf-8') as fB:
            jsonB = json.load(fB)
        with open(self.Cjson_path, mode='r', encoding='utf-8') as fC:
            jsonC = json.load(fC)

        Alabel_img1 = jsonA['Picture_2']['Label']
        Alabel_img2 = jsonA['Picture_3']['Label']
        Alabel_img3 = jsonA['Picture_4']['Label']
        Alabel_img4 = jsonA['Picture_5']['Label']

        Blabel_img1 = jsonB['Picture_2']['Label']
        Blabel_img2 = jsonB['Picture_3']['Label']
        Blabel_img3 = jsonB['Picture_4']['Label']
        Blabel_img4 = jsonB['Picture_5']['Label']

        Clabel_img1 = jsonC['Picture_2']['Label']
        Clabel_img2 = jsonC['Picture_3']['Label']
        Clabel_img3 = jsonC['Picture_4']['Label']
        Clabel_img4 = jsonC['Picture_5']['Label']

        img1_label = [Alabel_img1, Blabel_img1, Clabel_img1]
        img2_label = [Alabel_img2, Blabel_img2, Clabel_img2]
        img3_label = [Alabel_img3, Blabel_img3, Clabel_img3]
        img4_label = [Alabel_img4, Blabel_img4, Clabel_img4]

        imgo_label = 'Original'
        imgo_label_num = 3

        # Give final label to every image in a set, the final label is also called the effective label.
        if len(img1_label) == len(set(img1_label)):
            img1_effective_label = 'Similar'
            img1_label_num = 1
        else:
            img1_effective_label, img1_label_num = find_max_repetition(*img1_label)

        if len(img2_label) == len(set(img2_label)):
            img2_effective_label = 'Similar'
            img2_label_num = 1
        else:
            img2_effective_label, img2_label_num = find_max_repetition(*img2_label)

        if len(img3_label) == len(set(img3_label)):
            img3_effective_label = 'Similar'
            img3_label_num = 1
        else:
            img3_effective_label, img3_label_num = find_max_repetition(*img3_label)

        if len(img4_label) == len(set(img4_label)):
            img4_effective_label = 'Similar'
            img4_label_num = 1
        else:
            img4_effective_label, img4_label_num = find_max_repetition(*img4_label)

        label_list = [imgo_label, img1_effective_label, img2_effective_label, img3_effective_label, img4_effective_label]
        label_list_array = []

        # Since label string could not be converted into torch.tensor or np.array, then we firstly convert the labels into the formal of int
        for str in label_list:
            if str == "Original":
                k = 0
            elif str == "Positive":
                k = 1
            elif str == "Similar":
                k = 2
            elif str == "Negative":
                k = 3
            else:
                print(f"wrong label value {str}.")
            label_list_array.append(k)
        # print(f"label_list is {label_list}")
        label_list_array = np.stack(label_list_array, axis=0)
        l = [item == 1 for item in label_list_array]
        index = np.where(np.array(l) == True)[0]
        idx = random.choice(index)

        img_random_H_enhance_path = img_list_path[idx]

        # ------------------------------------
        # get H image
        # ------------------------------------

        img_H = util.imread_uint(img_original_path, self.n_channels)
        img_name, ext = os.path.splitext(os.path.basename(img_original_path))
        H, W, C = img_H.shape

        img_H_enhance = util.imread_uint(img_random_H_enhance_path, self.n_channels)

        # if H < self.patch_size or W < self.patch_size:
        #     img_H = np.tile(np.random.randint(0, 256, size=[1, 1, self.n_channels], dtype=np.uint8),
        #                     (self.patch_size, self.patch_size, 1))
        #     img_H_enhance = np.tile(np.random.randint(0, 256, size=[1, 1, self.n_channels], dtype=np.uint8),
        #                     (self.patch_size, self.patch_size, 1))

        # ------------------------------------
        # if train, get L/H patch pair
        # ------------------------------------
        if self.opt['phase'] == 'train':

            H, W, C = img_H.shape

            rnd_h_H = random.randint(0, max(0, H - self.patch_size))
            rnd_w_H = random.randint(0, max(0, W - self.patch_size))
            img_H = img_H[rnd_h_H:rnd_h_H + self.patch_size, rnd_w_H:rnd_w_H + self.patch_size, :]

            img_H_enhance = img_H_enhance[rnd_h_H:rnd_h_H + self.patch_size, rnd_w_H:rnd_w_H + self.patch_size, :]

            if 'face' in img_name:
                mode = random.choice([0, 4])
                img_H = util.augment_img(img_H, mode=mode)
                img_H_enhance = util.augment_img(img_H_enhance, mode=mode)
            else:
                mode = random.randint(0, 7)
                img_H = util.augment_img(img_H, mode=mode)
                img_H_enhance = util.augment_img(img_H_enhance, mode=mode)

            img_H = util.uint2single(img_H)
            img_H_enhance = util.uint2single(img_H_enhance)
            if self.degradation_type == 'bsrgan':
                img_L, img_H, img_H_enhance = blindsr.degradation_bsrgan_ours_p(img_H, img_H_enhance, self.sf, lq_patchsize=self.lq_patchsize,
                                                          isp_model=None)
            elif self.degradation_type == 'bsrgan_plus':
                img_L, img_H = blindsr.degradation_bsrgan_plus(img_H, self.sf, shuffle_prob=self.shuffle_prob,
                                                               use_sharp=self.use_sharp, lq_patchsize=self.lq_patchsize)

        else:
            img_H = util.uint2single(img_H)
            if self.degradation_type == 'bsrgan':
                img_L, img_H = blindsr.degradation_bsrgan(img_H, self.sf, lq_patchsize=self.lq_patchsize,
                                                          isp_model=None)
            elif self.degradation_type == 'bsrgan_plus':
                img_L, img_H = blindsr.degradation_bsrgan_plus(img_H, self.sf, shuffle_prob=self.shuffle_prob,
                                                               use_sharp=self.use_sharp, lq_patchsize=self.lq_patchsize)

        # ------------------------------------
        # L/H pairs, HWC to CHW, numpy to tensor
        # ------------------------------------
        img_H, img_L = util.single2tensor3(img_H), util.single2tensor3(img_L)
        img_H_enhance = util.single2tensor3(img_H_enhance)

        if L_path is None:
            L_path = img_original_path

        return {'L': img_L, 'H': img_H_enhance, 'L_path': L_path, 'H_path': img_random_H_enhance_path}

    def __len__(self):
        return len(self.effective_json)

    def filter_no_effective_set(self, all_json_folder):
        Apath = os.path.join(all_json_folder, 'A')
        Bpath = os.path.join(all_json_folder, 'B')
        Cpath = os.path.join(all_json_folder, 'C')

        no_effective_set = []
        for json_file in os.listdir(Apath):
            Ajson_path = os.path.join(Apath, json_file)
            Bjson_path = os.path.join(Bpath, json_file)
            Cjson_path = os.path.join(Cpath, json_file)

            with open(Ajson_path, mode='r', encoding='utf-8') as fA:
                jsonA = json.load(fA)
            with open(Bjson_path, mode='r', encoding='utf-8') as fB:
                jsonB = json.load(fB)
            with open(Cjson_path, mode='r', encoding='utf-8') as fC:
                jsonC = json.load(fC)

            # Each set has four enhanced GTs
            Alabel_img1 = jsonA['Picture_2']['Label']
            Alabel_img2 = jsonA['Picture_3']['Label']
            Alabel_img3 = jsonA['Picture_4']['Label']
            Alabel_img4 = jsonA['Picture_5']['Label']

            Blabel_img1 = jsonB['Picture_2']['Label']
            Blabel_img2 = jsonB['Picture_3']['Label']
            Blabel_img3 = jsonB['Picture_4']['Label']
            Blabel_img4 = jsonB['Picture_5']['Label']

            Clabel_img1 = jsonC['Picture_2']['Label']
            Clabel_img2 = jsonC['Picture_3']['Label']
            Clabel_img3 = jsonC['Picture_4']['Label']
            Clabel_img4 = jsonC['Picture_5']['Label']

            # One image has three labels from three different people
            img1_label = [Alabel_img1, Blabel_img1, Clabel_img1]
            img2_label = [Alabel_img2, Blabel_img2, Clabel_img2]
            img3_label = [Alabel_img3, Blabel_img3, Clabel_img3]
            img4_label = [Alabel_img4, Blabel_img4, Clabel_img4]

            img1_p_num, _, _ = check_list(*img1_label)
            img2_p_num, _, _ = check_list(*img2_label)
            img3_p_num, _, _ = check_list(*img3_label)
            img4_p_num, _, _ = check_list(*img4_label)

            # p_num <= 1 means the final label of the image wouldn't be P, since the P label numbers are not more than 1, when three different people give labels to the same image.
            if img1_p_num <= 1 and img2_p_num <= 1 and img3_p_num <= 1 and img4_p_num <= 1:
                no_effective_set.append(json_file)

        return  no_effective_set


"""
.. module:: mast
   :synopsis: Methods for retrieving GALEX files from MAST. Aspect solution
       retrieval is largely deprecated by consolidated aspect tables, but raw6
       (L0 telemetry) retrieval is not. Most methods of this module operate
       via live queries to MAST resources and therefore require internet
       access.
"""
# NOTE: contains some functions previously -- and perhaps one day again! --
# housed in FileUtils & GQuery. Some related but conditionally-deprecated
# functions are housed in gPhoton.io._query.mast_query.

import os
import time
from typing import Optional, Literal

import requests

from gPhoton import TIME_ID
from gPhoton.io.netutils import chunked_download
from gPhoton.pretty import print_inline
from gPhoton.types import GalexBand, Pathlike

# The following global variables are used to construct properly-formatted
# queries to the MAST database. Don't change them unless you know what you're
# doing!

BASE_URL = (
    "https://mast.stsci.edu/portal/Mashup/MashupQuery.asmx/Galex"
    "PhotonListQueryTest?query="
)
TSCALE = 1000
BASE_DB = "GPFCore.dbo"
MCAT_DB = "GR6Plus7.dbo"
FORMAT_URL = f" -- {TIME_ID}&format=extjs"

def truncate(n: float):
    """
    The photon event timestamps are stored in MAST's database at the level of
    SQL's BIGINT in order to save space. This is accomplished by
    multiplying the raw timestamps by 1000. This truncates (not rounds) some
    timestamps at the level of 1ms. Most timestamps have a resolution of only
    5ms except for rare high resolution visits, and even in that case the
    extra precision does not matter for science. For consistency with the
    database, we truncate times at 1ms for queries.
    """
    return str(n * TSCALE).split(".")[0]

def get_raw_paths(eclipse: int, verbose: int = 0) -> dict[str, Optional[str]]:
    """
    query MAST for URLs to the NUV and FUV raw6 (L0 telemetry) and scst
    (aspect solution) files associated with a particular eclipse.
    """
    url = raw_data_paths(eclipse)
    if verbose > 1:
        print(url)
    response = manage_networked_sql_request(url)
    out = {"NUV": None, "FUV": None, "scst": None}
    for f in response.json()["data"]["Tables"][0]["Rows"]:
        if (f[1].strip() == "NUV") or (f[1].strip() == "FUV"):
            out[f[1].strip()] = f[2]
        elif f[1].strip() == "BOTH":  # misnamed scst path
            out["scst"] = f[2]
    return out

def download_data(
    eclipse: int,
    ftype: Literal["raw6", "scst"],
    band: Optional[GalexBand] = None,
    datadir: str = "./",
    verbose: int = 0,
) -> Optional[str]:
    """
    download a raw6 (L0 telemetry) or scst (aspect solution) file for a given
    eclipse from MAST to datadir.
    """
    urls = get_raw_paths(eclipse, verbose=verbose)
    if ftype not in ["raw6", "scst"]:
        raise ValueError("ftype must be either raw6 or scst")
    if ftype == "raw6" and band not in ["NUV", "FUV"]:
        raise ValueError("band must be either NUV or FUV")
    url = urls[band] if ftype == "raw6" else urls["scst"]
    if not url:
        print(f"Unable to locate {ftype} file on MAST server.")
        return None
    if url[-6:] == ".gz.gz":  # Handling a very rare mislabeling of the URL.
        url = url[:-3]
    if verbose > 1:
        print(url)
    if not datadir:
        datadir = "."
    if datadir and datadir[-1] != "/":
        datadir += "/"
    filename = url.split("/")[-1]
    opath = f"{datadir}{filename}"
    if os.path.isfile(opath):
        print(f"Using {ftype} file already at {os.path.abspath(opath)}")
    else:
        # TODO: investigate handling options for different error cases
        try:
            chunked_download(url, opath, render_bar=verbose > 0)
        except Exception as ex:
            print(f"Unable to download data from {url}: {type(ex)}: {ex}")
            raise
    return opath

# -----------------------------------------------------------------------------
def raw_data_paths(eclipse):
    """
    Construct a URL that will retrieve a data structure containing MAST
    download paths for raw6 and scst files.

    :param eclipse: GALEX eclipse number.

    :type eclipse: int

    :returns: str -- URL to submit this query to the database.
    """
    return mast_url(f"spGetRawUrls {eclipse}")

def retrieve_raw6(eclipse: int, band: GalexBand, outbase: Pathlike) -> str:
    """retrieve raw6 (L0 telemetry) file from MAST and save it to outbase."""
    raw6file = download_data(
        eclipse, "raw6", band, datadir=os.path.dirname(outbase),
    )
    if raw6file is None:
        raise ValueError("Unable to retrieve raw6 file for this eclipse.")
    return raw6file

def mast_url(sql_string: str) -> str:
    """
    given a string containing a SQL query, construct a URL that will feed
    that query to MAST's GALEX database.
    """
    return f"{BASE_URL}{sql_string}{FORMAT_URL}"

def manage_networked_sql_request(
    query: str,
    maxcnt: int = 100,
    wait: int = 1,
    timeout: int = 60,
    verbose: int = 0
) -> Optional[requests.Response]:
    """
    Manage calls via `requests` to SQL servers behind HTTP endpoints,
    providing better feedback and making them more robust against network
    errors.

    :param query: The URL containing the query.

    :type query: str

    :param maxcnt: The maximum number of attempts to make before failure.

    :type maxcnt: int

    :param wait: The length of time to wait before attempting the query again.
        Currently a placeholder.

    :type wait: int

    :param timeout: The length of time to wait for the server to send data
        before giving up, specified in seconds.

    :type timeout: float

    :param verbose: If > 0, print additional messages to STDOUT. Higher value
        represents more verbosity.

    :type verbose: int

    :returns: requests.Response or None -- The response from the server. If the
        query does not receive a response, returns None.
    """
    # Keep track of the number of failures.
    cnt = 0

    # This will keep track of whether we've gotten at least one
    # successful response.
    successful_response = False

    while cnt < maxcnt:
        if cnt > 1:
            time.sleep(wait)
        try:
            response = requests.get(query, timeout=timeout)
            successful_response = True
        except KeyboardInterrupt:
            raise
        except requests.exceptions.ConnectTimeout:
            if verbose:
                print("Connection timed out.")
            cnt += 1
            continue
        except requests.exceptions.ConnectionError:
            if verbose:
                print("Domain did not resolve.")
            cnt += 1
            continue
        except Exception as ex:
            if verbose:
                print(f"bad query? {query}: {type(ex)}: {ex}")
            cnt += 1
            continue
        if response.json()["status"] == "EXECUTING":
            if verbose > 1:
                print_inline("EXECUTING")
            cnt = 0
            continue
        elif response.json()["status"] == "COMPLETE":
            if verbose > 1:
                print_inline("COMPLETE")
            break
        elif response.json()["status"] == "ERROR":
            print("ERROR")
            print(f"Unsuccessful query: {query}")
            raise ValueError(response.json()["msg"])
        else:
            print(f"Unknown return: {response.json()['status']}")
            cnt += 1
            continue

    if successful_response is not True:
        response = None

    # noinspection PyUnboundLocalVariable
    return response


import os, fnmatch, pickle
from tkinter import N
import numpy as np
import matplotlib.pyplot as plt
from climnet.grid import regular_lon_lat, regular_lon_lat_step, FeketeGrid
from climnet.myutils import *
import time
from collections import Counter
import networkx as nx
start_time = time.time()
#irun = int(os.environ['SLURM_ARRAY_TASK_ID'])
#print('Task: ', irun)
curr_time = time.time()

base_path = '../../climnet_output/'
distrib = 'igrf'

grid_type = 'fekete'
n_lat = 18*4
#typ ='threshold' # iterated over
#corr_method='spearman'
#weighted = False # iterate over
#ranks = False
# if corr_method in ['BI-KSG', 'binMI', 'HSIC']:
#     ranks = True

if not os.path.exists(base_path+'fdr/'):
    os.mkdir(base_path+'fdr/')

ar = 0
ar2 = None
var = 10

num_runs = 30
n_time = 100
nus = [0.5,1.5]
len_scales = [0.05,0.1,0.2]
nu = 0.5
len_scale = 0.1

denslist = [0.001,0.0025, 0.005,0.0075,0.01,0.015,0.02,0.025,0.03,0.035,0.04,0.045,0.05,0.1,0.2]
ks = [6, 60, 300, 600]#,1200]
#robust_tolerance = 0.5

filter_string = f'_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_'

# %%
#grid_helper and calc_true
exec(open("grid_helper.py").read())
#exec(open("calc_true.py").read())

eps2 = 2 * dist_equator
eps3 = 3 * dist_equator
alpha1 = 0.95
alpha2 = 0.99
robust_tolerance = 0.2

if ar2 is None:
    ar_coeff = ar * np.ones(num_points)
else:
    raise RuntimeError('Arbitrary AR values not implemented.')

# just to be detected as known variables in VSC
dists, numth, degbins, llbins, splbins, nbhds2, cov, all_degs, all_lls,all_ccs, all_ccws, all_spls, all_betw, all_eigc, all_dens, all_tele1, all_tele2, all_robusttele2, all_llquant1, all_llquant2, all_telequant, all_telequant2, all_mad, all_shufflemad, plink,dist_equator,num_points,truecorrbins = dists, numth, degbins, llbins, splbins, nbhds2, cov, all_degs, all_lls,all_ccs, all_ccws, all_spls, all_betw, all_eigc, all_dens, all_tele1, all_tele2, all_robusttele2, all_llquant1, all_llquant2, all_telequant, all_telequant2, all_mad, all_shufflemad, plink,dist_equator,num_points,truecorrbins
#true_dens, true_degs, true_lls, true_ccs, true_spls, true_llquant1,true_llquant2,true_tele1,true_tele2,true_robusttele2, true_mad, true_shufflemad = true_dens, true_degs, true_lls, true_ccs, true_spls, true_llquant1,true_llquant2,true_tele1,true_tele2,true_robusttele2, true_mad, true_shufflemad
# %%
# # load empcorrs, construct graphs, calc graph stats and save them
curr_time = time.time()
# empcorrs = find(f'{distrib}_{corr_method}_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_*',base_path+'empcorrs/')
# empdatas = find(f'*_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_*',base_path+'empdata/')
#for w in [True,False]:
#igrf_ES_matern_nu0.5_len0.1_ar0_fekete72_time1000_var10_seed1643248866.txt

for corr_method in ['BI-KSG','pearson', 'spearman','LWlin', 'binMI','HSIC', 'LW', 'ES']: # ['pearson', 'spearman', 'binMI', 'LW', 'ES', 'HSIC','BI-KSG']:
    print('Starting ', corr_method, time.time()-curr_time)
    curr_time = time.time()
    if corr_method == 'ES':
        empcorrs = find(f'{distrib}_{corr_method}_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time1000_*',base_path+'empcorrs/')
        empdatas = find(f'*_nu{nu}_{len_scale}_ar{ar}_{grid_type}{n_lat}_time1000_*',base_path+'empdata/')
    elif corr_method == 'BI-KSG':
        empcorrs = [nam for nam in find(f'empcorrdict_*_BI-KSG_matern_nu{nu}_len{len_scale}_*',base_path+'empcorrs/') if not fnmatch.fnmatch(nam,'*_part*')]
        empdatas = find(f'*_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_*',base_path+'empdata/')
    else:
        empcorrs = find(f'{distrib}_{corr_method}_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_*',base_path+'empcorrs/')
        empdatas = find(f'*_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_*',base_path+'empdata/')
    #outfilename = f'graphstats_{distrib}_{corr_method}_{typ}_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_{num_runs}.txt'
    all_fdr = np.zeros((numth))
    if irun < len(empcorrs):
        emp_corr = myload(empcorrs[irun])
    else:
        if irun < len(empdatas):
            orig_data = myload(empdatas[irun])
        else:
            seed = int(time.time())
            np.random.seed(seed)
            orig_data = diag_var_process(ar_coeff, cov, n_time)
            mysave(base_path+'empdata/', f'data_matern_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_var1_seed{seed}.txt',orig_data)
        data = np.sqrt(var) * orig_data
        # save with nam in name
        for j in range(len(lat)):
            data[:,j] -= orig_data[:,j].mean()
            data[:,j] /= orig_data[:,j].std()
        emp_corr = compute_empcorr(data, similarity=corr_method)
    for i, density in enumerate(denslist):
        adj = get_adj(emp_corr, density, weighted=False)
        # accumulate over iruns: plink[i,ibin] = (number of links at dens and bin)/(number of possible links at bin)
        distadj = get_adj(5-dists, density,weighted=True)
        maxdist = 5-distadj[distadj!=0].min()
        all_fdr[i] = adj[np.logical_and(adj != 0, dists > maxdist)].sum() / adj[adj!=0].sum() #np.logical_and(adj != 0, dists > maxdist).sum() / (adj != 0).sum()
        #all_fdr[i,irun] = np.logical_and(adj != 0, dists > maxdist).sum() / (adj != 0).sum()

    if corr_method == 'ES':
        outfilename = f'fdr/fdr_part{irun}_{corr_method}_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time1000_{num_runs}_numdens{len(denslist)}.txt'
    else:
        outfilename = f'fdr/fdr_part{irun}_{corr_method}_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_{num_runs}_numdens{len(denslist)}.txt'
    with open(base_path + outfilename, "wb") as fp:   #Pickling
        pickle.dump(all_fdr, fp)

# %%
# import matplotlib.pyplot as plt
# plt.clf()
# from tueplots import bundles
# plt.rcParams.update(bundles.icml2022())
# plt.rcParams.update({"figure.dpi": 150})

# fig,ax = plt.subplots()
# for i in range(len(denslist)):
#     ax.plot(truecorrbins[:-1], plink[i,:])
# %%
#knn_adj(np.abs(emp_corr), ks[-5],weighted=False).mean()

# %%
# try variogram estimator
corr_method='pearson'
irun = 0
empcorrs = find(f'{distrib}_{corr_method}_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_*',base_path+'empcorrs/')
empdatas = find(f'*_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_*',base_path+'empdata/')
#outfilename = f'graphstats_{distrib}_{corr_method}_{typ}_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_{num_runs}.txt'
all_fdr = np.zeros((numth))
if irun < len(empcorrs):
    emp_corr = myload(empcorrs[irun])
else:
    if irun < len(empdatas):
        orig_data = myload(empdatas[irun])
    else:
        seed = int(time.time())
        np.random.seed(seed)
        orig_data = diag_var_process(ar_coeff, cov, n_time)
        mysave(base_path+'empdata/', f'data_matern_nu{nu}_len{len_scale}_ar{ar}_{grid_type}{n_lat}_time{n_time}_var1_seed{seed}.txt',orig_data)
    data = np.sqrt(var) * orig_data
    # save with nam in name
    for j in range(len(lat)):
        data[:,j] -= orig_data[:,j].mean()
        data[:,j] /= orig_data[:,j].std()
    emp_corr = compute_empcorr(data, similarity=corr_method)
for i, density in enumerate(denslist):
    adj = get_adj(emp_corr, density, weighted=False)
    # accumulate over iruns: plink[i,ibin] = (number of links at dens and bin)/(number of possible links at bin)
    distadj = get_adj(5-dists, density,weighted=True)
    maxdist = 5-distadj[distadj!=0].min()
    all_fdr[i] = adj[np.logical_and(adj != 0, dists > maxdist)].sum() / adj[adj!=0].sum() #np.logical_and(adj != 0, dists > maxdist).sum() / (adj != 0).sum()

# %%
# get Euclidean coordinates
n_time, num_points = data.shape

coords = np.zeros((n_time*num_points,3))
values = data.T.flatten()

for ipt in range(num_points):
    lo = grid.grid['lon'][ipt]
    la = grid.grid['lat'][ipt]
    coo = spherical2cartesian(lo, la)
    for i in range(n_time):
        coords[int(n_time*ipt)+i,:] = coo

# %%
# input to variogram estimator
rd_idcs = np.random.randint(0,num_points*n_time,10000)

#kernel crashes for long rd_idcs :(
import skgstat as skg
V = skg.Variogram(coords[rd_idcs,:], values[rd_idcs])
V.plot()
# %%
plt.plot(V.bins, V.experimental, '.b')

# for ipt in range(num_points):
#     for jpt in range(num_points):
#         # if dist in bin, predict variogram cov.


def is_palindrome(s):
    # Remove non-alphanumeric characters and convert to lowercase
    s = ''.join(char.lower() for char in s if char.isalnum())

    # Base case: if the string is empty or has one character, it's a palindrome
    if len(s) <= 1:
        return True

    # Check if the first and last characters are the same
    if s[0] == s[-1]:
        # Recursive call excluding the first and last characters
        return is_palindrome(s[1:-1])

    # If the first and last characters don't match, it's not a palindrome
    return False

# Test the function
print(is_palindrome("racecar"))  # True
# print(is_palindrome("hello"))    # False


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from rosgraph.names import is_private
import rospy
from naoqi_bridge_msgs.msg import JointAnglesWithSpeed

def dab():
    pub = rospy.Publisher('/joint_angles', JointAnglesWithSpeed, queue_size=10)
    rospy.init_node('make_it_dab', anonymous=True)
    rate = rospy.Rate(4)

    while not rospy.is_shutdown():
        msg = create_msg("LShoulder",["LShoulderRoll","LShoulderPitch"],[1.5,0.5])
        pub.publish(msg)
        msg = create_msg("LElbow",["LElbowRoll"],[0.0])
        pub.publish(msg)

        msg = create_msg("RShoulder",["RShoulderPitch"],[-0.4])
        pub.publish(msg)
        msg = create_msg("RElbow",["RElbowRoll","RElbowYaw"],[1.3,0.0])
        pub.publish(msg)

        msg = create_msg("Head",["HeadYaw","HeadPitch"],[-0.6,0.4])
        pub.publish(msg)

        msg = create_msg("Leg",["HipRoll"],[-0.5])
        pub.publish(msg)

        rate.sleep()

    msg = create_msg("Head",["HeadPitch","HeadYaw"],[0.15,0.00])
    pub.publish(msg)

def create_msg(frame_id,joint_names,joint_angles):
    trajectory = JointAnglesWithSpeed()
    trajectory.header.stamp = rospy.Time.now()
    trajectory.header.frame_id = frame_id

    trajectory.joint_names = joint_names
    trajectory.joint_angles = joint_angles

    trajectory.speed = 0.04
    trajectory.relative=0

    return trajectory

if __name__=='__main__':
    try:
        dab()
    except rospy.ROSInterruptException:
        pass


def indeks(c):
    if str.islower(c):
        return ord(c)-ord('a')+1
    return ord(c)-ord('A')+27

p = 33554393
b = 64

def karpRabin(w, t):
    bM_1 = 1
    Hw = 0
    Ht = 0

    for i in range(len(w)):
        Hw = (Hw*b+indeks(w[i]))%p
        Ht = (Ht*b+indeks(t[i]))%p

    for i in range(1, len(w)):
        bM_1 = (bM_1*b)%p

    i = 0
    while Hw != Ht:
        Ht = (Ht+b*p-indeks(t[i])*bM_1)%p
        Ht = (Ht*b+indeks(t[i+len(w)]))%p

        if i>len(t)-len(w):
            return -1
        i+=1

    return i

w = "rak"
t = "abrakadabra"

print(karpRabin(w, t))


from time import sleep

for i in range(5):
    print(i)
    # i += 1

names = ["chris", "iftach", "jay"]
for name in names:
    print(name)

fruit_inventory = {"apples": 5, "peers": 2, "orange": 8}
print(list(fruit_inventory.items()))
for fruit in fruit_inventory:
    print(fruit)

for fruit in fruit_inventory.items():
    print(fruit)

for fruit, quantity in fruit_inventory.items():
    print("You have {} {}.".format(quantity, fruit))

i = 0
while i < 5:
    print(i)
    i += 1

while True:
    try:
        print("polling")
        sleep(1)
    except KeyboardInterrupt:
        break


"""
BiMPM (Bilateral Multi-Perspective Matching) model implementation.
"""

from typing import Dict, List, Any

import torch

from allennlp.common.checks import check_dimensions_match
from allennlp.data import TextFieldTensors, Vocabulary
from allennlp.modules import FeedForward, Seq2SeqEncoder, Seq2VecEncoder, TextFieldEmbedder
from allennlp.models.model import Model
from allennlp.nn import InitializerApplicator
from allennlp.nn import util
from allennlp.training.metrics import CategoricalAccuracy

from allennlp.modules.bimpm_matching import BiMpmMatching

@Model.register("bimpm")
class BiMpm(Model):
    """
    This `Model` implements BiMPM model described in [Bilateral Multi-Perspective Matching
    for Natural Language Sentences](https://arxiv.org/abs/1702.03814) by Zhiguo Wang et al., 2017.
    Also please refer to the [TensorFlow implementation](https://github.com/zhiguowang/BiMPM/) and
    [PyTorch implementation](https://github.com/galsang/BIMPM-pytorch).

    Registered as a `Model` with name "bimpm".

    # Parameters

    vocab : `Vocabulary`
    text_field_embedder : `TextFieldEmbedder`
        Used to embed the `premise` and `hypothesis` `TextFields` we get as input to the
        model.
    matcher_word : `BiMpmMatching`
        BiMPM matching on the output of word embeddings of premise and hypothesis.
    encoder1 : `Seq2SeqEncoder`
        First encoder layer for the premise and hypothesis
    matcher_forward1 : `BiMPMMatching`
        BiMPM matching for the forward output of first encoder layer
    matcher_backward1 : `BiMPMMatching`
        BiMPM matching for the backward output of first encoder layer
    encoder2 : `Seq2SeqEncoder`
        Second encoder layer for the premise and hypothesis
    matcher_forward2 : `BiMPMMatching`
        BiMPM matching for the forward output of second encoder layer
    matcher_backward2 : `BiMPMMatching`
        BiMPM matching for the backward output of second encoder layer
    aggregator : `Seq2VecEncoder`
        Aggregator of all BiMPM matching vectors
    classifier_feedforward : `FeedForward`
        Fully connected layers for classification.
    dropout : `float`, optional (default=`0.1`)
        Dropout percentage to use.
    initializer : `InitializerApplicator`, optional (default=`InitializerApplicator()`)
        If provided, will be used to initialize the model parameters.
    """

    def __init__(
        self,
        vocab: Vocabulary,
        text_field_embedder: TextFieldEmbedder,
        matcher_word: BiMpmMatching,
        encoder1: Seq2SeqEncoder,
        matcher_forward1: BiMpmMatching,
        matcher_backward1: BiMpmMatching,
        encoder2: Seq2SeqEncoder,
        matcher_forward2: BiMpmMatching,
        matcher_backward2: BiMpmMatching,
        aggregator: Seq2VecEncoder,
        classifier_feedforward: FeedForward,
        dropout: float = 0.1,
        initializer: InitializerApplicator = InitializerApplicator(),
        **kwargs,
    ) -> None:
        super().__init__(vocab, **kwargs)

        self.text_field_embedder = text_field_embedder

        self.matcher_word = matcher_word

        self.encoder1 = encoder1
        self.matcher_forward1 = matcher_forward1
        self.matcher_backward1 = matcher_backward1

        self.encoder2 = encoder2
        self.matcher_forward2 = matcher_forward2
        self.matcher_backward2 = matcher_backward2

        self.aggregator = aggregator

        matching_dim = (
            self.matcher_word.get_output_dim()
            + self.matcher_forward1.get_output_dim()
            + self.matcher_backward1.get_output_dim()
            + self.matcher_forward2.get_output_dim()
            + self.matcher_backward2.get_output_dim()
        )

        check_dimensions_match(
            matching_dim,
            self.aggregator.get_input_dim(),
            "sum of dim of all matching layers",
            "aggregator input dim",
        )

        self.classifier_feedforward = classifier_feedforward

        self.dropout = torch.nn.Dropout(dropout)

        self.metrics = {"accuracy": CategoricalAccuracy()}

        self.loss = torch.nn.CrossEntropyLoss()

        initializer(self)

    def forward(
        self,  # type: ignore
        premise: TextFieldTensors,
        hypothesis: TextFieldTensors,
        label: torch.LongTensor = None,
        metadata: List[Dict[str, Any]] = None,
    ) -> Dict[str, torch.Tensor]:
        """

        # Parameters

        premise : `TextFieldTensors`
            The premise from a `TextField`
        hypothesis : `TextFieldTensors`
            The hypothesis from a `TextField`
        label : `torch.LongTensor`, optional (default = `None`)
            The label for the pair of the premise and the hypothesis
        metadata : `List[Dict[str, Any]]`, optional, (default = `None`)
            Additional information about the pair

        # Returns

        An output dictionary consisting of:

        logits : `torch.FloatTensor`
            A tensor of shape `(batch_size, num_labels)` representing unnormalised log
            probabilities of the entailment label.
        loss : `torch.FloatTensor`, optional
            A scalar loss to be optimised.
        """

        mask_premise = util.get_text_field_mask(premise)
        mask_hypothesis = util.get_text_field_mask(hypothesis)

        # embedding and encoding of the premise
        embedded_premise = self.dropout(self.text_field_embedder(premise))
        encoded_premise1 = self.dropout(self.encoder1(embedded_premise, mask_premise))
        encoded_premise2 = self.dropout(self.encoder2(encoded_premise1, mask_premise))

        # embedding and encoding of the hypothesis
        embedded_hypothesis = self.dropout(self.text_field_embedder(hypothesis))
        encoded_hypothesis1 = self.dropout(self.encoder1(embedded_hypothesis, mask_hypothesis))
        encoded_hypothesis2 = self.dropout(self.encoder2(encoded_hypothesis1, mask_hypothesis))

        matching_vector_premise: List[torch.Tensor] = []
        matching_vector_hypothesis: List[torch.Tensor] = []

        def add_matching_result(matcher, encoded_premise, encoded_hypothesis):
            # utility function to get matching result and add to the result list
            matching_result = matcher(
                encoded_premise, mask_premise, encoded_hypothesis, mask_hypothesis
            )
            matching_vector_premise.extend(matching_result[0])
            matching_vector_hypothesis.extend(matching_result[1])

        # calculate matching vectors from word embedding, first layer encoding, and second layer encoding
        add_matching_result(self.matcher_word, embedded_premise, embedded_hypothesis)
        half_hidden_size_1 = self.encoder1.get_output_dim() // 2
        add_matching_result(
            self.matcher_forward1,
            encoded_premise1[:, :, :half_hidden_size_1],
            encoded_hypothesis1[:, :, :half_hidden_size_1],
        )
        add_matching_result(
            self.matcher_backward1,
            encoded_premise1[:, :, half_hidden_size_1:],
            encoded_hypothesis1[:, :, half_hidden_size_1:],
        )

        half_hidden_size_2 = self.encoder2.get_output_dim() // 2
        add_matching_result(
            self.matcher_forward2,
            encoded_premise2[:, :, :half_hidden_size_2],
            encoded_hypothesis2[:, :, :half_hidden_size_2],
        )
        add_matching_result(
            self.matcher_backward2,
            encoded_premise2[:, :, half_hidden_size_2:],
            encoded_hypothesis2[:, :, half_hidden_size_2:],
        )

        # concat the matching vectors
        matching_vector_cat_premise = self.dropout(torch.cat(matching_vector_premise, dim=2))
        matching_vector_cat_hypothesis = self.dropout(torch.cat(matching_vector_hypothesis, dim=2))

        # aggregate the matching vectors
        aggregated_premise = self.dropout(
            self.aggregator(matching_vector_cat_premise, mask_premise)
        )
        aggregated_hypothesis = self.dropout(
            self.aggregator(matching_vector_cat_hypothesis, mask_hypothesis)
        )

        # the final forward layer
        logits = self.classifier_feedforward(
            torch.cat([aggregated_premise, aggregated_hypothesis], dim=-1)
        )
        probs = torch.nn.functional.softmax(logits, dim=-1)

        output_dict = {"logits": logits, "label_probs": probs}
        if label is not None:
            loss = self.loss(logits, label)
            for metric in self.metrics.values():
                metric(logits, label)
            output_dict["loss"] = loss

        return output_dict

    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        return {
            metric_name: metric.get_metric(reset) for metric_name, metric in self.metrics.items()
        }

    def make_output_human_readable(
        self, output_dict: Dict[str, torch.Tensor]
    ) -> Dict[str, torch.Tensor]:
        """
        Does a simple argmax over the probabilities, converts index to string label, and
        add `"label"` key to the dictionary with the result.
        """
        predictions = output_dict["label_probs"]
        if predictions.dim() == 2:
            predictions_list = [predictions[i] for i in range(predictions.shape[0])]
        else:
            predictions_list = [predictions]
        classes = []
        for prediction in predictions_list:
            label_idx = prediction.argmax(dim=-1).item()
            label_str = self.vocab.get_index_to_token_vocabulary("labels").get(
                label_idx, str(label_idx)
            )
            classes.append(label_str)
        output_dict["label"] = classes
        return output_dict

    default_predictor = "textual_entailment"


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right

class Solution:
    def levelOrderBottom(self, root: TreeNode) -> List[List[int]]:
        if root is None:
            return []

        this_level = [root]
        next_level = []
        stack = []
        stack.append([root.val])

        while this_level:
            temp =[]
            for n in this_level:
                if n.left is not None:
                    next_level.append(n.left)
                    temp.append(n.left.val)
                if n.right is not None:
                    next_level.append(n.right)
                    temp.append(n.right.val)

            if temp:
                stack.append(temp)

            this_level = next_level
            next_level = []

        stack.reverse()
        # stack[::-1]
        return stack


"""
Purpose:    Get CVSSv3 metrics for specified CVEs from the NIST National Vulnerability Database API
            Useful tool for Vulnerability Management
Author: Kassidy Hall (technodivergent)
Date: July 2022
"""
import json
import argparse
import csv
import requests
import os.path
import config
from ratelimit import limits, RateLimitException, sleep_and_retry

ONE_MIN = 60
MAX_CPM = 100

@sleep_and_retry
@limits(calls=MAX_CPM, period=ONE_MIN)
def get_json(cve_id: str) -> json:
    """ Fetch JSON data from the NVD """
    API_KEY = config.API_KEY
    try:
        url = 'https://services.nvd.nist.gov/rest/json/cve/1.0/' + cve_id + '?apiKey='+ API_KEY
        print(url)
        resp = requests.get(url)
    except requests.exceptions.RequestException as e:
        raise SystemExit('Unable to establish connection: %s' % e)
    return resp.json()

def get_cvss3_data(json: json) -> dict:
    """ Extract the CVSSv3 data from JSON and store it as a dictionary """
    try:
        result = json['result']['CVE_Items'][0]
        cvss3_dict = {}
        # impact
        cvss3_json = result['impact']['baseMetricV3']['cvssV3']
        cvss3_dict['Attack_Vector']        = cvss3_json['attackVector'].title()
        cvss3_dict['Attack_Complexity']    = cvss3_json['attackComplexity'].title()
        cvss3_dict['Privileges_Required']  = cvss3_json['privilegesRequired'].title()
        cvss3_dict['User_Interaction']     = cvss3_json['userInteraction'].title()
        cvss3_dict['Scope']               = cvss3_json['scope'].title()
        cvss3_dict['Confidentiality_Impact'] = cvss3_json['confidentialityImpact'].title()
        cvss3_dict['Integrity_Impact']     = cvss3_json['integrityImpact'].title()
        cvss3_dict['Availability_Impact']  = cvss3_json['availabilityImpact'].title()
        cvss3_dict['Base_Score']           = cvss3_json['baseScore']
        cvss3_dict['Base_Severity']        = cvss3_json['baseSeverity'].title()
    except KeyError:
        cvss3_dict = {}
    return cvss3_dict

def save_csv(filename: str, dictionary: dict) -> None:
    """ Save CVSSv3 dictionary as a CSV file """
    file_exists = os.path.isfile(filename)

    try:
        with open(filename, 'a') as fh:
            headers = dictionary.keys()
            csv_writer = csv.DictWriter(fh, delimiter=',', lineterminator='\n', fieldnames=headers)

            if not file_exists:
                csv_writer.writeheader()
            csv_writer.writerow(dictionary)
    except csv.Error as e:
        raise SystemExit('Unable to write CSV to file: %s' % e)

def main() -> None:
    """ Main Method """
    argp = argparse.ArgumentParser(description='Request CVE details from National Vulnerability Database')

    group = argp.add_mutually_exclusive_group(required=True)
    group.add_argument(
        '-c', '--cve-id',
        type=str,
        default=None,
        help='Specify the requested CVE')
    group.add_argument(
        '-f', '--cve-list',
        type=argparse.FileType('r'),
        default=None,
        help='Specify the requested CVEs from a list file (one CVE entry per line)')
    args = argp.parse_args()

    cve_list = []

    # if single CVE requested, append it to a list, otherwise import the list from a file
    # NOTE: Format of list must be a single CVE entry per line
    if(args.cve_list is None):
        cve_list.append(args.cve_id)
    else:
        cve_list = args.cve_list.read().splitlines()

    # pull CVSSv3 data for one or more CVEs
    for cve_id in cve_list:
        json = get_json(cve_id)
        cvssv3_data = get_cvss3_data(json)
        save_csv('cvss_vectors.csv', cvssv3_data)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# coding: utf-8

# In[94]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[26]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import scanpy as sc

# In[27]:

dat = sc.read_10x_mtx(r"C:\Users\Виталик\Documents\RNAproject\10x genomics", var_names="gene_symbols")

# In[28]:

dat.var_names_make_unique()

# In[29]:

sc.pl.highest_expr_genes(dat, n_top=20)

# In[30]:

sc.pp.filter_cells(dat, min_genes=200)

# In[31]:

sc.pp.filter_genes(dat, min_cells=3)

# In[32]:

dat.var['mt'] = dat.var_names.str.startswith('mt-') #annotate the group of mitochondrial genes as 'mt'

# In[33]:

dat

# In[34]:

sc.pp.calculate_qc_metrics(dat, qc_vars=['mt'], percent_top=None, log1p=False, inplace=True)

# In[35]:

sc.pl.violin(dat, ['n_genes_by_counts', 'total_counts', 'pct_counts_mt'],
             jitter=0.4, multi_panel=True)

# In[36]:

dat.obs

# In[37]:

#Remove cells that have too many mitochondrial genes expressed or too many total counts:
sc.pl.scatter(dat, x='total_counts', y='pct_counts_mt')
sc.pl.scatter(dat, x='total_counts', y='n_genes_by_counts')

# In[38]:

#filter out genes with counts < 4500 and percent of mitochondrial genes < 25
adata = dat[dat.obs.n_genes_by_counts < 4500, :]
adata = dat[dat.obs.pct_counts_mt < 25, :]

# In[39]:

#Total-count normalize (library-size correct) the data matrix X to 10,000 reads per cell, so that counts become comparable among cells.
sc.pp.normalize_total(adata, target_sum=1e4)

# In[40]:

sc.pp.normalize_total

# In[41]:

#Identify highly-variable gen
sc.pp.log1p(adata)
#Identify highly-variable genes with some conditions
sc.pp.highly_variable_genes(adata, min_mean=0.0125, max_mean=3, min_disp=0.5)

# In[42]:

#make the plot with highly variable genes
sc.pl.highly_variable_genes(adata)

# In[43]:

#select only highly variable genes
adata = adata[:, adata.var.highly_variable]

# In[18]:

#look at the plot
sc.pl.highly_variable_genes(adata)

# In[19]:

#Regress out effects of total counts per cell and the percentage of mitochondrial genes expressed. Scale the data to unit variance.
sc.pp.regress_out(adata, ['total_counts', 'pct_counts_mt'])

# In[44]:

#Scale each gene to unit variance. Clip values exceeding standard deviation 10.
sc.pp.scale(adata, max_value=10)

# In[45]:

sc.pl.highest_expr_genes(adata, n_top=20)

# In[46]:

#pca and observe some genes(B2m,Ly86,Kif5c)
sc.tl.pca(adata, svd_solver='arpack')
sc.pl.pca(adata, color=['B2m', 'Ly86', 'Kif5c'])

# In[47]:

#take a look at adata object names
adata.var_names
adata

# In[48]:

#elbow plot with PCAs
sc.pl.pca_variance_ratio(adata, log=True)

# In[49]:

#neighbors
sc.pp.neighbors(adata, n_neighbors=10, n_pcs=25)

# In[50]:

#umap method for dimencion reduction
sc.tl.umap(adata)

# In[51]:

sc.pl.umap(adata, color= ['B2m', 'Ly86', 'Kif5c'] )

# In[52]:

#using leiden algoritm for clusterization
sc.tl.leiden(adata)

# In[53]:

#observe clusters with umap
sc.pl.umap(adata, color='leiden')

# In[31]:

#Let us compute a ranking for the highly differential genes in each cluster.
#and select the firs 25
sc.tl.rank_genes_groups(adata, 'leiden', method='t-test')
sc.pl.rank_genes_groups(adata, n_genes=25, sharey=False)

# In[56]:

sc.settings.verbosity = 2  # reduce the verbosity

# In[57]:

#As an alternative, let us rank genes using logistic regression
sc.tl.rank_genes_groups(adata, 'leiden', method='logreg')
sc.pl.rank_genes_groups(adata, n_genes=25, sharey=False)

# In[58]:

#Show the 10 top ranked genes per cluster 0, 1, …, 7 in a dataframe.
pd.DataFrame(adata.uns['rank_genes_groups']['names']).head(5)

# In[66]:

#make dataframe with ranked genes
df = pd.DataFrame(adata.uns['rank_genes_groups']['names']).head(10)

# In[67]:

df

# In[61]:

df.info()

# In[70]:

#compare cluster1 genes, only stores top 100 by default
tt = sc.tl.rank_genes_groups(adata, 'leiden', method='t-test')

# In[71]:

#use wilcoxon method
wil = sc.tl.rank_genes_groups(adata, 'leiden', method='wilcoxon')

# In[72]:

ttov = sc.tl.rank_genes_groups(adata, 'leiden', method='t-test_overestim_var')

# In[79]:

#observe the genes with heatmap
sc.pl.rank_genes_groups_heatmap(adata, n_genes=5, show_gene_labels=True)

# In[80]:

#another type of plot which can show as the cell fractions by size of circle
#and gene expression by color
sc.pl.rank_genes_groups_dotplot(adata, n_genes=5)

# In[81]:

sc.pl.rank_genes_groups_matrixplot(adata, n_genes=5)

# In[83]:

#select top 20 genes from 1 cluster
sc.pl.rank_genes_groups(adata, groups=['1'], n_genes=20)

# In[87]:

#genes with same expression
# convert numpy.recarray to list
mynames = [x[0] for x in adata.uns['rank_genes_groups']['names'][:20]]
sc.pl.stacked_violin(adata, mynames, groupby="leiden")

# In[99]:

import gseapy as gp

# In[101]:

gene_set_names = gp.get_library_name(organism='Mouse')
print(gene_set_names)

# In[ ]:

# In[35]:

#Compare to a single cluster:
sc.tl.rank_genes_groups(adata, 'leiden', groups=['0'], reference='1', method='wilcoxon')
sc.pl.rank_genes_groups(adata, groups=['0'], n_genes=20)

# In[ ]:

get_ipython().system('jt -r')

# In[36]:

sc.pl.rank_genes_groups_violin(adata, groups='0', n_genes=8)

# In[64]:

#If you want to compare a certain gene across groups, use the following.
sc.pl.violin(adata, ['Palm', 'B2m'], groupby='leiden')

# In[65]:

# In[39]:

sc.pl.umap(adata, color='leiden', legend_loc='on data', title='', frameon=False, save='.pdf')

# In[40]:

sc.pl.dotplot(adata,['Lyz2', 'B2m'], groupby='leiden')

# In[42]:

sc.pl.stacked_violin(adata,['Lyz2', 'B2m', 'Atf3'] , groupby='leiden', rotation=90)

# In[ ]:


import functools
import json
import threading
import re
from watchdog.events import FileSystemEventHandler

from event_mappings import EVENT_MAPPINGS, EVENT_IGNORE

class EventHandler(FileSystemEventHandler):
    def __init__(self, on_modified, db_connector):
        self._db_connector = db_connector
        self._db_lock = threading.Lock()
        self._on_modified = on_modified

    def on_modified(self, event):
        self._db_lock.acquire()
        db = self._db_connector()
        self._on_modified(event, db)
        db.close()
        self._db_lock.release()

class Event:
    @classmethod
    def from_text(cls, event_line):
        """
        :type event_line: unicode
        :rtype list[DFDash.Event]
        """

        def extract_events(text, mapping):
            """
            :type mapping: EventMapping
            """
            maybe_events = []
            try:
                matches = mapping.pattern.match(text)
                if matches:
                    types = mapping.types
                    if not isinstance(types, (list, tuple)):
                        types = [types, ]
                    for event_type in types:
                        try:
                            origin = matches.group('origin')
                            event_type = matches.expand(event_type)
                        except IndexError:
                            origin = "unknown"
                            event_type = matches.expand(
                                re.sub('\\\\g<origin>', origin, event_type))
                        except Exception:
                            print("Error expanding template:")
                            print(matches.groups())
                            print(event_type)
                            raise
                        maybe_events.append(
                            cls(origin=origin, message=text,
                                event_type=event_type))
            except Exception:
                print("Error testing regex:")
                print(mapping.pattern.pattern)
                print(text)
                raise
            return maybe_events

        events = []
        for maybe_events in map(
                functools.partial(extract_events, event_line),
                EVENT_MAPPINGS):
            if maybe_events is not None:
                for event in maybe_events:
                    events.append(event)
        if events:
            if len(events) > 2:
                print("Got many events for line:")
                print("\t{}".format(event_line))
                for e in events:
                    print("\t{}".format(e.event_type))

            return events
        for pattern in EVENT_IGNORE:
            if pattern.match(event_line):
                return []
        # raise Exception("Unhandled event: {}".format(event_line))
        return [cls.unknown_event(event_line)]

    def __init__(self, origin, message, event_type):
        """
        :type origin: unicode
        :type message: unicode
        :type event_type: unicode
        """
        self.origin = origin
        self.message = message
        self.event_type = event_type

    def put(self, db, commit=False):
        """
        :type db: DFDash.DB
        """
        return db.execute(
            b"INSERT INTO events (message, type, json) VALUES (?, ?, ?)",
            # ("", self.event_type, ""),
            (self.message, self.event_type, self.json),
            commit)

    @property
    def json(self):
        return json.dumps({
            "origin": self.origin,
            "message": self.message,
            "type": self.event_type,
        })

    @classmethod
    def unknown_event(cls, event_line):
        return cls(message=event_line, event_type="_.unknown", origin="")