#!/usr/bin/env python
"""
    usage:
        svm.py unified_input.csv engine_score_column_name
    i.e. :
        svm.py omssa_2_1_6_unified.csv 'OMSSA:pvalue'

    Writes a new file with added column "SVMscore" which is the distance to
    the separating hyperplane of a Percolator-like support vector machine.
"""
import numpy as np
import sklearn
from sklearn import svm
from sklearn.cross_validation import StratifiedKFold
from sklearn.preprocessing import Imputer
from collections import Counter, defaultdict
from random import random
import csv
import re
import os
import argparse

from misc import (
    get_score_colname_and_order,
    field_to_float,
    unify_sequence,
    calc_FDR,
    scale_scores,
    row_is_decoy,
    field_to_bayes_float,
    get_mz_values,
)

SCALER = (
    sklearn.preprocessing.RobustScaler()
)  # RobustScaler() seems to be most robust ;)
PROTON = 1.00727646677

class SVMWrapper(dict):
    def __init__(self):
        self._svm_score_name = "SVMscore"
        self.counter = {  # counting the # of possible training PSMs
            "target": 0,
            "decoy": 0,
            "positive": 0,
            "negative": 0,
            "unknown": 0,
            "parsed PSMs": 0,
        }
        self.results = {}
        self.shitty_decoy_seqs = set()  # is overwritten by find_shitty_decoys()
        self.mgf_lookup = {}
        self.pep_to_mz = {}

        if __name__ == "__main__":
            self.parse_options()  # parse command line args and set options
            self.set_input_csv()

        self.observed_charges = set()
        self.used_extra_fields = set()
        self.decoy_train_prob = (
            None  # probability to include decoy PSMs as negative training examples
        )
        self.maximum_proteins_per_line = 0
        self.tryptic_aas = set(["R", "K", "-"])
        self.delim_regex = re.compile(
            r"<\|>|\;"
        )  # regex to split a line by both ";" and "<|>"
        return

    def parse_options(self):
        """
        parses the command line args for options/parameters
        """
        parser = argparse.ArgumentParser()
        parser.add_argument(
            "-i",
            "--input_csv",
            type=str,
            help="Input CSV path(s)",
            required=True,
            nargs="+",
        )
        parser.add_argument(
            "-o", "--output_csv", type=str, help="Output CSV path", required=True
        )
        parser.add_argument(
            "-k",
            "--kernel",
            type=str,
            default="rbf",
            help='SVM kernel type ("rbf", "linear", "poly" or "sigmoid")',
        )
        parser.add_argument(
            "-c", type=float, default=1.0, help="Penalty parameter C of the error term"
        )
        parser.add_argument(
            "-g",
            "--gamma",
            type=str,
            default="auto",
            help="Gamma parameter of the SVM.",
        )
        parser.add_argument(
            "-r",
            "--mb_ram",
            type=float,
            default=4000,
            help="Available RAM in megabytes, for SVM calculation",
        )
        parser.add_argument(
            "-f",
            "--fdr_cutoff",
            type=float,
            default=0.01,
            help="Target PSMs with a lower FDR will be used as a "
            "positive training set",
        )
        parser.add_argument(
            "-x",
            "--columns_as_features",
            type=str,
            nargs="+",
            default=[
                "MS-GF:RawScore",
                "MS-GF:DeNovoScore",
                "MS-GF:SpecEValue",
                "MS-GF:EValue",
                "OMSSA:evalue",
                "OMSSA:pvalue",
                "X\!Tandem:expect",
                "X\!Tandem:hyperscore",
            ],
            help="Columns that should be used as a feature directly "
            "(e.g. secondary scores). Will be converted to float",
        )
        parser.add_argument(
            "-d",
            "--dump_svm_matrix",
            type=str,
            default=False,
            help="Dump SVM matrix in PIN (Percolator input) format "
            "to the specified path, mostly for debugging "
            "and benchmarking.",
        )

        arg_dict = vars(parser.parse_args())  # convert to dict
        self.update(arg_dict)
        try:
            self["gamma"] = float(self["gamma"])
        except ValueError:
            assert (
                self["gamma"] == "auto"
            ), "Invalid gamma param: " '"{0}", using "auto" instead.'.format(
                self["gamma"]
            )

    def set_input_csv(self):
        """
        distinguishes one vs. many unified input csv files and either
        sets the single csv as input, or merges all csvs and sets
        the merged csv as input.
        """
        if len(self["input_csv"]) > 1:
            raise Exception("You must only specify *one* unified CSV file!")
        self.csv_path = self["input_csv"][0]
        print("Using input file", self.csv_path)

    def find_shitty_decoys(self):
        """
        Finds and notes decoys that share their sequence with a target PSM.

        Also counts the number of targets and decoys to get a quick estimate
        of how many positive/negative training examples can be "claimed".
        """
        target_seqs = set()
        decoy_seqs = set()
        with open(self.csv_path, "r") as f:

            reader = csv.DictReader(f)

            sorted_reader = sorted(
                reader,
                reverse=self["bigger_scores_better"],
                key=lambda d: float(d[self.col_for_sorting]),
            )

            for row in sorted_reader:
                self.observed_charges.add(int(row["Charge"]))
                if row_is_decoy(row):
                    decoy_seqs.add(unify_sequence(row["Sequence"]))
                    self.counter["decoy"] += 1
                else:
                    target_seqs.add(unify_sequence(row["Sequence"]))
                    self.counter["target"] += 1

        self.shitty_decoy_seqs = target_seqs.intersection(decoy_seqs)
        if len(self.shitty_decoy_seqs) > 0:
            print(
                "Warning! Found {0} sequences that are target AND decoy "
                "(immutable peptides?). These will not be used for training.\n".format(
                    len(self.shitty_decoy_seqs)
                )
            )
        return

    def determine_csv_sorting(self):
        with open(self.csv_path, "r") as in_file:
            reader = csv.DictReader(in_file)
            (
                self.col_for_sorting,
                self["bigger_scores_better"],
            ) = get_score_colname_and_order(reader.fieldnames)
        if self.col_for_sorting == self._svm_score_name:
            self._svm_score_name = self._svm_score_name + "2"

        print(
            "CSV will be sorted by column {0} (reverse={1}"
            ")".format(self.col_for_sorting, self["bigger_scores_better"])
        )

        for feat in self["columns_as_features"]:
            if feat in reader.fieldnames and feat != self.col_for_sorting:
                self.used_extra_fields.add(feat)

    def sort_by_rank(self, rowdict):
        score = float(rowdict[self.col_for_sorting])
        spec_title = rowdict["Spectrum Title"]
        return (spec_title, score)

    @staticmethod
    def parse_protein_ids(csv_field, sep="<|>"):
        """
        Turns the unified CSV column "Protein ID"
        into a set of all protein IDs.
        """
        clean = csv_field.replace("decoy_", "").strip()
        prot_id_set = set(clean.split(sep))
        return prot_id_set

    def count_intra_set_features(self):
        """
        intra-set features as calculated by Percolator:
        - num_pep:  Number of PSMs for which this is the best scoring peptide.
        - num_prot: Number of times the matched protein matches other PSMs.
        - pep_site: Number of different peptides that match this protein.

        own ideas:
        - pep_charge_states: in how many charge states was the peptide found?
        - seq_mods: in how many mod states was the AA-sequence found?
        - num_spec: Number of times the matched spectrum matches other peptides.
        """
        print("Counting intra-set features...")
        self.num_pep = defaultdict(int)
        self.num_prot = defaultdict(set)
        self.pep_site = defaultdict(set)
        self.score_list_dict = defaultdict(list)

        self.pep_charge_states = defaultdict(set)
        self.seq_mods = defaultdict(set)
        self.num_spec = defaultdict(set)

        with open(self.csv_path, "r") as f:
            reader = csv.DictReader(f)
            previous_spec_title = None
            rows_of_spectrum = []

            for row in sorted(
                reader, reverse=self["bigger_scores_better"], key=self.sort_by_rank
            ):

                if unify_sequence(row["Sequence"]) in self.shitty_decoy_seqs:
                    continue
                current_spec_title = row["Spectrum Title"]
                if current_spec_title != previous_spec_title:
                    # the next spectrum started, so let's process the info we
                    # collected for the previous spectrum:
                    score_list = [
                        field_to_bayes_float(r[self.col_for_sorting])
                        for r in rows_of_spectrum
                    ]
                    self.score_list_dict[previous_spec_title] = score_list

                    for rank, line in enumerate(rows_of_spectrum):
                        # print("\t".join([
                        # str(rank), line['Spectrum Title'], line[self.col_for_sorting]
                        # ]))
                        uni_sequence = unify_sequence(line["Sequence"])
                        peptide = (uni_sequence, line["Modifications"])

                        # multiple proteins are separated by <|>
                        # ignore start_stop_pre_post part since it depends on the peptide
                        # and not the protein (i.e. _233_243_A_R)
                        proteins = set(
                            line["Protein ID"].replace("decoy_", "").split(";")
                        )

                        # old unify csv format:
                        # proteins = self.parse_protein_ids(
                        #    line['proteinacc_start_stop_pre_post_;']
                        # )
                        if len(proteins) > self.maximum_proteins_per_line:
                            self.maximum_proteins_per_line = len(proteins)

                        if rank == 0:
                            # this is the 'best' peptide for that spectrum
                            self.num_pep[peptide] += 1
                        for protein in proteins:
                            self.num_prot[protein].add(
                                (
                                    line["Spectrum Title"],
                                    uni_sequence,
                                    line["Modifications"],
                                )
                            )
                            self.pep_site[protein].add(peptide)

                        self.pep_charge_states[peptide].add(int(row["Charge"]))
                        self.seq_mods[uni_sequence].add(row["Modifications"])
                        self.num_spec[line["Spectrum Title"]].add(peptide)

                    rows_of_spectrum = []

                rows_of_spectrum.append(row)
                previous_spec_title = current_spec_title

    def row_to_features(self, row):
        """
        Converts a unified CSV row to a SVM feature matrix (numbers only!)
        """
        sequence = unify_sequence(row["Sequence"])
        charge = field_to_float(row["Charge"])
        score = field_to_bayes_float(row[self.col_for_sorting])
        calc_mz, exp_mz, calc_mass, exp_mass = get_mz_values(row)
        # calc_mz = field_to_float( row['Calc m/z'] )  # calc m/z or uCalc?
        # exp_mz = field_to_float( row['Exp m/z'] )

        pre_aa_field = row["Sequence Pre AA"]
        post_aa_field = row["Sequence Post AA"]
        all_pre_aas = set(re.split(self.delim_regex, pre_aa_field))
        all_post_aas = set(re.split(self.delim_regex, post_aa_field))

        if any(pre_aa not in self.tryptic_aas for pre_aa in all_pre_aas):
            enzN = 0
        else:
            enzN = 1

        if any(post_aa not in self.tryptic_aas for post_aa in all_post_aas):
            enzC = 0
        else:
            enzC = 1

        n_missed_cleavages = len(
            [aa for aa in sequence[:-1] if aa in ["R", "K"]]
        )  # / len(sequence)

        missed_cleavages = [0] * 6
        try:
            missed_cleavages[n_missed_cleavages] = 1
        except IndexError:  # if a peptide has more than 6 missed cleavages
            missed_cleavages[-1] = 2

        spectrum = row["Spectrum Title"].strip()
        mass = (exp_mz * charge) - (charge - 1) * PROTON
        pep_len = len(sequence)
        # delta_mz = calc_mz - exp_mz
        delta_mass = calc_mass - exp_mass

        peptide = (sequence, row["Modifications"])
        proteins = self.parse_protein_ids(row["Protein ID"])
        num_pep = self.num_pep[peptide]
        pep_charge_states = len(self.pep_charge_states[peptide])
        seq_mods = len(self.seq_mods[sequence])
        num_spec = len(self.num_spec[row["Spectrum Title"]])
        num_prot = sum((len(self.num_prot[protein]) for protein in proteins))
        pep_site = sum((len(self.pep_site[protein]) for protein in proteins))

        user_specified_features = []
        for feat in self.used_extra_fields:
            if feat != self.col_for_sorting:
                try:
                    user_specified_features.append(field_to_float(row[feat]))
                except ValueError:
                    pass

        charges = defaultdict(int)
        for charge_n in sorted(self.pep_charge_states[peptide]):
            charges[charge_n] = 1

        if sequence in self.shitty_decoy_seqs:
            is_shitty = 1
        else:
            is_shitty = 0

        score_list = sorted(
            list(set(self.score_list_dict[spectrum])),
            reverse=self["bigger_scores_better"],
        )

        try:
            score_list_scaled = scale_scores(score_list)
            rank = score_list.index(score)
            deltLCn = (
                score_list_scaled[rank] - score_list_scaled[1]
            )  # Fractional difference between current and second best XCorr
            deltCn = (
                score_list_scaled[rank] - score_list_scaled[-1]
            )  # Fractional difference between current and worst XCorr
        except (ValueError, IndexError, AssertionError):
            # NaN values will be replaced by the column mean later
            # NaN values are entered when there is no ranking
            # e.g. when only one peptide was matched to the spectrum.
            rank, deltLCn, deltCn = np.nan, np.nan, np.nan

        features = [
            score,
            rank,
            deltCn,
            deltLCn,
            charge,
            # delta_mz,# / pep_len,
            delta_mass,  # / pep_len,
            # abs(delta_mz),# / pep_len,
            abs(delta_mass),  # / pep_len,
            n_missed_cleavages / pep_len,
            missed_cleavages[0],
            missed_cleavages[1],
            missed_cleavages[2],
            missed_cleavages[3],
            missed_cleavages[4],
            missed_cleavages[5],
            enzN,
            enzC,
            mass,
            pep_len,
            num_pep,
            num_prot,
            pep_site,
            is_shitty,
            pep_charge_states,
            num_spec,
            seq_mods,
        ]

        for charge_n in self.observed_charges:
            features.append(charges[charge_n])

        return features + user_specified_features

    def collect_data(self):
        """
        parses a unified csv file and collects features from each row
        """
        categories = []
        list_of_feature_lists = []
        feature_sets = set()
        with open(self.csv_path, "r") as f:
            reader = csv.DictReader(f)
            # collecting some stats for FDR calculation:
            self.PSM_count = 0
            self.decoy_count = 0

            if self["dump_svm_matrix"]:
                self.init_svm_matrix_dump()
                additional_matrix_info = []

            for i, row in enumerate(
                sorted(
                    reader,
                    reverse=self["bigger_scores_better"],
                    key=lambda d: float(d[self.col_for_sorting]),
                )
            ):

                features = self.row_to_features(row)

                if tuple(features) in feature_sets:
                    continue
                feature_sets.add(tuple(features))

                category, psm_FDR = self.get_psm_category(row)

                list_of_feature_lists.append(features)
                categories.append(category)

                if self["dump_svm_matrix"]:
                    label = -1 if row_is_decoy(row) else 1
                    sequence = "{0}.{1}#{2}.{3}".format(
                        row["Sequence Pre AA"].strip(),
                        row["Sequence"].strip(),
                        row["Modifications"].strip(),
                        row["Sequence Post AA"].strip(),
                    )
                    additional_matrix_info.append(
                        {
                            "psm_id": row["Spectrum Title"].strip(),
                            "label": label,
                            "scannr": row["Spectrum Title"].strip().split(".")[-2],
                            "peptide": sequence,
                            "proteins": self.parse_protein_ids(row["Protein ID"]),
                        }
                    )

                if i % 1000 == 0:
                    score_val = float(row[self.col_for_sorting])
                    msg = (
                        "Generating feature matrix from input csv "
                        "(line ~{0}) with score {1} and FDR "
                        "{2}".format(i, score_val, psm_FDR)
                    )
                    print(msg, end="\r")

        # All data points are collected in one big matrix, to make standardization possible
        print("\nConverting feature matrix to NumPy array...")
        X_raw = np.array(list_of_feature_lists, dtype=float)

        print("Replacing empty/NaN values with the mean of each column...")
        self.nan_replacer = Imputer()
        self.nan_replacer.fit(X_raw)
        X_raw = self.nan_replacer.transform(X_raw)
        # Standardize input matrix to ease machine learning! Scaled data has zero mean and unit variance
        print("Standardizing input matrix...")
        self.scaler = SCALER.fit(X_raw)
        self.X = self.scaler.transform(X_raw)
        self.categories = np.array(categories)
        print()

        if self["dump_svm_matrix"]:
            print("Dumping SVM matrix to", self["dump_svm_matrix"])

            for i, matrix_row in enumerate(self.X):
                matrix_row_info = additional_matrix_info[i]
                self.dump_svm_matrix_row(
                    row=list(matrix_row),
                    psm_id=matrix_row_info["psm_id"],
                    label=matrix_row_info["label"],
                    scannr=matrix_row_info["scannr"],
                    peptide=matrix_row_info["peptide"],
                    proteins=matrix_row_info["proteins"],
                )

            print("Dumped SVM matrix to", self["dump_svm_matrix"])
        return

    def init_svm_matrix_dump(self):
        from misc import FEATURE_NAMES

        colnames = ["PSMId", "label", "scannr"] + FEATURE_NAMES
        colnames += ["charge{0}".format(c) for c in self.observed_charges]
        for extra_field in sorted(self.used_extra_fields):
            colnames += [extra_field]
        colnames += ["peptide"]
        for n_proteins in range(self.maximum_proteins_per_line):
            colnames.append("proteinId{0}".format(n_proteins + 1))
        self.matrix_csv_path = self["dump_svm_matrix"]
        print("Dumping raw SVM input matrix to", self.matrix_csv_path)
        with open(self.matrix_csv_path, "w") as f:
            f.write("\t".join(colnames) + "\n")

    def dump_svm_matrix_row(
        self,
        row=None,
        psm_id=None,
        label=None,
        scannr=None,
        peptide=None,
        proteins=None,
    ):
        full_row = [psm_id, label, scannr] + row + [peptide] + list(proteins)
        with open(self.matrix_csv_path, "a") as f:
            row_str = "\t".join(str(x) for x in full_row) + "\n"
            f.write(row_str)

    def get_psm_category(self, row):
        """
        Determines whether a PSM (csv row) should be used as a negative or
        positive training example.

        returns
            1  - high-scoring target (positive training example)
            0  - not-high-scoring target (not usable for training)
           -1  - decoy (negative training example)

        """
        category = 0  # unknown (mix of true positives and false positives)
        self.PSM_count += 1  # for FDR calculation
        sequence = unify_sequence(row["Sequence"])
        psm_FDR = calc_FDR(self.PSM_count, self.decoy_count)

        if row_is_decoy(row):
            self.decoy_count += 1
            if psm_FDR <= 0.25 and sequence not in self.shitty_decoy_seqs:
                category = -1  # decoy (false positive hits)
                self.counter["negative"] += 1
            else:
                if not self.decoy_train_prob:
                    need_max = self.counter["positive"] * 2
                    have = self.counter["negative"]
                    still_there = self.counter["decoy"] - have
                    prob = need_max / still_there
                    if prob < 0.001:
                        prob = 0.001
                    self.decoy_train_prob = prob
                    print()
                    print(self.counter)
                    print("need max:", need_max)
                    print("have:", have)
                    print("still_there:", still_there)
                    print("probability:", self.decoy_train_prob)
                    print()
                if self.decoy_train_prob >= 1.0 or random() <= self.decoy_train_prob:
                    category = -1  # decoy (false positive hits)
                    self.counter["negative"] += 1

        else:  # row is target
            if psm_FDR <= self["fdr_cutoff"] and sequence not in self.shitty_decoy_seqs:
                category = 1  # high quality target (almost certainly true positives)
                self.counter["positive"] += 1

        if category == 0:
            self.counter["unknown"] += 1
        return (category, psm_FDR)

    def train(self, training_matrix, training_categories):
        counter = Counter(training_categories)
        msg = "Training {0} SVM on {1} target PSMs and {2} decoy PSMs" "...".format(
            self["kernel"], counter[1], counter[-1]
        )
        print(msg, end="\r")
        # specify the classification method (rbf and linear SVC seem to work best and are quite fast)
        classifier = svm.SVC(
            C=self["c"],
            kernel=self["kernel"],
            probability=False,  # we don't want to get probabilities later on -> faster
            cache_size=self["mb_ram"],  # available RAM in megabytes
            # decision_function_shape = 'ovr',  # doesn't seem to matter
            # class_weight= 'balanced',  # doesn't seem to matter
        )
        # train the SVC on our set of training data:
        classifier.fit(
            training_matrix,
            training_categories,
        )
        print(msg + " done!")
        return classifier

    def classify(self, classifier, psm_matrix):
        msg = "Classifying {0} PSMs...".format(len(psm_matrix))
        print(msg, end="\r")
        for i, row in enumerate(psm_matrix):
            # get the distance to the separating SVM hyperplane and use it as a score:
            svm_score = classifier.decision_function(np.array([row]))[0]

            features = tuple(row)
            if features not in self.results:
                self.results[features] = svm_score
            else:
                print(
                    "Warning! This combination of features already has a predicted probability! "
                    "Previous svm_score: {0:f} - Current svm_score: {1:f}"
                    "".format(self.results[tuple(row)], svm_score)
                )
                # take the mean value, no idea how to handle this better, but it never happened so far...
                self.results[features] = (self.results[features] + svm_score) / 2.0
        print(msg + " done!")
        return

    def add_scores_to_csv(self):
        outfname = os.path.basename(self["output_csv"])
        print("Writing output csv {0} ...".format(outfname))
        msg = "Writing output csv {0} (line ~{1})..."

        with open(self["output_csv"], "w", newline="") as out_csv, open(
            self.csv_path, "r"
        ) as in_csv:
            reader = csv.DictReader(in_csv)
            writer = csv.DictWriter(out_csv, reader.fieldnames + [self._svm_score_name])
            writer.writeheader()
            for i, row in enumerate(reader):
                if i % 1000 == 0:
                    print(msg.format(outfname, i), end="\r")
                features = self.nan_replacer.transform(
                    np.array([self.row_to_features(row)])
                )
                features_scaled = tuple(list(self.scaler.transform(features)[0]))
                SVMScore = self.results[features_scaled]
                row[self._svm_score_name] = SVMScore
                writer.writerow(row)
        print("\n")
        return

    def __str__(self):
        out_str = ["\n\tpyPercolator Options:"]
        for option, value in self.items():
            out_str.append("{0:·<25}{1}".format(option, value))
        return "\n".join(out_str)

if __name__ == "__main__":
    s = SVMWrapper()

    print(s)  # print parameter/settings overview

    s.determine_csv_sorting()
    s.find_shitty_decoys()

    print("\nCounter:")
    print(s.counter)
    print()
    s.count_intra_set_features()
    s.collect_data()

    print(
        "Splitting data in half to avoid training and testing on the same features..."
    )
    skfold = StratifiedKFold(s.categories, n_folds=2, shuffle=True)

    # use one half to score the other half, and vice versa:
    for i, (train_index, test_index) in enumerate(skfold):
        current_half = "1st" if i == 0 else "2nd"
        other_half = "2nd" if i == 0 else "1st"
        print(
            "\nUsing high-scoring PSMs and decoys of the {0} half to train...".format(
                current_half
            )
        )
        mask = s.categories[train_index] != 0
        train_categories = s.categories[train_index][mask]
        train_features = s.X[train_index][mask]
        svm_classifier = s.train(
            training_matrix=train_features,
            training_categories=train_categories,
        )

        print(
            "Using the trained SVM to classify all PSMs of the {0} half".format(
                other_half
            )
        )
        s.classify(
            svm_classifier,
            s.X[test_index],
        )
        if s["kernel"].lower() == "linear":
            print()  # print SVM coefficients (only works for linear kernel)
            print(svm_classifier.coef_)
            print()

    print("\nCounter:")
    print(s.counter)
    print()
    s.add_scores_to_csv()


import logging
import os
from types import *
from clientagent.common.platform_id import PlatformID
from clientagent.common.config import Config

class ClientAgentState:
    '''
    Very basic state machine used by various client agent modules.
    '''
    INIT_SETUP = False
    CLIENTAGENT_ROOT = ""
    CONFIG = None
    # The following are largely used in the Win32 service APIs, but provided
    # in case we find a use for them elsewhere
    SRV_NAME = "EILClientAgent"
    SRV_DISPLAY_NAME = "EIL Client Agent"
    SRV_DESCRIPTION = "EIL Portal Unified Client Service (Python) - resides on client - interfaces with CCMS"
    # Version information
    VERSION = "Undefined!"

    # Various Linux-isms we need for compatibility with the dispatcher scripts
    COMDIR = None
    BINDIR = None

def updateLogger():
    debug_level = 2
    if(ClientAgentState.CONFIG.C.has_option('main', 'log_level')):
        _debug_level = ClientAgentState.CONFIG.C.get('main', 'log_level')
        try:
            debug_level = int(_debug_level)
        except:
            pass

    Logger = logging.getLogger()

    if debug_level < 1:
        Logger.setLevel(logging.CRITICAL)
    elif debug_level == 1:
        Logger.setLevel(logging.ERROR)
    elif debug_level == 2:
        Logger.setLevel(logging.WARNING)
    elif debug_level == 3:
        Logger.setLevel(logging.INFO)
    else:
        # Anything higher will be debug to full
        Logger.setLevel(logging.DEBUG)

    Logger.info('Log level set to %d' % debug_level)

    # FIXME - Temp debugging suds client items
    logging.getLogger('suds.client').setLevel(logging.DEBUG)

if not ClientAgentState.INIT_SETUP:
    platformID = PlatformID()

    # Set up the logging filename
    # FIXME - What do we want for a debug mode? Would be nice if we could run
    # from command-line and have the logging go to stdout
    if platformID.IS_WINDOWS:
        # FIXME - Will want this to be the same log we have used previously
        ClientAgentState.CLIENTAGENT_ROOT = 'C:\\eil'
        logging.basicConfig(filename='%s\\clienagent.log' % ClientAgentState.CLIENTAGENT_ROOT,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    else:
        # Our root will be defined by previously issued LANANA/LSB requirements
        ClientAgentState.CLIENTAGENT_ROOT = '/opt/intel/eil/clientagent'
        fn = '%s/home/client-agent-base.log' % ClientAgentState.CLIENTAGENT_ROOT
        try:
            stream = os.popen('/usr/bin/clientagent-helper.sh --stdlog')
            output = stream.readlines()
            stream.close()

            if len(output) == 1:
                fn = output[0]
        finally:
            stream.close()

        logging.basicConfig(filename=fn,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        comdir = '%s/home/commands' % ClientAgentState.CLIENTAGENT_ROOT
        try:
            stream = os.popen('/usr/bin/clientagent-helper.sh --comdir')
            output = stream.readlines()
            stream.close()

            if len(output) == 1:
                comdir = output[0]
        finally:
            stream.close()

        ClientAgentState.COMDIR = comdir

        bindir = '%s/bin' % ClientAgentState.CLIENTAGENT_ROOT
        try:
            stream = os.popen('/usr/bin/clientagent-helper.sh --bin')
            output = stream.readlines()
            stream.close()

            if len(output) == 1:
                bindir = output[0]
        finally:
            stream.close()

        ClientAgentState.BINDIR = bindir

    ClientAgentState.CONFIG = Config(ClientAgentState.CLIENTAGENT_ROOT)

    try:
        verFile = os.path.join(ClientAgentState.CLIENTAGENT_ROOT, 'lib', 'VERSION')
        version = open(verFile, 'r')
        for rawline in version:
            verInfo = rawline.strip()
            if len(verInfo) > 0:
                ClientAgentState.VERSION = verInfo
        version.close()
    except:
        ClientAgentState.VERSION = 'Undefined'

    updateLogger()
    ClientAgentState.CONFIG.setCallback(updateLogger)

    ClientAgentState.INIT_SETUP = True

def get_config():
    '''
    Returns the config instance
    '''
    return ClientAgentState.CONFIG

# vim:set ai et sts=4 sw=4 tw=80:


import analysis

# pcap file path
file1 = 'example_1.pcap'
file2 = 'example_2.pcap'
file3 = 'example_3.pcap'
file4 = 'example_4.pcap'

# analysis parameter
mode = 'sec'
time_interval = 60
myplot = analysis.TracePlot(time_interval, mode)

# give pcap file
print('Now analysis pcap 1')
myplot.first_sep_analysis(file1)

print('Now analysis pcap 2')
myplot.mid_sep_analysis(file2)

print('Now analysis pcap 3')
myplot.mid_sep_analysis(file3)

print('Now analysis pcap 4')
myplot.last_sep_analysis(file4)

# give attack list
myplot.import_attack_list('attack_list_example.txt')

# plot picture
myplot.entropy_one_plot(['entropy_src_ip', 'entropy_dst_ip', 'entropy_sport', 'entropy_dport',
                            'entropy_pkt_len', 'entropy_proto'])
myplot.entropy_seperate_plot(['entropy_src_ip', 'entropy_dst_ip', 'entropy_sport', 'entropy_dport',
                                'entropy_pkt_len', 'entropy_proto'])
myplot.count_one_plot(['count_pkt_cnt', 'count_total_pkt_len',
                        'distinct_src_ip', 'distinct_dst_ip', 'distinct_sport', 'distinct_dport',
                            'distinct_pkt_len', 'distinct_proto', 'count_average_pkt_len'])
myplot.csv_output()


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import seaborn as sns
import pandas as pd

#set seaborn style
_ = sns.set(style="whitegrid", palette="colorblind")

# reading data from the xls
file = 'D://S2//MGPS//1 First Semester//Methods_Ji Ma//Datasets//Code//Jamaica_cleaned2.xlsx'
Jamaica_data = pd.read_excel(file)

#Generate box plot, assign it as ax
ax = sns.boxplot(x = 'Teen_Marital_Union', y ='Yrs_Edu', data = Jamaica_data)

labels = [item.get_text() for item in ax.get_xticklabels()]
labels[0] = 'Non Adolescent marriage'
labels[1] = 'Adolescent marriage'
ax.set_xticklabels(labels)


import numpy as np
import cv2 as cv
import glob

chessboardSize = (10, 14)
criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
objp = np.zeros((chessboardSize[0] * chessboardSize[1], 3), np.float32)
objp[:, :2] = np.mgrid[0:chessboardSize[0], 0:chessboardSize[1]]
prev_img_shape = None

objpoints = []
imgpoints = []
images = glob.glob('*.jpg')
for image in images:
    print(image)
    img = cv.imread(image)
    gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
    ret, corners = cv.findChessboardCorners(gray, chessboardSize)
    if ret:
        objpoints.append(objp)
        corners2 = cv.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
        imgpoints.append(corners)
        cv.drawChessboardCorners(img, chessboardSize, corners2, ret)

    cv.imshow('img', img)
    cv.imwrite('caliResult.png', img)
    cv.waitKey(100)
cv.destroyAllWindows()


# from bs4 import BeautifulSoup as BS
# import pandas as pd
#
# with open('suyunjin.txt', 'r', encoding='gbk') as f:
#     soup = BS(f.read(), 'html.parser')
#
# data = []
# div = soup.find('div', class_='tableContent')
# tr_list = div.table.find_all('tr')
# header = tr_list[0].find_all('th')
# for i in range(1, len(tr_list)):
#     for ii, td in enumerate(tr_list[i].find_all('td')):
#         for v in td.find_all('span', class_='jdt-box'):
#             d = v.span.string.split('/')
#             d.extend([header[ii].string, tr_list[i].td.text.strip()])
#             data.append(d)
#
# df = pd.DataFrame(data, columns=[u'某数据1', u'某数据2', u'某数据3', u'服务器名称', u'活动名称'])
# print(df)
# df.to_excel(u'苏韵锦你这里欠我的用什么还.xlsx', index=False)

import random

def fuck(s, c, b, t):
    r = []
    p = {}
    for i in range(c):
        if i < c // 2:
            x = random.randint(b, t)
            r.append(x)
            p[i] = x - s // c
        else:
            if i + 1 == c:
                r.append(s - sum(r))
                return r
            d = p[i - c // 2]
            r.append(random.randint(b, (s // c) - d) if d > 0 else random.randint((s // c) - d, t))
    return r

r = fuck(1200, 4, 100, 400)
print(r, sum(r))


# 2583 영역 구하기
# 문제
# 눈금의 간격이 1인 M × N(M, N ≤ 100)크기의 모눈종이가 있다. 이 모눈종이 위에 눈금에 맞추어 K개의 직사각형을 그릴 때, 이들 K개의 직사각형의 내부를 제외한 나머지 부분이
# 몇 개의 분리된 영역으로 나누어진다.

# 예를 들어 M=5, N=7 인 모눈종이 위에 <그림 1>과 같이 직사각형 3개를 그렸다면, 그 나머지 영역은 <그림 2>와 같이 3개의 분리된 영역으로 나누어지게 된다.

# <그림 2>와 같이 분리된 세 영역의 넓이는 각각 1, 7, 13이 된다.

# M, N과 K 그리고 K개의 직사각형의 좌표가 주어질 때, K개의 직사각형 내부를 제외한 나머지 부분이 몇 개의 분리된 영역으로 나누어지는지, 그리고 분리된 각 영역의 넓이가
# 얼마인지를 구하여 이를 출력하는 프로그램을 작성하시오.

# 입력
# 첫째 줄에 M과 N, 그리고 K가 빈칸을 사이에 두고 차례로 주어진다. M, N, K는 모두 100 이하의 자연수이다. 둘째 줄부터 K개의 줄에는 한 줄에 하나씩 직사각형의 왼쪽 아래
# 꼭짓점의 x, y좌표값과 오른쪽 위 꼭짓점의 x, y좌표값이 빈칸을 사이에 두고 차례로 주어진다. 모눈종이의 왼쪽 아래 꼭짓점의 좌표는 (0,0)이고, 오른쪽 위
# 꼭짓점의 좌표는(N,M)이다. 입력되는 K개의 직사각형들이 모눈종이 전체를 채우는 경우는 없다.

# 출력
# 첫째 줄에 분리되어 나누어지는 영역의 개수를 출력한다. 둘째 줄에는 각 영역의 넓이를 오름차순으로 정렬하여 빈칸을 사이에 두고 출력한다.

import sys

m, n, k = map(int, sys.stdin.readline().split())
visited = [[1]*n for _ in range(m)]
dx = [1, 0, -1 ,0, 0]
dy = [0, 1, 0, -1, 0]
cnt = 0
total_area = []

for _ in range(k): # 구역을 나누어준 다음
    rec_list = list(map(int, sys.stdin.readline().split()))

    for i in range(rec_list[1], rec_list[3]):
        for j in range(rec_list[0], rec_list[2]):
            visited[i][j] = 0

for r in range(m): # DFS 실행
    for c in range(n):
        if visited[r][c] == 1:
            stack = []
            stack.append(r)
            stack.append(c)
            cnt += 1
            area = 0

            while stack:
                x = stack.pop()
                y = stack.pop()

                for dir in range(5):
                    if 0 <= x + dx[dir] < n and 0 <= y + dy[dir] < m:
                        if visited[y + dy[dir]][x + dx[dir]] == 1:
                            area += 1
                            visited[y + dy[dir]][x + dx[dir]] = 2
                            stack.append(y + dy[dir])
                            stack.append(x + dx[dir])

            total_area.append(area)

print(cnt)
print(*sorted(total_area))

# 예제 입력 1
# 5 7 3

# 0 2 4 4
# 1 1 2 5
# 4 0 6 2

# 예제 출력 1
# 3
# 1 7 13


class heatmapfinal:
    registered = True  # Value to define db operator

    def __init__(self):
        self.n = 0
        #  self.mydata = []
        self.heatmap = {}

    def step(self, *args):

        if self.n == 0:
            # print args, len(args)

            self.nomydatacolumns = int(len(args) / 4.0)
            self.colnames = [None] * self.nomydatacolumns
            self.curvalues = [None] * self.nomydatacolumns
            self.minvalues = [0.0] * self.nomydatacolumns
            self.step = [0.0] * self.nomydatacolumns
            self.index = [0.0] * self.nomydatacolumns
            # print self.nomydatacolumns, self.colnames ,self.curvalues , self.minvalues ,self.step ,self.index

        try:
            for i in xrange(self.nomydatacolumns):
                self.curvalues[i] = float(args[i * 4 + 1])
                if self.n == 0:
                    self.minvalues[i] = float(args[i * 4 + 2])
                    self.colnames[i] = (args[i * 4])
                    self.step[i] = float(args[i * 4 + 3])
            self.n += 1

        except (ValueError, TypeError):
            raise

        # print  self.step #["A", self.curvalues, self.colnames, self.minvalues, self.step]

        for i in xrange(self.nomydatacolumns):
            # print self.step[i]
            self.index[i] = max(int(round((self.curvalues[i] - self.minvalues[i]) / self.step[i])), 0)

        if tuple(self.index) in self.heatmap.keys():
            self.heatmap[tuple(self.index)] += 1
        else:
            self.heatmap[tuple(self.index)] = 1

    def final(self):
        import itertools
        yield tuple(itertools.chain.from_iterable((tuple(itertools.chain.from_iterable(
            [("colname" + str(i), "id" + str(i), "minvalue" + str(i), "maxvalue" + str(i)) for i in
             xrange(self.nomydatacolumns)])), ['num'])))
        # yield ('colname0', 'minvalue0', 'maxvalue1', 'colname1', 'minvalue1', 'maxvalue1')

        if self.n == 0:
            result = []
            for i in xrange(self.nomydatacolumns):
                result.append("None")
                yield result
        else:
            for item in self.heatmap:
                result = []
                for i in xrange(self.nomydatacolumns):
                    result.append(self.colnames[i])
                    result.append(item[i])
                    result.append(self.minvalues[i] + item[i] * self.step[i])
                    result.append(self.minvalues[i] + (item[i] + 1) * self.step[i])
                result.append(self.heatmap[item])
                yield result

if not ('.' in __name__):
    """
    This is needed to be able to test the function, put it at the end of every
    new function you create
    """
    import sys
    from functions import *

    testfunction()
    if __name__ == "__main__":
        reload(sys)
        sys.setdefaultencoding('utf-8')
        import doctest

        doctest.testmod()


# Find the largest palindrome made from the product of two 3-digit numbers.

def isPalindrome(x):
    isPal = False
    if str(x) == str(x)[::-1]:
        isPal = True
    return isPal

numOfDigits = 3
minNum = 10 ** (numOfDigits - 1)
maxNum = 10 ** numOfDigits - 1
largestPalindrome = -1

# Find the largest palindrome product amongst all product combinations
while maxNum > minNum:
    for i in range(maxNum, minNum - 1, -1):
        product = i * maxNum
        if isPalindrome(product) and largestPalindrome < product:
            largestPalindrome = product

    maxNum -= 1

print("The largest palindrome product from the product of two " + str(numOfDigits) + "-digit numbers is ", end="")
print(largestPalindrome)


import glob
import pdf2docx
from . import trans_txt

import front.trans_docx
import front.trans_pptx
import front.trans_excel
from translit.models import input_storage, output_storage

def trans(filenomi, t):
	# dalolatnoma.docx
	path='front/files'
	file_path  = input_storage.path(filenomi)
	file_path2 = output_storage.path(filenomi)
	file_name  = filenomi.rsplit('.',1)[0]
	file_type  = filenomi.split('.')[-1]

	file_types =['pptx', 'docx', 'xlsx', 'txt']

	if file_type not in file_types:
		return "Xa'tolik, fayl formati xa'to"

	if file_type == file_types[0]:
		front.trans_pptx.transliterate(file_path, file_path2, t)
	elif file_type == file_types[1]:
		front.trans_docx.transliterate(file_path, file_path2, t)
	elif file_type == file_types[2]:
		front.trans_excel.transliterate(file_path, file_path2, t)
	elif file_type == file_types[3]:
		trans_txt.transliterate(file_path, file_path2, t)
	return filenomi


import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from tqdm import tqdm
from load_data import DataGetter
import pickle

def similarities_matrix(tfidf_data: pd.DataFrame) -> None:
    """
        Process and builds cosine similarities dataframe between chuncks of apps ids for
        memory optimization.
            inputs:
                    tfidf_data: TF-IDF map of apps text description.
    """
    store = pd.HDFStore('../data/raw/similiarities.h5')
    last_idx = 0
    for idx in tqdm(range(1000, tfidf_data.shape[0], 1000)):
       temp_data = tfidf_data.dot(tfidf_data.iloc[last_idx:idx].transpose())
       store.append(f'val_{int(idx/1000)}', temp_data)
       last_idx = idx
       del temp_data

    #last apps in the range
    temp_data = tfidf_data.dot(tfidf_data.iloc[last_idx:tfidf_data.shape[0]].transpose())
    store.append('val_final', temp_data)
    store.close()

def tfidf_map(data: pd.DataFrame, column: str) -> pd.DataFrame:
    """TF-IDF of corpus."""

    # build TF-IDF sparse matrix.
    tfidf_vector = TfidfVectorizer(ngram_range=(1,1),
                                   min_df=0.005,
                                   use_idf=True)
    tf_idf_data = tfidf_vector.fit_transform(data[column])

    # build dataframe of TF-IDF Map
    data = pd.DataFrame(tf_idf_data.toarray(),
                        columns = tfidf_vector.get_feature_names_out(),
                        index=data.item_id)
    return(data)

def get_top10_similiarities_from_h5_store() -> None:
    """Gets top 10 most similars apps for each app."""

    store = pd.HDFStore('../data/raw/similiarities.h5')
    similiarities = {}
    for key in tqdm(store.keys()):
        df = store.get(key=key)
        for column in df.columns:
            similiarities[column] = df[column].nlargest(11).iloc[1:10]

    with open('../data/final/similiarities.p', 'wb') as fp:
        pickle.dump(similiarities, fp)

def main() -> None:
    """Builds the pipeline of Similarities Matrix."""

    app_metadata = DataGetter(['app_metadata'], data_form='process').app_metadata_df
    tfidf_data = tfidf_map(app_metadata, 'description')
    similarities_matrix(tfidf_data)
    get_top10_similiarities_from_h5_store()

if __name__ == "__main__":
    main()


from collections import namedtuple

STATEMENT_TYPE_READ_ONLY = "r"
STATEMENT_TYPE_READ_WRITE = "rw"
STATEMENT_TYPE_WRITE_ONLY = "w"
STATEMENT_TYPE_SCHEMA_WRITE = "s"

class ResultSummary(object):
    """ A summary of execution returned with a :class:`.StatementResult` object.
    """

    #: The statement that was executed to produce this result.
    statement = None

    #: Dictionary of parameters passed with the statement.
    parameters = None

    #: The type of statement (``'r'`` = read-only, ``'rw'`` = read/write).
    statement_type = None

    #: A set of statistical information held in a :class:`.Counters` instance.
    counters = None

    #: A :class:`.Plan` instance
    plan = None

    #: A :class:`.ProfiledPlan` instance
    profile = None

    #: Notifications provide extra information for a user executing a statement.
    #: They can be warnings about problematic queries or other valuable information that can be
    #: presented in a client.
    #: Unlike failures or errors, notifications do not affect the execution of a statement.
    notifications = None

    def __init__(self, statement, parameters, **metadata):
        self.statement = statement
        self.parameters = parameters
        self.statement_type = metadata.get("type")
        self.counters = SummaryCounters(metadata.get("stats", {}))
        if "plan" in metadata:
            self.plan = make_plan(metadata["plan"])
        if "profile" in metadata:
            self.profile = make_plan(metadata["profile"])
            self.plan = self.profile
        self.notifications = []
        for notification in metadata.get("notifications", []):
            position = notification.get("position")
            if position is not None:
                position = Position(position["offset"], position["line"], position["column"])
            self.notifications.append(Notification(notification["code"], notification["title"],
                                                   notification["description"], notification["severity"], position))

class SummaryCounters(object):
    """ Set of statistics from a Cypher statement execution.
    """

    #:
    nodes_created = 0

    #:
    nodes_deleted = 0

    #:
    relationships_created = 0

    #:
    relationships_deleted = 0

    #:
    properties_set = 0

    #:
    labels_added = 0

    #:
    labels_removed = 0

    #:
    indexes_added = 0

    #:
    indexes_removed = 0

    #:
    constraints_added = 0

    #:
    constraints_removed = 0

    def __init__(self, statistics):
        for key, value in dict(statistics).items():
            key = key.replace("-", "_")
            setattr(self, key, value)

    def __repr__(self):
        return repr(vars(self))

    @property
    def contains_updates(self):
        return bool(self.nodes_created or self.nodes_deleted or
                    self.relationships_created or self.relationships_deleted or
                    self.properties_set or self.labels_added or self.labels_removed or
                    self.indexes_added or self.indexes_removed or
                    self.constraints_added or self.constraints_removed)

#: A plan describes how the database will execute your statement.
#:
#: operator_type:
#:   the name of the operation performed by the plan
#: identifiers:
#:   the list of identifiers used by this plan
#: arguments:
#:   a dictionary of arguments used in the specific operation performed by the plan
#: children:
#:   a list of sub-plans
Plan = namedtuple("Plan", ("operator_type", "identifiers", "arguments", "children"))

#: A profiled plan describes how the database executed your statement.
#:
#: db_hits:
#:   the number of times this part of the plan touched the underlying data stores
#: rows:
#:   the number of records this part of the plan produced
ProfiledPlan = namedtuple("ProfiledPlan", Plan._fields + ("db_hits", "rows"))

#: Representation for notifications found when executing a statement. A
#: notification can be visualized in a client pinpointing problems or
#: other information about the statement.
#:
#: code:
#:   a notification code for the discovered issue.
#: title:
#:   a short summary of the notification
#: description:
#:   a long description of the notification
#: severity:
#:   the severity level of the notification
#: position:
#:   the position in the statement where this notification points to, if relevant.
Notification = namedtuple("Notification", ("code", "title", "description", "severity", "position"))

#: A position within a statement, consisting of offset, line and column.
#:
#: offset:
#:   the character offset referred to by this position; offset numbers start at 0
#: line:
#:   the line number referred to by the position; line numbers start at 1
#: column:
#:   the column number referred to by the position; column numbers start at 1
Position = namedtuple("Position", ("offset", "line", "column"))

def make_plan(plan_dict):
    """ Construct a Plan or ProfiledPlan from a dictionary of metadata values.

    :param plan_dict:
    :return:
    """
    operator_type = plan_dict["operatorType"]
    identifiers = plan_dict.get("identifiers", [])
    arguments = plan_dict.get("args", [])
    children = [make_plan(child) for child in plan_dict.get("children", [])]
    if "dbHits" in plan_dict or "rows" in plan_dict:
        db_hits = plan_dict.get("dbHits", 0)
        rows = plan_dict.get("rows", 0)
        return ProfiledPlan(operator_type, identifiers, arguments, children, db_hits, rows)
    else:
        return Plan(operator_type, identifiers, arguments, children)


# -*- coding: utf-8 -*-
#harvest theses from Sussex U.
#FS: 2020-03-23
#FS: 2022-12-21

import sys
import os
import urllib.request, urllib.error, urllib.parse
from bs4 import BeautifulSoup
import re
import ejlmod3
import time

publisher = 'Sussex U.'
hdr = {'User-Agent' : 'Magic Browser'}

skipalreadyharvested = True
boring = ['Media and Film', 'Management', 'Media and Cultural Studies', 'Accounting and finance',
          'Accounting and Finance', 'American studies', 'Anthropology', 'Art History',
          'Biochemistry', 'Biology', 'Chemistry', 'Development Studies', 'Economics', 'Education',
          'Engineering', 'English', 'Genome stability', 'Geography', 'History',
          'Institute of Development Studies', 'International Development', 'International Relations',
          'Law', 'Life Sciences', 'Media and film', 'Music', 'Neuroscience', 'Philosophy',
          'Politics', 'Psychology', 'Science policy research unit', 'Social work', 'Social Work',
          'Sociology', '|SPRU - Science Policy Research Unit', 'SPRU - Science Policy Research Unit']
prerecs = []
jnlfilename = 'THESES-SUSSEX-%s' % (ejlmod3.stampoftoday())

if skipalreadyharvested:
    alreadyharvested = ejlmod3.getalreadyharvested(jnlfilename)

for year in [ejlmod3.year(), ejlmod3.year(backwards=1)]:
#    for (fc, dep) in [('', 'd234'), ('m', 'd235')]:
#        tocurl = 'http://sro.sussex.ac.uk/view/divisions/%s/%i.html' % (dep, year)
    tocurl = 'http://sro.sussex.ac.uk/view/type/thesis/%i.html' % (year)
    print(tocurl)
    try:
        req = urllib.request.Request(tocurl, headers=hdr)
        tocpage = BeautifulSoup(urllib.request.urlopen(req), features="lxml")
        time.sleep(2)
    except:
        continue
    for p in tocpage.find_all('p'):
        if re.search('PhD', p.text):
            for a in p.find_all('a'):
                if a.has_attr('href'):
                    rec = {'tc' : 'T', 'jnl' : 'BOOK', 'link' : a['href'], 'note' : []}
                    rec['tit'] = a.text.strip()
                    rec['doi'] = '20.2000/UCLodon/' + re.sub('\D', '', a['href'])
                    if ejlmod3.checkinterestingDOI(rec['doi']):
                        if not skipalreadyharvested or not rec['doi'] in alreadyharvested:
                            prerecs.append(rec)
    print('  %4i records so far' % (len(prerecs)))

recs = []
i = 0
for rec in prerecs:
    keepit = True
    i += 1
    ejlmod3.printprogress('-', [[i, len(prerecs)], [rec['link']], [len(recs)]])
    try:
        artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['link']), features="lxml")
        time.sleep(10)
    except:
        try:
            print('retry %s in 180 seconds' % (rec['link']))
            time.sleep(180)
            artpage = BeautifulSoup(urllib.request.build_opener(urllib.request.HTTPCookieProcessor).open(rec['link']), features="lxml")
        except:
            print('no access to %s' % (rec['link']))
            continue
    ejlmod3.metatagcheck(rec, artpage, ['eprints.creators_name', 'eprints.creators_orcid',
                                        'eprints.keywords', 'eprints.abstract',
                                        'eprints.date', 'eprints.doi', 'eprints.pages'])
    rec['autaff'][-1].append(publisher)
    for meta in artpage.head.find_all('meta'):
        if meta.has_attr('name'):
            #PDF
            if meta['name'] == 'DC.identifier':
                if re.search('pdf$', meta['content']):
                    rec['hidden'] = meta['content']
            #PDF
            elif meta['name'] == 'eprints.thesis_award':
                rec['note'].append(meta['content'])
            #department
            elif meta['name'] == 'eprints.department':
                dep = meta['content']
                if dep in boring:
                    keepit = False
                elif dep == 'Astronomy':
                    rec['fc'] = 'a'
                elif dep == 'Mathematics':
                    rec['fc'] = 'm'
                elif dep == 'Informatics':
                    rec['fc'] = 'c'
                elif not dep in ['Physics']:
                    rec['note'].append('DEP=' + dep)
    if keepit:
        ejlmod3.printrecsummary(rec)
        recs.append(rec)
    else:
        ejlmod3.adduninterestingDOI(rec['doi'])

ejlmod3.writenewXML(recs, publisher, jnlfilename)


import dns
import os
import time
from recursortests import RecursorTest

class testOOOTCP(RecursorTest):
    _confdir = 'OOOTCP'

    _config_template = """dnssec=validate
"""

    @classmethod
    def generateRecursorConfig(cls, confdir):
        super(testOOOTCP, cls).generateRecursorConfig(confdir)

    def testOOOVeryBasic(self):
        expected = {}
        queries = []
        for zone in ['5.delay1.example.', '0.delay2.example.']:
            expected[zone] = dns.rrset.from_text(zone, 0, dns.rdataclass.IN, 'TXT', 'a')
            query = dns.message.make_query(zone, 'TXT', want_dnssec=True)
            query.flags |= dns.flags.AD
            queries.append(query)

        ress = self.sendTCPQueries(queries)

        self.assertEqual(len(ress), len(expected))

        i = 0
        for exp in [expected['0.delay2.example.'], expected['5.delay1.example.']]:
            print('ress0')
            print(ress[i].answer[0].to_text())
            print('exp')
            print(exp.to_text())
            self.assertMessageIsAuthenticated(ress[i])
            self.assertRRsetInAnswer(ress[i], exp)
            self.assertMatchingRRSIGInAnswer(ress[i], exp)
            i = i + 1

    def testOOOTimeout(self):
        expected = {}
        queries = []
        for zone in ['25.delay1.example.', '1.delay2.example.']:
            query = dns.message.make_query(zone, 'TXT', want_dnssec=True)
            query.flags |= dns.flags.AD
            queries.append(query)

        ress = self.sendTCPQueries(queries)

        self.assertEqual(len(ress), 2)
        exp = dns.rrset.from_text('1.delay2.example.', 0, dns.rdataclass.IN, 'TXT', 'a')
        self.assertRRsetInAnswer(ress[0], exp)
        self.assertMatchingRRSIGInAnswer(ress[0], exp)
        self.assertRcodeEqual(ress[1], dns.rcode.SERVFAIL)

        # Let the auth timeout happen to not disturb other tests
        # this can happen if the auth is single-threaded
        time.sleep(1)


# -*- coding: utf-8 -*-

from common.forms import TelefonoForm
from common.models import Telefono, ViewPort
from common.utils import direct_response, json_response
from django.forms.models import modelformset_factory
from django.shortcuts import get_object_or_404
from django.views.decorators.csrf import csrf_exempt
from django.contrib.sites.models import Site
from django.db.models.aggregates import Min
from portal.forms import BusquedaForm, ContactoForm, ChatForm
from portal.models import Inmobiliaria, Area
from proyectos.models import Aviso, Proyecto, Desarrollado
from usuarios.forms import AreaInteresForm
from usuarios.models import AdminHelpDesk

def inicio(request):
    """
    Vista de inicio
    """
    return direct_response(request, 'portal/inicio.html',
                           {'destacados': Proyecto.get_destacados(4),
                            'form': BusquedaForm(),
                            'rango_renta': Proyecto.get_rango_precio(u"R"),
                            'rango_venta': Proyecto.get_rango_precio(u"V"),})

@csrf_exempt
def chat(request):
    """
    Chat con los administradores de servicio
    """
    if request.method == "POST":
        form_chat = ChatForm(request.POST)
        if form_chat.is_valid():

            return direct_response(request, "portal/iframes/chat.html",
                    {'form_chat': form_chat})
    else:
        form_chat = ChatForm()

    return direct_response(request, "portal/iframes/chat.html",
                           {'form_chat': form_chat})

def ayuda_en_linea(request, id_area, usuario):
    """
    Crea un canal de chat con uno de lso administradores y registra a ambos
    usuarios para comenzar el chat
    """
    area = get_object_or_404(Area, id=id_area)

    return direct_response(request, 'portal/ayuda_en_linea.html',
            {'port': 9000,
             'hostname': Site.objects.get_current().name,
             'area': area,
             'manager': AdminHelpDesk.get_area_manager(area),
             'usuario': usuario})

# TODO: Colocar las ayudas en flatpages
def ayuda(request, template):
    """
    Carga una página de ayuda
    """
    return direct_response(request, template)

def nosotros(request):
    """
    Vista informativa con datos de la inmobiliaria
    """
    return direct_response(request, 'portal/nosotros.html',
                           {'desarrollados': Desarrollado.objects.all(),
                            'inmobiliaria': Inmobiliaria.objects.latest("id")})

@csrf_exempt
def contacto(request):
    """
    Formulario de contacto
    """
    AreasFormSet = modelformset_factory(ViewPort, form=AreaInteresForm, extra=0)
    TelefonoFormSet = modelformset_factory(Telefono, form=TelefonoForm)
    if request.method == 'POST' and "contacto_submit" in request.POST:
        form = ContactoForm(request.POST)
        formset_areas = AreasFormSet(request.POST, prefix="area")
        formset_telefonos = TelefonoFormSet(request.POST, prefix="tel")
        if form.is_valid() and formset_telefonos.is_valid() and \
           formset_areas.is_valid():
            form.save(request, formset_telefonos, formset_areas)

            return direct_response(request, 'portal/contacto_exito.html')
    elif request.method == "POST" and "contacto_proyecto_id" in request.POST:
        proyecto = Proyecto.objects.get(id=request.POST["contacto_proyecto_id"])
        form = ContactoForm(initial={'proyecto': proyecto.id,
                                     "rubros_interes": [proyecto.rubro]})
    else:
        form = ContactoForm()
    formset_areas = AreasFormSet(queryset=ViewPort.objects.none(),
                                 prefix="area")
    formset_telefonos = TelefonoFormSet(queryset=Telefono.objects.none(),
        prefix="tel")

    return direct_response(request, 'portal/contacto.html',
                           {'form': form,
                            'formset_telefonos': formset_telefonos,
                            'formset_areas': formset_areas})

def json_get_aviso(request):
    """
    Devuelve un aviso según el indice enviado
    """
    if request.method == 'GET' and "index" in request.GET:
        total = Aviso.objects.all().count()
        min_id = Aviso.objects.aggregate(Min("id"))['id__min']
        if total:
            aviso = Aviso.objects.get(id=(int(request.GET['index'])%total)+min_id)
            data = {
                'url': aviso.proyecto.get_absolute_url(),
                'aviso': aviso.get_archivo_html(),
                'duracion': aviso.duracion,
                'status': True,
            }
            return json_response(data)
        else:
            return json_response({'status': False,})
    else:
        return json_response({})

def google_webmaster_verification(request):
    """
    Vista para verificar la propiedad del dominio de quimerainmobiliaria
    """
    return direct_response(request, "portal/google_verification.html")


import chevron
import pickle
from pathlib import Path
from .recipient import Recipient

from .common import print_err, parse_yaml, Config

SCHEME_ENV = 'COLORSCHEME'

class Colors:

    def __init__(self, dir):
        self.pickle_file = dir / '.current'
        self.cfg = self.__get_config(dir)
        self.schemes = self.__fetch_schemes(dir)
        self.templates = self.__fetch_templates(dir)
        self.current = self.__get_current()

    def __get_config(self, dir: Path):
        config_path = dir / 'config.yaml'
        if config_path.is_file():
            return Config(config_path)
        else:
            print_err(f'"config.yaml" not found in "{dir}"\nAborted')

    def __get_current(self):
        if self.pickle_file.is_file():
            with self.pickle_file.open("rb") as f:
                current = pickle.load(f)
        else:
            current = None

        return current

    def __fetch_schemes(self, dir: Path):
        scheme_dir = dir / 'schemes'
        if scheme_dir.is_dir():
            schemes = {}
            for child in scheme_dir.iterdir():
                items = parse_yaml(child)
                schemes[child.stem] = items

            return schemes
        else:
            print_err(f'No scheme directory found in "{dir}"')

    def __fetch_templates(self, dir):
        template_dir = dir / 'templates'
        if template_dir.is_dir():
            templates = {}
            for child in template_dir.iterdir():
                with child.open() as f:
                    templates[child.stem] = f.read()
            return templates
        else:
            print_err(f'No scheme directory found in "{dir}"')

    def get(self, name):
        if name in self.schemes:
            return self.schemes[name]
        else:
            print_err(f'Colorscheme {name} not found')

    def list(self):
        for name in self.schemes:
            if name == self.current:
                print(f'-> {name}')
            else:
                print(f'   {name}')

    def inject(self, name, verbose=False):
        completed = []
        failed = []

        if name in self.schemes:
            scheme = self.schemes[name]

            for program_name, content in self.templates.items():
                if program_name in self.cfg.PATHS:
                    recipient_path = Path(self.cfg.PATHS[program_name])
                    data = chevron.render(content, scheme)

                    r = Recipient(recipient_path, data)
                    r.write()

                    completed.append(program_name)
                else:
                    failed.append(program_name)

            if verbose:
                if len(completed) > 0:
                    print(f'Updated: {", ".join(completed)}')

                if len(failed) > 0:
                    print(f'No configuration file found: {", ".join(failed)}')
        else:
            print_err(f'Colorscheme "{name}" not found.')

        self.__save(name)

    def __save(self, name):
        if name in self.schemes:
            self.current = name
            print(f'-> {self.current}')

            with self.pickle_file.open("wb+") as f:
                pickle.dump(self.current, f)


# text menu for use inside games. demonstrate the use of Menu and Item class. see pygameMenu.py

class Item:

    def __init__(self, name="dummy", choices=[], cindex=0, rect=None) :
        """
        Menu Item. For use inside Menu.items list.
        """
        self.name = name
        self.choices = choices
        self.cindex = cindex
        self.rect = rect

class Menu:

    def __init__(self, name="root", items=[], rect=None):
        """
        Menu , a list of Menu Items.
        Every Submenu (except the 'root' main menu) get's automatically added an Item('back')
        """
        self.name = name
        self.items = items
        self.rect = rect
        # autoexec: add 'back' as first item if necessary
        self.add_back_item()

    def add_back_item(self):
        if self.name != "root":
            if "back" not in [i.name for i in self.items]:
                self.items.insert(0,Item("back"))

# create menus, begin with sub-sub menu and working the way up to root menu
# every menu except the "root" menu gets automatically addaed an 'back' Item
audiomenu = Menu(name="audio", items=[Item("sound volume"), Item("music volume")])
videomenu = Menu(name="video", items=[Item("fullscreen"), Item("screen resolution")])
settingsmenu = Menu(name="settings",items=[audiomenu, videomenu, Item(name="difficulty",choices=["easy", "medium", "hard", "ironman"],cindex=0)])
rootmenu = Menu("root", [Item("play"), Item("credits"), settingsmenu, Item("quit")])

history = [rootmenu]
while True:
    menu = history[-1]

    # ----- print history --------
    print("You are here:", ">".join([i.name for i in history]))
    # ----- print menu items -----
    for number, item in enumerate(menu.items):
        print(number, item.name,
              ">" if type(item) == Menu else
              ": " + item.choices[item.cindex]
              if len(item.choices) > 0 else ""  )
    command = input("number or name >>>")
    itemnames = [i.name for i in menu.items]
    if command in itemnames:
        i = itemnames.index(command)
    elif command in [str(i) for i in range(len(menu.items))]:
        i = int(command)
    else:
        print("unacceptable command")
        continue
    # ---- data processing -----
    # jump into submenu?
    if type(menu.items[i]) == Menu:
        history.append(menu.items[i])
        menu = menu.items[i]
        continue
    # jump back to previous menu
    print("name:", menu.items[i].name)
    if menu.items[i].name == "back":
        history.pop() # delete last entry
        if len(history) == 1:
            menu = rootmenu
        else:
            menu = history[-1]
        continue
    # -- change value of item ----
    if len(menu.items[i].choices) > 0:
        menu.items[i].cindex += 1
        if menu.items[i].cindex == len(menu.items[i].choices):
            menu.items[i].cindex = 0
        continue
    # -- selection was not a sub-menu---
    selection = menu.items[i].name
    print("your choice was:", selection)
    if selection == "quit":
        break
    elif selection == "play":
        print("playing a game.....")
    elif selection == "credits":
        print("*** (c) 2020 by Horst JENS ***")

print("bye-bye")


import numpy as np
import pandas as pd
from tqdm import tqdm
from collections import Counter

class AutoDatatyper(object):
    def __init__(self, vector_dim=300, num_rows=1000):
        self.vector_dim = vector_dim
        self.num_rows = num_rows
        self.decode_dict = {0: 'numeric', 1: 'character', 2: 'time', 3: 'complex'}

    def create_dataset_from_data_column(self, iterable, label):
        iterable_str = self.__remove_na_and_stringify_iterable(iterable)
        choice_range = len(iterable_str)

        vector_list = []
        for i in tqdm(list(range(self.num_rows))):
            try:
                vec = self.__get_sample_from_column_data(iterable_str, choice_range)
            except ValueError:
                raise ValueError('All data are NaNs.')
            vector_list.append(vec)

        return np.array(vector_list), np.array([label] * self.num_rows).reshape(-1, 1)

    def __remove_na_and_stringify_iterable(self, iterable):
        # Convert iterable to Series
        if not isinstance(iterable, pd.Series):
            iterable = pd.Series(iterable)

        # Drop NAs
        iterable.dropna(inplace=True)
        iterable = iterable.values
        iterable_str = iterable.astype(str)

        return iterable_str

    def __get_data_column_type(self, iterable, estimator, robustness):
        iterable_str = self.__remove_na_and_stringify_iterable(iterable)
        choice_range = len(iterable_str)

        vector_list = []

        for i in (range(int(100 * robustness))):
            try:
                vec = self.__get_sample_from_column_data(iterable_str, choice_range)
            except ValueError:
                return 'NaN', 1.0
            vector_list.append(vec)

        prediction = estimator.predict(np.array(vector_list))
        prediction_count = Counter(np.vectorize(lambda x: round(x, 1))(prediction))
        confidence = prediction_count.most_common(1)[0][1] / len(prediction)

        return self.decode_dict[round(prediction.mean())], confidence

    def get_data_column_type_df(self, data, estimator, robustness=0.1):
        result_dict = {}

        if isinstance(data, pd.DataFrame):
            column_names = data.columns.values

            for i, colname in tqdm(list(enumerate(column_names))):
                datatype, confidence = self.__get_data_column_type(data[colname], estimator, robustness=robustness)
                result_dict[colname] = datatype, confidence
        else:
            column_names = list(range(data.shape[1]))

            for i, colname in tqdm(list(enumerate(column_names))):
                datatype, confidence = self.__get_data_column_type(data[colname], estimator, robustness=robustness)
                result_dict[colname] = datatype, confidence

        return result_dict

    def __get_sample_from_column_data(self, iterable_str, choice_range):
        indices = np.random.choice(choice_range, self.vector_dim)
        stringified_data = iterable_str[indices]

        raw_feature_names = ['length', 'max', 'min', 'range', 'sum', 'avg', 'std', 'float', 'time', \
                             'nan', 'json1', 'json2', 'json3', 'array1', 'array2', 'array3', 'array4', \
                            'tag1', 'tag2', 'tag3', 'tag4', 'url']

        raw_feature_dict = {
            'length': np.vectorize(len)(stringified_data),
            'max': np.vectorize(lambda x: max([ord(char) for char in x]))(stringified_data),
            'min': np.vectorize(lambda x: min([ord(char) for char in x]))(stringified_data),
            'range': np.vectorize(lambda x: max([ord(char) for char in x]) - min([ord(char) for char in x]))(stringified_data),
            'sum': np.vectorize(lambda x: sum([ord(char) for char in x]))(stringified_data),
            'avg': np.vectorize(lambda x: sum([ord(char) for char in x]))(stringified_data) / np.vectorize(len)(stringified_data),
            'std': np.vectorize(lambda x: np.array([ord(char) for char in x]).std())(stringified_data),
            'float': np.vectorize(lambda x: x.count('.'))(stringified_data),
            'time': np.vectorize(self.__contains_time_characters)(stringified_data),
            'nan': np.vectorize(self.__is_nan)(stringified_data),
            'json1': np.vectorize(lambda x: x.count('{'))(stringified_data),
            'json2': np.vectorize(lambda x: x.count('}'))(stringified_data),
            'json3': np.vectorize(lambda x: x.count(':'))(stringified_data),
            'array1': np.vectorize(lambda x: x.count('['))(stringified_data),
            'array2': np.vectorize(lambda x: x.count(']'))(stringified_data),
            'array3': np.vectorize(lambda x: x.count(','))(stringified_data),
            'array4': np.vectorize(lambda x: x.count(';'))(stringified_data),
            'tag1': np.vectorize(lambda x: x.count('\\'))(stringified_data),
            'tag2': np.vectorize(lambda x: x.count('/'))(stringified_data),
            'tag3': np.vectorize(lambda x: x.count('|'))(stringified_data),
            'tag4': np.vectorize(lambda x: x.count('-'))(stringified_data),
            'url': np.vectorize(self.__contains_url_characters)(stringified_data)
        }

        range_feature_dict = {feature_name + '_range':
           np.array([raw_feature_dict[feature_name].max() - raw_feature_dict[feature_name].min()]) for feature_name in raw_feature_names
        }

        sum_feature_dict = {feature_name + '_sum':
           np.array([raw_feature_dict[feature_name].sum()]) for feature_name in raw_feature_names
        }

        avg_feature_dict = {feature_name + '_avg':
           np.array([raw_feature_dict[feature_name].mean()]) for feature_name in raw_feature_names
        }

        std_feature_dict = {feature_name + '_std':
           np.array([raw_feature_dict[feature_name].std()]) for feature_name in raw_feature_names
        }

        count_distinct_feature_dict = {feature_name + '_distinct':
           np.array([len(Counter(raw_feature_dict[feature_name]))]) for feature_name in raw_feature_names
        }

        concat_list = [value for key, value in raw_feature_dict.items()] \
        + [value for key, value in sum_feature_dict.items()] \
        + [value for key, value in avg_feature_dict.items()] \
        + [value for key, value in std_feature_dict.items()] \
        + [value for key, value in count_distinct_feature_dict.items()]

        vec = np.concatenate(concat_list)

        return vec

    def __contains_time_characters(self, string):
            time_chars = {':', '/', '-', '\\', '.', '+',
                         'hr', 'hour', 'min', 'minute', 'sec', 'second',
                         'day', 'week', 'year',
                         '年', '月', '日', '时', '分', '秒',
                         '年', '月', '日', '時', '分', '秒'}

            count = 0
            for char in time_chars:
                if char in string:
                    count += 1
            return count

    def __is_nan(self, string):
        return 1 if string.lower() == 'nan' else 0

    def __contains_url_characters(self, string):
            url_chars = {'http', '//', 'www', 'com', 'cn', '_'}

            count = 0
            for char in url_chars:
                if char in string:
                    count += 1
            return count

    def reduce_data_dict_to_ndarray(self, data_dict):
        return np.concatenate([value[0] for key, value in data_dict.items()], axis=0), np.concatenate([value[1] for key, value in data_dict.items()])

    def consolidate_data(self, foundation_features, new_features, foundation_label, new_label):
        return np.concatenate((foundation_features, new_features), axis=0), np.concatenate((foundation_label, new_label), axis=0)


# IDEA
import requests
import json

def fetch(text):
  res = requests.get(text)
  response = json.loads(res.text)
  return response

def printList(listname,l):
  print(listname)
  for x in l:
    print(x)
  print('-------------------')


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from proto_mdd.utils import euclidean_metric

class ScaledDotProductAttention(nn.Module):
    ''' Scaled Dot-Product Attention '''

    def __init__(self, temperature, attn_dropout=0.1):
        super().__init__()
        self.temperature = temperature
        self.dropout = nn.Dropout(attn_dropout)
        self.softmax = nn.Softmax(dim=2)

    def forward(self, q, k, v):

        attn = torch.bmm(q, k.transpose(1, 2))
        attn = attn / self.temperature
        log_attn = F.log_softmax(attn, 2)
        attn = self.softmax(attn)
        attn = self.dropout(attn)
        output = torch.bmm(attn, v)
        return output, attn, log_attn

class MultiHeadAttention(nn.Module):
    ''' Multi-Head Attention module '''

    def __init__(self, args, n_head, d_model, d_k, d_v, dropout=0.1):
        super().__init__()
        self.n_head = n_head
        self.d_k = d_k
        self.d_v = d_v

        self.w_qs = nn.Linear(d_model, n_head * d_k)
        self.w_ks = nn.Linear(d_model, n_head * d_k)
        self.w_vs = nn.Linear(d_model, n_head * d_v)
        nn.init.normal_(self.w_qs.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_k)))
        nn.init.normal_(self.w_ks.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_k)))
        nn.init.normal_(self.w_vs.weight, mean=0, std=np.sqrt(2.0 / (d_model + d_v)))

        self.attention = ScaledDotProductAttention(temperature=np.power(d_k, 0.5))
        self.layer_norm = nn.LayerNorm(d_model)

        self.fc = nn.Linear(n_head * d_v, d_model)
        nn.init.xavier_normal_(self.fc.weight)
        self.dropout = nn.Dropout(dropout)

    def forward(self, q, k, v):
        d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
        sz_b, len_q, _ = q.size()
        sz_b, len_k, _ = k.size()
        sz_b, len_v, _ = v.size()

        residual = q
        q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)
        k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)
        v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)

        q = q.permute(2, 0, 1, 3).contiguous().view(-1, len_q, d_k) # (n*b) x lq x dk
        k = k.permute(2, 0, 1, 3).contiguous().view(-1, len_k, d_k) # (n*b) x lk x dk
        v = v.permute(2, 0, 1, 3).contiguous().view(-1, len_v, d_v) # (n*b) x lv x dv

        output, attn, log_attn = self.attention(q, k, v)

        output = output.view(n_head, sz_b, len_q, d_v)
        output = output.permute(1, 2, 0, 3).contiguous().view(sz_b, len_q, -1) # b x lq x (n*dv)

        output = self.dropout(self.fc(output))
        output = self.layer_norm(output + residual)

        return output

class proto_attention_net(nn.Module):

    def __init__(self, args, dropout=0.2):
        super().__init__()
        if args.model_type == 'ConvNet':
            from proto_mdd.networks.convnet import ConvNet
            self.encoder = ConvNet()
            z_dim = 64
        elif args.model_type == 'ResNet':
            from proto_mdd.networks.resnet import ResNet
            self.encoder = ResNet()
            z_dim = 640
        elif args.model_type == 'ResNet18':
            hdim = 512
            z_dim = 512
            from proto_mdd.networks.resnet_pytorch import resnet18
            self.encoder = resnet18(pretrained = False)
        else:
            raise ValueError('')

        self.slf_attn = MultiHeadAttention(args, args.head, z_dim, z_dim, z_dim, dropout=dropout)
        self.z_dim = z_dim
        self.args = args

    def forward(self, support, query, input_type = "data"):

        if input_type == "data":
            # feature extraction
            support = self.encoder(support)
            query = self.encoder(query)
            proto = support.reshape(self.args.shot, -1, support.shape[-1]).mean(dim=0) # N x d

            # refine by attention machenism
            proto = proto.unsqueeze(0)  # 1 x N x d
            proto = self.slf_attn(proto, proto, proto)
            proto = proto.squeeze(0)

            # compute distance for all batches
            logitis = euclidean_metric(query, proto) / self.args.temperature

            return support, query, logitis

        elif input_type == "feature":
            # get mean of the support
            proto = support.reshape(self.args.shot, -1, support.shape[-1]).mean(dim=0) # N x d
            # refine by attention machenism
            proto = proto.unsqueeze(0)  # 1 x N x d
            proto = self.slf_attn(proto, proto, proto)
            proto = proto.squeeze(0)

            # compute distance for all batches
            logitis = euclidean_metric(query, proto) / self.args.temperature

            return logitis


from alembic import command
from alembic.config import Config

from config import env
from model import db_conn_str

def up_migrate(version="heads"):
    alembic_cfg = Config()
    alembic_cfg.set_main_option('script_location', env.get_str("db.migrations.path"))
    alembic_cfg.set_main_option('sqlalchemy.url', db_conn_str)
    command.upgrade(alembic_cfg, version)

def down_migrate(version="0000_base_line"):
    alembic_cfg = Config()
    alembic_cfg.set_main_option('script_location', env.get_str("db.migrations.path"))
    alembic_cfg.set_main_option('sqlalchemy.url', db_conn_str)
    command.downgrade(alembic_cfg, version)


from flask import Flask,request,Response,jsonify;
from configuration import Configuration;
from models import database,Participant,Election,ElectionParticipant,Vote;
from email.utils import parseaddr;
from re import match,search;
from flask_jwt_extended import JWTManager, create_access_token, jwt_required, create_refresh_token, get_jwt, get_jwt_identity;
from sqlalchemy import and_, or_;
from adminDecorater import roleCheck;
from datetime import datetime,timezone;
import pytz;
from sqlalchemy import desc;

application=Flask(__name__);
application.config.from_object(Configuration);
jwt = JWTManager(application);

@application.route("/createParticipant",methods=["POST"])
@roleCheck(role="administrator")
def createParticipant():
    name = request.json.get("name", "");
    individual = request.json.get("individual", None);

    if (len(name)==0):
        return jsonify({'message':'Field name is missing.'}),400;
    if (individual is None):
        return jsonify({'message':'Field individual is missing.'}),400;

    participant=Participant(name=name,individual=individual);

    database.session.add(participant);
    database.session.commit();

    return jsonify({'id':participant.id}),200;

@application.route("/getParticipants",methods=["GET"])
@roleCheck(role="administrator")
def getParticipants():
    participants=Participant.query.all();
    array=[];
    for participant in participants:
        array.append(participant.to_JSON());

    return jsonify(participants=array),200;

@application.route("/createElection",methods=["POST"])
@roleCheck(role="administrator")
def createElection():
    try:
        start=request.json.get("start","");
        end=request.json.get("end","");
        individual=request.json.get("individual",None);
        participants=request.json.get("participants",None);
    except:
        return jsonify(message='Field start is missing.'), 400;
    # check if some field is missing
    if (len(start)==0):
        return jsonify({'message':'Field start is missing.'}),400;
    if (len(end) == 0):
        return jsonify({'message': 'Field end is missing.'}),400;
    if (individual is None):
        return jsonify({'message':'Field individual is missing.'}),400;
    if (type(individual)!=bool):
        return jsonify({'message':'Field individual is missing.'}),400;
    if participants is None:
        return jsonify({'message':'Field participants is missing.'}),400;

    # check format of fields
    # check datas
    try:
        if len(start)==19 or len(end)==19:
            raise Exception();
        startDate = datetime.strptime(start, "%Y-%m-%dT%H:%M:%S%z");
        endDate = datetime.strptime(end, "%Y-%m-%dT%H:%M:%S%z");

    except:
        try:
            if len(start)==19:
                startDate=start+"+0200";
            else:
                startDate=start;
            if len(end)==19:
                endDate=end+"+0200";
            else:
                endDate=end;
            startDate = datetime.strptime(startDate, "%Y-%m-%dT%H:%M:%S%z");
            endDate = datetime.strptime(endDate, "%Y-%m-%dT%H:%M:%S%z");

        except:
            return jsonify({'message': 'Invalid date and time.'}),400;
    startDate = startDate.astimezone(pytz.timezone('Europe/Belgrade'));
    endDate = endDate.astimezone(pytz.timezone('Europe/Belgrade'));
    if (startDate>=endDate):
        return jsonify({'message':'Invalid date and time.'}),400;

    elections=Election.query.all();
    if len(elections)>0:
        number=Election.query.filter(or_(and_(startDate<=Election.start,Election.start<=endDate),and_(startDate>=Election.start,startDate<=Election.end))).count();
        if number>0:
            return jsonify({'message': 'Invalid date and time.'}),400;
    election = Election(start=startDate, end=endDate, individual=individual, votesNumber=0);
    database.session.add(election);

    # check participants
    if (len(participants)<2):
        return jsonify({'message': 'Invalid participants.'}),400;

    participantsNew=[];
    i=1;
    for participantId in participants:
        try:
            int(participantId);
        except:
            return jsonify(message='Invalid participants.'), 400;
        existP=Participant.query.filter(Participant.id==participantId).first();
        if (not existP):
            return jsonify({'message':'Invalid participants.'}),400;

        # if elections are parlamental and participant is individual
        # if elections are president and participant is not individual
        if ((existP.individual==False and individual==True) or (existP.individual==True and individual==False)):
            return jsonify({'message': 'Invalid participants.'}),400;
        participantInElection=ElectionParticipant(electionId=election.id,participantId=existP.id,poolNumber=i);
        database.session.add(participantInElection);
        participantsNew.append(i);
        i=i+1;

    database.session.commit();

    return jsonify(pollNumbers=participantsNew),200;

@application.route("/getElections",methods=["GET"])
@roleCheck(role="administrator")
def getElections():
    elections=Election.query.all();
    arr=[];
    for election in elections:
        arr.append(election.to_JSON());
    return jsonify(elections=arr),200;

@application.route("/getResults",methods=["GET"])
@roleCheck(role="administrator")
def getResults():
    try:
        electionId = request.args.get("id", None);
        if electionId is None:
            raise Exception;
    except:
        return jsonify(message="Field id is missing."), 400;

    election=Election.query.filter(Election.id==electionId).first();
    if election is None:
        return jsonify({'message':'Election does not exist.'}),400;

    timeNow = datetime.now(timezone.utc).astimezone(pytz.timezone('Europe/Belgrade'));
    startDate= pytz.utc.localize(election.start);
    endDate=pytz.utc.localize(election.end);
    if startDate<=timeNow and endDate>=timeNow:
        return jsonify({'message':'Election is ongoing.'}),400;

    participants=[];
    participantsInElection=ElectionParticipant.query.filter(ElectionParticipant.electionId==electionId);
    # calculate if elections are presidental
    if election.individual==1:
        for item in participantsInElection:
            participant=presidentalElection(item,election);
            participants.append(participant);
    else:
        participants=parlamentalElecion(participantsInElection,election);
    invalidVotes=Vote.query.filter(and_(Vote.electionId==electionId,Vote.valid==False));
    invalidData=[];
    for item in invalidVotes:
        invalidData.append({
            "electionOfficialJmbg": item.electionOfficialJmbg,
            "ballotGuid": item.ballotGuid,
            "pollNumber": item.poolNumber,
            "reason": item.reason
        });

    return jsonify(participants=participants,invalidVotes=invalidData),200;
def presidentalElection(item,election):
    part = Participant.query.filter(Participant.id == item.participantId).first();
    participant={
        "pollNumber": item.poolNumber,
        "name": part.name,
        "result": 0,
    }
    if election.votesNumber>0:
        res=item.result/election.votesNumber;
        res=float(str(round(res,2)));
        participant['result']=res;
    return participant;

def parlamentalElecion(participantsInElection,election):
    mandateNumbers=250;
    participantsPassed=[];

    for item in participantsInElection:
        percentage=item.result/election.votesNumber;
        part=Participant.query.filter(Participant.id==item.participantId).first();
        participantsPassed.append({
            'id':part.id,
            'pollNumber':item.poolNumber,
            'name':part.name,
            'voteNumber':item.result,
            'mandatesNumber':0
        });
        if percentage<0.05:
            participantsPassed[-1]['voteNumber']=0;
    while mandateNumbers>0:
        averages=[];
        for item in participantsPassed:
            res=item['voteNumber']/(item['mandatesNumber']+1);
            averages.append(res);
        maxValue=max(averages);
        index=averages.index(maxValue);
        participantsPassed[index]['mandatesNumber']+=1;
        averages.clear();
        mandateNumbers-=1;

    participants=[];
    for item in participantsPassed:
        participants.append({
        'pollNumber': item['pollNumber'],
        "name": item['name'],
        "result":int(item['mandatesNumber'])
        });
    return participants;

if ( __name__ == "__main__" ):
    database.init_app ( application );
    application.run (host = "0.0.0.0", port = 5001 );


import graphene
import pytest

from .....order import OrderStatus
from .....order.error_codes import OrderErrorCode
from ....tests.utils import get_graphql_content

def test_draft_order_delete(staff_api_client, permission_manage_orders, draft_order):
    order = draft_order
    query = """
        mutation draftDelete($id: ID!) {
            draftOrderDelete(id: $id) {
                order {
                    id
                }
            }
        }
        """
    order_id = graphene.Node.to_global_id("Order", order.id)
    variables = {"id": order_id}
    staff_api_client.post_graphql(
        query, variables, permissions=[permission_manage_orders]
    )
    with pytest.raises(order._meta.model.DoesNotExist):
        order.refresh_from_db()

def test_draft_order_delete_product(
    app_api_client, permission_manage_products, draft_order
):
    query = """
        mutation DeleteProduct($id: ID!) {
          productDelete(id: $id) {
            product {
              id
            }
          }
        }
    """
    order = draft_order
    line = order.lines.first()
    product = line.variant.product
    product_id = graphene.Node.to_global_id("Product", product.id)
    variables = {"id": product_id}
    response = app_api_client.post_graphql(
        query, variables, permissions=[permission_manage_products]
    )
    content = get_graphql_content(response)
    assert content["data"]["productDelete"]["product"]["id"] == product_id

@pytest.mark.parametrize(
    "order_status",
    [
        OrderStatus.UNFULFILLED,
        OrderStatus.UNCONFIRMED,
        OrderStatus.CANCELED,
        OrderStatus.PARTIALLY_FULFILLED,
        OrderStatus.FULFILLED,
        OrderStatus.PARTIALLY_RETURNED,
        OrderStatus.RETURNED,
    ],
)
def test_draft_order_delete_non_draft_order(
    staff_api_client, permission_manage_orders, order_with_lines, order_status
):
    order = order_with_lines
    order.status = order_status
    order.save(update_fields=["status"])
    query = """
        mutation draftDelete($id: ID!) {
            draftOrderDelete(id: $id) {
                order {
                    id
                }
                errors {
                    code
                    field
                }
            }
        }
        """
    order_id = graphene.Node.to_global_id("Order", order.id)
    variables = {"id": order_id}
    response = staff_api_client.post_graphql(
        query, variables, permissions=[permission_manage_orders]
    )
    content = get_graphql_content(response)
    account_errors = content["data"]["draftOrderDelete"]["errors"]
    assert len(account_errors) == 1
    assert account_errors[0]["field"] == "id"
    assert account_errors[0]["code"] == OrderErrorCode.INVALID.name

DRAFT_ORDER_DELETE_BY_EXTERNAL_REFERENCE = """
    mutation draftDelete($id: ID, $externalReference: String) {
        draftOrderDelete(id: $id, externalReference: $externalReference) {
            order {
                id
                externalReference
            }
            errors {
                field
                message
        }
    }
}
"""

def test_draft_order_delete_by_external_reference(
    staff_api_client, permission_manage_orders, draft_order
):
    # given
    order = draft_order
    query = DRAFT_ORDER_DELETE_BY_EXTERNAL_REFERENCE
    ext_ref = "test-ext-ref"
    order.external_reference = ext_ref
    order.save(update_fields=["external_reference"])
    variables = {"externalReference": ext_ref}

    # when
    response = staff_api_client.post_graphql(
        query, variables, permissions=[permission_manage_orders]
    )
    content = get_graphql_content(response)

    # then
    data = content["data"]["draftOrderDelete"]
    with pytest.raises(order._meta.model.DoesNotExist):
        order.refresh_from_db()
    assert graphene.Node.to_global_id("Order", order.id) == data["order"]["id"]
    assert data["order"]["externalReference"] == order.external_reference

def test_draft_order_delete_by_both_id_and_external_reference(
    staff_api_client, permission_manage_orders
):
    # given
    query = DRAFT_ORDER_DELETE_BY_EXTERNAL_REFERENCE
    variables = {"externalReference": "whatever", "id": "whatever"}

    # when
    response = staff_api_client.post_graphql(
        query, variables, permissions=[permission_manage_orders]
    )
    content = get_graphql_content(response)

    # then
    errors = content["data"]["draftOrderDelete"]["errors"]
    assert (
        errors[0]["message"]
        == "Argument 'id' cannot be combined with 'external_reference'"
    )

def test_draft_order_delete_by_external_reference_not_existing(
    staff_api_client, permission_manage_orders
):
    # given
    query = DRAFT_ORDER_DELETE_BY_EXTERNAL_REFERENCE
    ext_ref = "non-existing-ext-ref"
    variables = {"externalReference": ext_ref}

    # when
    response = staff_api_client.post_graphql(
        query, variables, permissions=[permission_manage_orders]
    )
    content = get_graphql_content(response)

    # then
    errors = content["data"]["draftOrderDelete"]["errors"]
    assert errors[0]["message"] == f"Couldn't resolve to a node: {ext_ref}"