"""
This file contains cleaning logic to use when loading the dataset.

See the analysis for more information about each function.
"""
import pipeline
import pandas as pd

def unnecessary_columns(df):
    return df.drop(columns=['projectid',
                            'teacher_acctid',
                            'schoolid',
                            'school_ncesid',
                            'school_city',
                            'school_state',
                            'school_metro',
                            'school_district',
                            'school_county'])

def fix_types(df):
    for colname in ['school_charter',
                    'school_magnet',
                    'eligible_double_your_impact_match']:
        df[colname] = (df[colname] == 't').astype(float)

    format = '%m/%d/%y'
    df['date_posted'] = pd.to_datetime(df['date_posted'], format=format)
    df['datefullyfunded'] = pd.to_datetime(df['datefullyfunded'], format=format)
    return df

def handle_missing(df):
    df = df.drop(columns=['secondary_focus_subject',
                          'secondary_focus_area'])
    df = df.dropna()
    return df

def handle_categorical(df):
    pipeline.dummify(df, 'teacher_prefix',
                         'primary_focus_subject',
                         'primary_focus_area',
                         'resource_type',
                         'poverty_level',
                         'grade_level')
    df = df.drop(columns=['teacher_prefix',
                          'primary_focus_subject',
                          'primary_focus_area',
                          'resource_type',
                          'poverty_level',
                          'grade_level'])
    return df

def label(df, label_colname):
    df[label_colname] = (
        (df.datefullyfunded - df.date_posted)
        .apply(lambda d: d.days > 60)
        .astype(float)
    )

    df = df.sort_values('date_posted')
    df = df.reset_index(drop=True)

    df = df.drop(columns=['date_posted',
                          'datefullyfunded'])
    return df

def clean(df):
    df = unnecessary_columns(df)
    df = fix_types(df)
    df = handle_missing(df)
    df = handle_categorical(df)
    df = label(df)
    return df


#!/usr/bin/env python

import sys, pysam, re, subprocess
import numpy
import my_seq

SAre = re.compile('([^ \t\n\r\f\v,]+),(\d+),([\-\+]),(\w+),(\d+),(\d+);')
cigarMDRe = re.compile('(\d+)([MD])')
cigarHIMSRe = re.compile('(\d+)([HIMS])')
cigarHSRe_right = re.compile('(\d+)([HS])$')
cigarHSRe_left = re.compile('^(\d+)([HS])')

def parse_bp_from_bam(input_bam, output_file, key_seq_size, min_major_clip_size, max_minor_clip_size):

    """
    function for getting breakpoints from BAM file
    input:bam file
    output:output_file + ".bp.tmp.txt"
    breakpoint info (juncChr, juncPos-1, juncPos, Dir (right clipping:+, left clipping:-), Juncseq, ID + ("/1" or "/2"), MAPQ, Clipping size, AlignmentSize, Base quality of juncseq)
    """

    bamfile = pysam.Samfile(input_bam, "rb")
    hout = open(output_file, "w")

    # maybe add the regional extraction of bam files
    for read in bamfile.fetch():

        # get the flag information
        flags = format(int(read.flag), "#014b")[:1:-1]

        # skip if not aligned
        if flags[2] == "1": continue

        # skip supplementary alignment
        if flags[8] == "1" or flags[11] == "1": continue

        # skip duplicated reads
        if flags[10] == "1": continue

        # no clipping
        if len(read.cigar) == 1: continue

        # get the clipping size in the both side
        left_clipping = (read.cigar[0][1] if read.cigar[0][0] in [4, 5] else 0)
        right_clipping = (read.cigar[len(read.cigar) - 1][1] if read.cigar[len(read.cigar) - 1][0] in [4, 5] else 0)

        if left_clipping < min_major_clip_size and right_clipping < min_major_clip_size: continue

        # get the alignment basic information
        chr_current = bamfile.getrname(read.tid)
        pos_current = int(read.pos + 1)
        dir_current = ("-" if flags[4] == "1" else "+")

        # when the right side is clipped...
        if right_clipping >= min_major_clip_size:
            clipLen_current = right_clipping
            alignmentSize_current = read.alen
            readLength_current = read.rlen
            juncChr_current = chr_current
            juncPos_current = pos_current + alignmentSize_current - 1
            juncDir_current = "+"
            juncseq_start = readLength_current - clipLen_current
            juncseq_end = readLength_current - clipLen_current + key_seq_size
            juncseq = read.seq[juncseq_start:juncseq_end]

            #filter if base qualities of junction seq is low
            #if numpy.mean(read.query_qualities[juncseq_start:juncseq_end]) < 10:
            #    continue

            print >> hout, '\t'.join([juncChr_current, str(juncPos_current-1), str(juncPos_current), juncDir_current, juncseq,
                                      read.qname + ("/1" if flags[6] == "1" else "/2"), str(read.mapq), str(right_clipping), str(alignmentSize_current)])

        if left_clipping >= min_major_clip_size:

            clipLen_current = left_clipping
            alignmentSize_current = read.alen
            readLength_current = read.rlen

            juncChr_current = chr_current
            juncPos_current = pos_current
            juncDir_current = "-"

            juncseq_end = clipLen_current
            juncseq_start = clipLen_current - key_seq_size
            juncseq = my_seq.reverse_complement(read.seq[juncseq_start:juncseq_end])

            #filter if base qualities of soft clipping part is low
            #if numpy.mean(read.query_qualities[juncseq_start:juncseq_end])<10:
            #  continue

            print >> hout, '\t'.join([juncChr_current, str(juncPos_current-1), str(juncPos_current), juncDir_current, juncseq,
                                      read.qname + ("/1" if flags[6] == "1" else "/2"), str(read.mapq), str(left_clipping), str(alignmentSize_current)])

    bamfile.close()
    hout.close()

def cluster_breakpoint(input_file, output_file, check_interval):

    """
    function for clustering breakpoints
    input:output_file + ".bp.tmp.txt"               result of "parse_bp_from_bam" function
    output:output_file + ".bp.clustered.tmp.txt"    breakpoint info (juncChr, juncPos-1, juncPos, Dir (right clipping:+, left clipping:-), Juncseq, IDs + ("/1" or "/2"), MAPQs, Clipping sizes, AlignmentSizes, Base quality of juncseq)
    """

    hout = open(output_file, 'w')

    tmp_chr = ""
    tmp_pos = 0

    key2juncseq = {}
    key2read = {}
    key2mapq = {}
    key2clipsize = {}
    key2alnsize = {}
    #key2baseq = {}

    with open(input_file, 'r') as hin:
        for line in hin:
            F = line.rstrip('\n').split('\t')

            if tmp_chr != F[0] or int(F[1]) - tmp_pos > check_interval:
                del_list = []
                for key in key2read:
                    print >> hout, key + '\t' + ';'.join(list(set(key2juncseq[key]))) + '\t' + ';'.join(key2read[key]) + '\t' + ';'.join(key2mapq[key]) + '\t' + \
                                   ';'.join(key2clipsize[key]) + '\t' + ';'.join(key2alnsize[key])
                    del_list.append(key)

                for key in del_list:
                    del key2juncseq[key]
                    del key2read[key]
                    del key2mapq[key]
                    del key2clipsize[key]
                    del key2alnsize[key]
                    #del key2baseq[key]

                tmp_chr = F[0]
                tmp_pos = int(F[1])

            key = F[0] + '\t' + F[1] + '\t' + F[2] + '\t' + F[3]
            if key not in key2juncseq: key2juncseq[key] = []
            if key not in key2read: key2read[key] = []
            if key not in key2mapq: key2mapq[key] = []
            if key not in key2clipsize: key2clipsize[key] = []
            if key not in key2alnsize: key2alnsize[key] = []
            #if key not in key2baseq: key2baseq[key] = []

            key2juncseq[key].append(F[4])
            key2read[key].append(F[5])
            key2mapq[key].append(F[6])
            key2clipsize[key].append(F[7])
            key2alnsize[key].append(F[8])
            #key2baseq[key].append(F[9])

    # final flush
    for key in key2read:
        print >> hout, key + '\t' + ';'.join(list(set(key2juncseq[key]))) + '\t' + ';'.join(key2read[key]) + '\t' + ';'.join(key2mapq[key]) + '\t' + \
                       ';'.join(key2clipsize[key]) + '\t' + ';'.join(key2alnsize[key])

    hout.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
###################################################################
# Author: Mu yanru
# Date  : 2019.2
# Email : muyanru345@163.com
###################################################################

import functools

from dayu_widgets.MCheckBox import MCheckBox
from dayu_widgets.MMenu import MMenu
from dayu_widgets.MPushButton import MPushButton
from dayu_widgets.MRadioButton import MRadioButton
from dayu_widgets.MToolButton import MToolButton
from dayu_widgets.mixin import property_mixin
from dayu_widgets.qt import *

@property_mixin
class MButtonGroupBase(QWidget):
    def __init__(self, orientation=Qt.Horizontal, parent=None):
        super(MButtonGroupBase, self).__init__(parent=parent)
        self._main_layout = QBoxLayout(
            QBoxLayout.LeftToRight if orientation == Qt.Horizontal else QBoxLayout.TopToBottom)
        self._main_layout.setContentsMargins(0, 0, 0, 0)
        self.setLayout(self._main_layout)
        self.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
        self._button_group = QButtonGroup()
        self._orientation = 'horizontal' if orientation == Qt.Horizontal else 'vertical'

    def set_spacing(self, value):
        self._main_layout.setSpacing(value)

    def get_button_group(self):
        return self._button_group

    def create_button(self, data_dict):
        raise NotImplementedError()

    def add_button(self, data_dict, index=None):
        if isinstance(data_dict, basestring):
            data_dict = {'text': data_dict}
        elif isinstance(data_dict, QIcon):
            data_dict = {'icon': data_dict}
        button = self.create_button(data_dict)
        button.setProperty('combine', self._orientation)
        if data_dict.get('data'):
            button.setProperty('data', data_dict.get('data'))
        if data_dict.get('checked'):
            button.setProperty('checked', data_dict.get('checked'))
        if data_dict.get('shortcut'):
            button.setProperty('shortcut', data_dict.get('shortcut'))
        if data_dict.get('tooltip'):
            button.setProperty('toolTip', data_dict.get('tooltip'))
        if data_dict.get('clicked'):
            button.clicked.connect(data_dict.get('clicked'))
        if data_dict.get('toggled'):
            button.toggled.connect(data_dict.get('toggled'))

        if index is None:
            self._button_group.addButton(button)
        else:
            self._button_group.addButton(button, index)
        self._main_layout.insertWidget(self._main_layout.count(), button)

    def set_button_list(self, button_list):
        for button in self._button_group.buttons():
            self._button_group.removeButton(button)
            self._main_layout.removeWidget(button)
            button.setVisible(False)
        for index, data_dict in enumerate(button_list):
            self.add_button(data_dict, index)

class MPushButtonGroup(MButtonGroupBase):
    def __init__(self, size=None, type=None, orientation=Qt.Horizontal, parent=None):
        super(MPushButtonGroup, self).__init__(orientation=orientation, parent=parent)
        self.set_spacing(1)
        self._type = type
        self._size = size
        self._button_group.setExclusive(False)

    def create_button(self, data_dict):
        return MPushButton(text=data_dict.get('text'),
                           icon=data_dict.get('icon'),
                           size=data_dict.get('size') or self._size,
                           type=data_dict.get('type') or self._type,
                           parent=self
                           )

class MCheckBoxGroup(MButtonGroupBase):
    sig_checked_changed = Signal(list)

    def __init__(self, orientation=Qt.Horizontal, parent=None):
        super(MCheckBoxGroup, self).__init__(orientation=orientation, parent=parent)
        self.set_spacing(15)
        self._button_group.setExclusive(False)

        self.setContextMenuPolicy(Qt.CustomContextMenu)
        self.customContextMenuRequested.connect(self._slot_context_menu)

        self._button_group.buttonClicked[int].connect(self._slot_map_signal)
        self.set_value([])

    def create_button(self, data_dict):
        return MCheckBox(text=data_dict.get('text'),
                         icon=data_dict.get('icon'),
                         parent=self
                         )

    @Slot(QPoint)
    def _slot_context_menu(self, point):
        context_menu = MMenu(parent=self)
        action_select_all = context_menu.addAction('Select All')
        action_select_none = context_menu.addAction('Select None')
        action_select_invert = context_menu.addAction('Select Invert')
        action_select_all.triggered.connect(functools.partial(self._slot_set_select, True))
        action_select_none.triggered.connect(functools.partial(self._slot_set_select, False))
        action_select_invert.triggered.connect(functools.partial(self._slot_set_select, None))
        context_menu.exec_(QCursor.pos() + QPoint(10, 10))

    @Slot(bool)
    def _slot_set_select(self, state):
        for check_box in self._button_group.buttons():
            if state is None:
                old_state = check_box.isChecked()
                check_box.setChecked(not old_state)
            else:
                check_box.setChecked(state)
        self._slot_map_signal()

    @Slot(int)
    def _slot_map_signal(self, state=None):
        self.sig_checked_changed.emit(
            [check_box.text() for check_box in self._button_group.buttons() if check_box.isChecked()])

    def set_value(self, value):
        if not isinstance(value, list):
            value = [value]
        self.setProperty('value', value)

    def _set_value(self, value):
        edit_from_code = False
        for check_box in self._button_group.buttons():
            flag = Qt.Checked if check_box.text() in value else Qt.Unchecked
            if flag != check_box.checkState():
                # 更新来自代码
                edit_from_code = True
                check_box.setCheckState(flag)
        if edit_from_code:
            self.sig_checked_changed.emit(value)

class MRadioButtonGroup(MButtonGroupBase):
    sig_checked_changed = Signal(int)

    def __init__(self, orientation=Qt.Horizontal, parent=None):
        super(MRadioButtonGroup, self).__init__(orientation=orientation, parent=parent)
        self.set_spacing(15)
        self._button_group.setExclusive(True)
        self._button_group.buttonClicked[int].connect(functools.partial(self.setProperty, 'checked'))
        self._button_group.buttonClicked[int].connect(self.sig_checked_changed)
        self.set_checked(-1)

    def create_button(self, data_dict):
        return MRadioButton(text=data_dict.get('text'),
                            icon=data_dict.get('icon'),
                            parent=self
                            )

    def _set_checked(self, value):
        assert isinstance(value, int)
        if value != self._button_group.checkedId():
            # 更新来自代码
            button = self._button_group.button(value)
            if button:
                button.setChecked(True)
            self.sig_checked_changed.emit(value)

    def set_checked(self, value):
        self.setProperty('checked', value)

class MToolButtonGroup(MButtonGroupBase):
    sig_checked_changed = Signal(int)

    def __init__(self, size=None, type=None, exclusive=False, orientation=Qt.Horizontal, parent=None):
        super(MToolButtonGroup, self).__init__(orientation=orientation, parent=parent)
        self.set_spacing(1)
        self._button_group.setExclusive(exclusive)
        self._size = size
        self._type = type
        self._button_group.buttonClicked[int].connect(self.set_checked)
        self._button_group.buttonClicked[int].connect(self.sig_checked_changed)
        self.set_checked(-1)

    def create_button(self, data_dict):
        button = MToolButton(text=data_dict.get('text'),
                             icon=data_dict.get('icon'),
                             type=data_dict.get('type') or self._type,
                             size=data_dict.get('size') or self._size,
                             parent=self
                             )
        button.setCheckable(data_dict.get('checkable', False))
        if data_dict.get('icon_checked'):
            button.setProperty('icon_checked', data_dict.get('icon_checked'))
            button.setProperty('icon_unchecked', data_dict.get('icon'))
        return button

    def _set_checked(self, value):
        assert isinstance(value, int)
        if value != self._button_group.checkedId():
            # 更新来自代码
            button = self._button_group.button(value)
            if button:
                button.setChecked(True)
            self.sig_checked_changed.emit(value)

    def set_checked(self, value):
        self.setProperty('checked', value)


#Calculation Module
# Recieves the below inputs tp calc function
# n = Number inputs
# op = Operator
# l  = length of n(input)

def addtn(n):                   #Addtion
    sumn = 0
    for x in n:
        sumn +=x
    return sumn

def sub(n,l):                   #Subtraction
    d = n[0]
    for i in range(1 , l):
        d = d - n[i]
    return d

def mul(n,l):                   #Multiplication
    m = n[0]
    for i in range(1 , l):
        m = m * n[i]
    return m

def divtn(n,l):                #Division
    dv = n[0]
    for i in range(1 , l):
        if n[i] != 0:
            dv = dv / n[i]
        else:
            dv = 'Divide by zero error'
    return dv

def calc(n, op, l):             # Call fn based on operator
    if   op == 1: return addtn(n)
    elif op == 2: return sub(n,l)
    elif op == 3: return mul(n, l)
    elif op == 4: return divtn(n, l)


import numpy as np
import pandas as pd

def donchian(prices, period):
    '''Calculate upper, lower, & middle Donchian lines.'''
    df = pd.DataFrame(prices.copy())
    df['upr'] = df.high.rolling(period).max().shift(periods=1)
    df['lwr'] = df.low.rolling(period).min().shift(periods=1)
    df['mid'] = 0.5 * (df.upr + df.lwr)
    return df

def rsi(close, period):
    """Relative Strength Index (RSI)."""
    df = pd.DataFrame(close)
    df['chg'] = df.close.diff()
    df['gain'] = df.chg.clip(lower=0)
    df['loss'] = df.chg.clip(upper=0).abs()
    df['avg_gain'] = np.nan
    df['avg_loss'] = np.nan
    df.avg_gain.iloc[period] = df.gain[0:period].sum() / period
    df.avg_loss.iloc[period] = df.loss[0:period].sum() / period

    for i in range(period + 1, df.shape[0]):
        df['avg_gain'].iloc[i] = (
            df['avg_gain'].iloc[i - 1] * (period - 1) + df['gain'].iloc[i]
        ) / period

        df['avg_loss'].iloc[i] = (
            df['avg_loss'].iloc[i - 1] * (period - 1) + df['loss'].iloc[i]
        ) / period

    df['rs'] = df.avg_gain / df.avg_loss
    df['rsi'] = 100 - 100 / (1 + df.rs)
    return df.rsi


from rest_framework import serializers
from  .models import therapycenter
from rest_framework.reverse import reverse
from django.contrib.auth import get_user_model

User = get_user_model()

class therapycenterSerializer(serializers.ModelSerializer):
    links = serializers.SerializerMethodField('get_links')
    class Meta:
        model = therapycenter
        fields = ('id','name','candidates','location','therapists','links')
        read_only = ('candidates','therapists')

    def get_links(self, obj):
        request = self.context['request']
        links = {
                'self': reverse('therapycenter-detail',
                kwargs = {'pk': obj.pk}, request=request
                    ),
                'candidates': None,
                'therapists': None
                }
        if obj.candidates:
            links['candidates'] = reverse('user-detail',
                kwargs = {User.USERNAME_FIELD: obj.candidates}, request=request
                    )
        if obj.therapists:
            links['therapists'] = reverse('user-detail',
                kwargs = {User.USERNAME_FIELD: obj.therapists}, request=request
                    )
        return links


"""
Module for querying the reference and test products that will be used
in the intercomparison evaluation.
As the test or reference datasets might be missing some records, we
need to perform an inner join such that there is a 1-1 relationship.
Product names can be different, database environments can be different;
However, both must have their level-1 ancestor uuid available in the
yaml document for a given dataset. If not, that dataset record will be
removed from the resulting query.
"""

from pathlib import Path
from typing import Any, Dict, Optional, Tuple
import datacube  # type: ignore
import pandas  # type: ignore
import structlog  # type: ignore

from gost.odc_documents import load_odc_metadata

_LOG = structlog.get_logger()

def query_db(
    env: str,
    product_name: str,
    time: Optional[Tuple[str, str]] = None,
    lon: Optional[Tuple[str, str]] = None,
    lat: Optional[Tuple[str, str]] = None,
    additional_filters: Optional[Dict[str, Any]] = None,
) -> pandas.DataFrame:
    """
    Generic datacube query wrapper.

    :param env:
        Name of the database environment to query.

    :param product_name:
        Name of the product to query.

    :param time:
        The time (earliest, latest) extents (optional).

    :param lon:
        The longitude (left, right) extents (optional).

    :param lat:
        The latitude (top, bottom) extents (optional).

    :param additional_filters:
        A dictionary containing {field: value} pairs that datacube can
        use to further filter datasets (optional).
        e.g. {"region_code": "092084"}
    """

    dc = datacube.Datacube(env=env)

    _LOG.info(
        "finding datasets",
        env=env,
        product_name=product_name,
        time=time,
        lon=lon,
        lat=lat,
    )
    datasets = dc.find_datasets(
        product_name, time=time, lon=lon, lat=lat, **additional_filters
    )

    uuid = []
    yaml_pathname = []
    proc_info_pathname = []

    for dataset in datasets:
        _LOG.info("processing dataset", dataset=str(dataset.local_path))
        doc = load_odc_metadata(dataset.local_path)

        uuid.append(doc.parent_uuid)
        yaml_pathname.append(str(dataset.local_path))

        # procssing info document
        proc_info_pathname = dataset.local_path.parent.joinpath(doc.proc_info)
        proc_info_pathname.append(str(proc_info_pathname))

    dataframe = pandas.DataFrame(
        {
            "level1_uuid": uuid,
            "yaml_pathname": yaml_pathname,
            "proc_info_pathname": proc_info_pathname,
        }
    )

    return dataframe

def query_products(
    product_name_test: str,
    product_name_reference: str,
    db_env_test: str,
    db_env_reference: str,
    time: Optional[Tuple[str, str]] = None,
    lon: Optional[Tuple[str, str]] = None,
    lat: Optional[Tuple[str, str]] = None,
    additional_filters: Optional[Dict[str, Any]] = None,
) -> pandas.DataFrame:
    """
    Queries an ODC Database for both the test and reference products,
    then merges the results based on a common ancestor uuid.
    Assumes products have been indexed with lineage.

    :param product_name_test:
        Product name for the test datasets.

    :param product_name_reference:
        Product name for the reference datasets.

    :param db_env_test:
        Name of the database environment containing the test datasets.

    :param db_env_reference:
        Name of the database environment containing the reference
        datasets.

    :param time:
        The time (earliest, latest) extents (optional).

    :param lon:
        The longitude (left, right) extents (optional).

    :param lat:
        The latitude (top, bottom) extents (optional).

    :param additional_filters:
        A dictionary containing {field: value} pairs that datacube can
        use to further filter datasets.
        e.g. {"region_code": "092084"}
    """

    _LOG.info("querying the test datasets")
    test_dataframe = query_db(
        db_env_test, product_name_test, time, lon, lat, **additional_filters
    )

    _LOG.info("querying the reference datasets")
    reference_dataframe = query_db(
        db_env_reference, product_name_reference, time, lon, lat, **additional_filters
    )

    _LOG.info("filtering test and reference datasets for common ancestor")
    merged = pandas.merge(
        test_dataframe,
        reference_dataframe,
        on="uuid",
        how="inner",
        suffixes=["_test", "_reference"],
    )

    return merged

def query_filepath(path: Path, pattern: str) -> pandas.DataFrame:
    """
    Find datasets by globbing the filesystem.

    :param path:
        The full pathname to the directory containing the product.

    :param pattern:
        A string containing pattern used to glob the filesystem.
        eg `*/*/2019/05/*/*.odc-metadata.yaml`
    """
    files = list(path.rglob(pattern))

    uuid = []
    yaml_pathname = []
    proc_info_pathname = []

    _LOG.info(
        "finding datasets",
        path=path,
        pattern=pattern,
    )

    for fname in files:
        _LOG.info("processing dataset", dataset=str(fname))

        doc = load_odc_metadata(fname)
        uuid.append(doc.parent_uuid)
        yaml_pathname.append(str(fname))

        # procssing info document
        pathname = fname.parent.joinpath(doc.proc_info)
        proc_info_pathname.append(str(pathname))

    dataframe = pandas.DataFrame(
        {
            "level1_uuid": uuid,
            "yaml_pathname": yaml_pathname,
            "proc_info_pathname": proc_info_pathname,
        }
    )

    return dataframe

def query_via_filepath(
    test_directory: str,
    reference_directory: str,
    test_pattern: str,
    reference_pattern: str,
) -> pandas.DataFrame:
    """
    Find the test and reference datasets by globbing the filesystem.

    :param test_directory:
        The full pathname to the directory containing the test product.

    :param reference_directory:
        The full pathname to the directory containing the reference product.

    :param test_pattern:
        A string containing pattern used to glob the filesystem.
        eg `*/*/2019/05/*/*.odc-metadata.yaml`

    :param reference_pattern:
        A string containing pattern used to glob the filesystem.
        eg `*/*/2019/05/*/*.odc-metadata.yaml`
    """

    test_dir = Path(test_directory)
    reference_dir = Path(reference_directory)

    _LOG.info("querying the test datasets")
    test_dataframe = query_filepath(test_dir, test_pattern)

    _LOG.info("querying the reference datasets")
    reference_dataframe = query_filepath(reference_dir, reference_pattern)

    _LOG.info("filtering test and reference datasets for common ancestor")
    merged = pandas.merge(
        test_dataframe,
        reference_dataframe,
        on="level1_uuid",
        how="inner",
        suffixes=["_test", "_reference"],
    )

    return merged


from rest_framework.response import Response
from snuba_sdk import Column, Function, LimitBy

from sentry import features
from sentry.api.api_publish_status import ApiPublishStatus
from sentry.api.base import region_silo_endpoint
from sentry.api.bases.organization_events import OrganizationEventsEndpointBase
from sentry.api.endpoints.organization_events_spans_performance import EventID, get_span_description
from sentry.api.helpers.span_analysis import span_analysis
from sentry.search.events.builder import QueryBuilder
from sentry.search.events.types import QueryBuilderConfig
from sentry.search.utils import parse_datetime_string
from sentry.snuba.dataset import Dataset
from sentry.snuba.metrics_performance import query as metrics_query
from sentry.utils.snuba import raw_snql_query

DEFAULT_LIMIT = 50
SNUBA_QUERY_LIMIT = 10000

def query_spans(transaction, regression_breakpoint, params):
    selected_columns = [
        "count(span_id) as span_count",
        "sumArray(spans_exclusive_time) as total_span_self_time",
        "array_join(spans_op) as span_op",
        "array_join(spans_group) as span_group",
        # want a single event id to fetch from nodestore for the span description
        "any(id) as sample_event_id",
    ]

    builder = QueryBuilder(
        dataset=Dataset.Discover,
        params=params,
        selected_columns=selected_columns,
        equations=[],
        query=f"transaction:{transaction}",
        orderby=["span_op", "span_group", "total_span_self_time"],
        limit=SNUBA_QUERY_LIMIT,
        config=QueryBuilderConfig(
            auto_aggregations=True,
            use_aggregate_conditions=True,
            functions_acl=[
                "array_join",
                "sumArray",
                "percentileArray",
            ],
        ),
    )

    builder.columns.append(
        Function(
            "if",
            [
                Function("greaterOrEquals", [Column("timestamp"), regression_breakpoint]),
                "after",
                "before",
            ],
            "period",
        )
    )
    builder.columns.append(Function("countDistinct", [Column("event_id")], "transaction_count"))
    builder.groupby.append(Column("period"))
    builder.limitby = LimitBy([Column("period")], SNUBA_QUERY_LIMIT // 2)

    snql_query = builder.get_snql_query()
    results = raw_snql_query(snql_query, "api.organization-events-root-cause-analysis")

    return results.get("data", [])

@region_silo_endpoint
class OrganizationEventsRootCauseAnalysisEndpoint(OrganizationEventsEndpointBase):
    publish_status = {
        "GET": ApiPublishStatus.UNKNOWN,
    }

    def get(self, request, organization):
        if not features.has(
            "organizations:statistical-detectors-root-cause-analysis",
            organization,
            actor=request.user,
        ):
            return Response(status=404)

        # TODO: Extract this into a custom serializer to handle validation
        transaction_name = request.GET.get("transaction")
        project_id = request.GET.get("project")
        regression_breakpoint = request.GET.get("breakpoint")
        if not transaction_name or not project_id or not regression_breakpoint:
            # Project ID is required to ensure the events we query for are
            # the same transaction
            return Response(status=400)

        regression_breakpoint = parse_datetime_string(regression_breakpoint)

        params = self.get_snuba_params(request, organization)

        with self.handle_query_errors():
            transaction_count_query = metrics_query(
                ["count()"],
                f"event.type:transaction transaction:{transaction_name} project_id:{project_id}",
                params,
                referrer="api.organization-events-root-cause-analysis",
            )

        if transaction_count_query["data"][0]["count"] == 0:
            return Response(status=400, data="Transaction not found")

        span_data = query_spans(
            transaction=transaction_name,
            regression_breakpoint=regression_breakpoint,
            params=params,
        )

        span_analysis_results = span_analysis(span_data)

        for result in span_analysis_results:
            result["span_description"] = get_span_description(
                EventID(project_id, result["sample_event_id"]),
                result["span_op"],
                result["span_group"],
            )

        limit = int(request.GET.get("per_page", DEFAULT_LIMIT))
        return Response(span_analysis_results[:limit], status=200)


from django import forms
from django.core.exceptions import ValidationError
from site_escola.models import Quiz, QuizQuestion

class QuizForm(forms.ModelForm):
    class Meta:
        model = Quiz
        fields = ('name',)

class QuizQuestionForm(forms.ModelForm):
    class Meta:
        model = QuizQuestion
        fields = ('content', 'n_options')

    def clean_n_options(self):
        options = self.cleaned_data.get('n_options')

        if options <= 0:
            raise ValidationError(
                'escreve maior q zero',
                code='invalid'
            )

        if options > 9:
            raise ValidationError(
                'escreve mmenor q 10',
                code='invalid'
            )


import logging
import multiprocessing
import os
import sys
from .make_process import make_process

__all__ = 'Logger'

class QueueLogHandler(logging.Handler):
    def __init__(self, message_queue):
        super().__init__(logging.INFO)
        fmt = logging.Formatter('%(name)s: %(process)d: %(levelname)s: '
                                '%(message)s')
        super().setFormatter(fmt)
        self.message_queue = message_queue

    def emit(self, record):
        message = super().format(record)
        self.message_queue.put(message)

@make_process(daemon=True)
def print_messages(message_queue):
    print(f'Starting logger on PID {os.getpid()}')

    while True:
        message = message_queue.get()
        print(message, file=sys.stderr)

def instantiate(cls):
    return cls()

@instantiate
class Logger(logging.Logger):
    def __init__(self):
        super().__init__('pyt')
        self.message_queue = multiprocessing.Queue()
        handler = QueueLogHandler(self.message_queue)
        super().addHandler(handler)

    def start(self):
        print_messages(self.message_queue)


from ._version import get_versions
__version__ = get_versions()['version']
del get_versions

__doc__ = """
   Epics Channel Access Python module

   version: %s
   Principal Authors:
      Matthew Newville <newville@cars.uchicago.edu> CARS, University of Chicago
      Ken Lauer, David Chabot, Angus Gratton

== License:
   Except where explicitly noted, this file and all files in this distribution
   are licensed under the Epics Open License See LICENSE in the top-level
   directory of this distribution.

== Overview:
   Python Interface to the Epics Channel Access protocol of the Epics control system.

""" % (__version__)

import time
import sys
import threading
from . import ca
from . import dbr
from . import pv
from . import alarm
from . import device
from . import motor
from . import multiproc

PV    = pv.PV
Alarm = alarm.Alarm
Motor = motor.Motor
Device = device.Device
poll  = ca.poll

get_pv = pv.get_pv

CAProcess = multiproc.CAProcess
CAPool = multiproc.CAPool

# some constants
NO_ALARM = 0
MINOR_ALARM = 1
MAJOR_ALARM = 2
INVALID_ALARM = 3

_PVmonitors_ = {}

def caput(pvname, value, wait=False, timeout=60.0, connection_timeout=5.0):
    """caput(pvname, value, wait=False, timeout=60, connection_timeout=5.0)

    put a value to an epics Process Variable (PV).

    Arguments
    ---------
     pvname (str):   name of PV
     value (any):    value to put to PV (see notes)
     wait (bool):    whether to wait for processing to complete [False]
     timeout (float):  maximum time to wait for the processing to complete [60]
     connection_timeout (float): maximum time to wait for connection [5]

    Returns
    --------
      1  on succesful completion
      -1 or other negative value on failure of the low-level CA put.
      None  on a failure to connect to the PV.

    Notes
    ------
      1. Epics PVs are typically limited to an appropriate Epics data type,
         int, float, str, and enums or homogeneously typed lists or arrays.
         Numpy arrays or Python strings are generally coeced as appropriate,
         but care may be needed when mapping Python objects to Epics PV values.
      2. If not already connected, the PV will first attempt to connect to
         the networked variable. As the initial connection can take some time
         (typically 30 msec or so), if a successful connection is made, a rich
         PV object with will be stored internally for later use.  Use
         connection_timeout to control how long to wait before declaring that
         the PV cannot be connected (mispelled name, inactive IOC, improper
         network configuration)
      3. Since some PVs can take a long time to process (perhaps physically
         moving a motor or telling a detector to collect and not return until
         done), it is impossible to tell what a "reasonable" timeout for a put
         should be.
      4. All time in seconds.

    Examples
    ---------
       to put a value to a PV and return as soon as possible:
       >>> caput('xx.VAL', 3.0)

      to wait for processing to finish, use 'wait=True':
      >>> caput('xx.VAL', 3.0, wait=True)

    """
    start_time = time.time()
    thispv = get_pv(pvname, timeout=connection_timeout, connect=True)
    if thispv.connected:
        timeout -= (time.time() - start_time)
        return thispv.put(value, wait=wait, timeout=timeout)

def caget(pvname, as_string=False, count=None, as_numpy=True,
          use_monitor=True, timeout=5.0, connection_timeout=5.0):
    """caget(pvname, as_string=False, count=None, as_numpy=True,
             use_monitor=True, timeout=5.0, connection_timeout=5.0)

    get the current value to an epics Process Variable (PV).

    Arguments
    ---------
     pvname (str):   name of PV
     as_string (bool): whether to get the string representation [False]
     count (int or None): maximum number of elements of array values [None]
     use_monitor (bool): whether to use the value cached by the monitor [True]
     timeout (float):  maximum time to wait for the processing to complete [60]
     connection_timeout (float): maximum time to wait for connection [5]

    Returns
    --------
      the PV value on success, or `None`on a failure.

    Notes
    ------
      1. If not already connected, the PV will first attempt to connect to
         the networked variable. As the initial connection can take some time
         (typically 30 msec or so), if a successful connection is made, a rich
         PV object with will be stored internally for later use.
      2. as_string=True will return a string: float values will formatted
         according to the PVs precision, enum values will return the approriate
         enum string, etc.
      3. count can be used to limit the number of elements fetched for array PVs.
      4. `use_monitor=False` will return the most recently cached value from the
         internal monitor placed on the PV. This will be the latest value unless
         the value is changing very rapidly. `use_monitor=False` will ignore the
         cached value and ask for an explicit value.  Of course, if a PV is
         changing rapidly enough for that to make a difference, it may also
         change between getting the value and downstream code using it.
      5. All time values are in seconds.

    Examples
    ---------
       get of a PV's value
       >>> x = caget('xx.VAL')

       to get the character string representation (formatted double,
       enum string, etc):
       >>> x = caget('xx.VAL', as_string=True)

       to get a truncated amount of data from an array, you can specify
       the count with
       >>> x = caget('MyArray.VAL', count=1000)
    """
    start_time = time.time()
    thispv = get_pv(pvname, timeout=connection_timeout, connect=True)
    if thispv.connected:
        if as_string:
            thispv.get_ctrlvars()
        timeout -= (time.time() - start_time)
        val = thispv.get(count=count, timeout=timeout,
                         use_monitor=use_monitor,
                         as_string=as_string,
                         as_numpy=as_numpy)
        poll()
        return val

def cainfo(pvname, print_out=True, timeout=5.0, connection_timeout=5.0):
    """cainfo(pvname,print_out=True,timeout=5.0, connection_timeout=5.0)

    return printable information about PV

    Arguments
    ---------
     pvname (str):   name of PV
     print_out (bool): whether to print the info to standard out [True]
     timeout (float):  maximum time to wait for the processing to complete [60]
     connection_timeout (float): maximum time to wait for connection [5]

    Returns
    --------
       if `print_out` is True, returns the printable text
       otherwise return `None`.

    Notes
    ------
      1. If not already connected, the PV will first attempt to connect to
         the networked variable. As the initial connection can take some time
         (typically 30 msec or so), if a successful connection is made, a rich
         PV object with will be stored internally for later use.
      2. All time in seconds.

    Examples
    ---------
       to print a status report for the PV:
       >>>cainfo('xx.VAL')

       to get the multiline text, use
       >>>txt = cainfo('xx.VAL', print_out=False)

    """
    start_time = time.time()
    thispv = get_pv(pvname, timeout=connection_timeout, connect=True)
    if thispv.connected:
        conn_time = time.time() - start_time
        thispv.get(timeout=timeout-conn_time)
        get_time = time.time() - start_time
        thispv.get_ctrlvars(timeout=timeout-get_time)
        if print_out:
            ca.write(thispv.info)
        else:
            return thispv.info

def camonitor_clear(pvname):
    """clear monitors on a PV"""
    if pvname in _PVmonitors_:
        _PVmonitors_[pvname].remove_callback(index=-999)
        _PVmonitors_.pop(pvname)

def camonitor(pvname, writer=None, callback=None, connection_timeout=5.0):
    """ camonitor(pvname, writer=None, callback=None, connection_timeout=5)

    sets a monitor on a PV.

    Arguments
    ---------
     pvname (str):   name of PV
     writer (callable or None): function used to send monitor messages [None]
     callback (callback or None): custom callback function [None]
     connection_timeout (float): maximum time to wait for connection [5]

    Notes
    ------
     1. To write the result to a file, provide the writer option a
        write method to an open file or some other callable that
        accepts a string.
     2. To completely control where the output goes, provide a callback
         method and you can do whatever you'd like with them.
     3. A custom callback can be provieded.  This will be sent keyword
        arguments for pvname, value, char_value, and more.
        Important: use **kws!!

    Examples
    --------
      to write a message with the latest value for that PV each
      time the value changes and when ca.poll() is called.
      >>>camonitor('xx.VAL')
    """

    if writer is None:
        writer = ca.write
    if callback is None:
        def callback(pvname=None, value=None, char_value=None, **kwds):
            "generic monitor callback"
            if char_value is None:
                char_value = repr(value)
            writer("%.32s %s %s" % (pvname, pv.fmt_time(), char_value))

    thispv = get_pv(pvname, timeout=connection_timeout, connect=True)
    if thispv.connected:
        thispv.get()
        thispv.add_callback(callback, index=-999, with_ctrlvars=True)
        _PVmonitors_[pvname] = thispv

def caget_many(pvlist, as_string=False, as_numpy=True, count=None,
               timeout=1.0, connection_timeout=5.0, conn_timeout=None):
    """caget_many(pvlist, as_string=False, as_numpy=True, count=None,
                 timeout=1.0, connection_timeout=5.0, conn_timeout=None)
    get values for a list of PVs, working as fast as possible

    Arguments
    ---------
     pvlist (list):        list of pv names to fetch
     as_string (bool):     whether to get values as strings [False]
     as_numpy (bool):      whether to get values as numpy arrys [True]
     count  (int or None): max number of elements to get [None]
     timeout (float):      timeout on *each* get()  [1.0]
     connection_timeout (float): timeout for *all* pvs to connect [5.0]
     conn_timeout (float):  back-compat alias or connection_timeout

    Returns
    --------
      list of values, with `None` signifying 'not connected' or 'timed out'.

    Notes
    ------
       this does not cache PV objects.

    """
    chids, connected, out = [], [], []
    for name in pvlist:
        chids.append(ca.create_channel(name, auto_cb=False, connect=False))

    all_connected = False
    if conn_timeout is not None:
        connection_timeout = conn_timeout
    expire_time = time.time() + connection_timeout
    while (not all_connected and (time.time() < expire_time)):
        connected = [dbr.CS_CONN==ca.state(chid) for chid in chids]
        all_connected = all(connected)
        poll()

    for (chid, conn) in zip(chids, connected):
        if conn:
            ca.get(chid, count=count, as_string=as_string, as_numpy=as_numpy,
                   wait=False)

    poll()
    for (chid, conn) in zip(chids, connected):
        val = None
        if conn:
            val = ca.get_complete(chid, count=count, as_string=as_string,
                                  as_numpy=as_numpy, timeout=timeout)
        out.append(val)
    return out

def caput_many(pvlist, values, wait=False, connection_timeout=5.0,
               put_timeout=60):
    """caput_many(pvlist, values, wait=False, connection_timeout=5.0,
               put_timeout=60)

    put values to a list of PVs, as quickly as possible

    Arguments
    ----------
     pvlist (iterable):   list of PV names
     values (iterable):   list of values for corresponding PVs
     wait  (bool or string):   whether to wait for puts (see notes) [False]
     put_timeout (float):  maximum time to wait for the put to complete [60]
     connection_timeout (float): maximum time to wait for connection [5]

    Returns
    --------
     a list of ints or `None`, with values of 1 if the put was successful,
     or a negative number if the put failed (say, the timeout was exceeded),
     or `None` if the connection failed.

    Notes
    ------
      1. This does not maintain the PV objects.

      2. If wait is 'each', *each* put operation will block until it is
         complete or until the put_timeout duration expires.
         If wait is 'all', this method will block until *all* put
         operations are complete, or until the put_timeout expires.

         This 'wait' only applies to the put timeout, not the
         connection timeout.

    """
    if len(pvlist) != len(values):
        raise ValueError("List of PV names must be equal to list of values.")
    out = []
    pvs = [PV(name, auto_monitor=False, connection_timeout=connection_timeout) for name in pvlist]
    conns = [p.connected for p in pvs]
    wait_all = (wait == 'all')
    wait_each = (wait == 'each')
    for p, v in zip(pvs, values):
        out.append(p.put(v, wait=wait_each, timeout=put_timeout, use_complete=wait_all))
    if wait_all:
        start_time = time.time()
        while not all([(p.connected and p.put_complete) for p in pvs]):
            ca.poll()
            elapsed_time = time.time() - start_time
            if elapsed_time > put_timeout:
                break
        return [1 if (p.connected and p.put_complete) else -1 for p in pvs]
    else:
        return [o if o == 1 else -1 for o in out]


#
# Please note that this file has been modified by Unbound Tech
#
import logging

import pytest

from pypkcs11.default_templates import \
    (CKM_ECDSA_KEY_PAIR_GEN_PRIVTEMP, CKM_ECDSA_KEY_PAIR_GEN_PUBTEMP,
     curve_list, get_default_key_template, get_default_key_pair_template,
     MECHANISM_LOOKUP_EXT, CKM_ECDH_KEY_PAIR_GEN_PUBTEMP,
     CKM_ECDH_KEY_PAIR_GEN_PRIVTEMP)
from pypkcs11.defines import *
from pypkcs11.encryption import c_encrypt, c_decrypt
from pypkcs11.key_generator import c_generate_key, c_generate_key_pair, c_derive_key, c_destroy_object
from pypkcs11.mechanism import NullMech
from pypkcs11.object_attr_lookup import c_get_attribute_value, c_find_objects
from pypkcs11.lookup_dicts import ret_vals_dictionary
from pypkcs11.test_functions import verify_object_attributes
from .util import get_session_template

logger = logging.getLogger(__name__)

KEYS = [CKM_DES3_KEY_GEN, CKM_AES_KEY_GEN]

def pair_params(key_gen):
    """ Return the params tuple given the key_gen mech """
    return (key_gen,) + get_default_key_pair_template(key_gen)

KEY_PAIRS = [pair_params(CKM_RSA_PKCS_KEY_PAIR_GEN),
             pair_params(CKM_ECDSA_KEY_PAIR_GEN)]

DERIVE_PARAMS = {
    CKM_SHA256_KEY_DERIVATION: "SHA256",
    CKM_SHA384_KEY_DERIVATION: "SHA384",
    CKM_SHA512_KEY_DERIVATION: "SHA512",
    CKM_SHA3_256_KEY_DERIVATION: "SHA3_256",
    CKM_SHA3_384_KEY_DERIVATION: "SHA3_384",
    CKM_SHA3_512_KEY_DERIVATION: "SHA3_512",
}
DERIVE_KEYS = {CKM_DES3_KEY_GEN: "DES3",
               CKM_AES_KEY_GEN: "AES"}
DRV_TOO_LONG = {CKM_SHA1_KEY_DERIVATION: "SHA1"}
TOO_LONG_KEY = {CKM_DES3_KEY_GEN: "DES3",
                CKM_AES_KEY_GEN: "AES"
                }

DATA = b"1234567812345678"

class TestKeys(object):
    """
    Tests Key & Key pair generation
    """

    def verify_ret(self, ret, expected_ret):
        """ Verify ret check and len > 0"""
        assert ret == expected_ret, "Function should return: " + ret_vals_dictionary[expected_ret] \
                                    + ".\nInstead returned: " + \
            ret_vals_dictionary[ret]

    def verify_key_len(self, k1, k2):
        """ Verify that key > 0"""
        assert k1 > 0, "Key should be > 0"
        assert k2 > 0, "Priv key should be > 0"

    @pytest.fixture(autouse=True)
    def setup_teardown(self, auth_session):
        self.h_session = auth_session

    @pytest.mark.parametrize("key_type", KEYS, ids=[MECHANISM_LOOKUP_EXT[k][0] for k in KEYS])
    def test_generate_key(self, key_type, valid_mechanisms):
        """
        Test generation of keys for sym. crypto systems
        :param key_type: key generation mechanism
        """
        key_template = get_session_template(get_default_key_template(key_type))
        ret, key_handle = c_generate_key(
            self.h_session, key_type, key_template)

        try:
            if key_type not in valid_mechanisms:
                self.verify_ret(ret, CKR_MECHANISM_INVALID)
            else:
                self.verify_ret(ret, CKR_OK)
                self.verify_key_len(key_handle, key_handle)
        finally:
            c_destroy_object(self.h_session, key_handle)

    @pytest.mark.parametrize(("key_type", "pub_key_temp", "prv_key_temp"), KEY_PAIRS,
                             ids=[MECHANISM_LOOKUP_EXT[k[0]][0] for k in KEY_PAIRS])
    def test_generate_key_pair(self, key_type, pub_key_temp, prv_key_temp, valid_mechanisms):
        """
        Test generation of key pairs for asym. crypto systems
        :param key_type: key generation mechanism
        :param pub_key_temp: public key template
        :param prv_key_temp: private key template
        """
        ret, pub_key, prv_key = c_generate_key_pair(self.h_session, key_type,
                                                    get_session_template(
                                                        pub_key_temp),
                                                    get_session_template(prv_key_temp))
        try:
            if key_type not in valid_mechanisms:
                self.verify_ret(ret, CKR_MECHANISM_INVALID)
            else:
                self.verify_ret(ret, CKR_OK)
                self.verify_key_len(pub_key, prv_key)
        finally:
            c_destroy_object(self.h_session, prv_key)
            c_destroy_object(self.h_session, pub_key)

    @pytest.mark.parametrize("curve_type", list(curve_list.keys()))
    def test_generate_ecdsa_key_pairs(self, curve_type):
        """
        Test generate ECDSA key pairs
        :param curve_type:
        """
        pub_temp = CKM_ECDSA_KEY_PAIR_GEN_PUBTEMP.copy()
        pub_temp[CKA_ECDSA_PARAMS] = curve_list[curve_type]
        data = c_generate_key_pair(self.h_session,
                                   CKM_ECDSA_KEY_PAIR_GEN,
                                   get_session_template(pub_temp),
                                   get_session_template(CKM_ECDSA_KEY_PAIR_GEN_PRIVTEMP))
        ret, public_key_handle, private_key_handle = data
        try:
            self.verify_ret(ret, CKR_OK)
            self.verify_key_len(public_key_handle, private_key_handle)
        finally:
            if public_key_handle:
                c_destroy_object(self.h_session, public_key_handle)
            if private_key_handle:
                c_destroy_object(self.h_session, private_key_handle)

    @pytest.mark.parametrize("d_type", list(DERIVE_PARAMS.keys()), ids=list(DERIVE_PARAMS.values()))
    @pytest.mark.parametrize("key_type", list(DERIVE_KEYS.keys()), ids=list(DERIVE_KEYS.values()))
    def test_derive_key(self, key_type, d_type, valid_mechanisms):
        """
        Test derive key for using parametrized hash
        :param key_type: Key-gen mechanism
        :param d_type: Hash mech
        """
        if key_type not in valid_mechanisms:
            pytest.skip("Not a valid mechanism on this product")
        key_template = get_session_template(get_default_key_template(key_type))
        ret, h_base_key = c_generate_key(
            self.h_session, key_type, key_template)
        self.verify_ret(ret, CKR_OK)
        mech = NullMech(d_type).to_c_mech()

        derived_key_template = key_template.copy()
        del derived_key_template[CKA_VALUE_LEN]

        ret, h_derived_key = c_derive_key(self.h_session, h_base_key,
                                          key_template,
                                          mechanism=mech)
        try:
            self.verify_ret(ret, CKR_OK)
            verify_object_attributes(
                self.h_session, h_derived_key, key_template)
        finally:
            if h_base_key:
                c_destroy_object(self.h_session, h_base_key)
            if h_derived_key:
                c_destroy_object(self.h_session, h_derived_key)

    @pytest.mark.parametrize("d_type", list(DRV_TOO_LONG.keys()), ids=list(DRV_TOO_LONG.values()))
    @pytest.mark.parametrize("key_type", list(TOO_LONG_KEY.keys()), ids=list(TOO_LONG_KEY.values()))
    def test_too_long_length_derives(self, key_type, d_type, valid_mechanisms):
        """
        Verify that trying to derive a key that is too long for the given derivation function
        will return CKR_KEY_SIZE_RANGE
        :param key_type:
        :param d_type:
        """
        if key_type not in valid_mechanisms:
            pytest.skip("Not a valid mechanism on this product")
        key_template = get_session_template(get_default_key_template(key_type))
        ret, h_base_key = c_generate_key(
            self.h_session, key_type, key_template)
        self.verify_ret(ret, CKR_OK)
        mech = NullMech(d_type).to_c_mech()

        derived_key_template = key_template.copy()
        del derived_key_template[CKA_VALUE_LEN]

        ret, h_derived_key = c_derive_key(self.h_session, h_base_key,
                                          key_template,
                                          mechanism=mech)
        try:
            self.verify_ret(ret, CKR_KEY_SIZE_RANGE)
        finally:
            if h_base_key:
                c_destroy_object(self.h_session, h_base_key)
            if h_derived_key:
                c_destroy_object(self.h_session, h_derived_key)

    @pytest.mark.parametrize("d_type", list(DERIVE_PARAMS.keys()), ids=list(DERIVE_PARAMS.values()))
    @pytest.mark.parametrize("key_type", list(TOO_LONG_KEY.keys()), ids=list(TOO_LONG_KEY.values()))
    def test_long_length_derive_key(self, key_type, d_type, valid_mechanisms):
        """
        Test deriving a key
        :param key_type: key generation mechanism
        :param d_type: derive mechanism
        """
        key_template = get_session_template(get_default_key_template(key_type))
        if key_type not in valid_mechanisms:
            pytest.skip("Not a valid mechanism on this product")
        ret, h_base_key = c_generate_key(
            self.h_session, key_type, key_template)
        mech = NullMech(d_type).to_c_mech()

        derived_key_template = key_template.copy()
        del derived_key_template[CKA_VALUE_LEN]

        ret, h_derived_key = c_derive_key(self.h_session,
                                          h_base_key,
                                          key_template,
                                          mechanism=mech)
        try:
            self.verify_ret(ret, CKR_OK)
            verify_object_attributes(
                self.h_session, h_derived_key, key_template)
        finally:
            if h_base_key:
                c_destroy_object(self.h_session, h_base_key)
            if h_derived_key:
                c_destroy_object(self.h_session, h_derived_key)

    @pytest.mark.parametrize("curve_type", sorted(list(curve_list.keys())))
    def test_x9_key_derive(self, auth_session, curve_type):
        """
        Test we can do X9 key derivation
        """
        derived_key2 = derived_key1 = pub_key1 = pub_key2 = prv_key2 = prv_key1 = None
        derived_template = {
            CKA_CLASS: CKO_SECRET_KEY,
            CKA_KEY_TYPE: CKK_AES,
            CKA_ENCRYPT: True,
            CKA_DECRYPT: True,
            CKA_PRIVATE: True,
            CKA_SENSITIVE: True,
            CKA_VALUE_LEN: 16
        }
        pub_temp, priv_temp = (CKM_ECDH_KEY_PAIR_GEN_PUBTEMP,
                               CKM_ECDH_KEY_PAIR_GEN_PRIVTEMP)
        priv_temp = get_session_template(priv_temp)
        pub_temp = get_session_template(pub_temp)
        pub_temp[CKA_ECDSA_PARAMS] = curve_list[curve_type]

        ret, pub_key1, prv_key1 = c_generate_key_pair(auth_session,
                                                      CKM_ECDSA_KEY_PAIR_GEN,
                                                      pbkey_template=pub_temp,
                                                      prkey_template=priv_temp)
        self.verify_ret(ret, CKR_OK)
        try:
            ret, pub_key2, prv_key2 = c_generate_key_pair(auth_session,
                                                          CKM_ECDSA_KEY_PAIR_GEN,
                                                          pbkey_template=pub_temp,
                                                          prkey_template=priv_temp)
            self.verify_ret(ret, CKR_OK)
            ret, pub_key1_raw_ = c_get_attribute_value(auth_session,
                                                       pub_key1,
                                                       {CKA_EC_POINT: None})
            pub_key1_raw = pub_key1_raw_[CKA_EC_POINT]
            self.verify_ret(ret, CKR_OK)
            ret, pub_key2_raw_ = c_get_attribute_value(auth_session,
                                                       pub_key2,
                                                       {CKA_EC_POINT: None})
            pub_key2_raw = pub_key2_raw_[CKA_EC_POINT]
            self.verify_ret(ret, CKR_OK)
            ret, derived_key1 = c_derive_key(auth_session,
                                             h_base_key=prv_key2,
                                             template=derived_template,
                                             mechanism={"mech_type": CKM_ECDH1_DERIVE,
                                                        "params": {"kdf": CKD_NULL,
                                                                   "sharedData": None,
                                                                   "publicData": pub_key1_raw}})
            self.verify_ret(ret, CKR_OK)
            ret, derived_key2 = c_derive_key(auth_session,
                                             h_base_key=prv_key1,
                                             template=derived_template,
                                             mechanism={"mech_type": CKM_ECDH1_DERIVE,
                                                        "params": {"kdf": CKD_NULL,
                                                                   "sharedData": None,
                                                                   "publicData": pub_key2_raw}})
            self.verify_ret(ret, CKR_OK)
            ret, cipher_data = c_encrypt(auth_session,
                                         derived_key1,
                                         data=DATA,
                                         mechanism=CKM_AES_ECB)
            self.verify_ret(ret, CKR_OK)
            ret, restored_text = c_decrypt(auth_session,
                                           derived_key2,
                                           cipher_data,
                                           mechanism=CKM_AES_ECB)
            self.verify_ret(ret, CKR_OK)
            assert DATA == restored_text.rstrip(b'\x00')
        finally:
            for key in (pub_key1, prv_key1, pub_key2, prv_key2, derived_key1, derived_key2):
                if key:
                    c_destroy_object(auth_session, key)

if __name__ == '__main__':
    for k in KEY_PAIRS:
        mech = MECHANISM_LOOKUP_EXT[k[0]][0]
        print(mech)


from datetime import timedelta

from django.utils import timezone
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers

from .models import Match

class MatchSerializer(serializers.ModelSerializer):
    class Meta:
        model = Match
        fields = (
            "id",
            "start_at",
            "finished_at",
            "game",
            "home_robot",
            "away_robot",
            "home_score",
            "away_score",
            "runner_log",
            "created_at",
            "updated_at",
        )
        read_only_fields = (
            "id",
            "finished_at",
            "game",
            "home_score",
            "away_score",
            "runner_log",
            "created_at",
            "updated_at",
        )

    def validate_home_robot(self, value):
        if value.owner != self.context["request"].user:
            raise serializers.ValidationError(
                _("home_robot must be one of your robots.")
            )
        return value

    def validate_away_robot(self, value):
        if value.owner == self.context["request"].user:
            raise serializers.ValidationError(
                _("away_robot must not be one of your robots.")
            )
        return value

    def validate_start_at(self, value):
        if value < timezone.now() + timedelta(minutes=1):
            raise serializers.ValidationError(
                _("start_at must be at least 1 minute from now.")
            )
        return value

    def validate(self, data):
        if data["home_robot"].game != data["away_robot"].game:
            raise serializers.ValidationError(
                _("home_robot and away_robot must play the same game")
            )
        data["game"] = data["home_robot"].game
        return data


#! /usr/bin/env python

from __future__ import print_function

import numpy as np
import pandas as pd

import ROOT
# disable cmd line parsing before other ROOT deps are loaded
ROOT.PyConfig.IgnoreCommandLineOptions = True
from root_pandas import read_root

import matplotlib.pyplot as plt

from sklearn.cross_validation import KFold, train_test_split
from sklearn.grid_search import RandomizedSearchCV
from xgboost import XGBClassifier
from scripts.metrics import tagging_power_score
from uncertainties import ufloat

from scripts.calibration import PolynomialLogisticRegression
from scripts.data_preparation import NSplit

from collections import OrderedDict
from itertools import islice, product
from tqdm import tqdm
import argparse
import datetime
import json
from os import path, mkdir

def get_event_number(df, weight_column='SigYield_sw'):
    """ Use weighted sums
    """
    # max, min, mean, first should give the same values here
    return np.sum(df.groupby('event_id')[weight_column].first())

class CutBasedXGBClassifier(XGBClassifier):
    def __init__(self, max_depth=3, learning_rate=0.1, n_estimators=100,
                 silent=True, objective="reg:linear",
                 nthread=1, gamma=0, min_child_weight=1, max_delta_step=0,
                 subsample=1, colsample_bytree=1, colsample_bylevel=1,
                 reg_alpha=0, reg_lambda=1, scale_pos_weight=1,
                 base_score=0.5, seed=0, missing=None,
                 P_column='tp_partP', P_cut=0,
                 PT_column='tp_partPt', PT_cut=1.1,
                 phiDistance_column='tp_minPhiDistance', phiDistance_cut=0.005,
                 MuonPIDIsMuon_column='tp_MuonPIDIsMuon', MuonPIDIsMuon_cut=1,
                 mvaFeatures=None, only_max_pt=True,
                 event_identifier_column='event_id',
                 IsSignalDaughter_column='tp_IsSignalDaughter',
                 IsSignalDaughter_cut=0,
                 RecVertexIPs_column='tp_ABS_RecVertexIP',
                 RecVertexIPs_cut=0,
                 TRCHI2DOF_column='tp_partlcs', TRCHI2DOF_cut=3,
                 TRGHP_column='tp_ghostProb', TRGHP_cut=0.4,
                 PROBNNmu_column='tp_PIDNNm', PROBNNmu_cut=0.35,
                 PROBNNpi_column='tp_PROBNNpi', PROBNNpi_cut=0.8,
                 PROBNNe_column='tp_PROBNNe', PROBNNe_cut=0.8,
                 PROBNNk_column='tp_PROBNNk', PROBNNk_cut=0.8,
                 PROBNNp_column='tp_PROBNNp', PROBNNp_cut=0.8,
                 IPPUs_column='tp_IPPU', IPPUs_cut=3,
                 polynomial_pow=3,
                 ):
        self.cut_parameters = ['P', 'PT', 'phiDistance', 'MuonPIDIsMuon',
                               'IsSignalDaughter', 'TRCHI2DOF', 'TRGHP',
                               'PROBNNmu', 'PROBNNpi', 'PROBNNe', 'PROBNNk',
                               'PROBNNp', 'IPPUs', 'RecVertexIPs',
                               ]
        for cp in self.cut_parameters:
            setattr(self,
                    '{}_cut'.format(cp),
                    locals()['{}_cut'.format(cp)])
            setattr(self,
                    '{}_column'.format(cp),
                    locals()['{}_column'.format(cp)])
        self.calibrator = PolynomialLogisticRegression(power=polynomial_pow,
                                                       solver='lbfgs',
                                                       n_jobs=nthread)
        self.mvaFeatures = mvaFeatures
        self.only_max_pt = only_max_pt
        self.event_identifier_column = event_identifier_column
        self.fit_successful = False
        (super(CutBasedXGBClassifier, self)
         .__init__(max_depth=max_depth, learning_rate=learning_rate,
                   n_estimators=n_estimators,
                   silent=silent, objective=objective,
                   nthread=nthread, gamma=gamma,
                   min_child_weight=min_child_weight,
                   max_delta_step=max_delta_step,
                   subsample=subsample, colsample_bytree=colsample_bytree,
                   colsample_bylevel=colsample_bylevel,
                   reg_alpha=reg_alpha, reg_lambda=reg_lambda,
                   scale_pos_weight=scale_pos_weight,
                   base_score=base_score, seed=seed, missing=None))

    def select(self, X, y=None):
        len_before = get_event_number(X)
        selection = ((X[self.P_column] > self.P_cut) &
                     (X[self.PT_column] > self.PT_cut) &
                     (X[self.phiDistance_column] > self.phiDistance_cut) &
                     (X[self.MuonPIDIsMuon_column] == self.MuonPIDIsMuon_cut) &
                     (X[self.IsSignalDaughter_column] == self.IsSignalDaughter_cut) &
                     (X[self.TRCHI2DOF_column] < self.TRCHI2DOF_cut) &
                     (X[self.TRGHP_column] < self.TRGHP_cut) &
                     (X[self.PROBNNmu_column] > self.PROBNNmu_cut) &
                     (X[self.PROBNNpi_column] < self.PROBNNpi_cut) &
                     (X[self.PROBNNe_column] < self.PROBNNe_cut) &
                     (X[self.PROBNNk_column] < self.PROBNNk_cut) &
                     (X[self.PROBNNp_column] < self.PROBNNp_cut) &
                     (X[self.IPPUs_column] > self.IPPUs_cut) &
                     (X[self.RecVertexIPs_column] > self.RecVertexIPs_cut))
        X = X[selection]
        if y is not None:
            y = y[selection]

        if self.only_max_pt:
            X.reset_index(drop=True, inplace=True)
            max_pt_indices = (X
                              .groupby(self.event_identifier_column)[self.PT_column]
                              .idxmax())
            X = X.iloc[max_pt_indices]
            if y is not None:
                y.reset_index(drop=True, inplace=True)
                y = y.iloc[max_pt_indices]

        len_after = get_event_number(X)
        self.efficiency_ = len_after / len_before

        if self.mvaFeatures:
            X = X[self.mvaFeatures]

        if y is not None:
            return X, y
        else:
            return X

    def get_params(self, deep=False):
        params = super(CutBasedXGBClassifier, self).get_params(deep=deep)
        for cp in self.cut_parameters:
            params['{}_cut'.format(cp)] = getattr(self,
                                                  '{}_cut'.format(cp))
            params['{}_column'.format(cp)] = getattr(self,
                                                     '{}_column'.format(cp))
        params['mvaFeatures'] = self.mvaFeatures
        params['only_max_pt'] = self.only_max_pt
        params['event_identifier_column'] = self.event_identifier_column
        return params

    def set_params(self, **kwargs):
        for cp in self.cut_parameters:
            cutname = '{}_cut'.format(cp)
            if cutname in kwargs and kwargs[cutname] is not None:
                setattr(self, cutname, kwargs.pop(cutname))
        for other_param in ['mvaFeatures',
                            'only_max_pt',
                            'event_identifier_column']:
            if other_param in kwargs and kwargs[other_param] is not None:
                setattr(self, other_param, kwargs.pop(other_param))
        super(CutBasedXGBClassifier, self).set_params(**kwargs)
        return self

    def fit(self, X, y, X_calib, y_calib, eval_set=None, **kwargs):
        if eval_set is not None:
            raise "Passing an eval_set to xgb ist not yet implemented"
        X, y = self.select(X, y)
        if len(X) < 2:
            return self
        if X_calib is None or y_calib is None:
            X, X_calib, y, y_calib = train_test_split(X, y, test_size=0.5)
        else:
            X_calib, y_calib = self.select(X_calib, y_calib)
        super(CutBasedXGBClassifier, self).fit(X, y, **kwargs)
        probas = super(CutBasedXGBClassifier, self).predict_proba(X_calib)[:, 1]
        self.calibrator.fit(probas.reshape(-1, 1), y_calib)
        self.fit_successful = True
        return self

    def predict_proba(self, data, **kwargs):
        data = self.select(data)
        try:
            uncalibrated = (super(CutBasedXGBClassifier, self)
                            .predict_proba(data, **kwargs))
            return self.calibrator.predict_proba(uncalibrated[:, 1])
        except:
            return np.array([[0.5, 0.5]])

    def score(self, X, y, sample_weight=None):
        probas = self.predict_proba(X)[:, 1]
        sc = tagging_power_score(probas, efficiency=self.efficiency_,
                                 sample_weight=sample_weight)
        return sc

def setup_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--slices', type=int, default=None)
    parser.add_argument('-n', '--n-folds', type=int, default=2)
    parser.add_argument('-x', '--xgboost-jobs', type=int, default=1)
    parser.add_argument('-c', '--cv-jobs', type=int, default=1)
    parser.add_argument('-i', '--iter', type=int, default=100)
    parser.add_argument('-y', '--year', type=int, default=2011)
    return parser.parse_args()

def main():
    args = setup_args()

    # style setup
    plt.style.use('ggplot')
    plt.rcParams['figure.figsize'] = (10, 10)
    plt.rcParams['font.size'] = 14

    # just define some keyword arguments for read_root in a separate dict
    chunksize = 5000
    data_kwargs = dict(
        key='DecayTree',  # the tree name
        columns=['B_OS_Muon*',  # all branches that should be read
                 'B_ID',
                 'B_PT',
                 'SigYield_sw',
                 'runNumber',
                 'eventNumber',
                 ],
        chunksize=chunksize,  # this will create a generator, yielding subsets
                              # with 'chunksize' of the data
        where='(B_LOKI_MASS_JpsiConstr_NoPVConstr>0)',  # a ROOT where
                                                        # selection, does not
                                                        # work with
                                                        # array-variables
        flatten=True  # will flatten the data in the dimension of the first
                      # given column
    )

    data_dir = '/home/kheinicke/tank/flavourtagging/'
    year = args.year
    filenames = [
        data_dir + 'Bu2JpsiK_mu-k-e-TrainingTuple_' + str(year) + '_MD_sweighted_kheinick.root',
        data_dir + 'Bu2JpsiK_mu-k-e-TrainingTuple_' + str(year) + '_MU_sweighted_kheinick.root',
        data_dir + 'Bu2JpsiK_mu-k-e-TrainingTuple_' + str(year + 1) + '_MD_sweighted_kheinick.root',
        data_dir + 'Bu2JpsiK_mu-k-e-TrainingTuple_' + str(year + 1) + '_MU_sweighted_kheinick.root',
    ]
    chunksize = 5000

    # still have to use plain ROOT to get the number of entries...
    n_entries = 0
    for fn in filenames:
        f = ROOT.TFile(fn)
        t = f.Get('DecayTree')
        n_entries += t.GetEntries()

    # This will read chunks of the data inside a list comprehension and then
    # concat those to a big dataframe note that tqdm is just some boilerplate
    # to generate a progressbar
    if args.slices is None:
        total = n_entries / chunksize
    else:
        total = args.slices
    print('Reading {} chunks of {}-data'.format(total, year))
    df = pd.concat([df for df in tqdm(
        islice(read_root(filenames, **data_kwargs), args.slices),
        total=total)
    ])

    # prepare dataframe
    df['target'] = np.sign(df.B_ID) == np.sign(df.B_OS_Muon_ID)
    df.rename(columns=dict(zip(df.columns, [c.replace('B_OS_Muon', 'tp')
                                            for c in df.columns])),
              inplace=True)
    df['tp_ABS_RecVertexIP'] = np.abs(df.tp_RecVertexIP)
    df['event_id'] = df.runNumber.apply(str) + '_' + df.eventNumber.apply(str)

    # this is the list of BDT variables formerly used
    classic_MVA_features = ['tp_' + c for c in [
        'partP',
        'partPt',
        'IPPU',
        'ghostProb',
        'PIDNNm',
        'ABS_RecVertexIP',
        'mult',
        'ptB',
        'IPs',
    ]]

    # instead of using the sklearn randomized gridsearch, implement one that
    # handles 3-fold validation
    parameter_grid = OrderedDict([
            ('P_cut', np.linspace(1, 5, 21)),
            ('PT_cut', np.linspace(0, 3, 16)),
            ('phiDistance_cut', np.linspace(0, 0.5, 6)),
            ('TRGHP_cut', np.linspace(0, 0.6, 6)),
            ('IPPUs_cut', np.linspace(1, 4, 16)),
            ('TRCHI2DOF_cut', np.linspace(2, 5, 6)),
            ('PROBNNmu_cut', np.linspace(0.0, 0.6, 6)),
        ])

    # produce all possible grid points
    grid = []
    for cut_values in product(*parameter_grid.values()):
        grid.append({p: v for p, v in zip(parameter_grid.keys(), cut_values)})
    np.random.shuffle(grid)

    # iterate the grid points
    scores = []
    print('Starting grid search')
    for classifier_args in tqdm(grid[:args.iter]):
        # yield 3-fold split for this grid point
        df_sets = [df.iloc[indices] for indices in NSplit(df)]

        cv_scores = []
        for i in range(3):
            df1, df2, df3 = df_sets[i % 3], df_sets[(i + 1) % 3], df_sets[(i + 2) % 3]
            model = CutBasedXGBClassifier(nthread=args.xgboost_jobs,
                                          mvaFeatures=classic_MVA_features,
                                          n_estimators=300,
                                          **classifier_args)
            model.fit(df1, df1.target,
                      df2, df2.target)
            if model.fit_successful:
                cv_scores.append(model.score(df3, df3.target))
            else:
                cv_scores.append(0)

        scores.append((np.mean(cv_scores), cv_scores, classifier_args))

    scores = sorted(scores, key=lambda s: s[0], reverse=True)
    print('tagging power {}% with params\n{}'
          .format(100 * ufloat(scores[0][0], np.std(scores[0][1])),
                  scores[0][2]))
    if not path.isdir('build/'):
        mkdir('build/')
        print('build/ directory created')
    output_filename = ('build/search-{:%Y%m%d-%H%M%S}.json'
                       .format(datetime.datetime.now()))
    with open(output_filename, 'w') as f:
        json.dump({'settings': vars(args),
                   'scores': scores,
                   'parameter-grid': {k: list(v) for k, v in parameter_grid.items()},
                   }, f, indent=4)
        print('grid search results have been written to {}'
              .format(output_filename))

if __name__ == '__main__':
    main()


from typing import List

class Solution:
	def searchInsert(self, nums: List[int], target: int) -> int:
		ans, left, right = len(nums), 0, len(nums) - 1
		while left <= right:
			middle = ((right - left) >> 1) + left
			if target <= nums[middle]:
				right = middle - 1
				ans = middle
			else:
				left = middle + 1
		return ans

if __name__ == '__main__':
	print(Solution().searchInsert(nums=[1, 3, 5, 6], target=7))

LeetCode/Problems/35. Search Insert Position/python.py


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
import numpy as np

from mlp import MLPRegressor

# Data sets
N = 100
alpha = 0.8
num_train = int(np.ceil(alpha*N*N))

x = np.linspace(-10, 10, N)
y = np.linspace(-10, 10, N)
xc, yc = np.meshgrid(x, y)
zc = xc * yc

z = np.stack([xc, yc], 0)
z = np.transpose(np.reshape(z, (2, N*N)))
zc = np.reshape(zc, (N*N,))

zzc = np.concatenate([z, zc[:, np.newaxis]], axis=1)
np.random.shuffle(zzc)
zzc_train = zzc[:num_train, :]
zzc_test = zzc[num_train:, :]

x_train = zzc_train[:, :2]
z_train = zzc_train[:, 2:]

x_test = zzc_test[:, :2]
z_test = zzc_test[:, 2:]

# Create regressor
regressor = MLPRegressor([2, 10, 10, 1], 'relu', learning_rate=0.001,
                         optimizer='Adam')

# Fitting
regressor.fit(x=x_train, y=z_train, steps=100000)

# Evaluation
l2_error = regressor.eval(x=x_test, y=z_test)
print('l2 error: {}'.format(l2_error))


# Visual elements
DRAW_COLOR_BACKGROUND = (90, 0, 85)
DRAW_COLOR_INFOPANEL_BACKGROUND = (45, 0, 42)
DRAW_COLOR_INFOPANEL_TEXT = (100, 100, 100)
DRAW_PUNTO_INFOPANEL_TEXT = 11
DRAW_COLOR_BASE = (0xC1, 0x1D, 0x0A)
DRAW_COLOR_GOAL = (0xFD, 0x84, 0x1F)
DRAW_COLOR_NODE = (0x25, 0x1B, 0x37)
DRAW_COLOR_LINE = (0x85, 0x4B, 0xC7)
DRAW_COLOR_TRACE = (0xFF, 0x00, 0x00)

DRAW_RADIUS_GOAL = 15
DRAW_RADIUS_BASE = 10
DRAW_RADIUS_NODE = 5

DRAW_ACTIVE_SIZE = (500, 500)
DRAW_INFOPANEL_OFFSET_Y = 25

# Algorithm parameters
RRT_DISTANCE_MAX = 40  # in px
RRT_DISTANCE_MIN = 10  # in px

# https://colorhunt.co/palette/fd841fe14d2acd104d9c2c77


from django.contrib import admin
from django.urls import path, include
#from . import views
from . views import (
    PostListView,
    PostDetailView,
    PostCreateView,
    PostUpdateView,
    PostDeleteView,
    PostSearchView
)

'''urlpatterns = [
    path('', views.home,name='home'),
]'''
urlpatterns = [
    #path('',views.home,name='home')
    path('',PostListView.as_view(),name='home'),
    path('post/<int:pk>/', PostDetailView.as_view(), name='post-detail'),
    path('post/new',PostCreateView.as_view(),name='post-create'),
    path('post/<int:pk>/update/',PostUpdateView.as_view(),name='post-update'),
    path('post/<int:pk>/delete/',PostDeleteView.as_view(),name='post-delete'),
    path('search',PostSearchView.as_view(),name='search')

]


from django.db import models
Choices=[
    ("Select", "Select"),

    ("Entertinment",
     (
         ("Cinema", "Cinema"),
     ("Stories", "Stories"),
     ("Others", "Others"),
     )
     ),
    ("Business", "Business"),
    ("Sports", "Sports"),
    ("Education", "Education"),
    ("Careers",
      (
         ("IT jobs", "IT Jobs"),
         ("Non-IT jobs", "Non-IT Jobs"),
         ("Government jobs", "Government jobs"),
         ("Others", "Others"),
      )
    ),
    ("Hot News", "Hot News"),
]
# Create your models here.
class NewsModel(models.Model):
    news_type = models.CharField(max_length=50, choices=Choices, default="Select")
    news_headline = models.CharField(max_length=50)
    post_news_text = models.CharField(max_length=500)
    post_news_image = models.FileField(upload_to="app_news/static/Images",null=True, blank=True)
    def __str__(self):
        return self.news_type


from flask import Blueprint, render_template, current_app , session

from web_app.routes.wrappers import authenticated_route
from web_app.routes.forms.create_funding_form import CreateFundingForm

funding_routes = Blueprint("funding_routes", __name__)

@funding_routes.route("/create-funding", methods=['GET'])
@authenticated_route
def create_funding():
    service = current_app.config["FIREBASE_SERVICE"]
    fund_types = service.fetch_fund_types()
    form = CreateFundingForm()
    form.fundTypeSelect.choices = [(ft['type'], ft['type'].lower().replace("_"," ")) for ft in fund_types]
    return render_template("create_funding.html", form=form, fund_types=fund_types)

@funding_routes.route("/create-funding", methods=["POST"])
def handle_create_funding():
    form = CreateFundingForm()
    print("FORM: ", form)
    if form.validate_on_submit():
        title = form.fundraiserTitle.data
        print("TITLE: ", title)


"""
Given two straight line segments (represented as a start point and an end point),
compute the point of intersection, if any.
"""

def get_equation(arr1,arr2):
	x1,y1=arr1
	x2,y2=arr2
	m = (y2-y1)/(x2-x1)
	c = y1-m*x1
	return m,c

def is_on_segment(x,y,arr1,arr2):
	x1,y1=arr1
	x2,y2=arr2
	return min(x1, x2) <= x <= max(x1, x2) and min(y1, y2) <= y <= max(y1, y2)

def x_inter(arr1,arr2,arr3,arr4):
	m1,c1=get_equation(arr1,arr2)
	m2,c2=get_equation(arr3,arr4)
	try:
		x = (c2-c1)/(m1-m2)
		y = m1*x+c1
		if is_on_segment(x,y,arr1,arr2) and is_on_segment(x,y,arr3,arr4):
			return x,y
		else:
			print('Not on segment')
	except ZeroDivisionError:
		print('No intercept')

arr1,arr2,arr3,arr4 = [1,1],[2,2],[-1,1],[0,4]
print(x_inter(arr1,arr2,arr3,arr4))

arr1,arr2,arr3,arr4 = [0,1],[1,1],[0,3],[1,3]
print(x_inter(arr1,arr2,arr3,arr4))

# import matplotlib.pyplot as plt
# plt.plot([-10,2],[10,20])
# plt.plot([-1,0],[10,40])
# plt.plot([-0.7428571428571428],[17.714285714285715], 'rx')
# plt.show()

arr1,arr2,arr3,arr4 = [-10,10],[2,20],[-1,10],[0,40]
print(x_inter(arr1,arr2,arr3,arr4))


from django.db import migrations

def add_managers_group_permissions(apps, schema_editor):
    """
    Add permissions for "Managers" group.
    "Managers" have next permissions:
    - User - Can add/change/delete/view permission
    """
    group = apps.get_model("auth", "Group")
    permission = apps.get_model("auth", "Permission")
    db_alias = schema_editor.connection.alias
    permissions_code_names = [
        "add_user",
        "change_user",
        "delete_user",
        "view_user",
    ]
    permissions = permission.objects.using(db_alias).filter(
        codename__in=permissions_code_names
    )
    managers_group, created = group.objects.get_or_create(name="Managers")
    managers_group.permissions.set(permissions)

class Migration(migrations.Migration):
    dependencies = [
        ("user", "0005_add_permissions_groups"),
    ]
    operations = [
        migrations.RunPython(add_managers_group_permissions),
    ]


from pwn import *
from lib import *

# context.log_level = 'debug'

r = remote('cns.csie.org', 12805)

public_key = [None for i in range(4)]

r.recvuntil("The public key of server0 is ")
pk0 = f"{r.recvline()}"[3:-4]
public_key[0] = list(map(int, pk0.split(",")))
r.recvuntil("The public key of server1 is ")
pk1 = f"{r.recvline()}"[3:-4]
public_key[1] = list(map(int, pk1.split(",")))
r.recvuntil("The public key of server2 is ")
pk2 = f"{r.recvline()}"[3:-4]
public_key[2] = list(map(int, pk2.split(",")))
r.recvuntil("The public key of Bob is ")
pk_bob = f"{r.recvline()}"[3:-4]
public_key[3] = list(map(int, pk_bob.split(",")))

r.recvuntil("Send the message ")
message = r.recvline()[1:-9]

r.recvuntil("The route of the packet should be ")
route = f"{r.recvline()}"[3: -28]
route = list(map(int, route.split(",")))
route.reverse()

packet = Packet.create(message, 3, public_key[3])
for i in range(1, len(route)):
    packet.add_next_hop(route[i-1], public_key[route[i]])

r.sendlineafter("> ", packet.data.hex())
r.recvuntil("Bob: ")
print(r.recvline())


import torch

#########################################################
# Operation List:
#   1. gather
#   2. radius gaussian
#   3. closest_pool
#   4. max_pool
#   5. global_average
#########################################################

########################################
# 1. gather
########################################

def gather(x, idx, method=2):
    """
    implementation of a custom gather operation for faster backwards.
    :param x: input with shape [N, D_1, ... D_d]
    :param idx: indexing with shape [n_1, ..., n_m]
    :param method: Choice of the method
    :return: x[idx] with shape [n_1, ..., n_m, D_1, ... D_d]
    """

    if method == 0:
        return x[idx]
    elif method == 1:
        x = x.unsqueeze(1)
        x = x.expand((-1, idx.shape[-1], -1))
        idx = idx.unsqueeze(2)
        idx = idx.expand((-1, -1, x.shape[-1]))
        return x.gather(0, idx)
    elif method == 2:
        for i, ni in enumerate(idx.size()[1:]):
            x = x.unsqueeze(i+1)
            new_s = list(x.size())
            new_s[i+1] = ni
            x = x.expand(new_s)
        n = len(idx.size())
        for i, di in enumerate(x.size()[n:]):
            idx = idx.unsqueeze(i+n)
            new_s = list(idx.size())
            new_s[i+n] = di
            idx = idx.expand(new_s)
        return x.gather(0, idx)
    else:
        raise ValueError('Unkown method')

########################################
# 2. radius_gaussian
########################################

def radius_gaussian(sq_r, sig, eps=1e-9):
    """
    Compute a radius gaussian (gaussian of distance)
    :param sq_r: input radiuses [dn, ..., d1, d0]
    :param sig: extents of gaussians [d1, d0] or [d0] or float
    :return: gaussian of sq_r [dn, ..., d1, d0]
    """
    return torch.exp(-sq_r / (2 * sig**2 + eps))

########################################
# 3. closest_pool
########################################

def closest_pool(x, inds):
    """
    Pools features from the closest neighbors. WARNING: this function assumes the neighbors are ordered.
    :param x: [n1, d] features matrix
    :param inds: [n2, max_num] Only the first column is used for pooling
    :return: [n2, d] pooled features matrix
    """

    # Add a last row with minimum features for shadow pools
    x = torch.cat((x, torch.zeros_like(x[:1, :])), 0)

    # Get features for each pooling location [n2, d]
    return gather(x, inds[:, 0])

########################################
# 4. max_pool
########################################

def max_pool(x, inds):
    """
    Pools features with the maximum values.
    :param x: [n1, d] features matrix
    :param inds: [n2, max_num] pooling indices
    :return: [n2, d] pooled features matrix
    """

    # Add a last row with minimum features for shadow pools
    x = torch.cat((x, torch.zeros_like(x[:1, :])), 0)

    # Get all features for each pooling location [n2, max_num, d]
    pool_features = gather(x, inds)

    # Pool the maximum [n2, d]
    max_features, _ = torch.max(pool_features, 1)
    return max_features

########################################
# 5. max_pool
########################################

def global_average(x, batch_lengths):
    """
    Block performing a global average over batch pooling
    :param x: [N, D] input features
    :param batch_lengths: [B] list of batch lengths
    :return: [B, D] averaged features
    """

    # Loop over the clouds of the batch
    averaged_features = []
    i0 = 0
    for b_i, length in enumerate(batch_lengths):

        # Average features for each batch cloud
        averaged_features.append(torch.mean(x[i0:i0 + length], dim=0))

        # Increment for next cloud
        i0 += length

    # Average features in each batch
    return torch.stack(averaged_features)


import cv2
import glob
import numpy as np
import math
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
from scipy import stats

img_path_pos = glob.glob("D:/winuser/Desktop/HW3/dataset/pos/*.jpg")
img_path_neg = glob.glob("D:/winuser/Desktop/HW3/dataset/neg/*.jpg")
pos_mean_matrix = np.zeros((10, 10), dtype=np.float)
neg_mean_matrix = np.zeros((10, 10), dtype=np.float)

for path1 in img_path_pos:
    img_pos = cv2.imread(path1)
    img_pos_resize = cv2.resize(img_pos, (10, 10))
    img_pos_gray = cv2.cvtColor(img_pos_resize, cv2.COLOR_BGR2GRAY)
    pos_mean_matrix += img_pos_gray / len(img_path_pos)
cv2.imwrite("D:/winuser/Desktop/DM and Vis/HW3/pos_mean_matrix.jpg", pos_mean_matrix)
for path1 in img_path_neg:
    img_neg = cv2.imread(path1)
    img_neg_resize = cv2.resize(img_neg, (10, 10))
    img_neg_gray = cv2.cvtColor(img_neg_resize, cv2.COLOR_BGR2GRAY)
    neg_mean_matrix += img_neg_gray / len(img_path_neg)

plt.hist(pos_mean_matrix.flatten(), normed=True, color='pink', edgecolor='k')
plt.hist(neg_mean_matrix.flatten(), normed=True, color='lightblue', edgecolor='k')

line1 = sns.kdeplot(pos_mean_matrix.flatten(), color='r', linewidth=2, label='pos')
line2 = sns.kdeplot(neg_mean_matrix.flatten(), color='b', linewidth=2, label='neg')
plt.tick_params(top='off', right='off')
plt.xlabel('Brightness distribution(0 ~ 255)')
plt.ylabel('Probability')
plt.legend([line1, line2], labels=['pos', 'neg'], loc='best')
plt.savefig('D:/winuser/Desktop/DM and Vis/HW3/pic.jpg')
plt.close('all')

gaussian1 = stats.norm(loc=pos_mean_matrix.mean(), scale=1.0)
gaussian2 = stats.norm(loc=neg_mean_matrix.mean(), scale=1.0)
x2 = np.linspace(113.0, 140.0, 1000)
y1 = gaussian1.pdf(x2)
y2 = gaussian2.pdf(x2)
plt.xlabel('Brightness distribution(0 ~ 255)')
plt.ylabel('Probability')
a1 = plt.plot(x2, y1, color='red', label="pos")
a2 = plt.plot(x2, y2, color='blue', label='neg')
plt.legend([a1, a2], labels=['pos', 'neg'], loc='best')
plt.savefig('D:/winuser/Desktop/DM and Vis/HW3/gaussian.jpg')

for k in range(2):
    test = cv2.imread("D:/winuser/Desktop/DM and Vis/HW3/test{}.jpg".format(k + 1))
    test_gray = cv2.cvtColor(test, cv2.COLOR_BGR2GRAY)
    test_h = test_gray.shape[0]
    test_w = test_gray.shape[1]
    for i in range(math.floor(test_h / 10) - 1):
        for j in range(math.floor(test_w / 10) - 1):
            crop_test = test_gray[i * 10: (i + 1) * 10, j * 10: (j + 1) * 10]
            pixel = 0
            for m in range(10):
                for n in range(10):
                    pixel += crop_test[m, n]
            pixel_mean = pixel / 100
            if pixel_mean >= (pos_mean_matrix.mean() + neg_mean_matrix.mean()) / 2:
                cv2.rectangle(test, (j * 10, i * 10), ((j + 1) * 10, (i + 1) * 10), (0, 255, 0), 2)
        cv2.imwrite("D:/winuser/Desktop/DM and Vis/HW3/test{}_result1.jpg".format(k + 1), test)


import os
import json
import socket
import time

import utils

from front_base.ip_manager import IpManagerBase

class IpManager(IpManagerBase):
    def __init__(self, config, default_domain_fn, domain_fn, speed_fn, logger):
        super(IpManager, self).__init__(config, None, logger, speed_fn)
        self.default_domain_fn = default_domain_fn
        self.domain_fn = domain_fn

        self.domain_map = self.load_domains()
        # top_domain -> {}

    def __str__(self):
        o = ""
        o += " domain_map: \r\n%s\r\n" % json.dumps(self.domain_map, indent=2)
        return o

    def load_domains(self):
        domain_map = {}  # top_domain -> {}
        for fn in [self.domain_fn, self.default_domain_fn]:
            if not os.path.isfile(fn):
                continue

            try:
                with open(fn, "r") as fd:
                    ds = json.load(fd)
                    for top in ds:
                        domain_map[str(top)] = {
                            "links": 0,
                            "fail_times": 0,
                            "last_try": 0.0
                        }
                self.logger.info("load %s success", fn)
                break
            except Exception as e:
                self.logger.warn("load %s for host failed:%r", fn, e)
        return domain_map

    def save_domains(self, domains):
        ns = []
        for top_domain, _ in self.domain_map.items():
            ns.append(top_domain)

        if ns == domains:
            self.logger.debug("save domains not changed, ignore")
            return
        else:
            self.logger.info("save domains:%s", domains)

        dat = {}
        for domain in domains:
            dat[domain] = ["www." + domain]

        with open(self.domain_fn, "w") as fd:
            json.dump(dat, fd)

        self.domain_map = self.load_domains()

    def get_ip_sni_host(self):
        now = time.time()
        for top_domain, info in self.domain_map.items():
            if info["links"] < 0:
                info["links"] = 0

            if info["links"] >= self.config.max_connection_per_domain:
                continue

            if info["fail_times"] and now - info["last_try"] < 60:
                continue

            sni = "www." + top_domain
            try:
                ip = socket.gethostbyname(sni)
            except Exception as e:
                self.logger.warn("get ip for %s fail:%r", sni, e)
                continue

            self.logger.debug("get ip:%s sni:%s", ip, sni)
            info["links"] += 1
            info["last_try"] = now
            return ip, sni, top_domain

        return None, None, None

    def _get_domain(self, top_domain):
        self.domain_map.setdefault(top_domain, {
                            "links": 0,
                            "fail_times": 0,
                            "last_try": 0.0
                        })
        return self.domain_map[top_domain]

    def report_connect_fail(self, ip_str, sni=None, reason="", force_remove=True):
        ip, _ = utils.get_ip_port(ip_str)
        ip = utils.to_str(ip)
        top_domain = ".".join(sni.split(".")[1:])

        info = self._get_domain(top_domain)
        info["fail_times"] += 1
        info["links"] -= 1
        self.logger.debug("ip %s sni:%s connect fail, reason:%s", ip, sni, reason)

    def update_ip(self, ip_str, sni, handshake_time):
        top_domain = ".".join(sni.split(".")[1:])

        info = self._get_domain(top_domain)
        info["fail_times"] = 0
        info["last_try"] = 0.0
        # self.logger.debug("ip %s sni:%s connect success, rtt:%f", ip, sni, handshake_time)

    def ssl_closed(self, ip_str, sni=None, reason=""):
        ip, _ = utils.get_ip_port(ip_str)
        ip = utils.to_str(ip)
        top_domain = ".".join(sni.split(".")[1:])

        try:
            info = self._get_domain(top_domain)
            info["links"] -= 1
            self.logger.debug("ip %s sni:%s connect closed reason %s", ip, sni, reason)
        except Exception as e:
            self.logger.warn("ssl_closed %s sni:%s reason:%s except:%r", ip_str, sni, reason, e)


'''
Cálculo de areas en base a clusterización utilizando k-means, en imagenes satelite
parametros:
-- file imagen satelite
-- area area en km2 representada en la imagen satelite
se utilizan 3 clusters por imagen, (conversion a niveles de gris) con salto amplio
en la polarización de la imagen

MIT License
'''
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
from sklearn.cluster import KMeans
import argparse
# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-f", "--file", required=True,
                help="satellite image")
ap.add_argument("-a", "--area", required=True,
	help="satellite image area in km2")

args = vars(ap.parse_args())
# open file name
filename = args["file"]
area = args["area"]
I = Image.open(filename)

#resize and paint with axis
plt.figure(figsize=(8,8))
plt.imshow(I)
plt.axis('on')
plt.show()

# paint in gray scale
I1 = I.convert('L')
I2 = np.asarray(I1,dtype=np.float)

plt.figure(figsize=(8,8))
plt.imshow(I2,cmap='gray')
plt.axis('on')
plt.show()

# define 3 clusters and grey scale (black, white, mix scale)
X = I2.reshape((-1, 1))
k_means = KMeans(n_clusters=3)
'''
k_means=KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,
    n_clusters=3, n_init=10, n_jobs=1, precompute_distances='auto',
    random_state=None, tol=0.0001, verbose=0)
'''

k_means.fit(X)

centroides = k_means.cluster_centers_
etiquetas = k_means.labels_

I2_compressed = np.choose(etiquetas, centroides)
I2_compressed.shape = I2.shape

plt.figure(figsize=(8,8))
plt.imshow(I2_compressed,cmap='gray')
plt.axis('on')
plt.show()

I2 = (I2_compressed-np.min(I2_compressed))/(np.max(I2_compressed)-np.min(I2_compressed))*255
I2 = Image.fromarray(I2.astype(np.uint8))
w, h =I2.size
colors = I2.getcolors(w * h)
print('density points:',colors)

print (u'Área = ',  float(area)*float(colors[0][0])/float(w*h), 'km2')