#! /usr/bin/env python
"""
Testing correlation of dis parameters
"""
from __future__ import print_function
import os
from itertools import product

import numpy as np
import numpy.ma as ma
from numpy.linalg import inv
import scipy.optimize as opt
import h5py

from pisa.utils.log import logging, set_verbosity
from uncertainties import ufloat, unumpy
import ROOT

import matplotlib as mpl
# headless mode
mpl.use('Agg')
# fonts
mpl.rcParams['mathtext.fontset'] = 'custom'
mpl.rcParams['mathtext.rm'] = 'Bitstream Vera Sans'
mpl.rcParams['mathtext.it'] = 'Bitstream Vera Sans:italic'
mpl.rcParams['mathtext.bf'] = 'Bitstream Vera Sans:bold'
from matplotlib import pyplot as plt
from matplotlib.offsetbox import AnchoredText

TYPE = 14
NAME = r'gevgen_{0}'.format(TYPE)
CACHE = '/data/icecube/data/NuTeV/{0}/cache.hdf5'.format(TYPE)
DATA_NU = '/data/icecube/data/NuTeV/nutev_data/data/nutevpack/NuVec.dat'
DATA_NUBAR= '/data/icecube/data/NuTeV/nutev_data/data/nutevpack/NubarVec.dat'
DATA_NU_COV = '/data/icecube/data/NuTeV/nutev_data/data/nutevpack/NuMatr.hdf5'
DATA_NUBAR_COV = '/data/icecube/data/NuTeV/nutev_data/data/nutevpack/NubarMatr.hdf5'
Y_BINNING = np.array([0, 0.001, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8,
                      0.9, 0.97, 1.0])
X_BINNING = np.array([0.0001, 0.03, 0.06, 0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5,
                      0.6, 0.7, 0.8])
E_BINNING = np.array([30, 40, 50, 60, 70, 80, 90, 100, 120, 140, 160, 180, 200,
                      230, 260, 290, 320, 360])
Y_CENTERS = np.array([0.0005, 0.026, 0.075, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65,
                      0.75, 0.85, 0.935, 0.985])
X_CENTERS = np.array([0.015, 0.045, 0.08, 0.125, 0.175, 0.225, 0.275, 0.35,
                      0.45, 0.55, 0.65, 0.75])
E_CENTERS = np.array([35, 45, 55, 65, 75, 85, 95, 110, 130, 150, 170, 190, 215,
                      245, 275, 305, 340])
WA_NU = 0.677E-38
WA_NUBAR = 0.334E-38

def get_bin_sizes(bin_edges):
    return np.abs(np.diff(bin_edges))

def load_genie_mc():
    """Load parameters from a GENIE gst ROOT file or from a cached HDF5
    file if it exists.
    """
    logging.info('Loading GENIE parameteters...')
    is_cache = False
    if os.path.isfile(CACHE):
        is_cache = True

    STR_R = ['pdg_array', 'energy_array', 'y_array', 'x_array', 'dis_array',
             'cc_array']
    def cache_events(params_dict):
        h5file = h5py.File(CACHE, 'w')
        for key in params_dict.iterkeys():
            h5file.create_dataset(key, data=params_dict[key])
        h5file.close()

    params_dict = {}
    if not is_cache:
        logging.info('Loading gst files from folder {0}'.format(DIR))
        infiles = []
        for file_name in os.listdir(DIR):
            file_path = DIR + '/' + file_name
            if os.path.isfile(file_path) and (file_path.endswith('.gst.root')):
                infiles.append(ROOT.TFile(file_path))
        for key in STR_R.iterkeys():
            params_dict[key] = []
        for r_file in infiles:
            for event in r_file.gst:
                params_dict['pdg_array'].append(event.neu)
                params_dict['energy_array'].append(event.Ev)
                params_dict['y_array'].append(event.y)
                params_dict['x_array'].append(event.x)
                params_dict['dis_array'].append(event.dis)
                params_dict['cc_array'].append(event.cc)
        for key in STR_R.iterkeys():
            params_dict[key] = np.array(params_dict[key])
        cache_events(params_dict)
    else:
        logging.info('Loading from cached file {0}'.format(CACHE))
        h5file = h5py.File(CACHE, 'r')
        for key in h5file.iterkeys():
            params_dict[key] = np.array(h5file[key][:])
        h5file.close()

    return params_dict

def only_discc(params_dict):
    x_array = params_dict['x_array']
    invalid_entries = x_array < 0
    discc = (params_dict['dis_array'] & params_dict['cc_array']).astype(bool)
    params_dict['x_array'] = x_array[~invalid_entries & discc]
    params_dict['energy_array'] = \
            params_dict['energy_array'][~invalid_entries & discc]
    params_dict['y_array'] = \
            params_dict['y_array'][~invalid_entries & discc]
    params_dict['pdg_array'] = \
            params_dict['pdg_array'][~invalid_entries & discc]
    params_dict['dis_array'] = \
            params_dict['dis_array'][~invalid_entries & discc]
    params_dict['cc_array'] = \
            params_dict['cc_array'][~invalid_entries & discc]

    return params_dict

def make_histo(array, weights):
    hist, edges = np.histogram(array, bins=Y_BINNING, weights=weights)
    hist_2, edges = np.histogram(array, bins=Y_BINNING, weights=weights**2)

    u_hist = unumpy.uarray(hist, np.sqrt(hist_2))
    return ma.masked_equal(u_hist, 0)

def evaluate(params_dict, systematics, nu=True, shape_only=False):
    a_sys = systematics
    # a_sys, b_sys, c_sys = systematics
    e_bin_sizes = get_bin_sizes(E_BINNING).astype(float)
    x_bin_sizes = get_bin_sizes(X_BINNING).astype(float)
    y_bin_sizes = get_bin_sizes(Y_BINNING).astype(float)

    nu_histograms = unumpy.uarray(
        np.zeros(map(len, (e_bin_sizes, x_bin_sizes,
                           y_bin_sizes))).astype(float),
        np.zeros(map(len, (e_bin_sizes, x_bin_sizes,
                           y_bin_sizes))).astype(float)
    )
    scaling = []

    logging.info('a_systematic {0}'.format(a_sys))
    # logging.info('b_systematic {0}'.format(b_sys))
    # logging.info('c_systematic {0}'.format(c_sys))
    # weights = c_sys + np.power(params_dict['x_array'], -a_sys) * b_sys
    weights = np.power(params_dict['x_array'], -a_sys)

    if nu: nu_mask = params_dict['pdg_array'] > 0
    else: nu_mask = params_dict['pdg_array'] < 0
    for e_idx, e_bin in enumerate(E_BINNING[:-1]):
        energy_array = params_dict['energy_array']
        energy_mask = (energy_array >= e_bin) & \
                (energy_array < E_BINNING[e_idx + 1])
        weights_e = weights[energy_mask]
        y_array_e = params_dict['y_array'][energy_mask]
        x_array_e = params_dict['x_array'][energy_mask]
        nu_mask_e = nu_mask[energy_mask]

        sigma_nu  = 0
        nu_histograms[e_idx] = {}
        for x_idx, x_bin in enumerate(X_BINNING[:-1]):
            x_mask = (x_array_e >= x_bin) & (x_array_e < X_BINNING[x_idx+1])
            y_array_e_x = y_array_e[x_mask]
            weights_e_x = weights_e[x_mask]
            nu_mask_e_x = nu_mask_e[x_mask]

            sigma_nu_x_nw = make_histo(
                y_array_e_x[nu_mask_e_x], np.ones(weights_e_x[nu_mask_e_x].shape)
            )
            sigma_nu_x = make_histo(
                y_array_e_x[nu_mask_e_x], weights_e_x[nu_mask_e_x]
            )
            sigma_nu_x_e_nw = sigma_nu_x_nw * (1 / E_CENTERS[e_idx].astype(float))
            sigma_nu_x_e = sigma_nu_x * (1 / E_CENTERS[e_idx].astype(float))

            factors = 1 / (y_bin_sizes * x_bin_sizes[x_idx])
            nu_histograms[e_idx][x_idx] = sigma_nu_x_e * factors
            if shape_only:
                sigma_nu += np.sum(unumpy.nominal_values(sigma_nu_x_e.data))
            else:
                sigma_nu += np.sum(unumpy.nominal_values(sigma_nu_x_e_nw.data))

        if nu: scaling.append((WA_NU / 1E-38) / sigma_nu)
        else: scaling.append((WA_NUBAR / 1E-38) / sigma_nu)
    scaling = ma.masked_invalid(scaling)
    nu_histograms = ma.masked_equal(nu_histograms, 0)

    scaled_histograms = nu_histograms * \
            scaling[:, np.newaxis][:, np.newaxis]
    return scaled_histograms

if __name__ == '__main__':
    set_verbosity(3)

    params_dict = load_genie_mc()
    params_dict = only_discc(params_dict)

    a_vals = np.linspace(0.01, 0.2, 40)

    nu = True
    # nu = False

    expectation_array = []
    b_vals_array = []
    for a in a_vals:
        if nu:
            opt_histogram = evaluate(
                params_dict, a, nu=True, shape_only=False
            )
        else:
            opt_histogram = evaluate(
                params_dict, a, nu=False, shape_only=False
            )

        if nu: expectation = 1 - 1.65125 * a
        else: expectation = 1 - 1.8073 * a
        expectation_array.append(expectation)

        x_bin_sizes = get_bin_sizes(X_BINNING).astype(float)
        y_bin_sizes = get_bin_sizes(Y_BINNING).astype(float)
        b_vals = []
        for e_idx, e_bin in enumerate(E_CENTERS):
            integral = 0
            for x_idx, x_bin in enumerate(X_CENTERS):
                integral += np.sum(opt_histogram[e_idx][x_idx].data * \
                                   y_bin_sizes) * x_bin_sizes[x_idx]
            # print(integral)
            if nu: b = (WA_NU / 1E-38) / integral
            else: b = (WA_NUBAR / 1E-38) / integral
            b_vals.append(unumpy.nominal_values(b))
        b_val = ufloat(np.mean(b_vals), np.std(b_vals))
        b_vals_array.append(b_val)

    fig = plt.figure(figsize=(9, 5))
    ax = fig.add_subplot(111)
    ax.set_xlim(np.min(a_vals)-0.02, np.max(a_vals)+0.02)
    ax.set_ylim(0.6, 1.0)
    ax.tick_params(axis='x', labelsize=14)
    ax.tick_params(axis='y', labelsize=12)
    ax.set_xlabel('a', fontsize=18)
    ax.set_ylabel('b', fontsize=15)
    for ymaj in ax.yaxis.get_majorticklocs():
        ax.axhline(y=ymaj, ls=':', color='gray', alpha=0.7, linewidth=1)
    for xmaj in ax.xaxis.get_majorticklocs():
        ax.axvline(x=xmaj, ls=':', color='gray', alpha=0.7, linewidth=1)

    ax.errorbar(a_vals, expectation_array, xerr=0, yerr=0, capsize=3,
                alpha=0.5, linestyle='--', markersize=2, linewidth=1,
                color='blue', label='expectation')
    ax.errorbar(a_vals, unumpy.nominal_values(b_vals_array), xerr=0,
                yerr=unumpy.std_devs(b_vals_array), capsize=3, alpha=0.5,
                linestyle='--', markersize=2, linewidth=1, color='red',
                label='calculated')
    legend = ax.legend(prop=dict(size=12))
    plt.setp(legend.get_title(), fontsize=18)
    if nu:
        out = 'correlation_nu.png'
        tex = r'$\nu$'
    else:
        out = 'correlation_nubar.png'
        tex = r'$\bar{\nu}$'
    at = AnchoredText(tex, prop=dict(size=20), frameon=True, loc=2)
    at.patch.set_boxstyle("round,pad=0.,rounding_size=0.5")
    ax.add_artist(at)
    fig.savefig(out, bbox_inches='tight', dpi=150)


#!/usr/bin/env python

connector_status = {
    'altmarkets': 'bronze',
    'ascend_ex': 'silver',
    'binance': 'gold',
    'binance_perpetual': 'gold',
    'binance_perpetual_testnet': 'gold',
    'binance_us': 'bronze',
    'bitfinex': 'bronze',
    'bitget_perpetual': 'bronze',
    'bitmart': 'bronze',
    'bittrex': 'bronze',
    'bitmex': 'bronze',
    'bitmex_perpetual': 'bronze',
    'bitmex_testnet': 'bronze',
    'bitmex_perpetual_testnet': 'bronze',
    'bybit_perpetual': 'bronze',
    'bybit_perpetual_testnet': 'bronze',
    'bybit_testnet': 'bronze',
    'bybit': 'bronze',
    'coinbase_pro': 'bronze',
    'crypto_com': 'bronze',
    'dydx_perpetual': 'silver',
    'gate_io': 'silver',
    'gate_io_perpetual': 'silver',
    'hitbtc': 'bronze',
    'huobi': 'bronze',
    'k2': 'bronze',
    'kraken': 'bronze',
    'kucoin': 'silver',
    'kucoin_testnet': 'silver',
    'lbank': 'bronze',
    'loopring': 'bronze',
    'mexc': 'bronze',
    'ndax': 'bronze',
    'ndax_testnet': 'bronze',
    'okx': 'bronze',
    'perpetual_finance': 'bronze',
    'probit': 'bronze',
    'whitebit': 'bronze',
    'ciex': 'bronze',
    'uniswap': 'gold',
    'uniswapLP': 'gold',
    'pancakeswap': 'silver',
    'sushiswap': 'silver',
    'traderjoe': 'bronze',
    'quickswap': 'bronze',
    'perp': 'bronze',
    'openocean': 'bronze',
    'pangolin': 'bronze',
    'defikingdoms': 'bronze',
    'defira': 'bronze',
    'mad_meerkat': 'bronze',
    'vvs': 'bronze',
    'ref': 'bronze',
}

warning_messages = {
}

def get_connector_status(connector_name: str) -> str:
    """
    Indicator whether a connector is working properly or not.
    UNKNOWN means the connector is not in connector_status dict.
    RED means a connector doesn't work.
    YELLOW means the connector is either new or has one or more issues.
    GREEN means a connector is working properly.
    """
    if connector_name not in connector_status.keys():
        status = "UNKNOWN"
    else:
        return f"&c{connector_status[connector_name].upper()}"
    return status


# Write a function that returns the elements on odd positions in a list.

def getFromOddPosition(list_of_number):
    odd_position_array = []
    for i in range(1, len(list_of_numbers), 2):
        odd_position_array.append(list_of_number[i])

    return odd_position_array

list_of_numbers = [3,2,4,5,6,12,78,65,34,21,144,99,8,67,66,87,100]

print(getFromOddPosition(list_of_numbers))


from django import template
from betinf.models import Match, Game
register = template.Library()

@register.filter
def matchgames(games, match_pk):
    out_games = Game.objects.all().filter(match=Match.objects.get(match_pk=match_pk))
    return out_games

@register.filter
def addstr(arg1, arg2):
    """concatenate arg1 & arg2"""
    return str(arg1) + str(arg2)

@register.filter
def set_team1_score(sets, game):
    game = Game.objects.get(game_pk=game.game_pk)
    a = sets.filter(game=game, set_num=game.num_sets)
    if len(list(a))>0:
        return a[0].team1_score
    return -1

@register.filter
def set_team2_score(sets, game):
    game = Game.objects.get(game_pk=game.game_pk)
    a = sets.filter(game=game, set_num=game.num_sets)
    if len(list(a))>0:
        return a[0].team2_score
    return -1


'''Process timeout pragma in internal (non-function) nodes
Turns expressions into functions and calls'''
from .node import Node
from .edge import Edge
from .ast_.function import Function
from .ast_.call import FunctionCall
from .port import Port
from collections import OrderedDict

def collect_pragma_ir(node):
    '''collect all nodes involved in calculations affected
    by the pragma'''
    nodes, internal_edges, input_edges, output_edges = [], [], [], []

    group = Node.get_node_paragma_group(node)

    for n in group:
        new_nodes, new_internal_edges, new_input_edges = n.trace_back()
        nodes += new_nodes
        internal_edges += new_internal_edges
        input_edges += new_input_edges

    for n in group:
        for o_p in n.out_ports:
            output_edges.extend(Edge.edges_from[o_p.id])

    return nodes, internal_edges, input_edges, output_edges

def process_timeout():
    '''Move expressions affected by a timeout pragma into a new service
       function (that is marked with the same pragma) and replace them with
       a call. Timed function execution is implemented in
       codegen/ast_/function.py. This is done at IR level'''

    for id_, node in Node.node_index.copy().items():
        if node.name != "Lambda" and node.get_pragma("timeout"):
            # nodes - all the nodes involved in calculation
            # internal edges - all the edges between those nodes
            # input edges - all the edges pointing to nodes from the outside
            # output_edges - edges that point from the nodes to the outside
            parent_node = node.get_parent_node()

            (nodes, internal_edges,
                input_edges, output_edges) = collect_pragma_ir(node)

            # create a new function for the calculation of the
            # pragma-affected expression
            new_function = Function(dict(
                                    id="node" + str(len(Node.node_index)),
                                    function_name=Function.get_service_name
                                    ("timed_expression"),
                                    name="Lambda",
                                    edges=[], nodes=[],
                                    pragmas=node.pragmas.copy()))

            new_call = FunctionCall(dict(
                                    callee=new_function.function_name,
                                    id="node" + str(len(Node.node_index)),
                                    name="FunctionCall"))

            for index, e in enumerate(output_edges):
                label = "result" + str(index)
                func_out_port = Port(new_function, e.from_.type,
                                     index, label, False)
                call_out_port = Port(new_call, e.from_.type,
                                     index, label, False)
                new_function.out_ports.append(func_out_port)
                new_call.out_ports.append(call_out_port)

                service_func_output_edge = Edge(
                                            e.from_,
                                            new_function.out_ports[index])
                new_function.edges.append(service_func_output_edge)
                e.attach_origin(new_call.out_ports[index])

            # collect ports from edges "froms",
            # save edges connected to each in a list assigned to the port
            # store them in an OrderedDict
            edges_from_port = OrderedDict()
            for edge_index, e in enumerate(input_edges):
                if e.from_.id not in edges_from_port:
                    edges_from_port[e.from_.id] = [e]
                    port_index = len(edges_from_port)
                    label = "arg_" + str(port_index)
                    function_in_port = Port(new_function, e.to.type,
                                            port_index, e.from_.label, True)
                    call_in_port = Port(new_call, e.to.type,
                                        port_index, label, True)
                    new_function.in_ports.append(function_in_port)
                    new_call.in_ports.append(call_in_port)
                else:
                    edges_from_port[e.from_.id].append(e)

            # use the ordered dict to reassign edges' targets
            # and create new corresponding edges in the service function
            for index, (_, edges) in enumerate(edges_from_port.items()):
                for e in edges:
                    target_port = e.to
                    e.attach_target(new_call.in_ports[index])
                    service_func_input_edge = Edge(
                                            new_function.in_ports[index],
                                            target_port)
                    new_function.edges.append(service_func_input_edge)

            # remove the timeout from all the involved nodes
            for n in nodes:
                n.remove_pragma("timeout")
                # move nodes into the service function
                new_function.nodes.append(n)
                parent_node.nodes.remove(n)

            parent_node.nodes.append(new_call)

            # move input edges to the service function
            for index, e in enumerate(internal_edges):
                new_function.edges.append(e)
                parent_node.edges.remove(e)


"""
modules from CGVAE (Wang et al., ICML2022)
https://github.com/wwang2/CoarseGrainingVAE/cgvae.py
modified to a chemically transferable version by Soojung Yang
"""

import torch
from torch import nn
from torch_scatter import scatter_mean, scatter_add

from conv import *
from e3nn_enc import *

class EquivariantPsuedoDecoder(nn.Module):
    def __init__(self, n_atom_basis, n_rbf, cutoff, num_conv, activation, breaksym=False):

        nn.Module.__init__(self)
        res_embed_dim=4
        self.res_embed = nn.Embedding(25, res_embed_dim)
        self.message_blocks = nn.ModuleList(
                [EquiMessagePsuedo(feat_dim=n_atom_basis+res_embed_dim,
                              activation=activation,
                              n_rbf=n_rbf,
                              cutoff=cutoff,
                              dropout=0.0)
                 for _ in range(num_conv)]
            )

        self.update_blocks = nn.ModuleList(
            [UpdateBlock(feat_dim=n_atom_basis+res_embed_dim,
                         activation=activation,
                         dropout=0.0)
             for _ in range(num_conv)]
        )

        self.breaksym = breaksym
        self.n_atom_basis = n_atom_basis

    def forward(self, cg_z, cg_xyz, CG_nbr_list, mapping, S):

        CG_nbr_list, _ = make_directed(CG_nbr_list)
        r_ij = cg_xyz[CG_nbr_list[:, 1]] - cg_xyz[CG_nbr_list[:, 0]]
        res_S = self.res_embed(cg_z)
        S = torch.cat([S, res_S], axis=-1)

        V = torch.zeros(S.shape[0], S.shape[1], 3 ).to(S.device)
        if self.breaksym:
            Sbar = torch.ones(S.shape[0], S.shape[1]).to(S.device)
        else:
            Sbar = torch.zeros(S.shape[0], S.shape[1]).to(S.device)
        Vbar = torch.zeros(S.shape[0], S.shape[1], 3 ).to(S.device)

        for i, message_block in enumerate(self.message_blocks):

            # message block
            dS, dSbar, dV, dVbar = message_block(s_j=S,
                                                   sbar_j = Sbar,
                                                   v_j=V,
                                                   vbar_j=Vbar,
                                                   r_ij=r_ij,
                                                   nbrs=CG_nbr_list,
                                                   edge_wgt=None
                                                   )
            S = S + dS
            Sbar = Sbar + dSbar
            V = V + dV
            Vbar = Vbar + dVbar

            # update block
            dS_update, dV_update = self.update_blocks[i](s_i=S,
                                                v_i=V)

            S = S + dS_update
            V = V + dV_update

        return S, V

class EquiEncoder(nn.Module):

    def __init__(self,
             n_conv,
             n_atom_basis,
             n_rbf,
             activation,
             cutoff,
             dir_mp=False,
             cg_mp=False):
        super().__init__()

        self.atom_embed = nn.Embedding(50, int(n_atom_basis/2), padding_idx=0)
        self.res_embed = nn.Embedding(25, int(n_atom_basis/2), padding_idx=0)

        # distance transform
        self.dist_embed = DistanceEmbed(n_rbf=n_rbf,
                                  cutoff=cutoff,
                                  feat_dim=n_atom_basis,
                                  dropout=0.0)

        self.message_blocks = nn.ModuleList(
            [EquiMessageBlock(feat_dim=n_atom_basis,
                          activation=activation,
                          n_rbf=n_rbf,
                          cutoff=cutoff,
                          dropout=0.0)
             for _ in range(n_conv)]
        )

        self.cgmessage_layers = nn.ModuleList(
        [ContractiveMessageBlock(feat_dim=n_atom_basis,
                                         activation=activation,
                                         n_rbf=n_rbf,
                                         cutoff=12.5,
                                         dropout=0.0)
         for _ in range(n_conv)])

        self.n_conv = n_conv
        self.dir_mp = dir_mp
        self.cg_mp = cg_mp
        self.n_atom_basis = n_atom_basis

    def forward(self, z, xyz, cg_z, cg_xyz, mapping, nbr_list, cg_nbr_list):

        # atomic embedding
        if not self.dir_mp:
            nbr_list, _ = make_directed(nbr_list)
        cg_nbr_list, _ = make_directed(cg_nbr_list)

        h_atom = self.atom_embed(z.long())
        h_res = self.res_embed(cg_z[mapping].long())
        h = torch.cat([h_atom, h_res], axis=-1)

        v = torch.zeros(h.shape[0], h.shape[1], 3).to(h.device)

        r_ij = xyz[nbr_list[:, 1]] - xyz[nbr_list[:, 0]]

        # edge features
        r_iI = (xyz - cg_xyz[mapping])

        for i in range(self.n_conv):
            ds_message, dv_message = self.message_blocks[i](s_j=h,
                                                   v_j=v,
                                                   r_ij=r_ij,
                                                   nbrs=nbr_list)
            h = h + ds_message
            v = v + dv_message

            # contruct atom messages
            if i == 0:
                H = scatter_mean(h, mapping, dim=0)
                V = scatter_mean(v, mapping, dim=0)

            # CG message passing
            dH, dV = self.cgmessage_layers[i](h, v, r_iI, mapping)

            H = H + dH
            V = V + dV

        return H, h

class CGprior(nn.Module):

    def __init__(self,
             n_conv,
             n_atom_basis,
             n_rbf,
             activation,
             cutoff,
             dir_mp=False):
        super().__init__()

        self.res_embed = nn.Embedding(25, n_atom_basis, padding_idx=0)
        # distance transform
        self.dist_embed = DistanceEmbed(n_rbf=n_rbf,
                                  cutoff=cutoff,
                                  feat_dim=n_atom_basis,
                                  dropout=0.0)

        self.message_blocks = nn.ModuleList(
            [EquiMessageBlock(feat_dim=n_atom_basis,
                          activation=activation,
                          n_rbf=n_rbf,
                          cutoff=cutoff,
                          dropout=0.0)
             for _ in range(n_conv)]
        )

        self.mu = nn.Sequential(nn.Linear(n_atom_basis, n_atom_basis), nn.Tanh(), nn.Linear(n_atom_basis, n_atom_basis))
        self.sigma = nn.Sequential(nn.Linear(n_atom_basis, n_atom_basis), nn.Tanh(), nn.Linear(n_atom_basis, n_atom_basis))

        self.n_conv = n_conv
        self.dir_mp = dir_mp

    def forward(self, cg_z, cg_xyz, cg_nbr_list):

        # atomic embedding
        # if not self.dir_mp:
        cg_nbr_list, _ = make_directed(cg_nbr_list)

        h = self.res_embed(cg_z.long())
        v = torch.zeros(h.shape[0], h.shape[1], 3).to(h.device)

        r_ij = cg_xyz[cg_nbr_list[:, 1]] - cg_xyz[cg_nbr_list[:, 0]]

        for i in range(self.n_conv):
            ds_message, dv_message = self.message_blocks[i](s_j=h,
                                                   v_j=v,
                                                   r_ij=r_ij,
                                                   nbrs=cg_nbr_list)
            h = h + ds_message
            v = v + dv_message

        H_mu = self.mu(h)
        H_sigma = self.sigma(h)

        H_std = 1e-9 + torch.exp(H_sigma / 2)

        return H_mu, H_std

class CGequiVAE(nn.Module):
    def __init__(self, encoder, equivaraintconv,
                     atom_munet, atom_sigmanet,
                     n_cgs, feature_dim,
                    prior_net=None,
                    det=False, equivariant=True, offset=True):
        nn.Module.__init__(self)
        self.encoder = encoder
        self.equivaraintconv = equivaraintconv
        self.atom_munet = atom_munet
        self.atom_sigmanet = atom_sigmanet

        self.n_cgs = n_cgs
        self.prior_net = prior_net
        self.det = det

        self.offset = offset
        self.equivariant = equivariant
        if equivariant == False:
            self.euclidean = nn.Linear(self.encoder.n_atom_basis, self.encoder.n_atom_basis * 3)

    def get_inputs(self, batch):

        xyz = batch['nxyz'][:, 1:]

        cg_xyz = batch['CG_nxyz'][:, 1:]

        cg_z = batch['CG_nxyz'][:, 0].long()
        z = batch['nxyz'][:, 0]

        mapping = batch['CG_mapping']

        nbr_list = batch['nbr_list']
        CG_nbr_list = batch['CG_nbr_list']

        num_CGs = batch['num_CGs']

        return z, cg_z, xyz, cg_xyz, nbr_list, CG_nbr_list, mapping, num_CGs

    def reparametrize(self, mu, sigma):
        eps = torch.randn_like(sigma)
        S_I = eps.mul(sigma).add_(mu)

        return S_I

    def CG2ChannelIdx(self, CG_mapping):

        CG2atomChannel = torch.zeros_like(CG_mapping).to("cpu")

        for cg_type in torch.unique(CG_mapping):
            cg_filter = CG_mapping == cg_type
            num_contri_atoms = cg_filter.sum().item()
            CG2atomChannel[cg_filter] = torch.LongTensor(list(range(num_contri_atoms)))#.to(CG_mapping.device)

        return CG2atomChannel.detach()

    def decoder(self, cg_z, cg_xyz, CG_nbr_list, S_I, s_i, mapping, num_CGs):

        cg_s, cg_v = self.equivaraintconv(cg_z, cg_xyz, CG_nbr_list, mapping, S_I)
        CG2atomChannel = self.CG2ChannelIdx(mapping)

        # implement an non-equivariant decoder
        if self.equivariant == False:
            dv = self.euclidean(cg_s).reshape(cg_s.shape[0], cg_s.shape[1], 3)
            xyz_rel = dv[mapping, CG2atomChannel, :]
        else:
            xyz_rel = cg_v[mapping, CG2atomChannel, :]

        #this constraint is only true for geometrical mean
        # need to include weights

        if self.offset:
          decode_offsets = scatter_mean(xyz_rel, mapping, dim=0)
          xyz_rel = xyz_rel - decode_offsets[mapping]

        # sequential generation
        xyz_recon = xyz_rel + cg_xyz[mapping]

        return xyz_recon

    def forward(self, batch):

        z, cg_z, xyz, cg_xyz, nbr_list, CG_nbr_list, mapping, num_CGs= self.get_inputs(batch)
        S_I, s_i = self.encoder(z, xyz, cg_z, cg_xyz, mapping, nbr_list, CG_nbr_list)

        # get prior based on CG conv
        if self.prior_net:
            H_prior_mu, H_prior_sigma = self.prior_net(cg_z, cg_xyz, CG_nbr_list)
        else:
            H_prior_mu, H_prior_sigma = None, None

        z = S_I

        mu = self.atom_munet(z)
        logvar = self.atom_sigmanet(z)
        sigma = 1e-12 + torch.exp(logvar / 2)

        if not self.det:
            z_sample = self.reparametrize(mu, sigma)
        else:
            z_sample = z

        S_I = z_sample # s_i not used in decoding
        xyz_recon = self.decoder(cg_z, cg_xyz, CG_nbr_list, S_I, s_i, mapping, num_CGs)

        return mu, sigma, H_prior_mu, H_prior_sigma, xyz, xyz_recon


import sys
T=int(input())
book1_num = int(input())
book1 = list(map(int, sys.stdin.readline().split()))
book2_num = int(input())
book2 = list(map(int, sys.stdin.readline().split()))
def findnum(target):
    start = min(book1)
    end = max(book1)
    correct = 0
    while start <= end:
        mid = start + end //2
        if(target == mid or target == end):
            correct = 1
            break
        elif(target < mid):
            end = mid -1
        elif(target > mid):
            start = mid + 1
    return correct

for i in range(book2_num):
    print(findnum((book2[i])))


class SortingRobot:
    def __init__(self, l):
        """
        SortingRobot takes a list and sorts it.
        """
        self._list = l          # The list the robot is tasked with sorting
        self._item = None       # The item the robot is holding
        self._position = 0      # The list position the robot is at
        self._light = "OFF"     # The state of the robot's light
        self._time = 0          # A time counter (stretch)

    def can_move_right(self):
        """
        Returns True if the robot can move right or False if it's
        at the end of the list.
        """
        return self._position < len(self._list) - 1

    def can_move_left(self):
        """
        Returns True if the robot can move left or False if it's
        at the start of the list.
        """
        return self._position > 0

    def move_right(self):
        """
        If the robot can move to the right, it moves to the right and
        returns True. Otherwise, it stays in place and returns False.
        This will increment the time counter by 1.
        """
        self._time += 1
        if self._position < len(self._list) - 1:
            self._position += 1
            return True
        else:
            return False

    def move_left(self):
        """
        If the robot can move to the left, it moves to the left and
        returns True. Otherwise, it stays in place and returns False.
        This will increment the time counter by 1.
        """
        self._time += 1
        if self._position > 0:
            self._position -= 1
            return True
        else:
            return False

    def swap_item(self):
        """
        The robot swaps its currently held item with the list item in front
        of it.
        This will increment the time counter by 1.
        """
        self._time += 1
        # Swap the held item with the list item at the robot's position
        self._item, self._list[self._position] = self._list[self._position], self._item

    def compare_item(self):
        """
        Compare the held item with the item in front of the robot:
        If the held item's value is greater, return 1.
        If the held item's value is less, return -1.
        If the held item's value is equal, return 0.
        If either item is None, return None.
        """
        if self._item is None or self._list[self._position] is None:
            return None
        elif self._item > self._list[self._position]:
            return 1
        elif self._item < self._list[self._position]:
            return -1
        else:
            return 0

    def set_light_on(self):
        """
        Turn on the robot's light
        """
        self._light = "ON"

    def set_light_off(self):
        """
        Turn off the robot's light
        """
        self._light = "OFF"

    def light_is_on(self):
        """
        Returns True if the robot's light is on and False otherwise.
        """
        return self._light == "ON"

    def sort(self):
        """
        Sort the robot's list.
        """
        # the robot grabs the first item
        self.swap_item()

        # I start a while loop with my base case checking if the robot is not at the end of the list
        while self.can_move_right():
            # Next I need to search and compare each item and move them to the end of the list
            while self.move_right():
                # check the item next in the list and compare the items to see which has a greater value, according to the method above if the return is -1 that item is greater than what is held
                if self.compare_item() == - 1:
                    # swap the item
                    self.swap_item()

            # Check the item to see if its none, if it is swap the current item
            if self.compare_item() == None:
                self.swap_item()
                return

            # Starting another while loop to go back left, it will break once the robot can no longer go left or if it returns false
            while self.move_left():

                # The first item will be none since we grabbed it and moved
                if self.compare_item() == None:
                    self.swap_item()
                    self.move_right()
                    self.swap_item()

            # checking to see if the next items value is greater
            if self.compare_item() == -1:
                self.swap_item()

if __name__ == "__main__":
    # Test our your implementation from the command line
    # with `python robot_sort.py`

    l = [15, 41, 58, 49, 26, 4, 28, 8, 61, 60, 65, 21, 78, 14, 35, 90, 54, 5, 0, 87, 82, 96, 43, 92, 62, 97, 69, 94, 99, 93, 76, 47, 2, 88, 51, 40, 95, 6, 23, 81, 30, 19, 25, 91, 18, 68, 71, 9, 66, 1,
         45, 33, 3, 72, 16, 85, 27, 59, 64, 39, 32, 24, 38, 84, 44, 80, 11, 73, 42, 20, 10, 29, 22, 98, 17, 48, 52, 67, 53, 74, 77, 37, 63, 31, 7, 75, 36, 89, 70, 34, 79, 83, 13, 57, 86, 12, 56, 50, 55, 46]

    robot = SortingRobot(l)

    robot.sort()
    print(robot._list)


import datetime as dt

from airflow import DAG
from airflow.contrib.operators.spark_submit_operator import SparkSubmitOperator
from airflow.models import Variable
from airflow.operators.bash_operator import BashOperator

default_args = {
    "owner": "tpp",
    "depends_on_past": False,
    "start_date": dt.datetime(2019, 9, 29),
    "email": ["gfrogat@gmail.com"],
    "email_on_failure": True,
    "email_on_retry": True,
    "retries": 1,
    "retry_delay": dt.timedelta(minutes=5),
}

dag_config = Variable.get("chembl_config", deserialize_json=True)

with DAG(
    "pipeline_chembl",
    default_args=default_args,
    schedule_interval=None,
    params=dag_config,
) as dag:

    # Create Folder Structure
    create_folder_structure_command = """
    mkdir -p {{ params.tpp_root }}/{{ run_id }}/{chembl,tmp}
    """
    create_folder_structure = BashOperator(
        task_id="create_folder_structure", bash_command=create_folder_structure_command
    )

    # Download ChEMBL data
    download_chembl_command = """
    bash {{ params.tpp_python_home }}/scripts/download_chembl.sh
    """
    download_chembl = BashOperator(
        task_id="download_chembl",
        bash_command=download_chembl_command,
        env={
            "CHEMBL_DIR": "{{ params.tpp_root }}/{{ run_id }}/chembl/data",
            "RELEASE": "{{ params.chembl_release }}",
        },
    )

    # Extract downloaded ChEMBL data
    extract_chembl_command = """
    pushd ${CHEMBL_RELEASE_DIR} && \
    gunzip chembl_*.sdf.gz && \
    tar --strip-components=1 -zxf chembl_*_sqlite.tar.gz && \
    popd
    """
    extract_chembl = BashOperator(
        task_id="extract_chembl",
        bash_command=extract_chembl_command,
        env={
            "CHEMBL_RELEASE_DIR": (
                "{{ params.tpp_root }}/{{ run_id }}/chembl/data/"
                "chembl_{{ params.chembl_release }}"
            )
        },
    )

    # Export SQLite Assays
    export_sqlite_command = """
    python {{ params.tpp_python_home }}/scripts/export_chembl_sqlite.py \
        --input {{ params.tpp_root }}/{{ run_id }}/{{ params.chembl_sqlite_path}} \
        --output {{ params.tpp_root }}/{{ run_id }}/{{ params.output_path }} \
        --export {{ params.export_type }}
    """
    export_sqlite_assays = BashOperator(
        task_id="export_sqlite_assays",
        bash_command=export_sqlite_command,
        params={"output_path": "chembl/assays.parquet", "export_type": "assays"},
    )

    # Shard SDF
    shard_sdf_command = """
    python {{ params.tpp_python_home }}/scripts/shard_sdf.py \
        --input {{ params.tpp_root }}/{{ run_id }}/{{ params.chembl_sdf_path }} \
        --output {{ params.tpp_root }}/{{ run_id }}/{{ params.chembl_sdf_shards_path }} \
        --num-shards {{ params.num_shards }} \
        --num-proc {{ params.num_proc }}
    """
    shard_sdf = BashOperator(
        task_id="shard_sdf",
        bash_command=shard_sdf_command,
        params={"num_shards": 100, "num_proc": 6},
    )

    # SDF to Parquet
    sdf_to_parquet_app = "{{ params.tpp_python_home }}/jobs/sdf_to_parquet.py"
    sdf_to_parquet_app_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.chembl_sdf_shards_path }}",
        "--output",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_path }}",
        "--dataset",
        "{{ params.dataset }}",
        "--num-partitions",
        "{{ params.num_partitions }}",
    ]
    sdf_to_parquet = SparkSubmitOperator(
        task_id="sdf_to_parquet",
        conn_id=dag_config["spark_master"],
        application=sdf_to_parquet_app,
        application_args=sdf_to_parquet_app_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "output_path": "chembl/compounds.parquet",
            "dataset": "ChEMBL",
            "num_partitions": 200,
        },
    )

    # Process Assays
    process_assays_app = "{{ params.tpp_python_home }}/jobs/chembl/process_assays.py"
    process_assays_app_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_path }}",
        "--num-partitions",
        "{{ params.num_partitions }}",
    ]
    process_assays = SparkSubmitOperator(
        task_id="process_assays",
        conn_id=dag_config["spark_master"],
        application=process_assays_app,
        application_args=process_assays_app_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "chembl/assays.parquet",
            "output_path": "chembl/assays_processed.parquet",
            "num_partitions": 200,
        },
    )

    # Merge Assays
    merge_assays_app = "{{ params.tpp_python_home }}/jobs/merge_assays.py"
    merge_assays_app_args = [
        "--merge-chembl",
        "--chembl-compounds",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.chembl_compounds_path }}",
        "--chembl-assays",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.chembl_assays_path}}",
        "--output-dir",
        "{{ params.tpp_root }}/{{ run_id }}",
    ]
    merge_assays = SparkSubmitOperator(
        task_id="merge_assays",
        conn_id=dag_config["spark_master"],
        application=merge_assays_app,
        application_args=merge_assays_app_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "chembl_compounds_path": "chembl/compounds.parquet",
            "chembl_assays_path": "chembl/assays_processed.parquet",
        },
    )

    # Compute Semisparse Features - Scala
    compute_semisparse_features_scala_app = "{{ params.tpp_scala_jar }}"
    compute_semisparse_features_scala_args = [
        "-i",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "-o",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_path}}",
        "--features",
        "{{ params.feature_type }}",
        "--npartitions",
        "{{ params.num_partitions }}",
    ]
    compute_semisparse_features_scala = SparkSubmitOperator(
        task_id="compute_semisparse_features_scala",
        conn_id=dag_config["spark_master"],
        application=compute_semisparse_features_scala_app,
        application_args=compute_semisparse_features_scala_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "merged_data.parquet",
            "output_path": "features_semisparse_partial.parquet",
            "feature_type": "semisparse",
        },
    )

    # Compute Semisparse Features - Python
    compute_semisparse_features_python_app = (
        "{{ params.tpp_python_home }}/jobs/compute_descriptors.py"
    )
    compute_semisparse_features_python_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_path }}",
        "--feature-type",
        "{{ params.feature_type }}",
        "--num-partitions",
        "{{ params.num_partitions }}",
    ]
    compute_semisparse_features_python = SparkSubmitOperator(
        task_id="compute_semisparse_features_python",
        conn_id=dag_config["spark_master"],
        application=compute_semisparse_features_python_app,
        application_args=compute_semisparse_features_python_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "features_semisparse_partial.parquet",
            "output_path": "features_semisparse.parquet",
            "feature_type": "semisparse",
        },
    )

    # Clean Semisparse Features
    clean_semisparse_features_app = (
        "{{ params.tpp_python_home }}/jobs/clean_features.py"
    )
    clean_semisparse_features_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output-dir",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_dir_path }}",
        "--temp-files",
        "{{ params.tpp_root }}/{{ run_id }}/tmp/{{ task.task_id }}",
        "--feature",
        "CATS2D",
        "--feature",
        "SHED",
    ]
    clean_semisparse_features = SparkSubmitOperator(
        task_id="clean_semisparse_features",
        conn_id=dag_config["spark_master"],
        application=clean_semisparse_features_app,
        application_args=clean_semisparse_features_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "features_semisparse.parquet",
            "output_dir_path": "features_semisparse_clean",
        },
    )

    # Compute Sparse Features - Scala
    compute_sparse_features_scala_app = "{{ params.tpp_scala_jar }}"
    compute_sparse_features_scala_args = [
        "-i",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "-o",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_path }}",
        "--features",
        "{{ params.feature_type }}",
        "--npartitions",
        "{{ params.num_partitions }}",
    ]
    compute_sparse_features_scala = SparkSubmitOperator(
        task_id="compute_sparse_features_scala",
        conn_id=dag_config["spark_master"],
        application=compute_sparse_features_scala_app,
        application_args=compute_sparse_features_scala_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "merged_data.parquet",
            "output_path": "features_sparse.parquet",
            "feature_type": "sparse",
        },
    )

    # Clean Sparse Features
    clean_sparse_features_app = "{{ params.tpp_python_home }}/jobs/clean_features.py"
    clean_sparse_features_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output-dir",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_dir_path }}",
        "--temp-files",
        "{{ params.tpp_root }}/{{ run_id }}/tmp/{{ task.task_id }}",
        "--feature",
        "DFS8",
        "--feature",
        "ECFC4",
        "--feature",
        "ECFC6",
    ]
    clean_sparse_features = SparkSubmitOperator(
        task_id="clean_sparse_features",
        conn_id=dag_config["spark_master"],
        application=clean_sparse_features_app,
        application_args=clean_sparse_features_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "features_sparse.parquet",
            "output_dir_path": "features_sparse_clean",
        },
    )

    # Compute Tox Features - Python
    compute_tox_features_python_app = (
        "{{ params.tpp_python_home }}/jobs/compute_descriptors.py"
    )
    compute_tox_features_python_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_path }}",
        "--feature-type",
        "{{ params.feature_type }}",
        "--num-partitions",
        "{{ params.num_partitions }}",
    ]
    compute_tox_features_python = SparkSubmitOperator(
        task_id="compute_tox_features_python",
        conn_id=dag_config["spark_master"],
        application=compute_tox_features_python_app,
        application_args=compute_tox_features_python_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "merged_data.parquet",
            "output_path": "features_tox_morgan_partial.parquet",
            "feature_type": "tox",
        },
    )

    # Compute Morgan Fingerprints - Python
    compute_morgan_features_python_app = (
        "{{ params.tpp_python_home }}/jobs/compute_descriptors.py"
    )
    compute_morgan_features_python_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_path }}",
        "--feature-type",
        "{{ params.feature_type }}",
        "--num-partitions",
        "{{ params.num_partitions }}",
    ]
    compute_morgan_features_python = SparkSubmitOperator(
        task_id="compute_morgan_features_python",
        conn_id=dag_config["spark_master"],
        application=compute_morgan_features_python_app,
        application_args=compute_morgan_features_python_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "features_tox_morgan_partial.parquet",
            "output_path": "features_tox_morgan.parquet",
            "feature_type": "morgan",
        },
    )

    export_tfrecords_semisparse_app = (
        "{{ params.tpp_python_home }}/jobs/export_tfrecords.py"
    )
    export_tfrecords_semisparse_app_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output-dir",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_dir_path }}",
        "--cluster-mapping",
        "{{ params.cluster_mapping_path }}",
        "--feature",
        "CATS2D_clean",
        "--feature",
        "SHED_clean",
        "--feature",
        "PubChem",
        "--feature",
        "maccs_fp",
        "--feature",
        "rdkit_fp",
    ]
    export_tfrecords_semisparse = SparkSubmitOperator(
        task_id="export_tfrecords_semisparse",
        conn_id=dag_config["spark_master"],
        application=export_tfrecords_semisparse_app,
        jars="{{ params.tf_spark_connector_jar }}",
        application_args=export_tfrecords_semisparse_app_args,
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "features_semisparse_clean/data_clean.parquet",
            "output_dir_path": "records_semisparse",
        },
    )

    export_tfrecords_sparse_app = (
        "{{ params.tpp_python_home }}/jobs/export_tfrecords.py"
    )
    export_tfrecords_sparse_app_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output-dir",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_dir_path }}",
        "--cluster-mapping",
        "{{ params.cluster_mapping_path }}",
        "--feature",
        "ECFC4_clean",
        "--feature",
        "ECFC6_clean",
        "--feature",
        "DFS8_clean",
    ]
    export_tfrecords_sparse = SparkSubmitOperator(
        task_id="export_tfrecords_sparse",
        conn_id=dag_config["spark_master"],
        application=export_tfrecords_sparse_app,
        application_args=export_tfrecords_sparse_app_args,
        jars="{{ params.tf_spark_connector_jar }}",
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "features_sparse_clean/data_clean.parquet",
            "output_dir_path": "records_sparse",
        },
    )

    export_tfrecords_tox_morgan_app = (
        "{{ params.tpp_python_home }}/jobs/export_tfrecords.py"
    )
    export_tfrecords_tox_morgan_app_args = [
        "--input",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.input_path }}",
        "--output-dir",
        "{{ params.tpp_root }}/{{ run_id }}/{{ params.output_dir_path }}",
        "--cluster-mapping",
        "{{ params.cluster_mapping_path }}",
        "--feature",
        "morgan_fp",
        "--feature",
        "tox_fp",
    ]
    export_tfrecords_tox_morgan = SparkSubmitOperator(
        task_id="export_tfrecords_tox_morgan",
        conn_id=dag_config["spark_master"],
        application=export_tfrecords_tox_morgan_app,
        application_args=export_tfrecords_tox_morgan_app_args,
        jars="{{ params.tf_spark_connector_jar }}",
        conf={"spark.pyspark.python": "{{ params.conda_prefix }}/bin/python"},
        params={
            "input_path": "features_tox_morgan.parquet",
            "output_dir_path": "records_tox_morgan",
        },
    )

    # Store results
    store_results_command = """
    cp -r {{ params.tpp_root }}/{{ run_id }} {{ params.results_root }}
    """
    store_results = BashOperator(
        task_id="store_results", bash_command=store_results_command
    )

# Preprocessing + Protocol
create_folder_structure >> download_chembl >> extract_chembl
extract_chembl >> export_sqlite_assays >> process_assays
extract_chembl >> shard_sdf >> sdf_to_parquet
[process_assays, sdf_to_parquet] >> merge_assays

# Compute Semisparse Features
merge_assays >> compute_semisparse_features_scala >> compute_semisparse_features_python
compute_semisparse_features_python >> clean_semisparse_features

# Compute Sparse Features
merge_assays >> compute_sparse_features_scala >> clean_sparse_features

# Compute Tox + Morgan Features
merge_assays >> compute_tox_features_python >> compute_morgan_features_python

clean_semisparse_features >> export_tfrecords_semisparse
clean_sparse_features >> export_tfrecords_sparse
compute_morgan_features_python >> export_tfrecords_tox_morgan

[
    export_tfrecords_semisparse,
    export_tfrecords_sparse,
    export_tfrecords_tox_morgan,
] >> store_results

if __name__ == "__main__":
    dag.cli()


def polinimio(n, grau, COEFICIENTES):
    return sum([COEFICIENTES[i]*n**i for i in range(0, grau+1)])

def simpson(INTERVALO_INICIAL, INTERVALO_FINAL, GRAU, COEFICIENTES):

    TAMANHO_DO_INTERVALO = (INTERVALO_FINAL - INTERVALO_INICIAL)/2

    f0 = polinimio(INTERVALO_INICIAL, GRAU, COEFICIENTES)

    f1 = polinimio(INTERVALO_INICIAL + TAMANHO_DO_INTERVALO, GRAU, COEFICIENTES)

    f2 = polinimio(INTERVALO_FINAL, GRAU, COEFICIENTES)

    return (TAMANHO_DO_INTERVALO / 3) * (f0 + 4 * f1 + f2)

def simpson_composto(INTERVALO_INICIAL, INTERVALO_FINAL, DISCRETIZACAO, GRAU, COEFICIENTES):

    soma = 0

    TAMANHO_DO_INTERVALO = (INTERVALO_FINAL - INTERVALO_INICIAL)/DISCRETIZACAO

    for i in range(DISCRETIZACAO):
        soma += simpson(INTERVALO_INICIAL + (i * TAMANHO_DO_INTERVALO), INTERVALO_INICIAL + (i + 1) * TAMANHO_DO_INTERVALO, GRAU, COEFICIENTES)

    return soma

def main():

    GRAU = int(input())

    COEFICIENTES = list(map(float, input().split()))
    COEFICIENTES.reverse()

    INTER_DE_INTEGRACAO_E_DISCRET = list(map(int, input().split()))

    VALUE = simpson_composto(INTER_DE_INTEGRACAO_E_DISCRET[0], INTER_DE_INTEGRACAO_E_DISCRET[1], INTER_DE_INTEGRACAO_E_DISCRET[2], GRAU, COEFICIENTES)

    print("%.5f" % VALUE)

if __name__ == '__main__':
    main()


# Contributions
#   Jackson Cashman:
#       __init__, checkMonsterSlots, displayActiveMonsters, discardActiveMonster, discardMonsterIndex, findMatchingMonster
#       lootPlay, startTurn, midTurn, endTurn, checkTreasureSlots, displayActiveTreasures, findMatchingTreasure
#   Ethan Sandoval:
#       __init__, checkMonsterSlots, addMonsterSlot, discardActiveMonster, discardActiveMonster, getMonster
#       itemPlay, addTreasureSlot, checkTreasureSlots, displayActiveTreasures
#   Daniel De Guzman:
#       getJsonObject()
'''
The Board Contains all Decks (including discard piles) and Monsters/Treasures in their deck slots
Board is an attribute of a Room
Board is accessible from a Player
'''
from Decks import Deck
from Cards import *
from Coins import *
from LootReward import *
from TreasureReward import TreasureReward
from DeclareAttack import DeclaredAttack
from DeclarePurchase import DeclaredPurchase
from SilverTreasureCards import PlainSilverTreasure
from SilverTreasureCards import StartTurnTreasure
from SilverTreasureCards import EndTurnTreasure
from Events import Event

def lootPlay(player):
    if player.getHand().getDeckLength() == 0:
        print("You have no loot cards in your hand!\n")
    else:
        player.getHand().printCardListNames()
        print(f"{player.getHand().getDeckLength() + 1}: Cancel\n")
        num = int(input("What loot card do you want to play?: "))
        print("")
        handLen = player.getHand().getDeckLength()
        # cancel the loot play
        if num > handLen:
            pass
        elif player.getTapped() < 1:
            print("You can't play another loot card this turn.\n")
        else:
            # remove the used card from hand
            loot = player.getHand().removeCardIndex(num - 1)
            # use the loot card
            player.addToStack(loot)
            player.getRoom().useTopStack(player.getNumber())
    return

def itemPlay(player):
    if player.getItems().getDeckLength() == 0:
        print("You have no treasure cards in your hand!\n")
    else:
        player.getItems().printCardListNames()
        print(f"{player.getItems().getDeckLength() + 1}: Cancel\n")
        num = int(input("What tresure card do you want to play?: "))
        print("")
        # cancel option
        if num > len(player.getItems().getCardList()):
            return
        treasure = player.getItems().getCardList()[num - 1]
        if treasure.getTapped() == True:
            print("That item can't be used again this turn.\n")
            return
        # use the treasure card
        player.addToStack(treasure)
        player.getRoom().useTopStack(player.getNumber())
    return

class Board:
    def __init__(self, monsterDeck, treasureDeck, lootDeck):
        # max----Slots is the number of slots for that card type on the board
        self.maxMonsterSlots = 20
        self.maxTreasureSlots = 20
        # active----- is the populated slots for that card type. it should always be a list of lists
        # (even if the inside lists only have a single element)
        self.activeMonsters = [[], []] # active monsters are stored on the -1 index
        self.activeTreasures = [[], []]
        # the deck of ----- cards
        self.monsterDeck = monsterDeck
        self.treasureDeck = treasureDeck
        self.lootDeck = lootDeck
        # discard piles for the appropriate decks
        self.discardLootDeck = Deck([])
        self.discardMonsterDeck = Deck([])
        self.discardTreasureDeck = Deck([])
        # effects that are on the board
        # each element in global effect is a list of [card object, the owner]
        # reduce damage does not go in globalEffects
        # TODO: when someone loses an item/steals an item make sure to update globalEffects
        self.globalEffects = []

    def getJsonObject(self):
        boardObject = {
            "maxMonsterSlotsNumber": self.maxMonsterSlots,
            "maxTreasureSlotsNumber": self.maxTreasureSlots,
            "activeMonsters": self.activeMonsters,
            "activeTreasures": self.activeTreasures,
            "monsterDeck": self.monsterDeck.getJsonObject(),
            "treasureDeck": self.treasureDeck.getJsonObject(),
            "lootDeck": self.lootDeck.getJsonObject(),
            "discardMonsterDeck": self.discardMonsterDeck.getJsonObject(),
            "discardTreasureDeck": self.discardTreasureDeck.getJsonObject(),
            "discardLootDeck": self.discardLootDeck.getJsonObject(),
            "globalEffects": self.globalEffects
        }
        return boardObject

    def getLootDeck(self):
        return self.lootDeck

    def getMonsterDeck(self):
        return self.monsterDeck

    def getTreasureDeck(self):
        return self.treasureDeck

    def getDiscardLootDeck(self):
        return self.discardLootDeck

    def getDiscardMonsterDeck(self):
        return self.discardMonsterDeck

    def getDiscardTreasureDeck(self):
        return self.discardTreasureDeck

    # returns the active monster in the specified slot
    def getMonster(self, slotNum):
        return self.activeMonsters[slotNum - 1][-1]

    # returns all monster slots
    def getMonsters(self):
        return self.activeMonsters

    # returns the Treasure in the specified slot
    def getTreasure(self, slotNum):
        return self.activeTreasures[slotNum - 1][-1]

    # returns all treasure slots
    def getTreasures(self):
        return self.activeTreasures

    def getMaxMonsterSlots(self):
        return self.maxMonsterSlots

    def getGlobalEffects(self):
        return self.globalEffects

    def startTurn(self, player):
        print(f"Player {player.getNumber()}'s Turn!\n")
        # recharge all items and character
        player.getCharacter().setTapped(2)
        player.getCharacter().setAttacksLeft(1)
        player.getCharacter().setPurchases(1)
        for i in range(player.getItems().getDeckLength()):
            if isinstance(player.getItems().getCardList()[i], GoldTreasure):
                player.getItems().getCardList()[i].setTapped(False)

        # check for "at start of your turn" effects in globalEffects
        for i in range(len(self.globalEffects)):
            if isinstance(self.globalEffects[i][0], StartTurnTreasure):
                # if it is the start of the itemUsers turn
                itemUser = self.globalEffects[i][1]
                if player.getNumber() == itemUser.getNumber():
                    itemUser.addToStack(self.globalEffects[i][0])
                    itemUser.getRoom().useTopStack(itemUser.getNumber())

        # active player loots
        # TODO: this should go on the stack
        player.loot(1)
        return self.midTurn(player)

    def midTurn(self, player):
        inp = 0
        # increment activePlayer in room
        player.getRoom().incrementActivePlayer()
        while inp != 6:
            # print player stats and ask them what they want to do
            print(f"Player {player.getNumber()}:\n  HP: {player.getHp()}\n  Attack: {player.getAttack()}"
                  f"\n  Coins: {player.getCoins()}\n  Loot Plays Remaining: {player.getTapped()}\n  Hand: {player.getHand().getDeckLength()}\n")
            print("What would you like to do?:\n1. Play a Loot Card\n2. Activate a Gold Treasure\n"
                  "3. Attack a Monster\n4. Purchase a Shop Item\n5. Show other Players\n6. End Turn")
            inp = int(input("  Choice: "))
            print("")
            if inp == 1:
                lootPlay(player)
            elif inp == 2:
                itemPlay(player)
            elif inp == 3:
                if player.getCharacter().getAttacksLeft() <= 0:
                    print("You aren't able to attack again this turn.\n")
                else:
                    # show active monsters and prompt an attack
                    self.displayActiveMonsters()
                    num = int(input("What monster do you want to attack?: "))
                    # cancel prompted attack
                    if num >= (len(self.activeMonsters) + 1):
                        pass
                    # add the declared attack to the stack
                    else:
                        player.addToStack(DeclaredAttack(self.getMonster(num)))
                        # clear anything at the top of the stack that isnt the DeclaredAttack
                        ''' # this commented version will make the game run but could make bugs harder to detect
                        try:
                            while isinstance(player.getRoom().getStack().getStack()[-1][0], DeclaredAttack) != True:
                                player.getRoom().getStack().useTop()
                            # use the declared attack
                            player.getRoom().getStack().useTop()
                        except: # monster killed with a bomb or similar during combat
                            # decrement the number of attacks user can initiate this turn
                            player.getCharacter().subtractAttacksLeft()
                            print("Combat ended without attack roll.")
                        '''
                        #'''
                        if len(player.getRoom().getStack().getStack()) > 0:
                            while isinstance(player.getRoom().getStack().getStack()[-1][0], DeclaredAttack) != True:
                                player.getRoom().getStack().useTop()
                            # use the declared attack
                            player.getRoom().getStack().useTop()
                        #'''
            elif inp == 4:
                if player.getCharacter().getPurchases() <= 0:
                    print("You aren't able to purchase again this turn.\n")
                elif player.getCoins() < 10:
                    print("You do not have enough coins to purchase an item.\n")
                else:
                    # show active treasures and prompt a purchase
                    self.displayActiveTreasures()
                    num = int(input("Which treasure would you like to purchase?: "))
                    # cancel prompted purchase on invalid input
                    if num >= (len(self.activeTreasures) + 1):
                        pass
                    # choose which treasure to buy
                    else:
                        player.addToStack(DeclaredPurchase(self.getTreasure(num)))
                        # clear anything at the top of the stack that isnt the DeclaredPurchase
                        while isinstance(player.getRoom().getStack().getStack()[-1][0], DeclaredPurchase) != True:
                            player.getRoom().getStack().useTop()
                        # use the declared attack
                        player.getRoom().getStack().useTop()
            elif inp == 5:
                player.getRoom().displayCharacters()
            else:
                if player.getCharacter().getMandatoryAttacks() < 1:
                    return self.endTurn(player)
                else:
                    print("You must attack another time this turn!\n")
                    inp = 0

    def endTurn(self, player):
        # check for "at end of your turn" effects in globalEffects
        for i in range(len(self.globalEffects)):
            if isinstance(self.globalEffects[i][0], EndTurnTreasure):
                # if it is the end of the itemUsers turn
                itemUser = self.globalEffects[i][1]
                if player.getNumber() == itemUser.getNumber():
                    itemUser.addToStack(self.globalEffects[i][0])
                    itemUser.getRoom().useTopStack(itemUser.getNumber())

        # reduce loot plays to 1 if above
        if player.getTapped() > 1:
            player.getCharacter().setTapped(1)
        players = player.getRoom().getPlayers()
        entities = player.getRoom().getEntities()
        # Resets the HP of all entities on the board
        for i in range(len(entities)):
            entities[i].setHp(entities[i].getMaxHp())
            entities[i].setAttack(entities[i].getMaxAttack())
        numPlayers = len(players)
        nextPlayer = (player.getNumber() + 1) % numPlayers
        return self.startTurn(players[nextPlayer - 1])

    # check each monster slot to look for any empty slots
    # if an empty slot is found, fill it with the top card of the monster deck
    # player is some player, not guaranteed to be the active player (necessary input for Events)
    def checkMonsterSlots(self, player):
        # fill new monster slots if there are less than the maximum slots
        if (len(self.activeMonsters)) < self.maxMonsterSlots:
            for i in range(self.maxMonsterSlots - (len(self.activeMonsters))):
                self.activeMonsters += [[]]
        # for every monster slot
        for i in range(len(self.activeMonsters)):
            # if the slot is empty, fill it with the top card of the monster deck
            if len(self.activeMonsters[i]) == 0:
                # TODO: this if statement should shuffle the deck instead
                if self.monsterDeck.getDeckLength() <= 0:
                    raise IndexError("Monster Deck is Empty")
                nextMonster = self.monsterDeck.deal()
                self.activeMonsters[i].append(nextMonster)
                # add event cards to the stack if drawn
                if isinstance(nextMonster, Event):
                    player.getRoom().addToStack([nextMonster, player.getRoom().getActivePlayer()])
                    player.getRoom().useTopStack(0)
        return

    def checkTreasureSlots(self):
        # fill new treasure slots if there are less than the maximum slots
        if (len(self.activeTreasures)) < self.maxTreasureSlots:
            for i in range(self.maxTreasureSlots - (len(self.activeTreasures))):
                self.activeTreasures += [[]]
        for i in range(len(self.activeTreasures)):
            # if the slot is empty, fill it with the top card of the treasure deck
            if len(self.activeTreasures[i]) == 0:
                # TODO: this if statement should shuffle the deck instead
                if self.treasureDeck.getDeckLength() <= 0:
                    raise IndexError("Treasure Deck is Empty")
                self.activeTreasures[i].append(self.treasureDeck.deal())
        return

    def addMonsterSlot(self):
        self.maxMonsterSlots += 1
        return

    def addTreasureSlot(self):
        self.maxTreasureSlots += 1
        return

    def coveringMonster(self):
        # TODO:add a monster on top of the monster card on the list
        pass

    # prints the name of each active monster
    def displayActiveMonsters(self):
        print("Here are the monsters on the Board:")
        for i in range(len(self.activeMonsters)):
            if len(self.activeMonsters[i]) == 0:
                print("Empty")
            # print the name of the -1'th index of each monster slot
            else:
                print(f'{i + 1}: {self.activeMonsters[i][-1].getName()}')
                print(f'  HP: {self.activeMonsters[i][-1].getHp()}')
                print(f'  Dice: {self.activeMonsters[i][-1].getDiceValue()}')
                print(f'  Attack: {self.activeMonsters[i][-1].getAttack()}')
                # get the rewards of the monster into a string to display
                rewardString = ""
                for j in range(len(self.activeMonsters[i][-1].getReward())):
                    if isinstance(self.activeMonsters[i][-1].getReward()[j], CoinStack):
                        rewardString = rewardString + "    Coins: " + str(self.activeMonsters[i][-1].getReward()[j].getCount()) + "\n"
                    elif isinstance(self.activeMonsters[i][-1].getReward()[j], LootReward):
                        rewardString = rewardString + "    Loot: " + str(self.activeMonsters[i][-1].getReward()[j].getCount()) + "\n"
                    elif isinstance(self.activeMonsters[i][-1].getReward()[j], TreasureReward):
                        rewardString = rewardString + "    Treasure: " + str(self.activeMonsters[i][-1].getReward()[j].getCount()) + "\n"
                    elif isinstance(self.activeMonsters[i][-1].getReward()[j], LootXReward):
                        rewardString = rewardString + "    Loot: X" + "\n"
                    elif isinstance(self.activeMonsters[i][-1].getReward()[j], CoinXReward):
                        rewardString = rewardString + "    Coins: X" + "\n"
                print(f'  Reward(s): \n{rewardString}')
        print(f"{len(self.activeMonsters) + 1}: CANCEL\n")
        return

    def displayActiveTreasures(self):
        print("Here are the treasures in the Shop (10 cents):")
        for i in range(len(self.activeTreasures)):
            if len(self.activeTreasures[i]) == 0:
                print("Empty")
            else:
                print(f'{i + 1}: {self.activeTreasures[i][-1].getName()}\n')
        print(f"{len(self.activeTreasures) + 1}: CANCEL\n")
        return

    def discardLoot(self, player, handIndex):
        discard = player.getHand().removeCardIndex(handIndex)
        self.discardLootDeck.addCardTop(discard)
        return

    # a special case will need to be added for Sack of Pennies and The Poop
    # because they are gold treasures with passive effects
    def discardTreasure(self, player, itemIndex):
        discard = player.getItems().getCard(itemIndex)
        # if the card being discarded has a global effect
        plainSilver = isinstance(player.getItems().getCardList()[itemIndex], PlainSilverTreasure)
        plainGold = isinstance(player.getItems().getCardList()[itemIndex], GoldTreasure)
        if (plainSilver is False) and (plainGold is False):
            # remove that global effect from the list
            for i in range(len(self.globalEffects)):
                if self.globalEffects[i][0] == discard:
                    self.globalEffects.remove(self.globalEffects[i])
                    break
        player.getItems().removeCardIndex(itemIndex)
        self.discardTreasureDeck.addCardTop(discard)
        return

    # this function asks the player for input on what monster to discard
    def discardActiveMonster(self):
        self.displayActiveMonsters()
        index = int(input("Which monster do you want to get rid of: "))
        index -= 1
        discard = (self.activeMonsters[index]).pop(-1)
        self.discardMonsterDeck.addCardTop(discard)
        return

    # this function discards a monster in the matching index passed in as a parameter
    def discardMonsterIndex(self, index):
        discard = (self.activeMonsters[index]).pop(-1)
        self.discardMonsterDeck.addCardTop(discard)
        return

    # clears a treasure in the matching index (used when a player purchases an item)
    def clearTreasureSlot(self, slotNum):
        (self.activeTreasures[slotNum-1]).pop()
        return

    # returns the index of a monster with the provided name
    def findMatchingMonster(self, name):
        for i in range(len(self.activeMonsters)):
            if self.activeMonsters[i][-1].getName() == name:
                return i

    # returns the index of a treasure with the provided name
    def findMatchingTreasure(self, name):
        for i in range(len(self.activeTreasures)):
            if self.activeTreasures[i][-1].getName() == name:
                return i

    # used to display a single monster, kinda not working correctly (testing with input 0-2), don't know if needed
    def displayMonster(self, index):
        print(self.activeMonsters[index - 1][-1].getName())


import os,sys,json,bisect,shutil
current_script_path = __file__
sys.path.append(os.path.realpath(os.path.join(current_script_path,os.path.pardir,os.path.pardir)))
project_root = os.path.realpath(os.path.join(current_script_path,os.path.pardir,os.path.pardir))

from utils import serifxml as serifxml
import collections
import multiprocessing
import re
import subprocess

TokenSpanModel = collections.namedtuple('TokenSpanModel',['tokenIdxInSentence','tokenDocumentStartChar','tokenDocumentEndChar','tokenOriginalText','tokenOriginalPostag'])

def to_tokens(sentence):
    """
    :type sentence: serifxml.Sentence
    :rtype: list[nlplingo.text.text_span.Token]
    """
    ret = []
    """:type: list[nlplingo.text.text_span.Token]"""

    root = sentence.parse.root
    if root is None:
        return ret

    """:type: serifxml.SynNode"""
    for i, t in enumerate(root.terminals):
        t_text = t.text
        t_start = t.start_char
        t_end = t.end_char
        t_pos_tag = t.parent.tag
        ret.append(TokenSpanModel(i,t_start,t_end,t_text,t_pos_tag))
    return ret

def fuzzy_alignment(serif_doc_theory,sentStartOffset,sentEndOffset,triggerStartOffset,triggerEndOffset):
    candidates = dict()
    for idx,sentence in enumerate(serif_doc_theory.sentences):
        nlplingo_token = to_tokens(sentence)
        token_starts = tuple(token.tokenDocumentStartChar for token in nlplingo_token)
        token_ends = tuple(token.tokenDocumentEndChar for token in nlplingo_token)
        if len(token_starts) < 1 or len(token_ends) < 1:
            continue
        if (token_ends[-1] - token_starts[0]) != sentEndOffset - sentStartOffset:
            continue
        else:
            offset = nlplingo_token[0].tokenDocumentStartChar - sentStartOffset
            new_trigger_start_offset = triggerStartOffset + offset
            new_trigger_end_offset = triggerEndOffset + offset
            if new_trigger_start_offset in token_starts and new_trigger_end_offset in token_ends:
                candidates[nlplingo_token[0].tokenDocumentStartChar - sentStartOffset] = sentence
            elif new_trigger_start_offset in token_starts and new_trigger_end_offset - 1 in token_ends:
                candidates[nlplingo_token[0].tokenDocumentStartChar - sentStartOffset] = sentence
    for _,sentence in sorted(candidates.items(),key=lambda k:abs(k[0])):
        return sentence,"Candidate set size is:{}".format(len(candidates.keys()))
    return None,None

def worker_process(docId,docPath,event_argument_list,s3_trigger_queue,err_queue,strict=True):
    try:
        serif = serifxml.Document(docPath)
        sentence_tokensequence_start_to_sentence_idx_mapping = dict()
        sentence_start_to_sentence_idx_mapping = dict()
        for idx,sentence in enumerate(serif.sentences):
            if len(sentence.token_sequence) > 0:
                sentence_tokensequence_start_to_sentence_idx_mapping[sentence.token_sequence[0].start_char] = idx
                sentence_start_to_sentence_idx_mapping[sentence.start_char] = idx
        for _,sentStartOffset,sentEndOffset,triggerStartOffset,triggerEndOffset,argumentStartOffset,argumentEndOffset,argumentType in event_argument_list:
            sentence_start = sentStartOffset
            sentence_end = sentEndOffset
            trigger_start = triggerStartOffset
            trigger_end = triggerEndOffset
            argument_start = argumentStartOffset
            argument_end = argumentEndOffset
            argument_type = argumentType
            if sentence_tokensequence_start_to_sentence_idx_mapping.get(sentence_start,None) is not None:
                sentence = serif.sentences[sentence_tokensequence_start_to_sentence_idx_mapping.get(sentence_start)]
                sentence_start = sentence.start_char
                sentence_end = sentence.end_char
            elif sentence_start_to_sentence_idx_mapping.get(sentence_start,None) is not None:
                sentence = serif.sentences[sentence_start_to_sentence_idx_mapping.get(sentence_start)]
                sentence_start = sentence.start_char
                sentence_end = sentence.end_char
            else:
                if strict is True:
                    err_msg = "Misalignment:{} {} {} {} {} {} {}".format(docId , sentence_start, sentence_end,
                                                                      trigger_start, trigger_end,argument_start,argument_end)
                    # print(err_msg)
                    err_queue.put(err_msg)
                    continue
                fuzzy_sentence, msg = fuzzy_alignment(serif, sentence_start, sentence_end, trigger_start, trigger_end)
                if fuzzy_sentence is None:
                    err_msg = "Misalignment:{} {} {} {} {} {} {}".format(docId, sentence_start, sentence_end,
                                                                      trigger_start, trigger_end,argument_start,argument_end)
                    # print(err_msg)
                    err_queue.put(err_msg)
                    continue
                else:
                    # print(msg)
                    offset = fuzzy_sentence.start_char - sentence_start
                    sentence_start = sentence_start + offset
                    sentence_end = sentence_end + offset
                    trigger_start = trigger_start + offset
                    trigger_end = trigger_end + offset
                    argument_start = argument_start + offset
                    argument_end = argument_end + offset
                    sentence = fuzzy_sentence
            nlplingo_token = to_tokens(sentence)
            token_text = [token.tokenOriginalText for token in nlplingo_token]
            token_starts = [token.tokenDocumentStartChar for token in nlplingo_token]
            token_ends = [token.tokenDocumentEndChar for token in nlplingo_token]
            trigger_start_idx = bisect.bisect(token_starts, trigger_start) - 1
            trigger_end_idx = bisect.bisect(token_ends, trigger_end) - 1
            trigger_idx = list()
            for i in range(trigger_start_idx, trigger_end_idx + 1):
                trigger_idx.append(i)
            argument_idx = list()
            argument_start_idx = bisect.bisect(token_starts, argument_start) - 1
            argument_end_idx = bisect.bisect(token_ends, argument_end) - 1
            for i in range(argument_start_idx,argument_end_idx+1):
                argument_idx.append(i)
            if (len(trigger_idx) < 1):
                err_msg = "Misalignment:{} {} {} {} {} {} {}".format(docId, sentence_start, sentence_end,
                                                                     trigger_start, trigger_end, argument_start,
                                                                     argument_end)
                # print(err_msg)
                err_queue.put(err_msg)
                continue
            if token_starts[trigger_idx[0]] != trigger_start or token_ends[trigger_idx[-1]] != trigger_end:
                if token_ends[trigger_idx[-1]] == trigger_end - 1:
                    trigger_end = trigger_end - 1
                else:
                    err_msg = "Misalignment:{} {} {} {} {} {} {}".format(docId, sentence_start, sentence_end,
                                                                      trigger_start, trigger_end,argument_start,argument_end)
                    # print(err_msg)
                    err_queue.put(err_msg)
                    continue
            trigger_word = " ".join(token_text[trigger_idx[0]:trigger_idx[-1] + 1])
            if trigger_idx[0] - 2 < 0:
                key = token_text[0:5]
            elif trigger_idx[-1] + 2 >= len(token_text):
                key = token_text[-5:]
            else:
                key = token_text[trigger_idx[0] - 2:trigger_idx[-1] + 3]
            key = " ".join(key)
            s3_trigger_queue.put(json.dumps({'key':"{}.{}".format(docId,str((sentence_start,sentence_end))),
                                                                                'type':'sentence',
                                                                                'aux':{'instanceId':"{} {} {} {} {} {} {} {}".format(docId,sentStartOffset,sentEndOffset,triggerStartOffset,triggerEndOffset,argumentStartOffset,argumentEndOffset,argumentType),
                                                                                'sentence':token_text,
                                                                                'displayablekey': key,
                                                                                'tags':{
                                                                                argument_type:argument_idx,
                                                                                'trigger':trigger_idx
                                                                                },
                                                                                'argument_type':argument_type,
                                                                                'trigger_lemma':trigger_word,
                                                                                'annotated':False,
                                                                                'positive':True
                                                                                }}))
    except:
        import traceback
        traceback.print_exc()
        err_queue.put(traceback.format_exc())
    finally:
        s3_trigger_queue.close()
        err_queue.close()

def main(input_candidate_file,serifxml_list_file,output_folder):
    docId_to_docPath_mapping = dict()
    shutil.rmtree(output_folder,ignore_errors=True)
    os.makedirs(output_folder,exist_ok=True)
    with open(serifxml_list_file,'r') as fp:
        for i in fp:
            i = i.strip()
            docId = os.path.basename(i)
            docId = docId.replace(".xml","")
            docId_to_docPath_mapping[docId] = i
    manager = multiprocessing.Manager()
    s3_trigger_queue = manager.Queue()
    err_queue = manager.Queue()
    docId_to_eventArgument_mapping = dict()
    with open(input_candidate_file,'r') as fp:
        for idx,i in enumerate(fp):
            i = i.strip()
            docId,sentStart,sentEnd,triggerStart,triggerEnd,argStart,argEnd,argType = i.split(" ")
            docId_to_eventArgument_mapping.setdefault(docId,manager.list()).append(manager.list([docId,int(sentStart),int(sentEnd),int(triggerStart),int(triggerEnd),int(argStart),int(argEnd),argType]))
            if idx % 500 == 0:
                print("Read {} lines of event argument lines.".format(idx))
    with manager.Pool(processes=multiprocessing.cpu_count() - 1) as pool:
        print("Begin parallel processing.")
        work_list = list()
        for docId,entry_arr in docId_to_eventArgument_mapping.items():
            work_list.append(pool.apply_async(worker_process, args=(docId,
                                                                    docId_to_docPath_mapping.get(
                                                                        docId),
                                                                    entry_arr,
                                                                    s3_trigger_queue,
                                                                    err_queue, True,)))
        for idx,i in enumerate(work_list):
            if idx % 500 == 0:
                print("We're waiting {} workers to finish".format(len(work_list) - idx))
            i.wait()
    print("We're collecting error reports.")
    while err_queue.empty() is False:
        print(err_queue.get_nowait())
    print("We're writing results.")
    while s3_trigger_queue.empty() is False:
        rec = json.loads(s3_trigger_queue.get_nowait())
        with open(os.path.join(output_folder,rec['aux']['argument_type']+".ljson"),'a') as fp:
            fp.write(json.dumps(rec)+"\n")

def trigger_blacklist_filter(output_folder,word_blacklist):
    for file in os.listdir(output_folder):
        if os.path.isfile(os.path.join(output_folder,file)) and file.endswith(".ljson"):
            output_buf = list()
            with open(os.path.join(output_folder, file), 'r') as fp:
                for i in fp:
                    i = i.strip()
                    j = json.loads(i)
                    trigger_start_idx = j['aux']['tags']['trigger'][0]
                    trigger_end_idx = j['aux']['tags']['trigger'][-1]
                    normalized_trigger_word = " ".join(j['aux']['sentence'][trigger_start_idx:trigger_end_idx+1])
                    normalized_trigger_word = normalized_trigger_word.strip().lower()
                    if normalized_trigger_word in word_blacklist:
                        continue
                    else:
                        output_buf.append(i)
            if len(output_buf) < 1:
                os.remove(os.path.join(output_folder,file))
            else:
                with open(os.path.join(output_folder,file),'w') as fp:
                    for i in output_buf:
                        fp.write("{}\n".format(i))
def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i + n]

def spliter(output_folder,batch_size=200):
    booking = dict()
    for file in os.listdir(output_folder):
        if os.path.isfile(os.path.join(output_folder,file)) and file.endswith(".ljson"):
            current_folder = os.path.join(output_folder,file.replace(".ljson",""))
            current_arg_type_name = file.replace(".ljson","")
            shutil.rmtree(current_folder,ignore_errors=True)
            os.makedirs(current_folder,exist_ok=True)
            with open(os.path.join(output_folder,file),'r') as fp:
                buf = [i.strip() for i in fp]
                for idx,i in enumerate(chunks(buf,batch_size)):
                    booking[current_arg_type_name] = max(booking.get(current_arg_type_name,-1),idx)
                    with open(os.path.join(current_folder,str(idx)+".ljson"),'w') as fp2:
                        for j in i:
                            fp2.write("{}\n".format(j))
    with open(os.path.join(output_folder,'booking.json'),'w') as fp:
        json.dump(booking,fp)
    with open(os.path.join(output_folder,'type.json'),'w') as fp:
        json.dump({'type':'argument'},fp)

def argument_annotation_candidate_generator(config_ins,input_serifxml_list,output_file_path):
    subprocess.check_call([config_ins.BIN_EVENT_ARGUMENT_ANNOTATION_EXAMPLE_GENERATOR,input_serifxml_list,output_file_path])

if __name__ == "__main__":
    # from config import ReadAndConfiguration
    # c = ReadAndConfiguration(os.path.join("/hat_data", "config_default.json"))
    #
    # SCRATCH_PLACE = c.SCRATCH_PLACE
    # input_serifxml_list = c.serifxml_list_path_for_argument
    # big_ljson_buffer = os.path.join(SCRATCH_PLACE,'argument_annotation_big.ljson')
    # argument_annotation_candidate_generator(c,input_serifxml_list,big_ljson_buffer)
    # main(big_ljson_buffer, input_serifxml_list, c.FOLDER_SESSION(c.init_argument_session_id))
    # spliter(c.FOLDER_SESSION(c.init_argument_session_id))
    input_serifxml_list = "/home/hqiu/wm_m12_for_trigger_and_argument_annotation.list"
    big_ljson_buffer = "/home/hqiu/massive/tmp/annotation.span"
    output_folder = "/home/hqiu/massive/tmp/wm_m12_wm_intervention_4_arg"
    main(big_ljson_buffer, input_serifxml_list, output_folder)
    spliter(output_folder)


import pytest

from apps.appointment.models import Appointment
from apps.client.models import Client

pytestmark = [
    pytest.mark.django_db
]

@pytest.fixture
def category(mixer):
    return mixer.blend('service.Category')

@pytest.fixture
def service(mixer):
    return mixer.blend('service.SalonService')

@pytest.fixture
def appointment(mixer, employee):
    return mixer.blend('appointment.Appointment', company=employee.company)

@pytest.fixture
def payload(service, employee, category):
    return {
        'service': {
            'id': service.pkid
        },
        'employee': {
            'id': employee.pkid
        },
        'category': {
            'id': category.pkid
        },
        'client': {
            'name': 'Benedikt Kaberbetch',
            'phone': '+48731331105'
        },
        'date': {
            'day': '2022-12-02',
            'time_step': 45
        }
    }

def test_appointment_update(api, payload, service, employee, appointment):
    api.put(f'/api/v2/appointment/{appointment.pkid}/update', payload, expected_status_code=200)

    appointment = Appointment.objects.last()
    client = Client.objects.last()

    assert appointment.service == service
    assert appointment.employee == employee
    assert appointment.client == client
    assert appointment.day.strftime('%Y-%m-%d') == payload['date']['day']
    assert appointment.time_step == payload['date']['time_step']


# -*- coding: utf-8 -*-
"""
Created on May 18, 2018

@author: lisu, Bill

Document:

load_audio(filepath, sr=None, mono=True, dtype='float32')
    Parameters:
        sr:(number>0) sample rate;
            default = None(use raw audio sample rate)
        mono:(bool) convert signal to mono;
            default = True
        dtype:(numeric type) data type of x;
            default = 'float32'
    Returns:
        x:(np.ndarray) audio time series
        sr:(number>0) sample rate of x
feature_extraction(x, sr, Hop=320, Window=2049, StartFreq=80.0, StopFreq=1000.0, NumPerOct=48)
    Parameters:
        x:(np.ndarray) audio time series
        sr:(number>0) sample rate of x
        Hop: Hop size
        Window: Window size
        StartFreq: smallest frequency on feature map
        StopFreq: largest frequency on feature map
        NumPerOct: Number of bins per octave
    Returns:
        Z: mix cfp feature
        time: feature map to time
        CenFreq: feature map to frequency
        tfrL0: STFT spectrogram
        tfrLF: generalized cepstrum (GC)
        tfrLQ: generalized cepstrum of spectrum (GCOS)

get_CenFreq(StartFreq=80, StopFreq=1000, NumPerOct=48)
get_time(fs, Hop, end)
midi2hz(midi)
hz2midi(hz)

"""
import soundfile as sf
import numpy as np
np.seterr(divide='ignore', invalid='ignore')
import scipy
import scipy.signal
import pandas as pd

def STFT(x, fr, fs, Hop, h):
    t = np.arange(Hop, np.ceil(len(x)/float(Hop))*Hop, Hop)
    N = int(fs/float(fr))
    window_size = len(h)
    f = fs*np.linspace(0, 0.5, np.round(N/2), endpoint=True)
    Lh = int(np.floor(float(window_size-1) / 2))
    tfr = np.zeros((int(N), len(t)), dtype=np.float)

    for icol in range(0, len(t)):
        ti = int(t[icol])
        tau = np.arange(int(-min([round(N/2.0)-1, Lh, ti-1])), \
                        int(min([round(N/2.0)-1, Lh, len(x)-ti])))
        indices = np.mod(N + tau, N) + 1
        tfr[indices-1, icol] = x[ti+tau-1] * h[Lh+tau-1] \
                                /np.linalg.norm(h[Lh+tau-1])

    tfr = abs(scipy.fftpack.fft(tfr, n=N, axis=0))
    return tfr, f, t, N

def nonlinear_func(X, g, cutoff):
    cutoff = int(cutoff)
    if g!=0:
        X[X<0] = 0
        X[:cutoff, :] = 0
        X[-cutoff:, :] = 0
        X = np.power(X, g)
    else:
        X = np.log(X)
        X[:cutoff, :] = 0
        X[-cutoff:, :] = 0
    return X

def Freq2LogFreqMapping(tfr, f, fr, fc, tc, NumPerOct):
    StartFreq = fc
    StopFreq = 1/tc
    Nest = int(np.ceil(np.log2(StopFreq/StartFreq))*NumPerOct)
    central_freq = []

    for i in range(0, Nest):
        CenFreq = StartFreq*pow(2, float(i)/NumPerOct)
        if CenFreq < StopFreq:
            central_freq.append(CenFreq)
        else:
            break

    Nest = len(central_freq)
    freq_band_transformation = np.zeros((Nest-1, len(f)), dtype=np.float)
    for i in range(1, Nest-1):
        l = int(round(central_freq[i-1]/fr))
        r = int(round(central_freq[i+1]/fr)+1)
        #rounding1
        if l >= r-1:
            freq_band_transformation[i, l] = 1
        else:
            for j in range(l, r):
                if f[j] > central_freq[i-1] and f[j] < central_freq[i]:
                    freq_band_transformation[i, j] = (f[j] - central_freq[i-1]) / (central_freq[i] - central_freq[i-1])
                elif f[j] > central_freq[i] and f[j] < central_freq[i+1]:
                    freq_band_transformation[i, j] = (central_freq[i + 1] - f[j]) / (central_freq[i + 1] - central_freq[i])
    tfrL = np.dot(freq_band_transformation, tfr)
    return tfrL, central_freq

def Quef2LogFreqMapping(ceps, q, fs, fc, tc, NumPerOct):
    StartFreq = fc
    StopFreq = 1/tc
    Nest = int(np.ceil(np.log2(StopFreq/StartFreq))*NumPerOct)
    central_freq = []

    for i in range(0, Nest):
        CenFreq = StartFreq*pow(2, float(i)/NumPerOct)
        if CenFreq < StopFreq:
            central_freq.append(CenFreq)
        else:
            break
    f = 1/q
    Nest = len(central_freq)
    freq_band_transformation = np.zeros((Nest-1, len(f)), dtype=np.float)
    for i in range(1, Nest-1):
        for j in range(int(round(fs/central_freq[i+1])), int(round(fs/central_freq[i-1])+1)):
            if f[j] > central_freq[i-1] and f[j] < central_freq[i]:
                freq_band_transformation[i, j] = (f[j] - central_freq[i-1])/(central_freq[i] - central_freq[i-1])
            elif f[j] > central_freq[i] and f[j] < central_freq[i+1]:
                freq_band_transformation[i, j] = (central_freq[i + 1] - f[j]) / (central_freq[i + 1] - central_freq[i])

    tfrL = np.dot(freq_band_transformation, ceps)
    return tfrL, central_freq

def CFP_filterbank(x, fr, fs, Hop, h, fc, tc, g, NumPerOctave):
    NumofLayer = np.size(g)

    [tfr, f, t, N] = STFT(x, fr, fs, Hop, h)
    tfr = np.power(abs(tfr), g[0])
    tfr0 = tfr # original STFT
    ceps = np.zeros(tfr.shape)

    if NumofLayer >= 2:
        for gc in range(1, NumofLayer):
            if np.remainder(gc, 2) == 1:
                tc_idx = round(fs*tc)
                ceps = np.real(np.fft.fft(tfr, axis=0))/np.sqrt(N)
                ceps = nonlinear_func(ceps, g[gc], tc_idx)
            else:
                fc_idx = round(fc/fr)
                tfr = np.real(np.fft.fft(ceps, axis=0))/np.sqrt(N)
                tfr = nonlinear_func(tfr, g[gc], fc_idx)

    tfr0 = tfr0[:int(round(N/2)),:]
    tfr = tfr[:int(round(N/2)),:]
    ceps = ceps[:int(round(N/2)),:]

    HighFreqIdx = int(round((1/tc)/fr)+1)
    f = f[:HighFreqIdx]
    tfr0 = tfr0[:HighFreqIdx,:]
    tfr = tfr[:HighFreqIdx,:]
    HighQuefIdx = int(round(fs/fc)+1)

    q = np.arange(HighQuefIdx)/float(fs)

    ceps = ceps[:HighQuefIdx,:]

    tfrL0, central_frequencies = Freq2LogFreqMapping(tfr0, f, fr, fc, tc, NumPerOctave)
    tfrLF, central_frequencies = Freq2LogFreqMapping(tfr, f, fr, fc, tc, NumPerOctave)
    tfrLQ, central_frequencies = Quef2LogFreqMapping(ceps, q, fs, fc, tc, NumPerOctave)

    return tfrL0, tfrLF, tfrLQ, f, q, t, central_frequencies

def load_audio(filepath, sr=None, mono=True, dtype='float32'):

    if '.mp3' in filepath:
        from pydub import AudioSegment
        import tempfile
        import os
        mp3 = AudioSegment.from_mp3(filepath)
        _, path = tempfile.mkstemp()
        mp3.export(path, format="wav")
        del mp3
        x, fs = sf.read(path)
        os.remove(path)
    else:
        x, fs = sf.read(filepath)

    if mono and len(x.shape)>1:
        x = np.mean(x, axis = 1)
    if sr:
        x = scipy.signal.resample_poly(x, sr, fs)
        fs = sr
    x = x.astype(dtype)

    return x, fs

def feature_extraction(x, fs, Hop=512, Window=2049, StartFreq=80.0, StopFreq=1000.0, NumPerOct=48):

    fr = 2.0 # frequency resolution
    h = scipy.signal.blackmanharris(Window) # window size
    g = np.array([0.24, 0.6, 1]) # gamma value

    tfrL0, tfrLF, tfrLQ, f, q, t, CenFreq = CFP_filterbank(x, fr, fs, Hop, h, StartFreq, 1/StopFreq, g, NumPerOct)
    Z = tfrLF * tfrLQ
    time = t/fs
    return Z, time, CenFreq, tfrL0, tfrLF, tfrLQ

def midi2hz(midi):
    return 2**((midi-69)/12.0)*440
def hz2midi(hz):
    return 69+ 12*np.log2(hz/440.0)

def get_CenFreq(StartFreq=80, StopFreq=1000, NumPerOct=48):
    Nest = int(np.ceil(np.log2(StopFreq/StartFreq))*NumPerOct)
    central_freq = []
    for i in range(0, Nest):
        CenFreq = StartFreq*pow(2, float(i)/NumPerOct)
        if CenFreq < StopFreq:
            central_freq.append(CenFreq)
        else:
            break
    return central_freq

def get_time(fs, Hop, end):
    return np.arange(Hop/fs,end,Hop/fs)

def lognorm(x):
    return np.log(1+x)
def norm(x):
    return (x - np.min(x))/(np.max(x)-np.min(x))
def cfp_process(fpath, ypath=None, csv=False,sr=None, hop=256, model_type='vocal'):
    print('CFP process in '+str(fpath)+ ' ... (It may take some times)')
    y, sr = load_audio(fpath, sr=sr)
    if 'vocal' in model_type:
        Z, time, CenFreq, tfrL0, tfrLF, tfrLQ = feature_extraction(y, sr, Hop=hop, StartFreq=31.0, StopFreq=1250.0, NumPerOct=60)
    if 'melody' in model_type:
        Z, time, CenFreq, tfrL0, tfrLF, tfrLQ = feature_extraction(y, sr, Hop=hop, StartFreq=20.0, StopFreq=2048.0, NumPerOct=60)
    tfrL0 = norm(lognorm(tfrL0))[np.newaxis,:,:]
    tfrLF = norm(lognorm(tfrLF))[np.newaxis,:,:]
    tfrLQ = norm(lognorm(tfrLQ))[np.newaxis,:,:]
    W = np.concatenate((tfrL0,tfrLF,tfrLQ),axis=0)
    print('Done!')
    print('Data shape: '+str(W.shape))
    if ypath:
        if csv:
            ycsv = pd.read_csv(ypath, names = ["time", "freq"])
            gt0 = ycsv['time'].values
            gt0 = gt0[1:,np.newaxis]

            gt1 = ycsv['freq'].values
            gt1 = gt1[1:,np.newaxis]
            gt = np.concatenate((gt0, gt1), axis=1)
        else:
            gt = np.loadtxt(ypath)
        return W, gt, CenFreq, time
    else:
        return W, CenFreq, time