from jax.api import jit, grad, eval_shape, jacobian, jvp, vjp

import jax.numpy as np

from jax import random

# from jax.experimental import optimizers

from jax.experimental.stax import logsoftmax

from functools import partial

import tempfile

import itertools as IT

import os

from jax.experimental import optimizers

optimizer_dict = {}
optimizer_dict['adam'] = optimizers.adam
optimizer_dict['sgd'] = optimizers.sgd

def accuracy(params, f, images, targets, key = None):
    target_class = np.argmax(targets, axis=1)

    if key is not None:
        predicted_class = np.argmax(f(params, images, rng = key), axis=1)
    else:
        predicted_class = np.argmax(f(params, images), axis=1)
    return np.mean(predicted_class == target_class)

# def mean_and_variation_acc_across_trials(results_mat):
#     """
#     Args:
#     results_mat: A `np.ndarray` of shape [n_trials, n_data_points_evaluated]

#     Returns:
#     av_results_mat_across_trials: averaged results across trials (axis 0).

#     max_min_diff_ntk_distance: the upper/lower limit results obtaiend in different trials centered acround the results of av_results_mat_across_trials

#     """
#     av_results_mat_across_trials =  np.float32(np.mean(results_mat, axis = 0))

#     max_min_in_trials = np.vstack( (np.amin(results_mat, axis = 0 ), np.amax(results_mat, axis = 0 ) ) )

#     max_min_centered_diff = np.abs(max_min_in_trials - av_results_mat_across_trials)

#     return av_results_mat_across_trials, max_min_centered_diff

def _read_keys(keys):
    if keys is None or (isinstance(keys, np.ndarray) and keys.shape == (2,)):
        key1 = key2 = keys
    elif isinstance(keys, tuple):
        # assuming x1 and x2 using key1 and key2, resp.
        key1, key2 = keys
    elif isinstance(keys, np.ndarray) and keys.shape == (2, 2):
        key1, key2 = keys[0], keys[1]
    else:
        raise ValueError('`keys` must be one of the following: `None`, a PRNG '
                     'key, a tuple of PRNG keys or a (2, 2) array and dtype '
                     'unint32')
    return key1, key2

def empirical_ntk_fn(f):
    """Computes the ntk without batching for inputs x1 and x2.
    The Neural Tangent Kernel is defined as J(X_1)^T J(X_2) where J is the
    jacobian df/dparams. Computing the NTK directly involves directly
    instantiating the jacobian which takes
    O(dataset_size * output_dim * parameters) memory. It turns out it is
    substantially more efficient (especially as the number of parameters grows)
    to compute the NTK implicitly.
    This involves using JAX's autograd to compute derivatives of linear functions
    (which do not depend on the inputs). Thus, we find it more efficient to refer
    to fx_dummy for the outputs of the network. fx_dummy has the same shape as
    the output of the network on a single piece of input data.
    TODO(schsam): Write up a better description of the implicit method.
    Args:
    f: The function whose NTK we are computing. f should have the signature
       f(params, inputs) and should return an `np.ndarray` of outputs with shape
       [|inputs|, output_dim].
    Returns:
    A function ntk_fn that computes the empirical ntk.
    """

    def ntk_fn(x1, x2, params, keys=None):
        """Computes the empirical ntk.
        Args:
          x1: A first `np.ndarray` of inputs, of shape [n1, ...], over which we
            would like to compute the NTK.
          x2: A second `np.ndarray` of inputs, of shape [n2, ...], over which we
            would like to compute the NTK.
          params: A PyTree of parameters about which we would like to compute the
            neural tangent kernel.
          keys: None or a PRNG key or a tuple of PRNG keys or a (2, 2) array and
            dtype uint32. If `key == None`, then the function `f` is deterministic
            and requires no PRNG key; else if `keys` is a single PRNG key, then x1
            and x2 must be the same and share the same PRNG key; else x1 and x2 use
            two different PRNG keys.
        Returns:
          A `np.ndarray` of shape [n1, n2] + output_shape + output_shape.
        """
        key1, key2 = _read_keys(keys)
        # TODO(xlc): find a good way to check utils.x1_is_x2(x1, x2) == (key1==key2)
        if x2 is None:
            x2 = x1

        f_dummy = partial(f, rng=random.PRNGKey(1))
        fx2_struct = eval_shape(f_dummy, params, x2)
        fx_dummy = np.ones(fx2_struct.shape, fx2_struct.dtype)
        def delta_vjp_jvp(delta):
            def delta_vjp(delta):
                return vjp(lambda p: f(p, x2, rng=key2), params)[1](delta)
            return jvp(lambda p: f(p, x1, rng=key1), (params,), delta_vjp(delta))[1]

        ntk = jacobian(delta_vjp_jvp)(fx_dummy)
        ndim = len(fx2_struct.shape)
        ordering = (0, ndim) + tuple(range(1, ndim)) + \
            tuple(x + ndim for x in range(1, ndim))
        return np.transpose(ntk, ordering)

    return ntk_fn

def get_masks_from_jax_params(params, nn_density_level, magnitude_base_bool = True, global_bool = False, reshuffle_seed = 0):
    """ Assemble a collection of 0-1 valued masks which are of the same sizes and shapes as layers' weight tensors
        Note that this function ignores bias parameters.

    Args:
        params: parameters in a jax.experimental.stax format.
        nn_density_level: the desired density level for weight parameters.
        magnitude_base_bool: a boolean variable that decides whether to prune the network by magnitude or randomly prune the network

    Returns:
        masks: a collection of 0-1 valued masks which are of the same sizes and shapes as the layers' weight tensors.
    """

    if (type(magnitude_base_bool) != bool) or (type(global_bool) != bool):
        raise ValueError("magnitude_base_bool and global_bool should be boolean variables")

    masks = []

    if global_bool:
        weight_magnitudes_pooled = np.concatenate([ np.abs(layer_params[0].flatten()) for layer_params in params if len(layer_params) > 1])
        idx = int( (1 - nn_density_level) * np.size(weight_magnitudes_pooled) )
        global_thres = np.sort(weight_magnitudes_pooled)[idx]

    for layer_index in range( len(params)):

        if len(params[layer_index]) < 2:
            # In this the case, the layer does not contain weight and bias parameters.
            masks.append( [] )

        elif len(params[layer_index]) == 2:
            # In this case, the layer contains a tuple of parameters for weights and biases

            weights = params[layer_index][0]

            weight_magnitudes = np.abs(weights)

            if global_bool and magnitude_base_bool:

                this_mask = np.float32(weight_magnitudes > global_thres)

            else:
                # index: number of pruned parameters
                idx = int( (1 - nn_density_level) * np.size(weights) )

                # threshold: entries which below the thredhold will be removed
                thres = np.sort(np.reshape(weight_magnitudes, [-1] ))[idx]

                # 0 selected for weight parameters with magnitudes smaller than the threshold, 1 otherwise
                this_mask = np.float32(weight_magnitudes > thres)

                if magnitude_base_bool == False:
                    # in the case of random pruning: randomly shuffle the mask
                    this_mask = random.shuffle(random.PRNGKey(0), this_mask )

            masks.append(this_mask )

        else:
            raise NotImplementedError

    return masks

def get_sparse_params_filtered_by_masks(params, masks):

    """ Filter jax.experimental.stax parameters according to a collection of masks

    Args:
        params: parameters in a stax format.
        masks: a collection of masks for parameter pruning.

    Returns:
        sparse_params: sparse parameters acquired by passing params through the binay masks.
    """

    sparse_params = []
    for layer_index in range( len(params)):

        if len(params[layer_index]) < 2:
            # In this the case, the layer does not contain weight or bias parameters
            sparse_params.append( () )

        elif len(params[layer_index]) == 2:
            # In this case, the layer contains a tuple of parameters for weights and biases
            weights  = params[layer_index][0]

            biases = params[layer_index][1]

            mask_this_layer = masks[layer_index]

            # sparse weights gated by masks
            sparse_weights = np.multiply(mask_this_layer, weights )

            # weight-bias tuple
            sparse_params_this_layer = (sparse_weights, biases )

            sparse_params.append( sparse_params_this_layer )
        else:
            raise NotImplementedError

    return sparse_params

def stax_params_l2_square(params, mask = None, regularize_bias_bool = False):

    """ Compute squared l2 norm of stax parameters

    Args:
        params: parameters in a stax format.
        regularize_bias_bool: a boolean var deciding wether biases are regularized or not.
    Returns:
        params_norm_squared: squared l2 norm of stax parameters.
    """
    if (type(regularize_bias_bool) != bool):
        raise ValueError("regularize_bias_bool should be a boolean variable")

    if regularize_bias_bool == False:
        # assemble a list of weight parmeters; discard bias parameters
        list_params = [layer_param[0] for layer_param in params if len(layer_param) == 2 ]
    else:
        # remove the empty tuple from the list of paramters.
        list_params = list(sum(params, ()))

    params_norm_squared = np.sum([ np.sum(np.square( list_params[i] ))  for i in range(len(list_params)) ]  )

    return params_norm_squared

def softmax_cross_entropy_with_logits_l2_reg(params, f, x, targets, masks = None, L2_REG_COEFF = 0.0, key = None):

    """ cross entropy loss + weighted l2 regularization.

    Args:
        params: parameters in a stax format.
        apply_fn: a function that maps a set of network-parameters together with a set of network-inputs to network-outputs.
        x: network inputs
        targets: the target outputs
        masks: the sparsity-inducing mask
        L2_REG_COEFF: l2 regularization constant.
    Returns:
        The cross entropy loss + weighted l2 regularization
    """

    if masks is not None:
        masked_params = get_sparse_params_filtered_by_masks(params, masks)
    else:
        masked_params = params

    if key is not None:
        dense_outputs = f(masked_params, x, rng = key)
    else:
        dense_outputs = f(masked_params, x)

    preds = logsoftmax(dense_outputs)

    params_norm_squared = stax_params_l2_square(masked_params)

    return -np.mean(np.sum(preds * targets, axis=1)) + L2_REG_COEFF * params_norm_squared

def simpleaxis(ax):
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()

# def get_snip_masks(params, nn_density_level, predict, snip_batch, batch_input_shape):

#     init_grads = grad(softmax_cross_entropy_with_logits_l2_reg)(params, predict, snip_batch[0].reshape(batch_input_shape), snip_batch[1])

#     cs = [abs( init_grads[idx][0] *  params[idx][0]).flatten() for idx in range(len(params)) if len(params[idx]) == 2 ]

#     pooled_cs = np.hstack(cs)

#     idx = int( (1 - nn_density_level) * len(pooled_cs) )

#     # threshold: entries which below the thredhold will be removed
#     thres = np.sort(pooled_cs)[idx]

#     masks = []
#     for layer_index in range( len(params)):

#         if len(params[layer_index]) < 2:
#             # In this the case, the layer does not contain weight and bias parameters.
#             masks.append( [] )

#         elif len(params[layer_index]) == 2:
#             # In this case, the layer contains a tuple of parameters for weights and biases

#             weights = params[layer_index][0]

#             weights_grad = init_grads[layer_index][0]

#             layer_cs = np.abs(weights * weights_grad)

#             # 0 selected for weight parameters with magnitudes smaller than the threshold, 1 otherwise
#             this_mask = np.float32(layer_cs >= thres)

#             masks.append(this_mask )

#         else:
#             raise NotImplementedError

#     return masks

def get_snip_masks(params, nn_density_level, predict, snip_batch, batch_input_shape, GlOBAL_PRUNE_BOOL = True):

    init_grads = grad(softmax_cross_entropy_with_logits_l2_reg)(params, predict, snip_batch[0].reshape(batch_input_shape), snip_batch[1])

    thres_list = [None] * len(params)

    if GlOBAL_PRUNE_BOOL == True: # global pruning

        cs = [abs( init_grads[idx][0] *  params[idx][0]).flatten() for idx in range(len(params)) if len(params[idx]) == 2 ]

        pooled_cs = np.hstack(cs)

        idx = int( (1 - nn_density_level) * len(pooled_cs) )

        # threshold: entries which below the thredhold will be removed
        thres = np.sort(pooled_cs)[idx]
        thres_list = [thres] * len(params)

    else: # layerwise pruning
        for layer_index in range( len(params)):
            if len(params[layer_index]) == 2:

                cs = abs( init_grads[layer_index][0] *  params[layer_index][0]).flatten()
                idx = int( (1 - nn_density_level) * len(cs) )
                # threshold: entries which below the thredhold will be removed
                thres = np.sort(cs)[idx]
                thres_list[layer_index] = thres

    masks = []
    for layer_index in range( len(params)):

        if len(params[layer_index]) < 2:
            # In this the case, the layer does not contain weight and bias parameters.
            masks.append( [] )

        elif len(params[layer_index]) == 2:
            # In this case, the layer contains a tuple of parameters for weights and biases

            weights = params[layer_index][0]

            weights_grad = init_grads[layer_index][0]

            layer_cs = np.abs(weights * weights_grad)

            # 0 selected for weight parameters with magnitudes smaller than the threshold, 1 otherwise
            this_mask = np.float32(layer_cs >= thres_list[layer_index])

            masks.append(this_mask )

        else:
            raise NotImplementedError

    return masks

def get_logit_snip_masks(params, nn_density_level, predict, x_batch, batch_input_shape, GlOBAL_PRUNE_BOOL = True):

    def norm_square_logits(params, f, x):
        return np.sum(f(params, x) **2)

    init_grads = grad(norm_square_logits)(params, predict, x_batch.reshape(batch_input_shape) )

    thres_list = [None] * len(params)

    if GlOBAL_PRUNE_BOOL == True: # global pruning

        cs = [abs( init_grads[idx][0] *  params[idx][0]).flatten() for idx in range(len(params)) if len(params[idx]) == 2 ]

        pooled_cs = np.hstack(cs)

        idx = int( (1 - nn_density_level) * len(pooled_cs) )

        # threshold: entries which below the thredhold will be removed
        thres = np.sort(pooled_cs)[idx]
        thres_list = [thres] * len(params)

    else: # layerwise pruning
        for layer_index in range( len(params)):
            if len(params[layer_index]) == 2:

                cs = abs( init_grads[layer_index][0] *  params[layer_index][0]).flatten()
                idx = int( (1 - nn_density_level) * len(cs) )
                # threshold: entries which below the thredhold will be removed
                thres = np.sort(cs)[idx]
                thres_list[layer_index] = thres

    masks = []
    for layer_index in range( len(params)):

        if len(params[layer_index]) < 2:
            # In this the case, the layer does not contain weight and bias parameters.
            masks.append( [] )

        elif len(params[layer_index]) == 2:
            # In this case, the layer contains a tuple of parameters for weights and biases

            weights = params[layer_index][0]

            weights_grad = init_grads[layer_index][0]

            layer_cs = np.abs(weights * weights_grad)

            # 0 selected for weight parameters with magnitudes smaller than the threshold, 1 otherwise
            this_mask = np.float32(layer_cs >= thres_list[layer_index])

            masks.append(this_mask )

        else:
            raise NotImplementedError

    return masks

def mean_and_var_across_trials(results_mat):
    """
    Args:
    results_mat: A `np.ndarray` of shape [n_trials, n_data_points_evaluated]

    Returns:
    av_results_mat_across_trials: averaged results across trials (axis 0).

    var_results_mat_across_trials: the variance across trials
    """

    av_results_mat_across_trials =  np.float32(np.mean(results_mat, axis = 0))
    var_results_mat_across_trials = np.var(results_mat, axis = 0)
#     max_min_in_trials = np.vstack( (np.amin(results_mat, axis = 0 ), np.amax(results_mat, axis = 0 ) ) )

#     max_min_centered_diff = np.abs(max_min_in_trials - av_results_mat_across_trials)

    return av_results_mat_across_trials, var_results_mat_across_trials

def load_supervised_training_result(supervised_result_file, wiring_str, nn_density_list, num_plots = -1, supervised_result_path = '/tungstenfs/scratch/gzenke/liutian/nt_transfer/saved_data/supervised_results/'):
    result_dict = {}
    final_result_dict = {}

#     if wiring_str == 'rand':
        # layerwise and global random pruning is the same thing because each parameter will be pruned with an identitical chance.
#         supervised_result_file=supervised_result_file.replace("global", "layerwise")

    for density_level in nn_density_list:

        model_path = supervised_result_path + supervised_result_file
        model_wiring_dir = supervised_result_file +  '_' + wiring_str

        loadFile =  model_path + '/' + model_wiring_dir + '/density_' + str(density_level) + '/supervised_trained_' + supervised_result_file + '_' + 'density_' + str(density_level)

        density_result = list(np.load(loadFile + '.npy' ,allow_pickle='TRUE'))

        density_str = str(density_result[0])

        av_train, trial_variation_train = mean_and_var_across_trials(density_result[1][:, :num_plots] )

        av_test, trial_variation_test = mean_and_var_across_trials(density_result[2][:, :num_plots])

#         av_train, trial_variation_train = mean_and_var_across_trials(density_result[1])

#         av_test, trial_variation_test = mean_and_var_across_trials(density_result[2])

        result_density_dict = {'train': {'av': av_train, 'var': trial_variation_train}, 'test': {'av': av_test, 'var': trial_variation_test}}

#         final_result_density_dict = {'train': {'av': av_train[-1], 'var': trial_variation_train[:, -1]}, 'test': {'av': av_test[-1], 'var': trial_variation_test[:, -1]}}
        final_result_density_dict = {'train': {'av': av_train[-1], 'var': trial_variation_train[ -1]}, 'test': {'av': av_test[-1], 'var': trial_variation_test[ -1]}}

        result_dict[str(density_level) ] = result_density_dict

        final_result_dict[str(density_level) ] = final_result_density_dict

    return result_dict, final_result_dict


import math

def primeiraFunc(nome):
    print('Primeira Funcao, ola ' + nome)

print(primeiraFunc('Matheus'))

def segundaFunc(number, number2):
    print('Segunda Funcao, o numero eh ' + str(number))
    print('Segunda Funcao, o numero eh ' + str(number2))
    print('A soma eh ' + str(number + number2))

print(segundaFunc(10, 20))

def numerosPares(number):
    if number % 2 == 0:
        return 'O numero eh par'
    else:
        return 'O numero eh impar'

print(numerosPares(215))

def numeroPrimo(number):
    """Verificando se o número é primo"""

    if (number % 2) == 0 and number > 2:
        return "O número é par e não primo"
    for i in range(3, int(math.sqrt(number)) + 1, 2):
        if (number % i) == 0:
            return "Este número não é primo"
    return "Este número é primo"

print(numeroPrimo(119))

# Fazendo split dos dados

def split_string(text):
    return text.split(" ")

texto = "Como a democracia seria algo bom, se a maioria das pesoas não sabem nem o básico sobre política? "

print(split_string(texto))

token = split_string(texto)

print(token)


import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
from matplotlib.gridspec import GridSpec
import matplotlib.patches as mpatches
from os.path import join
import random
# from trial.plot_scripts.secondary_results import per_user_results

from trial.plot_scripts.util import collect_all_user_data, get_usefull_data_from_df

def subplot_overview_correct_false(data, ax=None):
    if ax is None:
        ax = plt.gca()

    num_correct = sum(data.Correct)
    num_total = len(data.Correct)
    num_wrong = num_total - num_correct
    labels = 'Correct', 'Wrong'
    plotdata = [num_correct, num_wrong]
    subplt = ax.pie(plotdata, explode=(0.1, 0.1), shadow=True, startangle=80, labels=labels, autopct='%1.1f%%',textprops={'fontsize': 14, 'fontweight':'bold'}, colors=('C0', 'red'), wedgeprops={"edgecolor":"k",'linewidth': 2, 'antialiased': True})
    ax.set_title("Top 1")

    for autotext in subplt[2]:
        autotext.set_color('white')
    return subplt

def subplot_confusion_matrix(data, ax=None):
    class ConfusionMatrix(object):
        def __init__(self, n_classes, labels=None):
            self.n_classes = n_classes
            self.mat = torch.zeros(n_classes, n_classes, requires_grad=False)
            self.labels = labels

        def update(self, preds, labels):
            for p, t in zip(preds, labels):
                self.mat[t, p] += 1

    predictions, gtlabels, labels, n_classes = get_usefull_data_from_df(data)
    label_to_idx = {l: idx for idx, l in enumerate(labels)}
    confusionmatrix = ConfusionMatrix(n_classes, labels)

    if ax is None:
        ax = plt.gca()

    for p,g in zip(data.Prediction, data.GroundTruth):
        confusionmatrix.update([label_to_idx[p]], [label_to_idx[g]])

    fontsize=15
    label_angle=45
    mat = confusionmatrix.mat

    total_samples = mat.sum(axis=1)
    mat = mat / total_samples[:, None]
    mat = torch.nan_to_num(mat, nan=0)

    subplt = ax.matshow(mat, cmap=plt.cm.Blues)

    threshold = mat.max() / 2.
    for (i, j), z in np.ndenumerate(mat):
        color = "white" if z > threshold else "black"
        ax.text(j, i, '{:.2f}'.format(z), ha='center', va='center', color=color, fontsize=fontsize)

    ax.set_xticks(range(0, n_classes))
    ax.xaxis.set_ticks_position('bottom')
    ax.set_xticklabels(labels, rotation=label_angle, fontsize=fontsize)
    ax.set_yticks(range(0, n_classes))
    ax.set_yticklabels(labels, rotation=label_angle, fontsize=fontsize)

    ax.set_ylabel('True Class')
    ax.yaxis.set_label_position("right")
    ax.set_xlabel('Predicted Class')
    ax.set_title("Confusion Matrix")
    return subplt

def subplot_confusion_histogram(data, ax=None):
    predictions, gtlabels, labels, n_classes = get_usefull_data_from_df(data)

    if ax is None:
        ax = plt.gca()

    num_correct_per_class = []
    num_wrong_per_class = []

    for lbl in labels:
        d = data.loc[data.GroundTruth==lbl]
        num_correct_per_class.append(len(d.loc[d.GroundTruth==d.Prediction]))
        num_wrong_per_class.append(len(d.loc[d.GroundTruth!=d.Prediction]))

    width = 0.5       # the width of the bars: can also be len(x) sequence

    subplt = ax.bar(labels, num_correct_per_class, width, label='Correct', edgecolor='black')
    ax.bar(labels, num_wrong_per_class, width, bottom=num_correct_per_class, label='Wrong', color='red', edgecolor='black')
    ax.set_xticklabels(labels, rotation=45, fontsize=15)
    bottom, top = ax.get_ylim()  # return the current ylim
    ax.set_ylim([bottom, top+(top*0.1)])
    ax.set_ylabel('# Videos')
    ax.set_title('Class confusion Histogram')
    ax.legend()

    return subplt

def subplot_decision_duration_chronological(data, ax=None):
    duration = list(data.Duration)
    correct = list(data.Correct)

    colors = []
    for i in correct:
        c = 'C0' if i==1 else 'red'
        colors.append(c)

    if ax is None:
        ax = plt.gca()

    subplt = ax.scatter(range(len(duration)), duration, c=colors)
    ax.set_ylabel('Viewing Duration (seconds)')
    ax.set_xlabel('Video-number shown')
    ax.set_title('How long to decide')
    ax.legend()

    return subplt

def subplot_decision_duration_per_class(data, ax=None, ylimtop=None):
    predictions, gtlabels, labels, n_classes = get_usefull_data_from_df(data)

    if ax is None:
        ax = plt.gca()

    duration_per_class = {}
    for i, lbl in enumerate(labels):
        durations = list(data.loc[data.GroundTruth==lbl].Duration)
        duration_per_class[lbl] = durations

        tmp = list(data.loc[data.GroundTruth==lbl].Correct)
        colors = []
        for tt in tmp:
            c = 'C0' if tt==1 else 'red'
            colors.append(c)

        x = np.random.normal(i+1, 0.08, size=len(durations))
        ax.scatter(x, durations, alpha=0.15,  s=20, c=colors)
        # ax.plot(x, durations, 'bo')

    ax.boxplot(duration_per_class.values(), widths=0.3, vert=True, showfliers=False, boxprops=dict(linewidth=2, color='black'), whiskerprops=dict(linestyle='-',linewidth=2.0, color='black'))
    ax.set_xticklabels(duration_per_class.keys(), fontsize=15)

    red_patch = mpatches.Patch(color='red', label='Wrong')
    blue_patch = mpatches.Patch(color='C0', label='Correct')

    ax.legend(handles=[blue_patch, red_patch], loc='upper right')
    ax.set_ylim([0, ylimtop])
    ax.set_ylabel('Viewing Duration (seconds)')
    # ax.set_xlabel('Video-number shown')
    ax.set_title('Decision speed')
    ax.set_xticklabels(labels, rotation=45)

    # ax.legend()

    return

def main_from_csv(filename):
    df_all = pd.read_csv(filename)
    main(df_all)

def main(dataframe, save_dir=None):

    df_all = dataframe.copy()
    df_all = df_all.loc[df_all.Trainmode==0]

    # TODO: SET THIS MAX YLIM TO SOMETHING REASONABLE
    max_decision_time = round(max(list(df_all.Duration))) + 2
    max_decision_time = 30
    modalities = set(df_all.Modality)

    modalities_nicename = {
        "ucf5": "UCF5",
        "afd5": "AFD5",
        # "ucf5rgbflow": "DONT USE RGBFLOW"
    }

    for modality in ['ucf5', 'afd5']:
        df = df_all.loc[df_all.Modality==modality]
        f, axes = plt.subplots(1, 4, figsize=(16, 5))
        (ax1, ax2, ax3, ax4) = axes.ravel()
        subplot_overview_correct_false(df, ax=ax1)
        subplot_confusion_matrix(df, ax=ax2)
        subplot_decision_duration_per_class(df, ax=ax3, ylimtop=max_decision_time)
        subplot_confusion_histogram(df, ax=ax4)

        f.suptitle(f"{modalities_nicename[set(df.Modality).pop()]}")
        plt.show(block=False)

        if save_dir != None:
            f.savefig(f'{join(save_dir, modality + ".pdf")}')
        f.tight_layout()
    plt.tight_layout()
    plt.show(block=True)

if __name__ == "__main__":
    root_dir = '/home/f/projects/MotionPaper/trial/output/'
    root_output_dir = '/home/f/projects/MotionPaper/trial/output'
    df = collect_all_user_data(root_dir)

    main(df, save_dir=root_dir)
    # per_user_results()


#!/usr/bin/env python3

import numpy as np
import cv2

def main():

    image = cv2.imread("../Aula2/lake.jpg")

    for i in np.arange(1, 0.2, -0.01):

        image2 = image.copy()

        image2[:, int(image.shape[1]/2):] = (image2[:, int(image.shape[1]/2):] * i).astype(np.uint8)

        #cv2.imshow('window', image)
        cv2.imshow('window2', image2)

        cv2.waitKey(50)

    cv2.waitKey(0)

if __name__ == "__main__":
    main()


import planetary_data as pd
import tools as t_
from OrbitPropagator import OrbitPropagator as OP
from OrbitPropagator import null_perts

#Cd coefficient of drag, 2.2 spacecraft
#A area
cb = pd.mars

# time parameterss
tspan = 3600*6 # hour
dt=100.0

if __name__ == '__main__':
    #perturbations dictionary, arbitrary

    # initial mass of spacecraft
    mass0=50.0 # kg

    # perigee and apogee
    rp=300+cb['radius'] # km
    ra=300+cb['radius'] # km

    # calculate other orbital elements
    a=(rp+ra)/2.0 # km
    e=(ra-rp)/(ra+rp)

    # calculate initial state vector
    state0=[a,e,i,ta,aop,raan]

    op=OP(state0,tspan,dt,deg=True,coes=True,mass0=mass0,perts=perts)
    op.plot_alts(show_plot=True,hours=True)
    op.plot_3d(show_plot=True)
    op.calculate_coes()
    op.plot_coes(show_plot=True,hours=True)
    op.calculate_apoapse_periapse()
    op.plot_apoapse_periapse(show_plot=True,hours=True)


#!/usr/bin/env python3

# import of built-in modules
import os

# import of third-party modules

# import of local modules
import ccf.one_subject_run_status_checker as one_subject_run_status_checker
import ccf.archive as ccf_archive

# authorship information
__author__ = "Timothy B. Brown"
__copyright__ = "Copyright 2017, The Connectome Coordination Facility"
__maintainer__ = "Timothy B. Brown"

class OneSubjectRunStatusPerScanChecker(one_subject_run_status_checker.OneSubjectRunStatusChecker):
	"""Determine the run status for a CCF pipeline that is run on each scan.

	This class is an abstract run status checker for pipelines that are run
	separately on a set of scans for a subject/session. See the superclass for
	further documentation.
	"""

	def _path_to_running_marker_file(self, subject_info):

		file_name = self.PIPELINE_NAME
		file_name += '.' + subject_info.subject_id
		file_name += '_' + subject_info.classifier
		file_name += '_' + subject_info.extra
		file_name += '.' + 'RUNNING'

		archive = ccf_archive.CcfArchive()
		self.running_status_dir = archive.running_status_dir_full_path(subject_info)
		path = self.running_status_dir + os.sep + file_name
		#print("path: " + path)
		return path


import sys # sys нужен для передачи argv в QApplication
from PyQt5 import QtWidgets, QtCore, QtGui
from des import *
from check_db import *

class Interface(QtWidgets.QMainWindow, Ui_Form):
    def __init__(self):
        # Это здесь нужно для доступа к переменным, методам
        # и т.д. в файле design.py
        super().__init__()
        self.ui = Ui_Form()
        self.ui.setupUi(self)  # Это нужно для инициализации нашего дизайна

        self.ui.pushButton.clicked.connect(self.register)
        self.base_line_Edit = [self.ui.lineEdit, self.ui.lineEdit_2, self.ui.lineEdit_3, self.ui.lineEdit_4]

        self.check_db = CheckThread()
        self.check_db.mysignal.connect(self.signal_handler)

    def check_input(funct):
         def wrapper(self):
            for line_edit in self.base_line_edit:
                if len(line_edit.text()) == 0:
                  return
            funct(self)
         return wrapper

    def signal_handler(self, value):
        QtWidgets.QMessageBox.about(self,'Notification', value)

    def register(self):
        name = self.ui.lineEdit.text()
        lastname = self.ui.lineEdit_2.text()
        mail = self.ui.lineEdit_3.text()
        time = self.ui.lineEdit_4.text()
        self.check_db.thr_register(name, lastname, mail, time)

def main():
    app = QtWidgets.QApplication(sys.argv)  # Новый экземпляр QApplication
    window = Interface()  # Создаём объект класса ExampleApp
    window.setWindowFlags(window.windowFlags() & ~QtCore.Qt.WindowMaximizeButtonHint) # Так мы убираем возможность нажать кнопку развернуть окно
    window.setFixedSize(window.size()) # а так устанавливаем фиксированный размер окна, без возможности изменить размер мышкой
    window.show()  # Показываем окнa
    app.exec_()  # и запускаем приложение

if __name__ == '__main__':  # Если мы запускаем файл напрямую, а не импортируем
    main()  # то запускаем функцию main()


import time
from CollectionModule.mongo_db_handler import collection_manager
from bson import ObjectId

class clean_data(object):

    #Gallery card transaction note
    '''
    Title => Account transfer of xxx$
    Subtitle => xxx$ transfered from sender_name to recieved_name
    Button => Click here to view more
    username,password => [{'save_variables': [{'user_id': 'bla_bla', 'balance': ''}]}]
    '''

    def generate_gallery_card_from_list(self, list_name, data):
        gal_card =   {
            "attachment":{
            "type":"template",
            "payload":{
                "template_type":"generic",
                    "elements":[
                    ]
                }
            }
        }
        for entry in data:
            element_list = gal_card["attachment"]["payload"]["elements"]
            element = {
                "title": "{0}",
                "buttons": [
                    {
                        "type": "web_url",
                        "url": "https://google.com",
                        "title": "click here to view more"
                    }
                ]
            }

            element["title"] = entry
            element_list.append(element)

        return [gal_card]

    def generate_gallery_card_transaction(self, transactions):
        gal_card =   {
            "attachment":{
            "type":"template",
            "payload":{
                "template_type":"generic",
                    "elements":[
                    ]
                }
            }
        }

        for trans in transactions:
            element_list = gal_card["attachment"]["payload"]["elements"]
            title = "Account transfer of ${0}".format(trans["amount"])
            subtitle = "${0} transfered from {1} to {2} on {3} at {4}".format(
                trans["amount"],
                trans["sender_name"],
                trans["receiver_name"],
                trans["date"],
                trans["time"])

            element = {
                "title": "{0}",
                "subtitle": "{1}",
                "buttons": [
                    {
                        "type": "web_url",
                        "url": "https://google.com",
                        "title": "click here to view more"
                    }
                ]
            }

            element["title"] = title
            element["subtitle"] = subtitle
            element_list.append(element)
        return [gal_card]

    def get_transactions(self, transactions, amounts):
        trans = []
        db = collection_manager("banks", "users")
        db_trans = collection_manager("banks", "transactions")
        if amounts == 0:
            amounts = 10
        for transaction in reversed(transactions):
            amounts = amounts - 1
            transaction = db_trans.find_query({"_id": ObjectId(transaction)})[0]
            sender_identity = transaction["sender"]
            receiver_identity = transaction["receiver"]
            amount = transaction["amount"]
            send_Obj = db.find_query({"_id": sender_identity})[0]
            receive_obj = db.find_query({"_id": receiver_identity})[0]
            send_name = send_Obj["fullname"]
            receive_name = receive_obj["fullname"]
            data_dict = {}
            data_dict["sender_name"] = send_name
            data_dict["receiver_name"] = receive_name
            data_dict["amount"] = amount
            data_dict["date"] = time.strftime('%Y-%m-%d', time.localtime(transaction["timestamp"]))
            data_dict["time"] = time.strftime('%H:%M:%S', time.localtime(transaction["timestamp"]))
            trans.append(data_dict)
            if amounts == 0:
                break
        return self.generate_gallery_card_transaction(trans)

    def generate_branch_card(self, branchArray):
        gal_card =   {
            "attachment":{
            "type":"template",
            "payload":{
                "template_type":"generic",
                    "elements":[
                    ]
                }
            }
        }
        for branch in branchArray:
            element_list = gal_card["attachment"]["payload"]["elements"]
            title = "{0} branch located at of {1}".format(branch["name"], branch["location"])
            subtitle = "branch ID is: {0}".format(str(branch["branch_id"]))

            element = {
                "title": "{0}",
                "subtitle": "{1}",
                "buttons": [
                    {
                        "type": "web_url",
                        "url": "https://google.com",
                        "title": "click here to view more"
                    }
                ]
            }

            element["title"] = title
            element["subtitle"] = subtitle
            element_list.append(element)
        return [gal_card]


import requests
import redis

from bs4 import BeautifulSoup
from datetime import datetime

class Download:
    '下载类'

    def __init__(self, dict_header=None, int_retries=3, int_delay=2, int_timeout=30):
        '''【初始化】
        dict_header：标头
        int_retries：重试
        delay：延迟
        timeout：超时'''
        self.dict_header = dict_header
        self.int_retries = int_retries
        self.int_delay = int_delay
        self.int_timeout = int_timeout

    def download_all(self, str_url):
        '''【下载页面】
        str_url：地址
        bool_json：是否json类型'''
        try:
            response_obj = requests.get(str_url, headers=self.dict_header, timeout=self.int_timeout)
            if response_obj.status_code == 200:
                return response_obj.content
        except Exception as e:
            str_html = ''
        return str_html

class Crawler:
    '爬虫类'

    def __init__(self, dict_header=None, int_retries=3, int_delay=2, int_timeout=30):
        '''【初始化】
        dict_header：标头
        int_retries：重试
        delay：延迟
        timeout：超时'''
        self.dict_header = dict_header
        self.int_retries = int_retries
        self.int_delay = int_delay
        self.int_timeout = int_timeout
        self.download = Download(dict_header, int_retries, int_delay, int_timeout)
        self.redis = Redis()

    def crawler_operate(self, str_url, int_page_start, int_page_end, int_page_step):
        str_date = ''
        int_page_num = 1
        for int_page in range(int_page_start, int_page_end, int_page_step):
            str_date += self.crawler_data(str_url.format(int_page))
            print('第%i页'.center(30, '*') % int_page_num)
            int_page_num += 1
        self.redis.redis_save(str_date)

    def crawler_data(self, str_url):
        '【处理数据】，str_url：地址'
        str_page = self.download.download_all(str_url)
        # 解析内容
        soup_obj = BeautifulSoup(str_page, 'lxml')
        soup_all_items = soup_obj.find_all('div', attrs={'class': 't_l_bd'})
        str_date = ''
        for soup_item in soup_all_items:
            # print('='*30)
            str_en = soup_item.find('p', attrs={'class': 't_l_en'}).find('a').text
            str_cn = soup_item.find('p', attrs={'class': 't_l_cn'}).find('a').text
            str_date += ('{}{}_'.format(str_en, str_cn))
        return str_date

    # def crawler_re_chars(self, str_text):
    #     '【过滤特殊符号】，str_text：文本'
    #     return re.sub('[^\u4e00-\u9fa5a-zA-Z0-9\'\",.，。!@#$%&*()]', '', str_text)

class Redis:

    def __init__(self, str_host='127.0.0.1', int_port=6379):
        '【初始化】'
        self.obj_clinet = redis.StrictRedis(str_host, int_port)

    def redis_save(self, str_value):
        '【记录】'
        self.obj_clinet.set(datetime.now().strftime("%Y%m%d%H%M"), str_value)

    def redis_query(self, str_key, str_count):
        '【查询】'
        try:
            list_value = self.obj_clinet.get(str_key).decode().split('_')
            for i in range(0, int(str_count)):
                print(list_value[i])
        except Exception as e:
            return 0

def main():
    # 地址
    str_url = 'http://news.iciba.com/appv3/wwwroot/ds.php?action=tags&id=4&ob=1&order=2&page={}'

    # 标头
    dict_header = {
        'User-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
        'referer': 'http://my.iciba.com/index.php'
    }
    # 实例化
    crawler_obj = Crawler(dict_header=dict_header)
    # 获取3页数据
    # crawler_obj.crawler_operate(str_url, 1, 4, 1)
    Redis().redis_query('201810272251', 6)

if __name__ == '__main__':
    main()


# -*- coding: UTF-8 -*-
import sys
from operator import itemgetter
import numpy as np
import cv2
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F

'''
Function:
    change rectangles into squares (matrix version)
Input:
    rectangles:  torch.Tensor
        rectangles[i][0:3] is the position, rectangles[i][4] is score
Output:
    squares: same as input
'''
def rect2square(rectangles):
    w = rectangles[:,2] - rectangles[:,0]
    h = rectangles[:,3] - rectangles[:,1]
    l, max_index = torch.max(torch.stack([w, h], dim=1), dim=1)
    rectangles[:,0] = rectangles[:,0] + w*0.5 - l*0.5
    rectangles[:,1] = rectangles[:,1] + h*0.5 - l*0.5
    rectangles[:,2:4] = rectangles[:,0:2] + l.view(-1, 1).repeat(1, 2)
    return rectangles

def NMS(rectangles, threshold, type):
    """
    Function:
        apply NMS(non-maximum suppression) on ROIs in same scale(matrix version)
    Input:
        rectangles: torch.floattensor. rectangles[i][0:3] is the position, rectangles[i][4] is score
    Output:
        rectangles: same as input
    """
    if rectangles.size(0)==0:
        return rectangles

    x1 = rectangles[:, 0]
    y1 = rectangles[:, 1]
    x2 = rectangles[:, 2]
    y2 = rectangles[:, 3]
    score = rectangles[:, 4]
    area = torch.mul(x2 - x1 + 1.0, y2 - y1 + 1.0)
    _, order = torch.sort(score, descending=True)
    keep = []
    while order.numel() > 0:
        idx = order[0]  #---highest score
        keep.append(idx)
        if order.size(0) ==1: break
        xx1 = torch.clamp(x1[order[1:]], min=x1[idx])
        yy1 = torch.clamp(y1[order[1:]], min=y1[idx])
        xx2 = torch.clamp(x2[order[1:]], max=x2[idx])
        yy2 = torch.clamp(y2[order[1:]], max=y2[idx])

        w = torch.clamp(xx2 - xx1 + 1, min=0.0)
        h = torch.clamp(yy2 - yy1 + 1, min=0.0)
        inter  = torch.mul(w, h)
        if type == 'iom':
            o = inter / torch.clamp(area[order[1:]], max=area[idx])
        else:
            o = inter / (area[idx] + area[order[1:]] - inter)
        order = order[1:][(o < threshold)]

    keep = torch.LongTensor(keep) #---convert list to torch.LongTensor
    result_rectangles = torch.index_select(rectangles, 0, keep)
    return result_rectangles

def detect_face_pnet(img, imgPad, net, thres, device_id=0):
    """
    img: original image
    imgPad: padded original image
    net: net model
    thres: threshold for net
    """
    img_pad_rows, img_pad_cols, _ = imgPad.shape
    img_rows, img_cols, _ = img.shape
    delta_row = (img_pad_rows - img_rows) / 2
    delta_col = (img_pad_cols - img_cols) /2

    netSize = 24 #-----size of input
    minFace = 28
    stride = 8
    scale_ = 1.414
    curScale = minFace * 1.0 / netSize
    imgResized = cv2.resize(img, (int(img.shape[1]/curScale), int(img.shape[0]/curScale)), interpolation=cv2.INTER_CUBIC)

    result = []
    count = 0
    while min(imgResized.shape[:2]) >= netSize:
        #print("curScale:\t", curScale)
        imgResized_input = imgResized.copy()
        imgResized_input = imgResized_input.transpose((2, 0, 1)) / 255.0 #---[H, W, 3] ----> [3, H, W]
        imgResized_input = torch.from_numpy(imgResized_input)
        imgResized_input = torch.unsqueeze(imgResized_input, 0) #----[1, 3, H, W]
        imgResized_input = Variable(imgResized_input).float()
        prob, rotate_prob, bbox_reg = net(imgResized_input)
        #prob = F.sigmoid(prob)
        #rotate_prob = F.sigmoid(rotate_prob)
        #prob: [1, 1, h, w]; rotate_prob: [1, 1, h, w]; bbox_reg: [1, 4, h, w]
        prob = prob[0, 0]; rotate_prob = rotate_prob[0, 0];  bbox_reg = bbox_reg[0]
        prob = prob.t(); rotate_prob = rotate_prob.t(); bbox_reg = bbox_reg.permute(0, 2, 1); #--[4, w, h]
        prob_mask = (prob > thres)
        indexes = prob_mask.nonzero().float()
        if indexes.numel() > 0:
            bb1 = torch.round((stride * indexes + 0) * curScale) #--[x1, y1]
            bb2 = torch.round((stride * indexes + 24) * curScale) #--[x2, y2]
            boundingbox = torch.cat([bb1, bb2], dim=1)
            dx1, dy1, dx2, dy2 = bbox_reg[0][prob_mask], bbox_reg[1][prob_mask], bbox_reg[2][prob_mask], bbox_reg[3][prob_mask]
            offset = torch.stack([dx1, dy1, dx2, dy2], dim=1)

            boundingbox = boundingbox + offset * 24.0 * curScale
            #boundingbox[:, 0::2] = boundingbox[:, 0::2] + delta_col
            #boundingbox[:, 1::2] = boundingbox[:, 1::2] + delta_row
            score = prob[prob_mask].unsqueeze(1)
            rotate_score = rotate_prob[prob_mask]
            rotate_positive_mask = (rotate_score >= 0.5)
            rotate_negative_mask = ~rotate_positive_mask
            rotate_score[rotate_positive_mask] = 0. #----0 degree
            rotate_score[rotate_negative_mask]  = 180. #----180 degree
            rotate_score = rotate_score.unsqueeze(1) #---change shape
            rectangles = torch.cat([boundingbox, score, rotate_score], dim=1)
            rectangles = rect2square(rectangles) #---turn into square

            rectangles[:, :2] = torch.clamp(rectangles[:, :2], min=0)
            rectangles[:, 2] = torch.clamp(rectangles[:, 2], max=img_cols-1)
            rectangles[:, 3] = torch.clamp(rectangles[:, 3], max=img_rows-1)

            index1 = (rectangles[:, 2] > rectangles[:, 0] + 12) #---x2 > x1
            index2 = (rectangles[:, 3] > rectangles[:, 1] + 12) #---y2 > y1
            index = (index1 & index2).nonzero().squeeze()
            rectangles = rectangles.index_select(0, index)

            if rectangles.numel() > 0:
                rectangles = rectangles.data
                rectangles = NMS(rectangles, 0.5, 'iou')
                result.append(rectangles)

        #cv2.imwrite("resized_"+str(count) + ".jpg", imgResized)
        count += 1
        imgResized = cv2.resize(imgResized, (int(imgResized.shape[1]/scale_), int(imgResized.shape[0]/scale_)), interpolation=cv2.INTER_CUBIC)
        curScale = img_rows * 1.0 / imgResized.shape[0]

    result = torch.cat(result, dim=0)
    return result

def filter_face_rnet(prob, angle_prob, bbox_reg, rectangles, img, img180, threshold):
    """
    prob.size: [N, 1]
    angle_prob.size: [N, 3]
    bbox_reg.size: [N, 4]
    """
    height, width, _ = img.shape
    prob = F.sigmoid(prob) #---sigmoid classification prob
    angle_prob = F.softmax(angle_prob) #---softmax for angle prob
    prob = prob.data; #---variable to tensor
    angle_prob = angle_prob.data; #---variable to tensor
    bbox_reg = bbox_reg.data #---variable to tensor

    #---if face up: index = 0, angle = 90
    #               index = 1, angle = 0
    #               index = 2, angle = -90
    #---if face down:index = 0, angle = 90
    #               index = 1, angle = 180
    #               index = 2, angle = -90
    _, angle_max_index = torch.max(angle_prob, dim=1) #----find corresponding angle
    binary_tensor = (prob>=threshold)
    indexes = binary_tensor.nonzero()[:,0]
    if indexes.numel() < 0: return []
    angle_max_index = torch.index_select(angle_max_index, 0, indexes) #---find positive samples
    print("angle_max_index")
    print(angle_max_index)
    rectangles = torch.index_select(rectangles, 0, indexes) #----select rectangles of positive samples
    rectangles[:, 4] = torch.index_select(prob, 0, indexes) #----replace classification score

    face_down_mask = torch.gt(rectangles[:, 5], 10.0) #----mask for face down
    face_down_index = face_down_mask.nonzero()

    face_up_mask = ~face_down_mask #----mask for face up
    face_up_index = face_up_mask.nonzero()

    if face_up_index.numel() > 0:
        face_up_index = face_up_index[:, 0]
        rectangles_up = torch.index_select(rectangles, 0, face_up_index)
        angle_max_index_up = torch.index_select(angle_max_index, 0, face_up_index)
        rectangles_up[:, 5] = (angle_max_index_up - 1) * -90
        rectangles[face_up_index] = rectangles_up

    if face_down_index.numel() > 0:
        face_down_index = face_down_index[:, 0]
        rectangles_down = torch.index_select(rectangles, 0, face_down_index)
        angle_max_index_down = torch.index_select(angle_max_index, 0, face_down_index)
        rectangles_down[:, 5] = (angle_max_index_down - 1) * -90
        angle_max_index_down_1 = torch.eq(angle_max_index_down, 1) #----rotate class label 等于1
        angle_max_index_down_1 = angle_max_index_down_1.nonzero()
        if angle_max_index_down_1.numel() > 0:
            angle_max_index_down_1 = angle_max_index_down_1[:, 0]
            rectangles_down_tmp = torch.index_select(rectangles_down, 0, angle_max_index_down_1)
            rectangles_down_tmp[:, 5] = 180.0
            rectangles_down[angle_max_index_down_1] = rectangles_down_tmp

        rectangles_down[:, 1], rectangles_down[:, 3] = height - 1 - rectangles_down[:, 3], height -1 - rectangles_down[:, 1] #---img180中的坐标
        rectangles[face_down_index] = rectangles_down

    #-----deal [x1, y1, x2, y2]------
    bbox_reg = torch.index_select(bbox_reg, 0, indexes)
    w = (rectangles[:, 2] - rectangles[:, 0]).view(-1, 1) #---[N]--->[N, 1]
    h = (rectangles[:, 3] - rectangles[:, 1]).view(-1, 1)
    bbox_reg[:, 0::2] = torch.mul(bbox_reg[:, 0::2], w)
    bbox_reg[:, 1::2] = torch.mul(bbox_reg[:, 1::2], h)
    rectangles[:, :4] = rectangles[:, :4] + bbox_reg
    rectangles = rect2square(rectangles)

    #-----坐标换算到原来的图片
    if face_down_index.numel() > 0:
        rectangles_down = torch.index_select(rectangles, 0, face_down_index)
        rectangles_down[:, 1], rectangles_down[:, 3] = height-1 - rectangles_down[:, 3], height-1 - rectangles_down[:, 1]
        rectangles[face_down_index] = rectangles_down

    #----legal judgement-----
    rectangles[:, :2] = torch.clamp(rectangles[:, :2], min=0)
    rectangles[:, 2] = torch.clamp(rectangles[:, 2], max=width-1)
    rectangles[:, 3] = torch.clamp(rectangles[:, 3], max=height-1)

    index1 = (rectangles[:, 2] > rectangles[:, 0]) #---x2 > x1
    index2 = (rectangles[:, 3] > rectangles[:, 1]) #---y2 > y1
    index = (index1 & index2).nonzero().squeeze()
    rectangles = rectangles.index_select(0, index)
    rectangles = NMS(rectangles, 0.5, 'iou')
    return rectangles

def filter_face_onet(cls_prob, rotate_reg, bbox_reg, rectangles, img, img180, img90, imgneg90, threshold):
    """
    cls_prob.size: [N, 1]
    rotate_reg.size: [N, 1]
    bbox_reg.size: [N, 4]
    """
    height, width, _ = img.shape
    print("cls_prob")
    print(cls_prob)
    cls_prob = F.sigmoid(cls_prob).data
    #print("cls_prob")
    #print(cls_prob)
    rotate_reg = rotate_reg.data * -45
    bbox_reg = bbox_reg.data

    binary_tensor = (cls_prob>=threshold)
    indexes = binary_tensor.nonzero()[:, 0]
    if indexes.numel() < 0: return []
    rectangles = torch.index_select(rectangles, 0, indexes)
    rectangles[:, 4] = torch.index_select(cls_prob, 0, indexes) #--replace score
    rotate_reg = torch.index_select(rotate_reg, 0, indexes)
    print(rectangles)

    index_180 = torch.eq(rectangles[:, 5], 180).nonzero()
    index_negative_90 = torch.eq(rectangles[:, 5], -90).nonzero()
    index_positive_90 = torch.eq(rectangles[:, 5], 90).nonzero()
    index_0 = torch.eq(rectangles[:, 5], 0).nonzero()

    if index_180.numel()>0: #---angle = 180的情况
        index_180 = index_180[:,0]
        rectangle_tmp = torch.index_select(rectangles, 0, index_180)
        rectangle_tmp[:, 1], rectangle_tmp[:, 3] = height - 1 - rectangle_tmp[:, 3], height-1- rectangle_tmp[:, 1]
        rectangles[index_180] = rectangle_tmp

    if index_positive_90.numel()>0:
        index_positive_90 = index_positive_90[:, 0]
        rectangle_tmp = torch.index_select(rectangles, 0, index_positive_90)
        rectangle_tmp = rectangle_tmp[:, [1, 0, 3, 2, 4, 5]]
        rectangles[index_positive_90] = rectangle_tmp

    if index_negative_90.numel() > 0:
        index_negative_90 = index_negative_90[:, 0]
        rectangle_tmp = torch.index_select(rectangles, 0, index_negative_90)
        rectangle_tmp = rectangle_tmp[:, [1, 2, 3, 0, 4, 5]]
        rectangle_tmp[:, 1] = width - 1 - rectangle_tmp[:, 1]
        rectangle_tmp[:, 3] = width - 1 - rectangle_tmp[:, 3]
        rectangles[index_negative_90] = rectangle_tmp

    #if index_0.numel()>0:
    #    index_0 = index_0[:, 0]
    #    rectangle[index_0] = torch.index_select(rotate_reg, 0, index_0)
    rectangles[:, 5] = rotate_reg

    #-----deal [x1, y1, x2, y2]------
    bbox_reg = torch.index_select(bbox_reg, 0, indexes)
    w = (rectangles[:, 2] - rectangles[:, 0]).view(-1, 1) #---[N]--->[N, 1]
    h = (rectangles[:, 3] - rectangles[:, 1]).view(-1, 1)
    bbox_reg[:, 0::2] = torch.mul(bbox_reg[:, 0::2], w)
    bbox_reg[:, 1::2] = torch.mul(bbox_reg[:, 1::2], h)
    rectangles[:, :4] = rectangles[:, :4] + bbox_reg
    rectangles = rect2square(rectangles)

    #---现在换算到原来的空间-----
    if index_180.numel()>0:
        print("CCCCCCCCCCCCCCCCC")
        rectangle_tmp = torch.index_select(rectangles, 0, index_180)
        rectangle_tmp[:, 1], rectangle_tmp[:, 3] = height - 1 - rectangle_tmp[:, 3], height-1- rectangle_tmp[:, 1]
        rectangle_tmp[:, 5] = 180 - rectangle_tmp[:, 5]
        #rectangle_tmp[:, 5] = -(180 - rectangle_tmp[:, 5])
        rectangles[index_180] = rectangle_tmp

    if index_positive_90.numel()>0:
        rectangle_tmp = torch.index_select(rectangles, 0, index_positive_90)
        rectangle_tmp = rectangle_tmp[:,[1, 0, 3, 2, 4, 5]]
        rectangle_tmp[:, 5] =  90 - rectangle_tmp[:, 5]
        #rectangle_tmp[:, 5] =  -(90 - rectangle_tmp[:, 5])
        rectangles[index_positive_90] = rectangle_tmp

    if index_negative_90.numel()>0:
        rectangle_tmp = torch.index_select(rectangles, 0, index_negative_90)
        rectangle_tmp = rectangle_tmp[:, [3, 0, 1, 2, 4, 5]]
        rectangle_tmp[:, 0] = width - 1 - rectangle_tmp[:, 0]
        rectangle_tmp[:, 2] = width - 1 - rectangle_tmp[:, 2]
        rectangle_tmp[:, 5] = -90 + rectangle_tmp[:, 5]
        #rectangle_tmp[:, 5] = -(-90 + rectangle_tmp[:, 5])
        rectangles[index_negative_90] = rectangle_tmp

    rectangles[:, :2] = torch.clamp(rectangles[:, :2], min=0)
    rectangles[:, 2] = torch.clamp(rectangles[:, 2], max=width)
    rectangles[:, 3] = torch.clamp(rectangles[:, 3], max=height)

    index1 = (rectangles[:, 2] > rectangles[:, 0]) #---x2 > x1
    index2 = (rectangles[:, 3] > rectangles[:, 1]) #---y2 > y1
    index = (index1 & index2).nonzero().squeeze()
    rectangles = rectangles.index_select(0, index)
    rectangles = NMS(rectangles, 0.5, 'iou')
    return rectangles

def PadImg(img):
    row, col, _ = img.shape
    row = min(int(row * 0.2), 100)
    col = min(int(col * 0.2), 100)
    img = cv2.copyMakeBorder(img, row, row, col, col, cv2.BORDER_CONSTANT)
    return img


from abc import abstractmethod

from serif.model.document_model import DocumentModel
from serif.model.validate import *
from serif.theory.mention import Mention
from serif.theory.syn_node import SynNode
from serif.theory.proposition import Proposition

class DependencyModel(DocumentModel):

    def __init__(self, **kwargs):
        super(DependencyModel, self).__init__(**kwargs)

    @abstractmethod
    def add_dependencies_to_sentence(self, sentence):
        pass

    @staticmethod
    def add_new_proposition(dependency_set, pred_type, head_node, *, particle=None, adverb=None, negation=None,
                            modal=None, statuses=None):
        new_proposition = dependency_set.add_new_proposition(pred_type, head_node)
        new_proposition.particle = particle
        new_proposition.adverb = adverb
        new_proposition.negation = negation
        new_proposition.modal = modal
        new_proposition.statuses = statuses
        return [new_proposition]

    @staticmethod
    def add_new_argument(proposition, role, theory_obj):
        if isinstance(theory_obj, SynNode):
            return [proposition.add_new_synnode_argument(role, theory_obj)]
        elif isinstance(theory_obj, Mention):
            return [proposition.add_new_mention_argument(role, theory_obj)]
        elif isinstance(theory_obj, Proposition):
            return [proposition.add_new_proposition_argument(role, theory_obj)]
        else:
            raise NotImplementedError("Cannot support {}".format(type(theory_obj)))

    def process_document(self, serif_doc):
        for i, sentence in enumerate(serif_doc.sentences):
            validate_sentence_tokens(sentence, serif_doc.docid, i)
            # validate_sentence_mention_sets(sentence, serif_doc.docid, i)
            # Since we can set max_tokens, we can now end up without a
            # MentionSet
            if sentence.mention_set is None:
                sentence.add_new_mention_set()
            if sentence.dependency_set is None:
                sentence.add_new_dependency_set(sentence.mention_set)
            self.add_dependencies_to_sentence(sentence)


import dill as pickle
from datetime import datetime
from constants import PREPOSITIONS
from collections import defaultdict
import re

caps_matcher = re.compile(r'[A-ZÄÖÜ]')
lemma_map = {}
lemmas = set()
lemma_file = open('lemmatization-de.txt')
for line in lemma_file.readlines():
    [lemma, token] = line.split()
    lemma_map[token] = lemma
    lemmas.add(lemma)

def pick_lemma_from_pair(matches):
    [a, b] = matches
    if a in lemma_map and lemma_map[a] == b:
        return b
    if b in lemma_map and lemma_map[b] == a:
        return a
    else:
        return min(matches, key=len)

# generalizes the above
# bad. bad. just returns the first match that is the lemma of another match
# probably will usually just return the shortest?
# could be more than one. no time to fix.
# ideally i'd use some kind of semantic analysis to sort candidates
def pick_lemma_from_many(matches):
    for match in matches:
        without_that_one = list(matches)
        without_that_one.remove(match)
        if match in lemma_map and lemma_map[match] in without_that_one:
            return lemma_map[match]
    shortest = min(matches, key=len)
    return shortest

def old_lemma_fn(yc):
    for fn, candidate in yc.fuge_candidates[0].items():
        if candidate == yc.old_lemmas[0]:
            return fn, True
    return 'NULL', False

t0 = datetime.now()
for year in range(1995, 2023):
    t1 = datetime.now()
    elapsed = t1 - t0
    t0 = t1
    print(f'{year}\t{elapsed}')
    ycs = pickle.load(open(f'dumps/named_ents_filtered/{year}ycs', 'rb'))
    countsfile = open(f'corpora/deu_news_{year}_1M/deu_news_{year}_1M-words.txt')
    counts = defaultdict(int)
    for line in countsfile.readlines():
        if len(line.split()) != 3:
            continue
        _, word, count = line.split()
        counts[word] = int(count)
    # messy af, out of time
    for text, yc in ycs.items():
        matches = []
        for fn, candidate in yc.fuge_candidates[0].items():
            if candidate in lemmas:
                matches.append(candidate)
        if len(matches) == 0:
            fn, found = old_lemma_fn(yc)
            if found:
                yc.lemma_found = True
                lemma = yc.fuge_candidates[0][fn]
            elif yc.splits[0].lower() not in PREPOSITIONS:
            #     dictionary lookup
                yc.lemma_found = False
                for fn, candidate in yc.fuge_candidates[0].items():
                    if candidate in lemma_map:
                        matches.append(candidate)
            else:
                yc.lemma_found = False
        elif len(matches) == 2:
            yc.lemma_found = True
            lemma = pick_lemma_from_pair(matches)
        elif len(matches) > 2:
            yc.lemma_found = True
            lemma = pick_lemma_from_many(matches)
        else:
            yc.lemma_found = True
            lemma = matches[0]

        if yc.lemma_found:
            candidates = yc.fuge_candidates[0].items()
            for fn, defuged in candidates:
                if lemma == defuged:
                    yc.chosen_fn = fn
                    break #?
        # pick best dictionary match
        elif len(matches) > 0:
            yc.chosen_fn = 'NULL'
            chosen_word = max(matches, key=lambda x: counts[x])
            candidates = yc.fuge_candidates[0].items()
            for fn, defuged in candidates:
                if chosen_word == defuged:
                    yc.chosen_fn = fn
                    if caps_matcher.match(yc.text[0]):
                        pass
                    break #?
        else:
            yc.chosen_fn = 'NULL'
    with open(f'dumps/lemma_lookup_trial/{year}ycs', 'wb') as out_file:
        pickle.dump(ycs, out_file)

# print(f'single matches: {single_match}')
# print(f'no matches: {no_match}')
# print(f'multi matches: {multi_match}')


class Database:
    def __init__(self, db, database_name):
        # Cursor to navigate database and database name
        self.cursor = db.cursor()
        self.database_name = database_name.upper()

        # List of table names, dictionary of table sizes
        # Also dictionary of table columns nested with a dictionary of column name and values
        self.table_names = []
        self.table_sizes = {}
        self.table_columns = {}
        #
        self.query = ""

        # Get the tables, table sizes, table's column names, and table's columns' contents
        self.get_tables()
        self.get_table_sizes()
        self.get_column_names()
        self.get_column_contents()

    def get_tables(self):
        # Execute result of showing tables
        self.cursor.execute("SHOW TABLES;")
        my_result = self.cursor.fetchall()

        # Split by ' (used to find table name) and append the capitalized table name
        for x in my_result:
            line = str(x).split("'")
            table_name = line[1].upper()
            self.table_names.append(table_name)

    def get_table_sizes(self):
        for table_name in self.table_names:
            # Fetch query of table name
            self.cursor.execute(f'SELECT * FROM {table_name};')
            my_result = self.cursor.fetchall()

            # Get size of table and append to dictionary of table sizes
            table_len = len(my_result)
            self.table_sizes[table_name] = table_len

    def get_column_names(self):
        for table_name in self.table_names:
            # Fetch query of table name and make a empty dictionary of column names
            self.cursor.execute(f'SELECT * FROM {table_name};')
            my_result = self.cursor.fetchall()
            col_dict = {}

            for row in self.cursor.description:
                # Get name of column and put into column name dictionary with empty list value
                col_name = row[0]
                col_dict[col_name] = []

            # Sub dictionary is value of table name
            self.table_columns[table_name] = col_dict

    def get_column_contents(self):
        for table_name in self.table_names:
            # Fetch query of table name and nested dict value of table of such name
            self.cursor.execute(f'SELECT * FROM {table_name};')
            my_result = self.cursor.fetchall()
            cols = self.table_columns[table_name]

            # Iterator to traverse column names and store it in array by index ordering
            col_int = 0
            col_arr = []

            for col in cols:
                # Store in array of column names with 0th indexing
                col_arr.append(col)
                col_int += 1

            for res in my_result:
                # For each tuple value from query, find necessary index corresponding column
                # Get that corresponding column name and add tuple value to that corresponding column
                for j in range(0, len(col_arr)):
                    into_col = col_arr[j]
                    cols[into_col].append(res[j])

    def delete_top(self, table_name):
        # Table name to delete rows in
        table_name = table_name.upper()

        if table_name in self.table_names:
            # Table size and columns if table name is valid
            table_size = self.table_sizes[table_name]
            table_value = self.table_columns[table_name]

            # Table has to have elements
            if table_size > 0:
                # New table size, then delete first row
                self.table_sizes[table_name] = table_size - 1
                self.cursor.execute(f"DELETE FROM {table_name} LIMIT {1};")

                # Substring each column of table by leaving off first 'n' values
                for column in table_value:
                    values = table_value[column]
                    values = values[1:table_size]

    def delete_table(self, table_name):
        # Table name to delete rows in
        table_name = table_name.upper()

        if table_name in self.table_names:
            # Table size and columns if table name is valid
            table_size = self.table_sizes[table_name]
            table_value = self.table_columns[table_name]

            # Table should have elements to be cleared
            if table_size > 0:
                # Table size is 0 and all elements deleted
                self.table_sizes[table_name] = 0
                self.cursor.execute(f"DELETE FROM {table_name};")

            # Clear each column of its values
            for column in table_value:
                values = table_value[column]
                values.clear()

    def reset(self):
        # Reset all data structures
        self.table_names = []
        self.table_columns = {}
        self.table_lengths = {}


from half_life.dice import Dice

def run_experiment(nr_of_dice, nr_throws):
    dice = [Dice() for _ in range(nr_of_dice)]

    nr_remaining = [len(dice)]

    for _ in range(nr_throws):
        values = [die.roll() for die in dice]
        nr_not_six = len([val for val in values if val != 6])
        dice = dice[:nr_not_six]
        nr_remaining.append(len(dice))
    return nr_remaining


sampleTime = 0.0001
carrierFrequency = 10
messageFrequency = 1
multiplexCarrierFrequency = 10 * carrierFrequency
multiplexCarrierAmplitude = 1 #Am modulation can break if carrier amplitude is smaller then whats being carried
#Keep carrier frequency a multiple of message frequency for simplicity purposes
imageFolderPath = "images/"
#maximum bits that should be sent to the transmission creator function to reduce risk of RAM problems
#To work with all constellations, needs to be divisible by 8 and 12
maxBitsPerTransmission = 48


# Naive Bayes Classification
# Made by: Khalid Qubbaj, Taras Derewecki, Raj Zaveri

# Import Libraries
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import cross_val_score

# Load Data Set
df2 = pd.read_csv("train.csv", index_col="RecipeId")
df2.target = df2.Target.astype('category')

# Load Training Set
dfTrain = pd.read_csv("train.csv", index_col="RecipeId")
trainY = dfTrain.Target.astype('category')
dfTrain = dfTrain.drop(["Target"],axis=1)
df = pd.concat([dfTrain, pd.read_csv("test.csv", index_col="RecipeId")])

# Stemming - Imports punkt
import nltk
from nltk.stem.porter import PorterStemmer
nltk.download('punkt')

def tokenize(text):
    tokens = nltk.word_tokenize(text)
    stems = []
    for item in tokens:
        stems.append(PorterStemmer().stem(item))
    return stems

# Fit Classifier the training data
v = TfidfVectorizer(max_features=1800, tokenizer=tokenize)
df_x = v.fit_transform(dfTrain.Ingredients.values.astype('U')).todense()

#print(v.get_feature_names())

            # INDIAN
# ndx=v.get_feature_names().index("indian")
# df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index("ghee")
df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index("chutney")
# df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index('masala')
# df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index('turmer')
# df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index("masa")
df_x[:,ndx] = df_x[:,ndx] * 20

            # FRENCH
ndx=v.get_feature_names().index('french')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index('baguett')
df_x[:,ndx] = df_x[:,ndx] * 20

            # CHINESE
ndx=v.get_feature_names().index('chines')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index("szechwan")
df_x[:,ndx] = df_x[:,ndx] * 20

            # GREEK
ndx=v.get_feature_names().index('greek')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index('feta')
df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index("yoghurt")
# df_x[:,ndx] = df_x[:,ndx] * 20

            # ITALIAN
ndx=v.get_feature_names().index('italian')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index("bertolli")
df_x[:,ndx] = df_x[:,ndx] * 20

            # MEXICAN
ndx=v.get_feature_names().index('mexican')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index('salsa')
df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index("burrito")
# df_x[:,ndx] = df_x[:,ndx] * 20
#ndx=v.get_feature_names().index("tequila")
#df_x[:,ndx] = df_x[:,ndx] * 20

            # THAI
ndx=v.get_feature_names().index('thai')
df_x[:,ndx] = df_x[:,ndx] * 20

    # OTHERS THAT SEEM TO IMPROVE ACC
# ndx=v.get_feature_names().index("epazot")
# df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index("haricot")
# df_x[:,ndx] = df_x[:,ndx] * 20

# Import Naive Bayes
from sklearn.naive_bayes import MultinomialNB

# Get the best alpha value [0.5, 1, 10]
bestAlpha = None
#for alpha in [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 10]:
for alpha in [0.3]:
    nb = MultinomialNB(alpha=alpha, fit_prior=True, class_prior=None)
    scores = cross_val_score(nb, df_x, trainY, cv=10)

    print("For alpha =", alpha, " mean accuracy accross folds =", scores.mean(), " standard deviation across folds =",
          scores.std())
    if bestAlpha == None or scores.mean() > bestAlphaMeanAcc:
        bestAlpha = alpha
        bestAlphaMeanAcc = scores.mean()

print("Best k =", bestAlpha, "with estimated accuracy of", bestAlphaMeanAcc)

# Fit data into Naive Bayes
nb = MultinomialNB(alpha=bestAlpha, fit_prior=True, class_prior=None)
df_x = v.fit_transform(df.Ingredients.values.astype('U')).todense()
trainX = df_x[:len(dfTrain)]
testX = df_x[len(dfTrain):]
nb.fit(trainX,trainY)

# Generate Submission file
predictions = nb.predict(testX)
submissions = pd.read_csv("sampleSubmission.csv", index_col="RecipeId")
submissions.Target = predictions
submissions.to_csv("submission.csv")