#!/usr/bin/env python
# coding:utf-8
# 写函数，检查用户传入的对象（字符串、列表、元组）的每一个元素是否含有空内容。

def isblank(arg):
    """
    :param arg: list | str | tuple
    :return: None
    """
    if isinstance(arg, str) or isinstance(arg, list) or isinstance(arg, tuple):
        string = "".join(arg)
        for i in string:
            if i.isspace():
                print("对象中含有空格")
                break
            else:
                continue
    else:
        print("参数不正确")
ret = ["15454    dasdas", "24"]
isblank(ret)


#!/usr/bin/env python3
"""
@Filename:    utils.py
@Author:      dulanj
@Time:        2021-09-23 00.27
"""
import collections

import numpy as np
from matplotlib import gridspec
from matplotlib import pyplot as plt

DatasetInfo = collections.namedtuple(
    'DatasetInfo',
    'num_classes, label_divisor, thing_list, colormap, class_names')

def _cityscapes_label_colormap():
    """Creates a label colormap used in CITYSCAPES segmentation benchmark.

    See more about CITYSCAPES dataset at https://www.cityscapes-dataset.com/
    M. Cordts, et al. "The Cityscapes Dataset for Semantic Urban Scene Understanding." CVPR. 2016.

    Returns:
      A 2-D numpy array with each row being mapped RGB color (in uint8 range).
    """
    colormap = np.zeros((256, 3), dtype=np.uint8)
    colormap[0] = [128, 64, 128]
    colormap[1] = [244, 35, 232]
    colormap[2] = [70, 70, 70]
    colormap[3] = [102, 102, 156]
    colormap[4] = [190, 153, 153]
    colormap[5] = [153, 153, 153]
    colormap[6] = [250, 170, 30]
    colormap[7] = [220, 220, 0]
    colormap[8] = [107, 142, 35]
    colormap[9] = [152, 251, 152]
    colormap[10] = [70, 130, 180]
    colormap[11] = [220, 20, 60]
    colormap[12] = [255, 0, 0]
    colormap[13] = [0, 0, 142]
    colormap[14] = [0, 0, 70]
    colormap[15] = [0, 60, 100]
    colormap[16] = [0, 80, 100]
    colormap[17] = [0, 0, 230]
    colormap[18] = [119, 11, 32]
    return colormap

def _cityscapes_class_names():
    return ('road', 'sidewalk', 'building', 'wall', 'fence', 'pole',
            'traffic light', 'traffic sign', 'vegetation', 'terrain', 'sky',
            'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle',
            'bicycle')

def cityscapes_dataset_information():
    return DatasetInfo(
        num_classes=19,
        label_divisor=1000,
        thing_list=tuple(range(11, 19)),
        colormap=_cityscapes_label_colormap(),
        class_names=_cityscapes_class_names())

def perturb_color(color, noise, used_colors, max_trials=50, random_state=None):
    """Pertrubs the color with some noise.

    If `used_colors` is not None, we will return the color that has
    not appeared before in it.

    Args:
      color: A numpy array with three elements [R, G, B].
      noise: Integer, specifying the amount of perturbing noise (in uint8 range).
      used_colors: A set, used to keep track of used colors.
      max_trials: An integer, maximum trials to generate random color.
      random_state: An optional np.random.RandomState. If passed, will be used to
        generate random numbers.

    Returns:
      A perturbed color that has not appeared in used_colors.
    """
    if random_state is None:
        random_state = np.random

    for _ in range(max_trials):
        random_color = color + random_state.randint(
            low=-noise, high=noise + 1, size=3)
        random_color = np.clip(random_color, 0, 255)

        if tuple(random_color) not in used_colors:
            used_colors.add(tuple(random_color))
            return random_color

    print('Max trial reached and duplicate color will be used. Please consider '
          'increase noise in `perturb_color()`.')
    return random_color

def color_panoptic_map(panoptic_prediction, dataset_info, perturb_noise):
    """Helper method to colorize output panoptic map.

    Args:
      panoptic_prediction: A 2D numpy array, panoptic prediction from deeplab
        model.
      dataset_info: A DatasetInfo object, dataset associated to the model.
      perturb_noise: Integer, the amount of noise (in uint8 range) added to each
        instance of the same semantic class.

    Returns:
      colored_panoptic_map: A 3D numpy array with last dimension of 3, colored
        panoptic prediction map.
      used_colors: A dictionary mapping semantic_ids to a set of colors used
        in `colored_panoptic_map`.
    """
    if panoptic_prediction.ndim != 2:
        raise ValueError('Expect 2-D panoptic prediction. Got {}'.format(
            panoptic_prediction.shape))

    semantic_map = panoptic_prediction // dataset_info.label_divisor
    instance_map = panoptic_prediction % dataset_info.label_divisor
    height, width = panoptic_prediction.shape
    colored_panoptic_map = np.zeros((height, width, 3), dtype=np.uint8)

    used_colors = collections.defaultdict(set)
    # Use a fixed seed to reproduce the same visualization.
    random_state = np.random.RandomState(0)

    unique_semantic_ids = np.unique(semantic_map)
    for semantic_id in unique_semantic_ids:
        semantic_mask = semantic_map == semantic_id
        if semantic_id in dataset_info.thing_list:
            # For `thing` class, we will add a small amount of random noise to its
            # correspondingly predefined semantic segmentation colormap.
            unique_instance_ids = np.unique(instance_map[semantic_mask])
            for instance_id in unique_instance_ids:
                instance_mask = np.logical_and(semantic_mask,
                                               instance_map == instance_id)
                random_color = perturb_color(
                    dataset_info.colormap[semantic_id],
                    perturb_noise,
                    used_colors[semantic_id],
                    random_state=random_state)
                colored_panoptic_map[instance_mask] = random_color
        else:
            # For `stuff` class, we use the defined semantic color.
            colored_panoptic_map[semantic_mask] = dataset_info.colormap[semantic_id]
            used_colors[semantic_id].add(tuple(dataset_info.colormap[semantic_id]))
    return colored_panoptic_map, used_colors

def vis_segmentation(image,
                     panoptic_prediction,
                     dataset_info,
                     perturb_noise=60,
                     output_name="show"):
    """Visualizes input image, segmentation map and overlay view."""
    plt.figure(figsize=(30, 20))
    grid_spec = gridspec.GridSpec(2, 2)

    ax = plt.subplot(grid_spec[0])
    plt.imshow(image)
    plt.axis('off')
    ax.set_title('input image', fontsize=20)

    ax = plt.subplot(grid_spec[1])
    panoptic_map, used_colors = color_panoptic_map(panoptic_prediction,
                                                   dataset_info, perturb_noise)
    plt.imshow(panoptic_map)
    plt.axis('off')
    ax.set_title('panoptic map', fontsize=20)

    ax = plt.subplot(grid_spec[2])
    plt.imshow(image)
    plt.imshow(panoptic_map, alpha=0.7)
    plt.axis('off')
    ax.set_title('panoptic overlay', fontsize=20)

    ax = plt.subplot(grid_spec[3])
    max_num_instances = max(len(color) for color in used_colors.values())
    # RGBA image as legend.
    legend = np.zeros((len(used_colors), max_num_instances, 4), dtype=np.uint8)
    class_names = []
    for i, semantic_id in enumerate(sorted(used_colors)):
        legend[i, :len(used_colors[semantic_id]), :3] = np.array(
            list(used_colors[semantic_id]))
        legend[i, :len(used_colors[semantic_id]), 3] = 255
        if semantic_id < dataset_info.num_classes:
            class_names.append(dataset_info.class_names[semantic_id])
        else:
            class_names.append('ignore')

    plt.imshow(legend, interpolation='nearest')
    ax.yaxis.tick_left()
    plt.yticks(range(len(legend)), class_names, fontsize=15)
    plt.xticks([], [])
    ax.tick_params(width=0.0, grid_linewidth=0.0)
    plt.grid('off')
    if output_name == "show":
        plt.show()
    else:
        plt.savefig(f'{output_name}.png')


import random

import math
import numpy as np
import pandas as pd

from matplotlib import pyplot as plt
import sklearn.gaussian_process as gp
from mpl_toolkits.mplot3d import Axes3D

from tensorflow import keras
import tensorflow as tf

import tensorflow.keras.optimizers as tko
import tensorflow.keras.activations as tka
import tensorflow.keras.losses as tkl
from tensorflow.keras.layers import Input, Dense
from tensorflow.keras.models import Model

##########################################################################################################

def __build_activation_function(af):
    if af.lower() == 'none':
        return None
    exp_af = 'lambda _ : tka.' + af
    return eval(exp_af)(None)

def __build_optimizer(optimizer):
    opt_init = optimizer
    exp_po = 'lambda _ : tko.' + opt_init
    optimizer = eval(exp_po)(None)
    return optimizer

def __build_loss(loss):
    exp_loss = 'lambda _ : tkl.' + loss
    return eval(exp_loss)(None)

def __build_model (hidden_layers_layout, activation_functions):
    inputs = Input(shape=(2,))
    hidden = inputs
    for i in range(0, len(hidden_layers_layout)):
        hidden = Dense(hidden_layers_layout[i], \
                activation= __build_activation_function(activation_functions[i]))(hidden)
    outputs = Dense(1)(hidden)
    model = Model(inputs=inputs, outputs=outputs)

    return model

##########################################################################################################

def build_model_NN_2 ():

    hidden_layers_layout = [100, 100]
    activation_functions = ["relu", "relu"]
    inoptimizer = "SGD(decay=1e-6, momentum=0.9, nesterov=True)"
    loss = 'MeanSquaredError()'

    model = __build_model (hidden_layers_layout, activation_functions)
    optimizer = __build_optimizer(inoptimizer)
    model.compile(loss= __build_loss(loss), optimizer=optimizer)
    #model.summary()

    return model

##########################################################################################################

def build_model_NN_3 ():

    model = keras.Sequential()
    model.add(keras.layers.Dense(units = 2, activation = 'linear', input_shape=[2]))
    model.add(keras.layers.Dense(units = 100, activation = 'relu'))
    model.add(keras.layers.Dense(units = 100, activation = 'relu'))
    model.add(keras.layers.Dense(units = 1, activation = 'linear'))
    model.compile(loss='mse', optimizer="adam", metrics='mse')

    return model

##########################################################################################################

def build_model_NN_1 ():

    model = keras.Sequential()
    model.add(keras.layers.Dense(units = 2, activation = 'linear', input_shape=[2]))
    model.add(keras.layers.Dense(units = 32, activation = 'relu'))
    model.add(keras.layers.Dense(units = 64, activation = 'relu'))
    model.add(keras.layers.Dense(units = 128, activation = 'relu'))
    model.add(keras.layers.Dense(units = 32, activation = 'relu'))
    #model.add(keras.layers.Dense(units = 16, activation = 'relu'))
    model.add(keras.layers.Dense(units = 1, activation = 'linear'))
    model.compile(loss='mse', optimizer="adam", metrics='mse')
    #model.summary()

    return model

##########################################################################################################3

def build_model_GP_1 (train_x, train_y):
    #kernel = gp.kernels.ConstantKernel(1.0, (1e-5, 1e5))* gp.kernels.RBF(length_scale=1)
    kernel = gp.kernels.ConstantKernel(1.0, (1e-3, 1e3)) * gp.kernels.RBF([5,5], (1e-2, 1e2))
    model = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, \
        normalize_y=False)
    print("Start training ")
    model.fit(train_x, train_y)
    print("Done ")

    return model

##########################################################################################################3

def build_model_GP_2 (train_x, train_y):
    #kernel = gp.kernels.ConstantKernel(1.0, (1e-5, 1e5))* gp.kernels.RBF(length_scale=1)
    nuval = 5.0/2.0
    kernel = 1.0 * gp.kernels.Matern(length_scale=1.0, nu=nuval)
    model = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, \
        normalize_y=False)
    print("Start training ")
    model.fit(train_x, train_y)
    print("Done ")

    return model

##########################################################################################################3

def build_model_GP_2D (train_x, train_y):
    #kernel = gp.kernels.ConstantKernel(1.0, (1e-5, 1e5))* gp.kernels.RBF(length_scale=1)
    nuval = 5.0/2.0
    kernel = 1.0 * gp.kernels.Matern(length_scale=1.0, nu=nuval)
    model = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, \
        normalize_y=False)
    print("Start training ")
    model.fit(train_x, train_y)
    print("Done ")

    return model

##########################################################################################################

def build_model_GP_3D (train_x, train_y):
    #kernel = gp.kernels.ConstantKernel(1.0, (1e-5, 1e5))* gp.kernels.RBF(length_scale=1)
    nuval = 5.0/2.0
    kernel = 1.0 * gp.kernels.Matern(length_scale=1.0, nu=nuval)
    model = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=50, \
        normalize_y=False)
    print("Start training ")
    model.fit(train_x, train_y)
    print("Done ")

    return model

#####################################################################################################

def filterinitialset (filename, headername = "vibrational level v\Temperature(K)",  \
    factor = 1.0, normalize = False):

    dfin = pd.read_excel(filename)

    #print(dfin)

    dfdict = {}

    tempvalues = list(dfin.columns[1:])
    vibvalues = list(dfin[headername].values)

    min = float("inf")
    max = float("-inf")
    for c in dfin.columns:
        dfdict[c] = []
        if c == headername:
            dfdict[c] = list(dfin[c].values)
        else:
            for v in dfin[c].values:
                val = math.log10(factor*v)
                if val > max:
                    max = val
                if val < min:
                    min = val

    for c in dfin.columns:
        if c != headername:
            for v in dfin[c].values:
                val = math.log10(factor*v)
                valp = (val - min) / (max - min)
                if normalize:
                    dfdict[c].append(valp)
                else:
                    dfdict[c].append(val)

    df = pd.DataFrame.from_dict(dfdict)

    return df, vibvalues, tempvalues, min, max

##############################################################################

def plotfull3dcurve (df, vib_values, temp_values):

    y = []
    x = []
    for t in temp_values:
        for idx in range(len(vib_values)):
            x.append([float(t), float(vib_values[idx])])
            y.append(df[t].values[idx])

    X = np.array(x)
    Y = np.array(y)

    xdim = len(temp_values)
    ydim = len(vib_values)

    Xp = np.zeros((xdim, ydim), dtype=float)
    Yp = np.zeros((xdim, ydim), dtype=float)
    Zp = np.zeros((xdim, ydim), dtype=float)
    for xidx in range(xdim):
        t = temp_values[xidx]
        for yidx in range(ydim):
            v =  vib_values[yidx]
            Xp[xidx, yidx] = float(t)
            Yp[xidx, yidx] = float(v)
            Zp[xidx, yidx] = df[t].values[yidx]

    #fig = plt.figure(figsize=(10,8))
    fig = plt.figure(figsize=plt.figaspect(2.))
    plt.gcf().set_size_inches(40, 30)
    ax = fig.add_subplot(2,1,1, projection='3d')
    surf = ax.plot_surface(Xp, Yp, Zp, rstride=1, cstride=1, cmap='jet', linewidth=0, antialiased=False)
    plt.show()

##########################################################################################################

def get_train_and_test_rmt (temp_values, vib_values, df, \
    removetemps=[]):

    maxt = max(temp_values)
    mint = min(temp_values)

    minv = min(vib_values)
    maxv = max(vib_values)

    train_xy = []
    train_z = []

    test_xy = []
    test_z = []

    maxz = float("-inf")
    minz = float("+inf")

    for tidx, t in enumerate(temp_values):
        for vidx, v in enumerate(vib_values):
            zval = df[t].values[vidx]

            if zval < minz:
                minz = zval
            elif zval > maxz:
                maxz = zval

    for t in temp_values:
        if t not in removetemps:
            tnorm = (t - mint)/(maxt - mint)

            for vidx, v in enumerate(vib_values):
                vnorm  = (v - minv)/(maxv - minv)
                train_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                train_z.append(znorm)
        else:
            tnorm = (t - mint)/(maxt - mint)
            for vidx, v in enumerate(vib_values):
                vnorm  = (v - minv)/(maxv - minv)
                test_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                test_z.append(znorm)

    train_xy = np.asarray(train_xy)
    train_z = np.asarray(train_z)

    test_xy = np.asarray(test_xy)
    test_z = np.asarray(test_z)

    return train_xy, train_z, test_xy, test_z

##########################################################################################################

def get_train_and_test_rmv (temp_values, vib_values, df, \
    removevibs=[]):

    maxt = max(temp_values)
    mint = min(temp_values)

    minv = min(vib_values)
    maxv = max(vib_values)

    train_xy = []
    train_z = []

    test_xy = []
    test_z = []

    maxz = float("-inf")
    minz = float("+inf")

    for tidx, t in enumerate(temp_values):
        for vidx, v in enumerate(vib_values):
            zval = df[t].values[vidx]

            if zval < minz:
                minz = zval
            elif zval > maxz:
                maxz = zval

    for t in temp_values:
        tnorm = (t - mint)/(maxt - mint)

        for vidx, v in enumerate(vib_values):
            if v not in removevibs:
                vnorm  = (v - minv)/(maxv - minv)
                train_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                train_z.append(znorm)
            else:
                vnorm  = (v - minv)/(maxv - minv)
                test_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                test_z.append(znorm)

    train_xy = np.asarray(train_xy)
    train_z = np.asarray(train_z)

    test_xy = np.asarray(test_xy)
    test_z = np.asarray(test_z)

    return train_xy, train_z, test_xy, test_z

##########################################################################################################

def get_train_and_test_random (temp_values, vib_values, df, \
    perc):

    maxt = max(temp_values)
    mint = min(temp_values)

    minv = min(vib_values)
    maxv = max(vib_values)

    train_xy = []
    train_z = []

    test_xy = []
    test_z = []

    maxz = float("-inf")
    minz = float("+inf")

    totnumber = 0;
    for t in temp_values:
        for vidx, v in enumerate(vib_values):
            zval = df[t].values[vidx]

            totnumber += 1

            if zval < minz:
                minz = zval
            elif zval > maxz:
                maxz = zval

    random.seed(1)

    for t in temp_values:
        tnorm = (t - mint)/(maxt - mint)
        for vidx, v in enumerate(vib_values):
            rv = random.uniform(0.0, float(totnumber))
            #print(rv, perc*float(totnumber))
            if rv > (perc*float(totnumber)):
                vnorm  = (v - minv)/(maxv - minv)
                train_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                train_z.append(znorm)
            else:
                vnorm  = (v - minv)/(maxv - minv)
                test_xy.append([tnorm, vnorm])

                z = df[t].values[vidx]
                znorm = (z - minz)/(maxz - minz)
                test_z.append(znorm)

    train_xy = np.asarray(train_xy)
    train_z = np.asarray(train_z)

    test_xy = np.asarray(test_xy)
    test_z = np.asarray(test_z)

    return train_xy, train_z, test_xy, test_z

##############################################################################

def get_train_full (temp_values, vib_values, df, vtopredict):

    maxt = max(temp_values)
    mint = min(temp_values)

    train_xy = []
    train_z = []

    maxz = float("-inf")
    minz = float("+inf")

    minv = min(vib_values)
    maxv = max(vib_values)

    totnumber = 0;
    for t in temp_values:
        for vidx, v in enumerate(vib_values):
            zval = df[t].values[vidx]

            totnumber += 1

            if zval < minz:
                minz = zval
            elif zval > maxz:
                maxz = zval

    for t in temp_values:
        tnorm = (t - mint)/(maxt - mint)
        for vidx, v in enumerate(vib_values):
            vnorm  = (v - minv)/(maxv - minv)
            train_xy.append([tnorm, vnorm])
            z = df[t].values[vidx]
            znorm = (z - minz)/(maxz - minz)
            train_z.append(znorm)

    test_xy = []
    for t in temp_values:
        tnorm = (t - mint)/(maxt - mint)
        for v in vtopredict:
            vnorm  = (v - minv)/(maxv - minv)
            test_xy.append([tnorm, vnorm])

    train_xy = np.asarray(train_xy)
    train_z = np.asarray(train_z)
    test_xy = np.asarray(test_xy)

    return train_xy, train_z, test_xy

##############################################################################

def plotfull3dcurve (columntorm, x, y):

    yv = []
    xv = []
    for i, v in enumerate(x):
        toappend = []
        for j in range(len(v)):
            if j != columntorm:
                toappend.append(v[j])
        xv.append(toappend)
        yv.append(y[i])

    X = np.array(xv)
    Y = np.array(yv)

    #for i in range(X.shape[0]):
    #    print("%4d %5d %10.5e"%(X[i,0], X[i,1], Y[i]))

    #print(X.shape)
    #print(Y.shape)

    x1set = sorted(list(set(X[:,0])))
    x2set = sorted(list(set(X[:,1])))

    x1dim = len(x1set)
    x2dim = len(x2set)

    Xp = np.zeros((x1dim, x2dim), dtype=float)
    Yp = np.zeros((x1dim, x2dim), dtype=float)
    Zp = np.zeros((x1dim, x2dim), dtype=float)
    for x1idx in range(x1dim):
        x1 = x1set[x1idx]
        for x2idx in range(x2dim):
            x2 =  x2set[x2idx]
            Xp[x1idx, x2idx] = float(x1)
            Yp[x1idx, x2idx] = float(x2)

            zval = None
            for i in range(X.shape[0]):
                if X[i,0] == x1 and X[i,1] == x2:
                    zval = Y[i]
                    break

            Zp[x1idx, x2idx] = zval

    #print(Xp.shape, Yp.shape, Zp.shape)

    #fig = plt.figure(figsize=(10,8))
    fig = plt.figure(figsize=plt.figaspect(2.))
    #plt.gcf().set_size_inches(40, 30)
    ax = fig.add_subplot(2, 1, 1, projection='3d')
    surf = ax.plot_surface(Xp, Yp, Zp, rstride=1, cstride=1, cmap='jet', linewidth=0, antialiased=True)
    plt.show()

#######################################################################

def plotfull3dscatter (columntorm, x, y):

    fig = plt.figure()
    ax = fig.add_subplot(projection='3d')

    for i, v in enumerate(x):
        toappend = []
        for j in range(len(v)):
            if j != columntorm:
                toappend.append(v[j])
        xs = toappend[0]
        ys = toappend[1]
        zs = y[i]
        ax.scatter(xs, ys, zs, marker='o')

    plt.show()

#######################################################################

def test_train_split (column, valuestotest, x, y):

    xtest = []
    ytest = []
    xtrain = []
    ytrain = []

    for i, xv in enumerate(x[:,column]):
        if xv in valuestotest:
            xtest.append(x[i,:])
            ytest.append(y[i])
        else:
            xtrain.append(x[i,:])
            ytrain.append(y[i])

    #print(len(xtest), len(ytest))
    #print(len(xtrain), len(ytrain))

    return np.asarray(xtrain), np.asarray(xtest), \
        np.asarray(ytrain), np.asarray(ytest)

#######################################################################

def buildmodel(modelshape, inputshape=3):
    model = keras.Sequential()
    model.add(keras.layers.Input(shape=(inputshape)))

    for n in modelshape:
        model.add(keras.layers.Dense(units = n, activation = 'relu'))

    model.add(keras.layers.Dense(units = 1, activation = 'linear'))
    model.compile(loss='mse', optimizer="adam", metrics='mse')

    return model

#######################################################################


from dyatel.base.element import Element
from dyatel.base.group import Group

class MainGroup(Group):

    def __init__(self, custom_locator='MainGroup'):
        super().__init__(custom_locator)

    el1 = Element('lcoator')
    el2 = Element('lcoator', parent=el1)

class Nested1(MainGroup):

    def __init__(self):
        super().__init__(custom_locator='Nested1')

class Nested2(MainGroup):

    def __init__(self):
        super().__init__(custom_locator='Nested2')

def test_group_2_parent_arg_in_other_group_updated(mocked_selenium_driver):
    nested1 = Nested1()
    nested2 = Nested2()
    assert nested1.el2.parent.__base_obj_id == nested2.el1.__base_obj_id
    assert nested1.el2.parent.parent == nested1

    assert nested2.el2.parent.__base_obj_id == nested2.el1.__base_obj_id
    assert nested2.el2.parent.parent == nested2


import numpy as np
import os
import pandas as pd
import glob

from torchtext.data.metrics import bleu_score
from jiwer import wer

def get_csv(root):
    root = os.path.join(root, 'phoenix2014-release/phoenix-2014-multisigner')
    root = os.path.join(root, 'annotations/manual/train.corpus.csv')
    return pd.read_csv(root)

def DatasetStatistic(root, mode='train'):
    '''
    analysis the statistic of the RWTH-PHOENIX-Weather 2014 dataset
    Args:
        root: the whole data root
        mode: 'train','dev', 'test'
    return:
        max_len: maximum video length
    '''

    root = os.path.join(root, 'phoenix2014-release/phoenix-2014-multisigner')
    video_root = os.path.join(root, 'features/fullFrame-210x260px/' + mode)
    csv_path = os.path.join(root, 'annotations/manual/' + mode + '.corpus.csv')

    csv_file = pd.read_csv(csv_path)
    video_paths = [os.path.join(video_root, csv_file.iloc[i, 0].split('|')[1])
                   for i in range(csv_file.shape[0])]

    video_lens = [len(glob.glob(path)) for path in video_paths]
    return [max(video_lens), min(video_lens),
            np.mean(video_lens), np.std(video_lens)]

def itos(idx_seq, TRG):
    '''
    Description:
        denumericalize: convert the index sequence to the text sequence
    '''
    return [TRG.vocab.itos[idx] for idx in idx_seq]

def bleu_count(outputs, targets, TRG):
    '''
    sentence level
    shape:
        outputs: [T, N, E]
        targets: [T, N]
    '''
    outputs = outputs.max(2)[1].transpose(0,1)
    targets = targets.transpose(0,1)
    candidate_corpus = [itos(idx_seq, TRG) for idx_seq in outputs]
    references_corpus = [[itos(idx_seq, TRG)] for idx_seq in targets]
    return bleu_score(candidate_corpus, references_corpus)

def wer_count(outputs, targets, TRG):
    '''
    word error rate
    Ref: https://pypi.org/project/jiwer/
    shape:
        outputs: [T, N, E]
        targets: [T, N]
    '''
    outputs = outputs.max(2)[1].transpose(0,1)
    targets = targets.transpose(0,1)
    candidate_corpus = [' '.join(itos(idx_seq, TRG)) for idx_seq in outputs]
    reference_corpus = [' '.join(itos(idx_seq, TRG)) for idx_seq in targets]
    return wer(reference_corpus, candidate_corpus, standardize = True)


from typing import TYPE_CHECKING, Any, Dict, List, Set

from apcsp.ui import form, selectable

if TYPE_CHECKING:
    from apcsp.ui import menu

class MenuContext(object):
    data: Dict[str, Any]
    selected = property(lambda self: self.menu.selected)

    def __init__(
        self,
        menu: "menu.Menu | None" = None,
        *,
        parent: "MenuContext | None" = None,
        data: Dict[str, Any] = None,
    ):
        self.menu = menu
        self.parent = parent
        self.data = data or parent.data if parent else {}
        self.exit_next = False

    def check_fields(self) -> Dict[str, form.FormField]:
        assert self.menu, "No menu available"
        objects: Dict[str, form.FormField] = {}
        for field in self.menu.fields:
            if field.id in objects:
                raise ValueError(
                    f'Field id "{field.id}" is shared by fields'
                    f" {objects[field.id].__class__} and {field.__class__}"
                )
            objects[field.id] = field
        return objects

    def run(self, menu: "menu.Menu | None" = None) -> None:
        self.menu = menu or self.menu

        assert self.menu, "No menu available"
        if self.exit_next:
            return

        self.check_fields()
        self.menu.run(self)

    def fork(self, menu: "menu.Menu | None" = None) -> "MenuContext":
        return self.__class__(menu, parent=self)

    def redraw(self, clear: bool = True) -> None:
        assert self.menu, "No menu available"
        self.menu.print(self, clear)

    def exit(self) -> None:
        self.exit_next = True

    def replace(self, menu: "menu.Menu") -> None:
        self.exit()
        if self.parent is None:
            raise ValueError("Cannot replace the root menu")
        new_context = self.parent.fork()
        new_context.run(menu)

    def submit(self) -> selectable.FormResult | None:
        assert self.menu, "No menu available"
        if not self.menu.fields:
            print("WARNING: No fields in form")
            return {}

        self.check_fields()
        success = True
        for field in self.menu.fields:
            result = field.validate(self)
            success = success and result

        if not success:
            self.redraw()
            return None

        data: selectable.FormResult = {}
        for field in self.menu.fields:
            data[field.id] = field

        return data


import json
import numpy as np
import os
import reconstruction as rct
from file_reader import roiread

'''
    A colections of points Point() are correlated in a manner that creates
    a perimeter Perimeter(). Every perimeter should be nice, i.e.:
        1) Not self-intersecting;
        2) No overlaping points;
        3) Have a prefered orientation.
    If we can garantee this properties, than we proceed to stitch a colection
    of perimeters in a surface Surface().
'''
with open('main.json', 'r') as settings:
    data = settings.read()
Data = json.loads(data)
FileDir = Data["FileDir"]
OutputDir = Data["OutputDir"]
DisplayConsoleStats = Data["DisplayConsoleStats"]
MeshObjOutput = Data["MeshObjOutput"]

print("Loading files\n\n")
print(FileDir)

def island_init(file_dir,f,subdivision=3):
    arq = roiread(file_dir+"/"+f)
    I = rct.Perimeter(arq)
    I.remove_overlap()
    I.remove_overlap()
    I.fix_intersection()
    I.fix_distance(subdivision=3)
    I.c_clockwise()
    return I

for block in Data["Stitches3D"]:
    for section in block:
        S = rct.Surface()
        for file in block[section]:
            try:
                if isinstance(file,list):
                    I = 0
                    for f in file:
                        I_s = island_init(FileDir,f,3)
                        if I == 0:
                            I = I_s
                        else:
                            I.islands_ensemble(I_s)
                    I.fix_intersection()
                else:
                    I = island_init(FileDir,file,3)
                S.add_island(I)
            except Exception:
                print("Failed to load "+file)

        print("\nBuilding surface: ",section)
        S.build_surface()

        with open("Kamilla_interna_"+section+".obj", "w") as out_file:
            out_file.write(S.surfaceV)
            out_file.write(S.surfaceE)


import urllib.request
from bs4 import BeautifulSoup
import os
import json
import requests
from .utils import get_params

class SniperSpider:
    """Used to get a single inofrmation on a single web page.
    """
    def __init__(self, params={}):
        self.url = None
        self.request = None

        self.http_string = None
        self.expected_value = None

        self.set_params_from_dict(params=params)

    def set_params_from_dict(self, params):
        self.url = params.get("url", "")
        self.request = params.get("request", [])
        self.expected_value = params.get("expected_value", "")

    # ==================================================================

    def shoot(self):
        """Load the web page and get the desired information from it
        Returns:
            str: info from web page
        """
        target = ""
        # Get the web page
        self._get_http_string()
        # Try to access the data in the web page
        try:
            target = self._aim()
        except:
            target = "None"
            print(f"The spider of {self.url} couldn't aim at the target")
        # determining whether the page was parsed and the spider succeeded
        success, parsed = self._pull_triger(target)
        return target, success, parsed

    def _get_http_string(self):
        """call the web page, format it to be readable by beatifulsoup
         and store the result in http_string
        """
        try:
            rep = requests.get(self.url, timeout=0.5)
            self.http_string = rep.content.decode("utf8")
            self.http_string = BeautifulSoup(self.http_string, 'html.parser')
        except:
            self.http_string = "None"
            print(f"Couldn't get {self.url} http string")

    def _aim(self):
        """Read the web page and get the information

        Args:
            request (dictionary): defines the cell information where the data is.

        Returns:
            str: info from web page
        """
        if self.http_string != "None":
            http_sel = self.http_string
            for step in self.request:
                # assign parameters of the parsing
                params = {"name": step["cell_type"]}
                if step.get("class", ""):
                    params["class_"]= step["class"]
                http_sel = http_sel.find(**params)
                # result is the class attribute of the cell if it has been found
                if step.get("test_presence", False) and http_sel:
                    result = step.get("class", "None")
                # result is the content of the cell
                elif step.get("string", False):
                    result = http_sel.string
                else:
                    result = "None"
        else:
            result = "None"
        return result

    def _pull_triger(self, parsing_result):
        """indicate wheter the page was parsed and the spider found the value expected
        Args: parsing_result (string): the value returned by the spider
        Returns: success, parsed (bool, bool)
        """
        if parsing_result == self.expected_value:
            success = True
            parsed = True
        elif parsing_result == "None":
            success = False
            parsed = False
        else:
            success = False
            parsed = True

        return success, parsed

if __name__ == "__main__":
    params = {
        "url": "",
        "request": {
            "cell_type" : "div",
            "class" : "product-page-description col-flex-lg-5 col-flex-sm-12",
            "string" : False
        }
    }
    params = get_params("spider_data")["processor_topachat"]
    #print(params)
    sniper = SniperSpider(params)

    result, success, parsed = sniper.shoot()

    print(result, success, parsed)


import torch
from code_slac.network.base import BaseNetwork
from code_slac.network.latent import ConstantGaussian

from diayn_seq_code_revised.networks.my_gaussian import MyGaussian as Gaussian

from self_supervised.utils.my_pytorch_util import tensor_equality

from latent_with_splitseqs.base.slac_latent_base import SlacLatentBase

# Adaption of the SLAC latent network architecture
class SlacLatentNetConditionedOnSingleSkill(SlacLatentBase):

    def __init__(self,
                 *args,
                 obs_dim,
                 skill_dim,
                 latent1_dim=32,
                 latent2_dim=256,
                 hidden_units=(256, 256),
                 leaky_slope=0.2,
                 dropout=0.,
                 **kwargs
                 ):
        super(SlacLatentNetConditionedOnSingleSkill, self).__init__(
            *args,
            latent1_dim=latent1_dim,
            latent2_dim=latent2_dim,
            **kwargs
        )
        # We use the observations as actions for this model
        # and the infered skill as observaton

        # p(z1(0)) = N(0, I)
        self.latent1_init_prior = ConstantGaussian(latent1_dim)
        # p(z2(0) | z1(0))
        self.latent2_init_prior = Gaussian(
            input_dim=latent1_dim,
            output_dim=latent2_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )
        # p(z1(t+1) | z2(t), a(t))
        self.latent1_prior = Gaussian(
            input_dim=latent2_dim + obs_dim,
            output_dim=latent1_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )
        # p(z2(t+1) | z1(t+1), z2(t), a(t))
        self.latent2_prior = Gaussian(
            input_dim=latent1_dim + latent2_dim + obs_dim,
            output_dim=latent2_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )

        # q(z1(0) | feat(0))
        self.latent1_init_posterior = Gaussian(
            input_dim=skill_dim,
            output_dim=latent1_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )
        # q(z2(0) | z1(0)) = p(z2(0) | z1(0))
        self.latent2_init_posterior = self.latent2_init_prior
        # q(z1(t+1) | z2(t), a(t))
        self.latent1_posterior_step = Gaussian(
            input_dim=latent2_dim + obs_dim,
            output_dim=latent1_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )
        # q(z2(t+1) | z1(t+1), z2(t), a(t)) = p(z2(t+1) | z1(t+1), z2(t), a(t))
        self.latent2_posterior = self.latent2_prior

        self.latent1_posterior_end = Gaussian(
            input_dim=latent2_dim + obs_dim + skill_dim,
            output_dim=latent1_dim,
            hidden_units=hidden_units,
            leaky_slope=leaky_slope,
            dropout=dropout,
        )

        ## p(skill | z1(end), z2(end))
        #self.classifier = Gaussian(
        #    input_dim=latent1_dim + latent2_dim,
        #    output_dim=skill_dim,
        #    hidden_units=hidden_units,
        #    leaky_slope=leaky_slope,
        #)

    def sample_prior(self,
                     obs_seq,
                     ):
        """
        Sample from prior feature encoding

        Args:
            obs_seq             : (N, S, obs_dim) tensor of observations
        Returns:
            latent1_samples     : (N, S+1, L1) tensor of latent samples
            latent2_samples     : (N, S+1, L2) tensor of latent samples
            latent1_dists       : (S+1) list of (N, L1) distributions
            latent2_dists       : (S+1) list of (N, L2) distributions
        """
        batch_dim = 0
        seq_dim = 1
        data_dim = -1
        seq_len = obs_seq.size(seq_dim)
        obs_seq_seqdim_first = torch.transpose(obs_seq, batch_dim, seq_dim)

        latent1_samples = []
        latent2_samples = []
        latent1_dists = []
        latent2_dists = []

        for t in range(seq_len + 1):
            if t == 0:
                # p(z1(0)) = N(0, I)
                latent1_dist = self.latent1_init_prior(obs_seq_seqdim_first[t])
                latent1_sample = latent1_dist.rsample()

                # p(z2(0) | z1(0))
                latent2_dist = self.latent2_init_prior(latent1_sample)
                latent2_sample = latent2_dist.rsample()

            else:
                # p(z1(t) | z2(t-1), a(t-1))
                latent1_dist = self.latent1_prior(
                    [latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent1_sample = latent1_dist.rsample()

                # p(z2(t) | z1(t), z2(t-1), a(t-1))
                latent2_dist = self.latent2_prior(
                    [latent1_sample,
                     latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent2_sample = latent2_dist.rsample()

            latent1_samples.append(latent1_sample)
            latent2_samples.append(latent2_sample)
            latent1_dists.append(latent1_dist)
            latent2_dists.append(latent2_dist)

        latent1_samples_stacked = torch.stack(latent1_samples, dim=seq_dim)
        latent2_samples_stacked = torch.stack(latent2_samples, dim=seq_dim)

        return {
            'latent1_samples': latent1_samples_stacked,
            'latent2_samples': latent2_samples_stacked,
            'latent1_dists': latent1_dists,
            'latent2_dists': latent2_dists,
        }

    def sample_posterior(self, skill, obs_seq):
        """
        Sample from posterior

        Args:
            skill               : (N, skill_dim) tensors
            obs_seq             : (N, S, obs_dim) tensors of observations
        Returns:
            latent1_samples     : (N, S+1, L1) tensor of sampled latent vectors
            latent2_samples     : (N, S+1, L2) tensor of sampled latent vectors
            latent1_dists       : (S+1) length list of (N, L1) distributions
            latent2_dists       : (S+1) length list of (N, L2) distributions
        """
        batch_dim = 0
        seq_dim = 1
        data_dim = -1
        seq_len = obs_seq.size(seq_dim)

        obs_seq_seqdim_first = torch.transpose(obs_seq, batch_dim, seq_dim)

        latent1_samples = []
        latent2_samples = []
        latent1_dists = []
        latent2_dists = []

        for t in range(seq_len + 1):
            if t == 0:
                # q(z1(0))
                # sample from normal dist
                # (observation is only used for batchsize and device)
                latent1_dist = self.latent1_init_posterior(skill)
                latent1_sample = latent1_dist.rsample()

                # q(z2(0) | z1(0))
                latent2_dist = self.latent2_init_posterior(latent1_sample)
                latent2_sample = latent2_dist.rsample()

            elif t < seq_len:
                # q(z1(t) | z2(t-1), obs(t-1))
                latent1_dist = self.latent1_posterior_step(
                    [latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent1_sample = latent1_dist.rsample()

                # q(z2(t) | z1(t), z2(t-1), obs(t-1))
                latent2_dist = self.latent2_posterior(
                    [latent1_sample,
                     latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent2_sample = latent2_dist.rsample()

            elif t == seq_len:
                # q(z1(end) | skill, z2(end-1), obs(end-1))
                assert tensor_equality(obs_seq_seqdim_first[-1],
                                       obs_seq_seqdim_first[t-1])
                latent1_dist = self.latent1_posterior_end(
                    [skill,
                     latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent1_sample = latent1_dist.rsample()

                # q(z2(t) | z1(t), z2(t-1), obs(t-1))
                latent2_dist = self.latent2_posterior(
                    [latent1_sample,
                     latent2_samples[t-1],
                     obs_seq_seqdim_first[t-1]]
                )
                latent2_sample = latent2_dist.rsample()

            else:
                raise ValueError

            latent1_samples.append(latent1_sample)
            latent2_samples.append(latent2_sample)
            latent1_dists.append(latent1_dist)
            latent2_dists.append(latent2_dist)

        latent1_samples_stacked = torch.stack(latent1_samples, dim=seq_dim)
        latent2_samples_stacked = torch.stack(latent2_samples, dim=seq_dim)

        return {
            'latent1_samples': latent1_samples_stacked,
            'latent2_samples': latent2_samples_stacked,
            'latent1_dists': latent1_dists,
            'latent2_dists': latent2_dists,
        }


import pdb
import random
import torch
from torch.optim import Adam
from torch.nn import functional as F
from tqdm import tqdm
from model import Model
from tokenizing import create_vocab

def main():

  print("building vocab")

  with open("data/all_notes.txt", "r") as f:
    full_corpus = f.read()

  token_map, char_map = create_vocab(full_corpus)

  print("loading splits")

  with open("data/train_split.txt", "r") as f:
    train_corpus = f.read()

  with open("data/test_split.txt", "r") as f:
    test_corpus = f.read()

  device = "cpu"  #"cuda" # cpu

  context_size = 8 #16 #64
  hidden_dim = 16 #128
  vocab_size = len(token_map.keys())

  print("initializing model")

  model = Model(vocab_size, context_size=context_size, hidden_dim=hidden_dim, layers=2).to(device)

  iterations = 100
  steps_per_iter = 20000 #10000
  test_steps = 100
  batch_size = 64

  opt = Adam(model.parameters())

  with torch.no_grad():
    init_inp, init_out = get_batch(test_corpus, token_map, batch_size, context_size, device)
    print(f"initial sample: \n{generate_sample(model, init_inp[0:1], char_map)}")

  print("begin training")

  for i in range(iterations):
    mean_train_loss = 0
    for e in range(steps_per_iter):
      input_t, output_t = get_batch(train_corpus, token_map, batch_size, context_size, device)
      opt.zero_grad()
      logits = model(input_t)
      # probs = F.softmax(logits, dim=1)
      #if i == 5:
      #  pdb.set_trace()
      loss = F.cross_entropy(logits, output_t)
      mean_train_loss += loss.cpu().item()
      loss.backward()
      opt.step()

    mean_train_loss /= steps_per_iter
    print(f"train loss: {mean_train_loss}")
    eval_in, eval_out = get_batch(test_corpus, token_map, batch_size, context_size, device)
    with torch.no_grad():
      mean_test_loss = 0
      for tb in range(test_steps):
        input_t, output_t = get_batch(test_corpus, token_map, batch_size, context_size, device)
        logits = model(input_t)
        loss = F.cross_entropy(logits, output_t)
        mean_test_loss += loss.cpu().item()
      mean_test_loss /= test_steps
      print(f"test loss: {mean_test_loss}")
      sampled_text = generate_sample(model, eval_in[0:1], char_map)
      print(f"{sampled_text}")

def generate_sample(model, curr_context, char_map):
  raw_chars = [char_map[t.item()] for t in curr_context[0][:16]]
  new_text = ""
  for c in raw_chars:
    new_text += c
  new_text += "|generation-start|"
  for i in range(100):
    logits = model(curr_context)
    #probs = F.softmax(logits, dim=1)[0]
    new_token = torch.distributions.Categorical(logits=logits).sample()
    new_text += char_map[new_token.item()]
    curr_context[0] = curr_context[0].roll(-1)
    curr_context[0][-1] = new_token
    #pdb.set_trace()

  return new_text + "|generation-end"

def get_batch(data, token_map, batch_size, context_size, device):
  max_idx = len(data) - context_size - 1
  offsets = [random.randint(0, max_idx) for _ in range(batch_size)]
  input_tokens = torch.tensor([[token_map[c] for c in data[off:off+context_size]] for off in offsets], device=device)
  #print(f"input chars : {input_tokens}")
  output_tokens = torch.tensor([token_map[data[off+context_size]] for off in offsets], device=device)
  #print(f"next chars : {output_tokens}")
  return input_tokens, output_tokens

if __name__ == "__main__":
  main()


import rrcli
import nfsp
import struct
import os
import errno
import time
import cache

import sys

# File access mode flags
O_RDONLY = 0
O_WRONLY = 1
O_RDWR = 2

# Open time flags
O_TRUNC = 4
O_CREAT = 8
O_EXCL = 16

# Values for whence (seek)
SEEK_SET = 0
SEEK_CUR = 1
SEEK_END = 2

# Chunk size for reading and writing
CHUNK_SIZE = 1300

class MyNFSClient:
	def __init__(self, server_ip, server_port, cacheblocks = 4, blocksize = 1300, fresh_t = 1.0):
		"""Initialize RR communication with server and
		NFS client's data structures."""
		self._rr = rrcli.RRClient(server_ip, server_port)

		# Read-write position for each fd
		self._rwpos = {}

		# Access mode for each fd
		self._mode = {}

		# fd to (server file id, session num) associations
		self._fd_to_id = {}

		# Used file descriptors
		self._used_fds = []

		# Most recent file size reported by server by fd
		self._fsize = {}

		# fd to filename associations
		self._fd_to_fname = {}

		# LRU client cache of <cacheblocks> blocks,
		# is used to store file blocks
		# so that they can be retrieved on a read,
		# in order to decrease client-server interactions.
		# Blocks are keyed by (filename, block_num) and
		# contain (block_data, tmod, tfresh)
		self._cache = cache.Cache(cacheblocks)

		# Store block size
		self._block_size = blocksize

		# Store freshness interval
		self._fresh_t = fresh_t

		# Average req-reply time metric
		self._avg_rr_time = 0.0

	def open(self, fname, flags):
		"""Communicates with MyNFS server in order to locate/create
		the file, and obtain the server-side file id. Returns a client-side fd."""

		# Check if the provided flag combination is valid,
		# and get access mode + open time flags
		mode, otime_flags = self._check_flags(flags)

		# Send open request to server and receives file data
		try:
			fid, session_num, fsize = self._open_file(fname, otime_flags)
		except Exception as e:
			raise e

		# Choose next fd
		fd = self._choose_fd()

		# Initialize all data for the fd
		self._rwpos[fd] = 0
		self._fd_to_id[fd] = (fid, session_num)
		self._fd_to_fname[fd] = fname
		self._used_fds.append(fd)
		self._fsize[fd] = fsize
		self._mode[fd] = mode

		# Print data structures (DEBUG)
		#self._print_data()

		return fd

	def read(self, fd, count):
		# """Reads and returns up to <count> bytes from the file
		global O_WRONLY

		# Check if fd exists
		if fd not in self._used_fds:
			raise OSError(errno.EBADF, os.strerror(errno.EBADF),
				          "Invalid file descriptor!")

		# Check if fd is open for reading
		elif self._mode[fd] == O_WRONLY:
			raise OSError(errno.EBADF, os.strerror(errno.EBADF),
				          "File descriptor not open for reading!")

		# The read data chunks
		total_data = []
		bytes_read = 0

		# Compute start and end block indices
		start_block = self._rwpos[fd] // self._block_size
		end_block = (self._rwpos[fd] + count - 1) // self._block_size
		#print(start_block, end_block)

		# For all blocks E [start_block, end_block]
		for k in range(start_block, end_block + 1):
			#print(k, file = sys.stderr)
			# Look in the cache for the block
			block = self._cache.lookup((self._fd_to_fname[fd], k))
			if block is not None:
				# If the block was found in the cache
				# check its freshness
				if time.time() > block[2]:
					#print("HIT-NOT_FRESH")
					# The block wasn't fresh so do the validity check
					block_data = self._send_read_req(fd, k, block[0], block[1])
				else:
					# The block was fresh, so just take the data
					# from the cache
					#print("HIT-FRESH")
					block_data = block[0]
			else:
				# If the block was NOT found in the cache,
				# fetch it from the server
				#print("MISS")
				block_data = self._send_read_req(fd, k, None, -1)

			# If the server didn't have the block
			# stop reading
			if not block_data:
				#print("{} EOF".format(k))
				break

			# Get as much data as needed from the retrieved block
			start_pos = max(self._rwpos[fd] - k * self._block_size, 0)
			bytes_remaining = count - bytes_read
			needed_data = block_data[start_pos:start_pos + bytes_remaining]

			# Insert the data to the total
			total_data.append(needed_data)
			bytes_read += len(needed_data)

			# If the data received from the block is less than
			# the block_size, the block was the last, stop
			# asking the server for more
			if len(needed_data) < self._block_size:
				#print("EOB")
				break

			# Print cache
			#self._print_cache()

		#print(bytes_read)

		# Update read/write position
		self._rwpos[fd] += bytes_read

		#print(total_data)

		# Assemble the received bytes and return them
		return b"".join(total_data)

	def _send_read_req(self, fd, block_idx, block_data, tmod):
		""" Sends a read request (for validity check or plain
		fetch) and then returns the requested block (after
		updating the cache properly). A tmod == -1 indicates
		fetch mode, while a tmod != -1 means validity check."""

		fname = self._fd_to_fname[fd]

		# Do as many RR as needed with the server
		# so that the request is sent in the correct session
		while True:
			# If read request (validity ver.) must be sent
			if tmod != -1:
				read_req = struct.pack("!BII?IQH",
					                   nfsp.READ,
					                   self._fd_to_id[fd][0],
					                   self._fd_to_id[fd][1],
					                   True,
					                   tmod,
					                   block_idx * self._block_size,
					                   self._block_size)
			# Else send read request (fetch ver.)
			else:
				read_req = struct.pack("!BII?QH",
					                   nfsp.READ,
					                   self._fd_to_id[fd][0],
					                   self._fd_to_id[fd][1],
					                   False,
					                   block_idx * self._block_size,
					                   self._block_size)
			reply = self._rr.req_reply(read_req)

			# Depending on the reply's type, do sth
			rtype = struct.unpack_from("!B", reply)[0]
			#print(rtype)
			if rtype == nfsp.DATA:
				# If the type is data, receive the new block
				fsize, tmod, data_len = struct.unpack_from("!QIH", reply, 1)
				new_data = struct.unpack_from("!" + str(data_len) + "s", reply, 15)[0]
				self._fsize[fd] = fsize

				# Update the block in the cache (new mod, new data,
				# new tfresh)
				self._cache.insert((fname, block_idx),
					               (new_data, tmod, time.time() + self._fresh_t))
				#self._print_cache()
				block_data = new_data
				break
			elif rtype == nfsp.OK:
				# If type is OK, the block in the
				# cache is valid, so update fresh time
				self._cache.insert((fname, block_idx),
					               (block_data, tmod, time.time() + self._fresh_t))
				#self._print_cache()
				break
			elif rtype == nfsp.ERR:
				# Raise an exception according to the error
				# code the server sent
				err_code = struct.unpack_from("!B", reply, 1)[0]
				raise OSError(err_code, os.strerror(err_code))
			elif rtype == nfsp.EOF:
				# If the server sent EOF, the block <block_idx> does
				# not exist after all in the file, so
				# stop trying to read blocks and remove the block
				# (or do nothing if you never inserted it)
				self._fsize[fd] = struct.unpack_from("!Q", reply, 1)[0]
				self._cache.remove((fname, block_idx))
				#self._print_cache()
				block_data = b''
				break
			elif rtype == nfsp.BAD_SNUM:
				# The server session changed, the file must be reopended
				# to obtain a new file id
				fid, session_num, fsize = self._open_file(self._fd_to_fname[fd], 0)
				self._fd_to_id[fd] = (fid, session_num)
				self._fsize[fd] = fsize
				#self._print_data()

		return block_data

	def write(self, fd, data):
		"""Writes <data> to the file corresponding to fd."""
		global O_RDONLY

		# Check if fd exists
		if fd not in self._used_fds:
			raise OSError(errno.EBADF, os.strerror(errno.EBADF),
				          "Invalid file descriptor!")
		# Check if fd is open for writing
		elif self._mode[fd] == O_RDONLY:
			raise OSError(errno.EBADF, os.strerror(errno.EBADF),
				          "File descriptor not open for writing!")

		data_len = len(data)

		# Compute number of chunks
		chunks = data_len // CHUNK_SIZE
		if data_len % CHUNK_SIZE:
			chunks += 1

		rwpos = self._rwpos[fd]

		# Write each chunks to the file in the server by
		# sending a write request for each. If the session in the
		# server changes, re-open the file to obtain a new id
		for k in range(0, chunks):
			chunk = data[k * CHUNK_SIZE:(k + 1) * CHUNK_SIZE]
			chunk_len = len(chunk)

			# Retry until there are no session changes
			while True:
				write_req = struct.pack("!BIIQH" + str(chunk_len) + "s",
					                    nfsp.WRITE,
					                    self._fd_to_id[fd][0],
					                    self._fd_to_id[fd][1],
					                    rwpos,
					                    chunk_len,
					                    chunk)
				# Send request and get reply
				reply = self._rr.req_reply(write_req)

				# Act depending on the type of the reply
				rtype = struct.unpack_from("!B", reply)[0]
				if rtype == nfsp.OK:
					# If the reply is OK, move on to the next chunk
					self._fsize[fd], tmod = struct.unpack_from("!QI", reply, 1)

					# But first, pass the changes to any block
					# in the cache they cover
					self._write_to_cache(fd, chunk, tmod, rwpos)

					rwpos += chunk_len
					break
				elif rtype == nfsp.ERR:
					# If the reply is ERR, raise an exception
					# using the error code the server provided
					err_code = struct.unpack_from("!B", reply, 1)[0]
					raise OSError(err_code, os.strerror(err_code), self._fd_to_fname[fd])
				elif rtype == nfsp.BAD_SNUM:
					# The reply type was BAD_SNUM, so open the file again in
					# the new session
					fid, session_num, fsize = self._open_file(self._fd_to_fname[fd], 0)
					self._fd_to_id[fd] = (fid, session_num)
					self._fsize[fd] = fsize
					#self._print_data()

		self._rwpos[fd] = rwpos

		# All of the chunks were written,
		# return the number of written bytes
		# (always data_len)
		return data_len

	def _write_to_cache(self, fd, chunk, tmod, rwpos):
		""" Passes a write done to the server to all
		blocks in the cache it corresponds to. If a block
		doesn't exist in the cache it is NOT fetched (no-write allocate)."""

		fname = self._fd_to_fname[fd]

		# Compute start and end block indices
		chunk_len = len(chunk)
		start_block = rwpos // self._block_size
		end_block = (rwpos + chunk_len - 1) // self._block_size

		# Chunk position
		chunk_pos = 0

		#print("\nstart block: {}, end_block: {}".format(start_block, end_block))

		for k in range(start_block, end_block + 1):
			# Look for the block in the cache
			block = self._cache.lookup((fname, k))
			if block is not None:
				# If the block was found, write the data to it
				data_start = max(rwpos - k * self._block_size, 0)
				data_end = min(self._block_size, rwpos + chunk_len - k * self._block_size)
				new_data = block[0][:data_start] + \
				           chunk[chunk_pos:chunk_pos + data_end - data_start] + \
				           block[0][data_end:]

				#print("data start: {}, data end: {}, chunk pos: {}".format(data_start,
				#	  data_end, chunk_pos))

				# Update chunk pos
				chunk_pos += data_end - data_start

				# Update the cache block in the cache
				#print(time.time())
				self._cache.insert((fname, k),
					               (new_data, tmod, time.time() + self._fresh_t))
				#self._print_cache()
			else:
				# Don't write to cache, but still advance chunk_pos
				chunk_pos += self._block_size

	def seek(self, fd, pos, whence):
		""" Sets the read/write position for fd according to pos and
		whence."""
		global SEEK_SET
		global SEEK_CUR
		global SEEK_END

		# Check if fd is valid
		if fd not in self._used_fds:
			raise OSError(errno.EBADF, os.strerror(errno.EBADF),
				          "Invalid file descriptor!")
		# Also check if the given whence is valid
		elif whence not in (SEEK_SET, SEEK_CUR, SEEK_END):
			raise OSError(errno.EINVAL, os.strerror(errno.EINVAL),
				          "Whence wasn't one of SEEK_SET, SEEK_CUR or SEEK_END!")

		# Compute the new rw position
		if whence == SEEK_SET:
			new_pos = pos
		elif whence == SEEK_CUR:
			new_pos = self._rwpos[fd] + pos
		else:
			new_pos = self._fsize[fd] + pos

		# If the new position is negative, raise exception
		if new_pos < 0:
			raise OSError(errno.EINVAL, os.strerror(errno.EINVAL),
				          "Given pos, whence gave negative offset!")

		# The new position is good, store it
		self._rwpos[fd] = new_pos

	def _open_file(self, fname, otime_flags):
		""" Sends an open request for <fname> to the
		server, and returns id, fsize, session_num. """
		fname_len = len(fname)
		open_req = struct.pack("!BBH" + str(fname_len) + "s",
			                   nfsp.OPEN, otime_flags, fname_len, fname.encode())

		# Send open request and wait for reply
		reply = self._rr.req_reply(open_req)

		# Handle the reply
		rtype = struct.unpack_from("!B", reply)[0]
		if rtype == nfsp.OK:
			# If the reply was OK, the server sent the id and session num
			fid, session_num, fsize = struct.unpack_from("!IIQ", reply,  1)
		elif rtype == nfsp.ERR:
			# If the reply was ERR, raise OSError with the sent error code
			err_code = struct.unpack_from("!B", reply, 1)[0]
			raise OSError(err_code, os.strerror(err_code), fname)

		print("{} -> {}, {}".format(fname, fid, session_num), file = sys.stderr)

		return fid, session_num, fsize

	def close(self, fd):
		""" Deletes all data for an fd, and frees the fd for future use."""

		# Check if the fd exists
		if fd not in self._used_fds:
			raise OSError(errno.EBADF, os.strerror(errno.EBADF),
				          "Invalid file descriptor!")

		# Delete all of the fd's state
		del self._rwpos[fd]
		del self._fd_to_id[fd]
		del self._fd_to_fname[fd]
		del self._fsize[fd]
		del self._mode[fd]

		# Free for future use
		self._used_fds.remove(fd)

		# Print data structures (DEBUG)
		#self._print_data()

	def metrics(self):
		metrics = self._rr.metrics()
		return metrics[0], metrics[1], self._avg_rr_time / metrics[0]

	def _check_flags(self, flags):
		"""Checks if the flag combination <flags> is invalid, and if not
		it returns the provided access mode and the OR combination of
		O_TRUNC, O_EXCL, and O_CREAT flags."""

		global O_RDONLY
		global O_WRONLY
		global O_RDWR
		global O_CREAT
		global O_EXCL
		global O_TRUNC

		# Only one access mode flag should be specified
		if (O_RDONLY in flags and O_WRONLY in flags) or \
		   (O_RDONLY in flags and O_RDWR in flags) or \
		   (O_WRONLY in flags and O_RDWR in flags):
		   raise ValueError("More than one access mode specified!")

		# O_TRUNC should be specified only on a file open for writing
		elif O_TRUNC in flags and O_RDONLY in flags:
			raise ValueError("Included O_TRUNC without write mode!")

		# Compute otime_flags
		otime_flags = 0
		if O_TRUNC in flags:
			otime_flags |= O_TRUNC
		if O_CREAT in flags:
			otime_flags |= O_CREAT
		if O_EXCL in flags:
			otime_flags |= O_EXCL

		# Get the access mode (if there isn't one -> error)
		if O_RDONLY in flags:
			mode = O_RDONLY
		elif O_WRONLY in flags:
			mode = O_WRONLY
		elif O_RDWR in flags:
			mode = O_RDWR
		else:
			raise ValueError("No access mode was specified!")

		return mode, otime_flags

	def _choose_fd(self):
		"""Returns the next available file descriptor."""
		fd = 0
		while fd in self._used_fds:
			fd += 1

		return fd

	def _print_data(self):
		"""Prints all data structures for debugging purposes."""
		print('')
		print(self._rwpos)
		print(self._fd_to_id)
		print(self._fd_to_fname)
		print(self._used_fds)
		print(self._fsize)
		print(self._mode)

	def _print_cache(self):
		print(self._cache._block_data)


import pandas as pd
import os
import matplotlib.pyplot as plt
import numpy as np

def plot_df(df_results, out_path=None):
    f1s = df_results['F1-score'].values
    plt.figure(figsize=(10, 10))
    plt.hist(f1s, bins='auto')
    plt.title("F1-distribution")
    plt.ylabel('Absolute frequency')
    plt.xlabel('F1-score')
    if out_path:
        plt.savefig(out_path)
        plt.close()
    else:
        plt.show()
        plt.close()

def make_results_table2(df_results, pipeline_name, round_dec=1):
    df_results = df_results[["Precision", "Recall", "F1-score"]]
    percentiles = np.transpose(np.quantile(df_results, q=[0.025, 0.5, 0.975], axis=0))
    percentiles = np.round(percentiles, 4) * 100
    precision = "{0} ({1},{2})".format(str(np.round(percentiles[0][1], round_dec)), str(np.round(percentiles[0][0], round_dec)),
                                       str(np.round(percentiles[0][2], round_dec)))
    recall = "{0} ({1},{2})".format(str(np.round(percentiles[1][1], round_dec)), str(np.round(percentiles[1][0], round_dec)),
                                    str(np.round(percentiles[1][2], round_dec)))
    f1 = "{0} ({1},{2})".format(str(np.round(percentiles[2][1], round_dec)), str(np.round(percentiles[2][0], round_dec)),
                                str(np.round(percentiles[2][2], round_dec)))

    iqv = np.round(percentiles[2][2], round_dec) - np.round(percentiles[2][0], round_dec)
    data = [[pipeline_name, precision, recall, f1, iqv]]
    out_dataframe = pd.DataFrame(data, columns=['Pipeline', 'Precision', 'Recall', 'F1-score', 'IQV'])
    return out_dataframe

def extract_f1(inp_text):
    return float(inp_text.split("(")[0])

def rename(inp_word):
    mapper = dict(res_title='Title', res_authors='Authors', res_journal='Journal', res_keywords='Keywords',
                  res_abstract='Abstract', res_chemical='Chemicals', res_mesh='MeSH', res_pub_type='Pub. Type',
                  res_affiliations='Affiliations', res_all='All fields', res_optimal='Opt. Fields',
                  res_optimal_idf='Opt. Fields IDF', res_unigrams='Unigrams', res_bigrams='Bigrams',
                  res_trigrams='Trigrams', res_specter='SPECTER', res_biobert_avg='BioBERT\nmean pooling',
                  res_biobert_all='BioBERT mean\n+ min&max pooling', res_biobert_all_bow='Unigrams\n+ BioBERT mean\n+ '
                                                                                'min&max pooling',
                  res_biobert_avg_bow='Unigrams\n+ BioBERT\nmean pooling')
    if inp_word in mapper.keys():
        return mapper[inp_word]
    else:
        return inp_word

def get_all_results(inp_result_files, input_dir, output_dir, convert_latex):
    all_results = []
    all_for_boxplot = []
    all_for_boxplot_names = []

    if 'res_unigrams.csv' in inp_result_files:
        inp_result_files = inp_result_files[-1:] + inp_result_files[:-1]

    print(inp_result_files)

    for inp_result in inp_result_files:
        instance_df = pd.read_csv(os.path.join(input_dir, inp_result))
        all_for_boxplot.append(instance_df["F1-score"].values)
        instance_name = inp_result.split(".")[0]
        all_for_boxplot_names.append(instance_name)
        results = make_results_table2(instance_df, instance_name)
        all_results.append(results)

    all_results_ready = pd.concat(all_results)
    all_results_ready['F1'] = all_results_ready['F1-score'].apply(extract_f1)
    all_results_ready = all_results_ready.sort_values(by="F1-score", ascending=False)
    print(all_results_ready)
    if convert_latex:
        print(all_results_ready.to_latex(index=False))

    all_results_ready.to_csv(os.path.join(output_dir, "all_results.csv"))

    idx_medians_sorted = np.asarray([np.median(x) for x in all_for_boxplot]).argsort()
    # idx_medians_sorted = range(0,len(idx_medians_sorted))
    all_for_boxplot = [all_for_boxplot[i] * 100 for i in idx_medians_sorted]
    all_for_boxplot_names = [rename(all_for_boxplot_names[i]) for i in idx_medians_sorted]

    fig7, ax7 = plt.subplots()
    fig7.set_figheight(40)
    fig7.set_figwidth(10)

    plt.ylim(55, 90)
    ax7.boxplot(all_for_boxplot, labels=all_for_boxplot_names, whis=[2.5, 97.5])
    plt.ylabel('F1-score (%)', fontsize=16)
    plt.xticks(rotation=65, fontsize=12)
    plt.yticks(fontsize=12)
    plt.gcf().subplots_adjust(bottom=0.2)
    plt.show()
    plt.close()


import csv
from datetime import datetime
from convert import convert
from matplotlib import pyplot as plt

def main(fl):
    filenames=fl  #16.1.1
    with open(filenames) as f:
        reader = csv.reader(f)
        header_row=next(reader)
        print(header_row)

        datas,highs,lows=[],[],[]
        for row in reader:
            try:
                current_date=datetime.strptime(row[0],"%Y-%m-%d")
                high=convert(float(row[1]))
                low= convert(float(row[3]))

            except ValueError:
                print(current_date,'missing data')

            else:
                datas.append(current_date)
                highs.append(high)
                lows.append(low)

    title=[]
    title.append(datas)
    title.append(highs)
    title.append(lows)
    return title

def high_lows(weather,weather_1=''):
    datas,highs,lows=weather
    if weather_1 != 0:
        datas_1,highs_1,lows_1=weather_1

    fig=plt.figure(dpi=128,figsize=(10,6))
    plt.plot(datas,highs,c='red',alpha=0.5)
    plt.plot(datas,lows,c='blue',alpha=0.5)
    plt.fill_between(datas,highs,lows,facecolor='blue',alpha=0.1)

    if weather_1 != 0:
        plt.plot(datas_1,highs_1,c='#C71585',alpha=0.5)
        plt.plot(datas_1,lows_1,c='#8A2BE2',alpha=0.5)
        plt.fill_between(datas_1,highs_1,lows_1,facecolor='#FFA500',alpha=0.1)

    plt.title("Daily tempretures, 2014",fontsize=24)
    plt.xlabel('',fontsize=16)
    fig.autofmt_xdate()
    plt.ylabel("Temperature(C)",fontsize=16)
    plt.tick_params(axis='both',which='major',labelsize=16)

    plt.show()

weather=main('sitka_weather_07-2014.csv')
weather_1=main('death_valley_2014.csv')
high_lows(weather,weather_1)


# -*- coding: utf-8 -*-
# Questions:
# 1 - when does this data become publicly available to trade on?
# 2- is this data 'as of' or was it later adjusted?
# 3 - why the big gaps in the data, and the big jumps in values?
# 4 - I'm missing the first available data point (2003 mid year, which could flow over into this dataset...)
"""
Created on Thu Oct 20 18:41:25 2016

@author: Richard
"""
############ IMPORTS
from typing import NamedTuple
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import xarray as xr
#import numpy as np
#import matplotlib.pyplot as plt
import os.path
#from scipy.stats import norm
import backtest_crypto as bt
#from scipy.stats.mstats import gmean
from pycoingecko import CoinGeckoAPI
import datetime
import time

############ INPUTS
#cost = 0.000
frequency = 12.0

cg = CoinGeckoAPI()
datetime1 = datetime.datetime(2021, 1, 1, 10, 20)
datetime2 = datetime.datetime(2021, 5, 15, 10, 20)
from_dateTime = time.mktime(datetime1.timetuple())
to_dateTime = time.mktime(datetime2.timetuple())
#print(from_dateTime)
#print(to_dateTime)
#print(cg.get_price(ids='bitcoin', vs_currencies='usd'))
portfolio = ['bitcoin', 'ethereum', 'solana', 'venus'] #'polkadot', 'cardano', 'chainlink', 'XRP', 'Binance Coin', 'Fantom', 'Matic', 'Aave'
data={}
for name in portfolio:
    try:
        data[name] = cg.get_coin_market_chart_range_by_id(id=name, vs_currency='usd', from_timestamp=from_dateTime , to_timestamp=to_dateTime )
    except:
        print(name, " not found")

# need to pull the list of tickers from coingecko, and then iterate through them to get all the coins imported and create the database...
#inputs = {}
#inputs.keys = data[name].keys()

for name in portfolio:
    for x in data[name].keys(): # from a dict of lists, create a dict of dataframes
        data[name][x] = pd.DataFrame.from_records(data[name][x], exclude=['Date'], columns=['Date', name], index=['Date'])
        data[name][x] = data[name][x].set_index(pd.to_datetime(data[name][x].index, unit='ms'))

#inputs = data[name][x].assign(name = data[name][x][name])
inputs = {}
for x in data[name].keys():
    inputs[x] = data[portfolio[0]][x]
    for i in range(len(portfolio)-1):
        inputs[x] = pd.concat([inputs[x], data[portfolio[i+1]][x]], axis=1)

print(inputs)

"""
    df_btc = {}
df_eth = {}

for x in data_eth.keys():
    df_eth[x] = pd.DataFrame.from_records(data_eth[x], exclude=['Date'], columns=['Date', 'ETH'], index=['Date'])
    df_eth[x] = df_eth[x].set_index(pd.to_datetime(df_eth[x].index, unit='ms'))
    merged_df[x] = df_eth[x].assign(BTC = df_btc[x]['BTC'])
"""
#print (df_btc['prices'])
#print (df_eth['prices'])
print(inputs)

"""
#print(cg.get_coin_market_chart_by_id(id='aave', vs_currency='usd', days=14))
for x in cg.get_coins_list():
    try:
        tmp_data = cg.get_coin_market_chart_by_id(id=x['id'], vs_currency='usd', days=14)
        tmp_df = pd.DataFrame(tmp_data['prices'], columns=['Date', x['id']])
        mergedDf = mergedDf.merge(tmp_df)
        print(x['id'], " included in dataframe")
    except:
        print(x['id'], " not found")
"""

inputs_not_to_shift = ['prices','market_caps','total_volumes']
inputs_to_shift=[item for item in inputs if item not in inputs_not_to_shift]

############ RUN CODE
# create a data object 'd' using backtest.py
d = bt.Data(inputs, inputs_to_shift, months_delay_data=3, start=0, delist_value=1)
#print(d.basic_data['prices'])
#data_present = d.accruals()*d.pfd()*d.pman()*d.market_cap()*d.mom()*d.mvi_weight()*d.value()*d.fip()
#data_present = data_present.notnull().astype(int) # would be more reassuring if I had na's here..
#data_present = data_present.replace(0,np.nan)

#print(d.basic_data['prices'].index)
#print(d.basic_data['prices'].columns)

# create a pandas Dataframe of all 1's, with dates for an index and the tickers for column headers
init_pos1 = pd.DataFrame(1, index=d.basic_data['prices'].index, columns=d.basic_data['prices'].columns)
print(init_pos1)
#init_pos1 = init_pos1 * data_present

def run_backtests():
    ret={}
    mc=bt.Mkt_cap_scr(threshold=1000)
    mw=bt.Mkt_cap_weights()

    #print("Creating QM object")
    #QM_object = bt.QM(threshold=1000, scr2_perc = 0.15, scr3_perc = 0.6, upper_limit=0.2)
    #QM_object.backtest(init_pos= init_pos1, data = d) #, rebalance = 'M', #Q-NOV, Q-DEC, Q-OCT
    #ret['QM'] = QM_object.calc_ret(price_data= d.basic_data['prices'])

    #ret['QM+QV'] = (QM_object.calc_ret(price_data= d.basic_data['prices']) + QV_object.calc_ret(price_data= d.basic_data['prices'])  )/2.0

    print("Creating BAH instance")
    BAH_instance = bt.BAH()
    BAH_instance.backtest(mc.run(init_pos1,data=d), data=d)
    print (BAH_instance.backtest(mc.run(init_pos1,data=d), data=d))
    ret['BAH'] = BAH_instance.calc_ret(d.basic_data['prices'])

    print("Creating mom instance")
    mom_instance = bt.Mom(scr_perc = 0.15)
    mom_instance.run(mc.run(init_pos1,data=d), data=d)
    print (mom_instance.run(mc.run(init_pos1,data=d), data=d))
    ret['mom'] = mom_instance.calc_ret(d.basic_data['prices'])

    #print("Creating fip object") - needs daily price!!
    #fip_object = bt.Fip(scr_perc = 0.6)
    #fip_object.run(mc.run(init_pos1,data=d), data=d)
    #ret['fip'] = fip_object.calc_ret(d.basic_data['prices'])

    metrics=(['BAH','mom']) #'QV', 'QV_fin',,'QM','alsi'

    print(" about to plot things")

    bt.plot_returns(ret, metrics)
    bt.tabulate_results(ret, metrics, frequency=12.0, risk_free = 0.07) # risk free rate is per annum
    bt.plot_CAGR(ret, metrics, 1)
    bt.plot_CAGR(ret, metrics, 5)

print("About to run_backtests()")
run_backtests()
print("All done")

def QM_mkt_cap(init_pos = init_pos1, threshold = 1000, scr_perc_mom = 0.2, scr_perc_fip = 0.6):
    mc=bt.Mkt_cap_scr(threshold = threshold)
    mom_object = bt.Mom(scr_perc = scr_perc_mom)
    fip_object = bt.Fip(scr_perc = scr_perc_fip)
    mw = bt.Mkt_cap_weights()

    mw.run(fip_object.run(mom_object.run(mc.run(init_pos,data=d), data=d), data=d),data=d)
    return mw

def QM_vmi(init_pos = init_pos1, threshold = 1000, scr_perc_mom = 0.2, scr_perc_fip = 0.6):
    mc=bt.Mkt_cap_scr(threshold = 1000)
    mom_object = bt.Mom(scr_perc = 0.2)
    fip_object = bt.Fip(scr_perc = 0.6)
    mw = bt.Mvi_weights()

    mw.run(fip_object.run(mom_object.run(mc.run(init_pos,data=d), data=d), data=d),data=d)
    return mw

def QM_parts():
    ret={}
    #mc=bt.Mkt_cap_scr(threshold=1000)

    QM_object = bt.QM(threshold=1000, scr2_perc = 0.2, scr3_perc = 0.6, upper_limit=0.2)
    QM_object.backtest(init_pos= init_pos1, data = d) #, rebalance = 'M', #Q-NOV, Q-DEC, Q-OCT
    ret['QM'] = QM_object.calc_ret()

#    BAH_object = bt.BAH()
#    BAH_object.backtest(mc.run(init_pos1,data=d), data=d)
#    ret['alsi'] = BAH_object.calc_ret(d.basic_data['prices'])
#
#    mom_object = bt.Mom(scr_perc = 0.2)
#    mom_object.run(mc.run(init_pos1,data=d), data=d)
#    ret['mom'] = mom_object.calc_ret(d.basic_data['prices'])
#
#    fip_object = bt.Fip(scr_perc = 0.6)
#    fip_object.run(mc.run(init_pos1,data=d), data=d)
#    ret['fip'] = fip_object.calc_ret(d.basic_data['prices'])

    metrics=(['QM']) #,'fip','QM','alsi'

    bt.plot_returns(ret, metrics)
    bt.tabulate_results(ret, metrics, frequency=12.0, risk_free = 0.07) # risk free rate is per annum
    bt.plot_CAGR(ret, metrics, 1)
    bt.plot_CAGR(ret, metrics, 5)

# this runs a random screen strategy, as a baseline comparison (as opposed to just comparing to holding the whole universe)
def random_test():
    ret={}
    mc=bt.Mkt_cap_scr(threshold=0)
    mw = bt.Mkt_cap_weights()
    #mw = bt.Mvi_weights()

    #QV_object = bt.QV(threshold=1000, scr2_perc=0.95, scr3_perc = 0.15, scr4_perc = 0.6, upper_limit=0.2)
    #QV_object.backtest(init_pos= init_pos1, data=d) #, rebalance = 'M'
   #ret['QV'] = QV_object.calc_ret(price_data= d.basic_data['prices'])

    #QV_fin_object = bt.QV_fin(banks, insurers, threshold=0, scr2_perc = 0.15, scr3_perc = 0.6, upper_limit=0.2)
    #QV_fin_object.backtest(init_pos= init_pos_fin, data=d) #, rebalance = 'M',
    #ret['QV_fin'] = QV_fin_object.calc_ret(price_data= d.basic_data['prices'])

    BAH_object = bt.BAH()
    mw.run(BAH_object.backtest(mc.run(init_pos_non_fins,data=d), data=d), data=d)
    mw.final_positions = bt.limit_pos_size(mw.final_positions)
    ret['alsi'] = mw.calc_ret(d.basic_data['prices'])

    val_object = bt.Value(scr_perc = 0.15)
    mw.run(val_object.run(mc.run(init_pos_non_fins,data=d), data=d), data=d)
    mw.final_positions = bt.limit_pos_size(mw.final_positions)
    ret['value'] = mw.calc_ret(d.basic_data['prices'])

    import numpy as np
    rs_obj = bt.Random_screen(scr_perc = 0.15)
    runs=100
    test=pd.Series(index=range(runs))
    for i in range(runs):
        mw.run(rs_obj.run(mc.run(init_pos_non_fins,data=d)), data=d)
        mw.final_positions = bt.limit_pos_size(mw.final_positions)
        test[i] = ((np.prod(1.+mw.calc_ret(d.basic_data['prices'])))**(frequency/len(mw.calc_ret(d.basic_data['prices']))))-1
    #random_CAGR = test.mean()
    value_CAGR = ((np.prod(1.+ret['value']))**(frequency/len(ret['value'])))-1
    idx = test  < value_CAGR # how many of the random results are less than our value result (CAGR)
    print ("Percentage of random outcomes that are less than our outcome:")
    print ("{:.2%}".format(idx.sum()/runs)) # what percentage of the random results does our value result outperform?
    # need to calculate what % of the random_CAGR's my value_CAGR beats
    metrics=range(10)

    #(['value','random','alsi']) #,'QV_fin'

    bt.plot_returns(ret, metrics)
    bt.tabulate_results(ret, metrics, frequency=12.0, risk_free = 0.07) # risk free rate is per annum

    bt.plot_CAGR(ret, metrics, 1)
    bt.plot_CAGR(ret, metrics, 5)

def testing_QM():
    ret={}

    QM_object={}
    mom_params=[0.05,0.1, 0.15,0.2,0.25]
    fip_params=[0.55,0.6,0.65,0.7,0.75]
    for i in (mom_params):
        for j in (fip_params):
            QM_object[i,j] = bt.QM(threshold=1000, scr2_perc = i, scr3_perc = j, upper_limit=0.2)
            QM_object[i,j].backtest(init_pos= init_pos1, data = d) #, rebalance = 'M', #Q-NOV, Q-DEC, Q-OCT
            ret[i,j] = QM_object[i,j].calc_ret(price_data= d.basic_data['prices'])

    metrics=[(x,y) for x in mom_params for y in fip_params]

    #bt.plot_returns(ret, metrics)
    bt.tabulate_results(ret, metrics, frequency=12.0, risk_free = 0.07) # risk free rate is per annum
    #bt.plot_CAGR(ret, metrics, 1)
    #bt.plot_CAGR(ret, metrics, 5)
    plotGPR_3d(mom_params, fip_params)

def plotGPR_3d(x_params, y_params):
    import numpy as np
    from mpl_toolkits.mplot3d import Axes3D
    import matplotlib.pyplot as plt
    from scipy.interpolate import griddata

    fig = plt.figure(figsize=plt.figaspect(0.5))
    ax = fig.add_subplot(1, 2, 1, projection='3d')
    # note this: you can skip rows!
    X,Y = np.meshgrid(x_params,y_params)
    X = X.flatten(order='F')
    Y = Y.flatten(order='F')
    Z = [sum(ret[(x,y)])/abs(sum(ret[(x,y)][ret[(x,y)]<0]))for x in mom_params for y in fip_params] #GPR

    xi = np.linspace(min(X),max(X),100)
    yi = np.linspace(min(Y),max(Y),100)
    # VERY IMPORTANT, to tell matplotlib how is your data organized
    zi = griddata((X, Y), Z, (xi[None,:], yi[:,None]), method='cubic')

    CS = plt.contour(xi,yi,zi,15,linewidths=0.5,color='k')
    ax = fig.add_subplot(1, 2, 2, projection='3d')

    xig, yig = np.meshgrid(xi, yi)

    surf = ax.plot_surface(xig, yig, zi,
            linewidth=0)

    plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 16 12:05:18 2022

@author: cbone
"""

#fichier refaisant la procédure de Gilett avec nos données pour comparaison

#import extraction_data as extr
import numpy as np
import detatt_mk as da
import torch
import matplotlib.pyplot as plt
from scipy import stats
def mkdir_p(mypath):
    '''Creates a directory. equivalent to using mkdir -p on the command line'''

    from errno import EEXIST
    from os import makedirs,path

    try:
        makedirs(mypath)
    except OSError as exc: # Python >2.5
        if exc.errno == EEXIST and path.isdir(mypath):
            pass
        else: raise

obs = torch.load('figures/Europe/rand_hist_IPSL-CM6A-LR.pt').detach().numpy()[0:115] * 1.06
print(f"obs shape : {obs.shape}")
#print(f"obs : {obs}")

liste_models = ['ACCESS', 'BCC', 'CESM2', 'CNRM', 'CanESM5', 'FGOALS',
                'GISS', 'HadGEM3', 'IPSL', 'MIRO', 'ESM2', 'NorESM2']
#liste_models = ['ACCESS-ESM1-5', 'BCC-CSM2-MR', 'CESM2',
 #               'CNRM-CM6-1', 'CanESM5', 'FGOALS-g3', 'GISS-E2-1-G',
  #              'HadGEM3-GC31-LL', 'IPSL-CM6A-LR', 'MIROC6', 'MRI-ESM2-0',
   #             'NorESM2-LM']

variab = np.array([0.1,0.13,0.15,0.11,0.17,0.10,0.11,0.13,0.1,0.15,0.13,0.15])

Mean_orig = np.load('figures/Europe/concatenated_IPSL-CM6A-LR.npy')
print(f"Mean_orig[3] : {Mean_orig[3]}")
print(f"Mean_orig shape : {Mean_orig.shape}")
mean_tot = np.mean(np.load('figures/Europe/combined_data.npy')[:,1],axis=0)
print(f"mean_tot shape : {mean_tot.shape}")
aer = torch.load('figures/Europe/aer.pt')
print(f"aer : {aer}")

for tensor in aer:
    print(tensor.shape)

mean = np.copy(Mean_orig)

#obs = obs - np.mean(obs)
# =============================================================================
# mean[0] -= np.mean(mean,axis=1)[0]
# mean[1] -= np.mean(mean,axis=1)[1]
# mean[2] -= np.mean(mean,axis=1)[2]
# mean[3] -= np.mean(mean,axis=1)[3]
# =============================================================================

#mean = np.subtract(mean - np.mean(mean,axis=1))
count = np.load("figures/Europe/count.npy")
print(f"count shape : {count.shape}")
print(f"count : {count}")
noise = np.load("figures/Europe/noise.npy")
print(f"noise shape : {noise.shape}")

mean = np.transpose(mean)
anom_index = noise.shape[0]
noise = np.transpose(noise)
model = 'IPSL-CM6A-LR'

(xr,yr,cn1,cn2)=da.reduce_dim(mean,obs[:,None],noise[:,list(range(1,anom_index,2))],noise[:,list(range(0,anom_index,2))])
result=da.tls(xr[:,0:3],yr,cn1,ne=count[0:3],cn2=cn2,rof_flag=1,RCT_flag=0)
name_exp = 'ROF/'+model+'/'
path = '/usr/home/habbar/Bureau/data_nc/'
mkdir_p(path+name_exp)
print(f"result Beta : {result['beta']}")
print(f"result Beta shape : {result['beta'].shape}")

mean_variab = np.mean(variab)
sum_combined = np.zeros((115))
for forc in range(3):
    ecart = (result['betaCI'][forc,1] - result['betaCI'][forc,0]) / 2

    ecart = ecart / 1.64
    print(ecart)
    x_inf=np.zeros((115))
    x_sup=np.zeros((115))
    nn=np.arange(115)
    for i in nn:
        Xn=stats.norm.rvs(size=10000,loc = Mean_orig[forc][i], scale = mean_variab/np.sqrt(count[forc]))
        betan=stats.norm.rvs(size=10000,loc = result['beta'][forc], scale = ecart)
        mult=Xn*betan
        x_inf[i]=np.percentile(mult,5)
        x_sup[i]=np.percentile(mult,95)

    plt.fill_between(nn,x_inf,x_sup,alpha=0.5)
    np.save(path+name_exp+str(forc)+'_result_mean',Mean_orig[forc]*result['beta'][forc])
    np.save(path+name_exp+str(forc)+'_result_min',x_inf)
    np.save(path+name_exp+str(forc)+'_result_max',x_sup)
    plt.plot(nn,Mean_orig[forc]*result['beta'][forc])
    plt.grid()
    plt.savefig(path+name_exp+str(forc)+'_result.png')

    sum_product = np.sum(result['beta'][forc] * Mean_orig[forc])

    print(f"Somme des produits pour le forçage {forc}: {sum_product}")

    # Ajouter la courbe calculée à la somme globale
    sum_combined += Mean_orig[forc] * result['beta'][forc]

# Plot de la somme des courbes combinées
plt.plot(nn, sum_combined, label='Somme des courbes combinées', color='black')
plt.legend()

# Plot de l'observation historique cible
plt.plot(nn, obs, label='Observation historique cible', color='red')
plt.legend()
# sauvegarde de la figure
plt.savefig(path+name_exp+'sum_combined.png')

plt.show()


import logging

from tg import tmpl_context as c

from ming.odm import ThreadLocalODMSession

from forgewiki import model as WM

log = logging.getLogger(__name__)

def main():
    c.project = None
    pages = WM.Page.query.find({'title': {'$regex': r'\/'}}).all()
    print('Found %s wiki titles containing "/"...' % len(pages))
    for page in pages:
        page.title = page.title.replace('/', '-')
        print('Updated: %s' % page.title)
    ThreadLocalODMSession.flush_all()

if __name__ == '__main__':
    main()


import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
from torchvision.transforms import transforms

# 设置随机种子，以便结果可重复
torch.manual_seed(42)

# 检查是否有可用的GPU设备
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# 数据预处理
transform = transforms.Compose([
    transforms.Resize((224, 224)),  # 调整图像大小为224x224像素
    transforms.ToTensor(),  # 转换为张量
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])  # 标准化图像
])

# 加载训练集和验证集
train_data = torchvision.datasets.ImageFolder('path_to_train_data', transform=transform)
val_data = torchvision.datasets.ImageFolder('path_to_validation_data', transform=transform)

# 创建数据加载器
train_loader = torch.utils.data.DataLoader(train_data, batch_size=32, shuffle=True)
val_loader = torch.utils.data.DataLoader(val_data, batch_size=32, shuffle=False)

# 定义CNN模型
class FoodClassifier(nn.Module):
    def __init__(self):
        super(FoodClassifier, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),
            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(256 * 28 * 28, 512),
            nn.ReLU(inplace=True),
            nn.Linear(512, 10)  # 假设有10个食物类别
        )

    def forward(self, x):
        x = self.features(x)
        x = torch.flatten(x, 1)
        x = self.classifier(x)
        return x

# 创建模型实例
model = FoodClassifier().to(device)

# 定义损失函数和优化器
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# 训练模型
num_epochs = 10

for epoch in range(num_epochs):
    model.train()
    train_loss = 0.0
    train_correct = 0

    for images, labels in train_loader:
        images = images.to(device)
        labels = labels.to(device)

        optimizer.zero_grad()
        outputs = model(images)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        _, predicted = torch.max(outputs.data, 1)
        train_correct += (predicted == labels).sum().item()
        train_loss += loss.item() * images.size(0)

    train_accuracy = train_correct / len(train_data)
    train_loss = train_loss / len(train_data)

    # 在验证集上评估模型
    model.eval()
    val_loss = 0.0
    val_correct = 0

    with torch.no_grad():
        for images, labels in val_loader:
            images = images.to(device)
            labels = labels.to(device)

            outputs = model(images)
            loss = criterion(outputs, labels)

            _, predicted = torch.max(outputs.data, 1)
            val_correct += (predicted == labels).sum().item()
            val_loss += loss.item() * images.size(0)

    val_accuracy = val_correct / len(val_data)
    val_loss = val_loss / len(val_data)

    print(f"Epoch [{epoch+1}/{num_epochs}], "
          f"Train Loss: {train_loss:.4f}, Train Accuracy: {train_accuracy:.4f}, "
          f"Val Loss: {val_loss:.4f}, Val Accuracy: {val_accuracy:.4f}")

# 保存模型
torch.save(model.state_dict(), 'food_classifier.pth')


from model_mommy import mommy
from django.urls import reverse
from mydentist.models import Doctor

def test_doctor_only_from_facility(doctor_client):
    mommy.make(Doctor, _quantity=2)
    url = reverse("doctors-list")
    response = doctor_client.get(url)
    data = response.data
    assert data
    assert Doctor.objects.count() == 3
    assert len(data) == 1


class Node:
    def __init__(self):
        self.data = None
        self.link = None

def print_nodes(start):
    current = start
    #if current == None :
    if current is None:
        return
    print(current.data, end=' ')
    while current.link is not None:
        current = current.link
        print(current.data, end=' ')
    print()

def insert_node(find_data, insert_data):
    global head, current, pre

    if head.data == find_data:  # 첫 번째 노드 삽입
        node = Node()
        node.data = insert_data
        node.link = head
        head = node
        return

    current = head
    while current.link is not None:  # 중간 노드 삽입
        pre = current
        current = current.link
        if current.data == find_data:
            node = Node()
            node.data = insert_data
            node.link = current
            pre.link = node
            return

    node = Node()                   # 마지막 노드 삽입
    node.data = insert_data
    current.link = node

head, current, pre = None, None, None
data_array = ["다현", "정연", "쯔위", "사나", "지효"]
#data_array = ["백현", "첸"]

if __name__ == "__main__":
    node = Node()
    node.data = data_array[0]
    head = node

    for i in range(1, len(data_array)):
        pre = node
        node = Node()
        node.data = data_array[i]
        pre.link = node

    print_nodes(head)

    insert_node("다현", "화사")
    print_nodes(head)

    insert_node("사나", "솔라")
    print_nodes(head)

    insert_node("재남", "문별")
    print_nodes(head)

    insert_node("쯔위", "휘인")
    print_nodes(head)