# -*- coding: utf-8 -*-
import os
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import copy
import random
import time

import torch
import torch.nn as nn
import torch.nn.functional as F

from torch.utils.data import Dataset, DataLoader
from torch.utils.tensorboard import SummaryWriter
from torchvision import transforms, utils, datasets

"""## Load Data"""

# 数据集导入
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.deterministic = True
# create transforms
# We will just convert to tensor and normalize
transforms_mnist = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.1307,), (0.3081,))
])
# Load Data
mnist_data_train = datasets.MNIST(
    './data/mnist/', train=True, download=True, transform=transforms_mnist)
mnist_data_test = datasets.MNIST(
    '../data/mnist/', train=False, download=True, transform=transforms_mnist)
classes = np.array(list(mnist_data_train.class_to_idx.values()))
classes_test = np.array(list(mnist_data_test.class_to_idx.values()))
num_classes = len(classes)
print("Classes: {} \tType: {}".format(classes, type(classes)))
print("Classes Test: {} \tType: {}".format(classes_test, type(classes)))
print("Image Shape: {}".format(mnist_data_train.data[0].size()))
print("Load Data Done!")
classes_mapper = {}
for item in classes:
    # numpy64转int
    classes_mapper[int(item)] = []

"""## Data Partitioning"""

class CustomDataset(Dataset):
    def __init__(self, dataset, idxs):
        self.dataset = dataset
        self.idxs = list(idxs)

    def __len__(self):
        return len(self.idxs)

    def __getitem__(self, item):
        image, label = self.dataset[self.idxs[item]]
        return image, label

def classesIdxsGenerator(classes, dataset):
    # 这里可以使用pickle模块进行dump，不用每次都跑一遍
    for i in range(len(dataset)):
        # tensor转int
        targetForOnce = int(dataset.targets[i])
        classes_mapper[targetForOnce].append(i)

    for key in classes_mapper.keys():
        # list转为numpy数组，所以mapper的k-v类型是int-numpy_array
        classes_mapper[key] = np.array(classes_mapper[key])
    return classes_mapper

classes_mapper = classesIdxsGenerator(
    classes=classes, dataset=mnist_data_train)

def customizedIdxsGenerator(Bias_array, local_datasize):
    """
    @Params:
    Bias_array即用于表示该客户端数据集类别偏好的数组（可能要改为字典存储）
    local_datasize用于表示该客户端有多少样本量
    """
    # list为可变对象，需要深拷贝
    Bias_num_array = copy.deepcopy(Bias_array)
    for i in range(len(Bias_array)):
        Bias_num_array[i] = int(local_datasize*Bias_array[i])

    local_dataitem_idxs = np.array([])
    # classes_mapper=classesIdxsGenerator(classes=classes,dataset=mnist_data_train)

    for key in classes_mapper.keys():
        # 先把classes_mapper[key] list类型转化numpy
        classes_mapper[key] = np.array(classes_mapper[key])
        # concatenate
        local_dataitem_idxs = np.concatenate((local_dataitem_idxs, np.random.choice(
            classes_mapper[key], int(Bias_num_array[key]), replace=False)), axis=0)

    np.random.shuffle(local_dataitem_idxs)
    local_dataitem_idxs = local_dataitem_idxs.astype(int)
    return local_dataitem_idxs.tolist()

def iid_partition(dataset, numOfClients):

    num_items_per_client = int(len(dataset)/numOfClients)
    client_dict = {}
    image_idxs = [i for i in range(len(dataset))]

    for i in range(numOfClients):
        client_dict[i] = set(np.random.choice(
            image_idxs, num_items_per_client, replace=False))
        image_idxs = list(set(image_idxs) - client_dict[i])
    return client_dict

def non_iid_partition(dataset, clients, total_shards, shards_size, num_shards_per_client):

    shard_idxs = [i for i in range(total_shards)]
    client_dict = {i: np.array([], dtype='int64') for i in range(clients)}
    idxs = np.arange(len(dataset))
    data_labels = dataset.targets.numpy()

    # sort the labels
    label_idxs = np.vstack((idxs, data_labels))
    label_idxs = label_idxs[:, label_idxs[1, :].argsort()]
    idxs = label_idxs[0, :]

    # divide the data into total_shards of size shards_size
    # assign num_shards_per_client to each client
    for i in range(clients):
        rand_set = set(np.random.choice(
            shard_idxs, num_shards_per_client, replace=False))
        shard_idxs = list(set(shard_idxs) - rand_set)

        for rand in rand_set:
            client_dict[i] = np.concatenate(
                (client_dict[i], idxs[rand*shards_size:(rand+1)*shards_size]), axis=0)

    return client_dict

"""## LeNet Model"""

class LeNet(nn.Module):
    def __init__(self):
        super(LeNet, self).__init__()
        # 1 input image channel, 6 output channels, 5x5 square convolution,padding=2 because MNIST is of 28*28
        self.conv1 = nn.Conv2d(1, 6, 5, padding=2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        # an affine operation: y = Wx + b
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2)
        x = x.view(x.size(0), -1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

"""## Client"""

class Client(object):
    def __init__(self, client_index, dataset, batchsize, learning_rate, local_epochs, data_idxs, bias_array):
        self.client_index = client_index
        self.bias_array = torch.Tensor(bias_array)
        self.data_idxs = data_idxs
        self.train_loader = DataLoader(CustomDataset(
            dataset, data_idxs), batch_size=batchsize, shuffle=True)
        self.learning_rate = learning_rate
        self.epochs = local_epochs
        self.local_datasize = len(data_idxs)

    def printProfile(self):
        print(
            f"ClientNumber: {self.client_index},learningRate: {self.learning_rate},local_epochs {self.epochs},and its bias is {self.bias_array}")
        print(
            f"Type of dataidxs is {type(self.data_idxs)}, and they are {self.data_idxs}")
        return

    def train(self, model):
        criterion = nn.CrossEntropyLoss()
        # Adam优化器可以自适应调整学习率
        optimizer = torch.optim.Adam(model.parameters(), lr=self.learning_rate)
        e_loss = []
        for _ in range(1, self.epochs+1):  # _ means epoch
            train_loss = 0.0
            model.train()
            for data, labels in self.train_loader:
                if torch.cuda.is_available():
                    data, labels = data.cuda(), labels.cuda()

                # clear the gradients
                optimizer.zero_grad()
                # make a forward pass
                output = model(data)
                # calculate the loss
                loss = criterion(output, labels)
                # do a backwards pass
                loss.backward()
                # perform a single optimization step
                optimizer.step()
                # update training loss
                train_loss += loss.item()*data.size(0)

            # average losses
            train_loss = train_loss/len(self.train_loader.dataset)
            e_loss.append(train_loss)

        total_loss = sum(e_loss)/len(e_loss)
        return model.state_dict(), total_loss

"""## Edge or Mediator"""

class Edge(object):
    def __init__(self, edge_index, region_clients_idxs, Cr):
        self.edge_index = edge_index
        self.region_clients_idxs = region_clients_idxs
        self.Cr = Cr
        self.Bias_array = torch.Tensor([0.0 for i in range(10)])
        # 建立client_index到对应weights和datasize的键值对，初始化占位即可
        self.cachedWeights, self.cachedDatasizes = {}, {}
        for i in range(len(self.region_clients_idxs)):
            self.cachedWeights[self.region_clients_idxs[i]] = 0
            self.cachedDatasizes[self.region_clients_idxs[i]] = 0

    def addClient(self, client_index):
        self.region_clients_idxs.append(client_index)
        return

    def printProfile(self):
        sumOfDistribution = torch.sum(self.Bias_array)
        print(f"Edge_index: {self.edge_index}, region clients: {self.region_clients_idxs},Cr: {self.Cr}, and its bias is {self.Bias_array}, which sums upto {sumOfDistribution}")
        return

    # edge端单次更新
    def edgeUpdate(self, model):
        self.selected_client_num = max(
            int(self.Cr*len(self.region_clients_idxs)), 1)
        self.selected_client_idxs = np.random.choice(
            self.region_clients_idxs, self.selected_client_num, replace=False)
        # print(f"Edge#{self.edge_index} this turn select {self.selected_client_idxs} ")
        self.tempTotalDatasize = 0
        local_Loss = []
        for k in self.selected_client_idxs:
            local_update = TotalClients[k]
            weights, _ = local_update.train(
                model=copy.deepcopy(model))  # _ 指本次loss
            local_Loss.append(copy.deepcopy(_))
            self.cachedWeights[k] = copy.deepcopy(weights)
            self.cachedDatasizes[k] = local_update.local_datasize
            self.tempTotalDatasize += local_update.local_datasize

        loss_avg = sum(local_Loss) / len(local_Loss)
        # print(f"Edge#{self.edge_index} this turn LossAvg is {loss_avg}, lenOf localLoss is{len(local_Loss)}")

    def edgeAggregation(self):
        weights_avg = copy.deepcopy(
            self.cachedWeights[self.selected_client_idxs[0]])
        for key in weights_avg.keys():
            weights_avg[key] *= self.cachedDatasizes[self.selected_client_idxs[0]
                                                     ]/self.tempTotalDatasize
        for key in weights_avg.keys():
            for k in self.selected_client_idxs[1:]:
                ratio = self.cachedDatasizes[k]/self.tempTotalDatasize
                # test print
                # if(self.edge_index==1):
                # 	print(' ',ratio,end='')
                weights_avg[key] += self.cachedWeights[k][key]*ratio
            # weights_avg[key]=torch.div(weights_avg[key], len(self.selected_client_idxs))

        return weights_avg, self.tempTotalDatasize

"""## HyBridFL protocol"""

def HyBridFL(model, rounds):
    global_weights = model.state_dict()
    # 全局更新轮数
    for i in range(rounds):
        print(f"\nGLOBAL ROUND {i}")
        temp_weights, temp_datasizes = [], []
        # 遍历每一个Edge
        for j in range(Edges):
            # 模型分发
            TotalEdges[j].edgeUpdate(model=copy.deepcopy(model))
            edge_weights, edge_datasize = TotalEdges[j].edgeAggregation()
            temp_weights.append(copy.deepcopy(edge_weights))
            temp_datasizes.append(edge_datasize)

        # Edge端聚合
        sumOfTempDatasizes = sum(temp_datasizes)
        weights_global_avg = copy.deepcopy(temp_weights[0])
        for key in weights_global_avg.keys():
            weights_global_avg[key] *= temp_datasizes[0]/sumOfTempDatasizes
        for key in weights_global_avg.keys():
            for k in range(1, len(temp_weights)):
                ratio = temp_datasizes[k]/sumOfTempDatasizes
                # print(f"Edge#{k} this turn datasize ratio {ratio} across all edges")
                weights_global_avg[key] += temp_weights[k][key]*ratio

            # weights_global_avg[key] = torch.div(weights_global_avg[key], len(temp_weights))

        global_weights = weights_global_avg
        print('\n')
        model.load_state_dict(weights_global_avg)
        Criterion = nn.CrossEntropyLoss()
        TestGlobalModel(model=model, dataset=mnist_data_test, criterion=Criterion,
                        test_batchsize=128, num_classes=num_classes, classes=classes)

"""## Testing"""

# TODO: Test Global Model

def TestGlobalModel(model, dataset, criterion, test_batchsize, num_classes, classes):
    # test loss
    test_loss = 0.0
    correct_class = list(0. for i in range(num_classes))
    total_class = list(0. for i in range(num_classes))

    test_loader = DataLoader(dataset, batch_size=test_batchsize)
    l = len(test_loader)
    model.eval()
    for data, labels in test_loader:
        if torch.cuda.is_available():
            data, labels = data.cuda(), labels.cuda()

        output = model(data)
        loss = criterion(output, labels)
        test_loss += loss.item()*data.size(0)

        _, pred = torch.max(output, 1)

        correct_tensor = pred.eq(labels.data.view_as(pred))
        correct = np.squeeze(correct_tensor.numpy()) if not torch.cuda.is_available(
        ) else np.squeeze(correct_tensor.cpu().numpy())

        # test accuracy for each object class
        for i in range(num_classes):
            label = labels.data[i]
            correct_class[label] += correct[i].item()
            total_class[label] += 1

    # avg test loss
    test_loss = test_loss/len(test_loader.dataset)
    print("Test Loss: {:.6f}\n".format(test_loss))

    # print test accuracy
    for i in range(10):
        if total_class[i] > 0:
            print('Test Accuracy of %5s: %2d%% (%2d/%2d)' %
                  (classes[i], 100 * correct_class[i] / total_class[i],
                   np.sum(correct_class[i]), np.sum(total_class[i])))
        else:
            print('Test Accuracy of %5s: N/A (no training examples)' %
                  (classes[i]))

    print('\nFinal Test  Accuracy: {:.3f} ({}/{})'.format(
        100. * np.sum(correct_class) / np.sum(total_class),
        np.sum(correct_class), np.sum(total_class)))

    test_accuracies.append(100. * np.sum(correct_class) / np.sum(total_class))

"""## Basic Experiment Settings"""

"""## 基本实验设置"""

#TODO: 基本实验设置
# global rounds
t_max = 100
# global client fraction
C = 0.1
# number of total clients
K = 500
# number of total edges
Edges = 10
# local_epochs
E = 5
# learning_rate&batchsize
lr = 0.0005
batchSize = 10
# local_datasize
local_datasize = len(mnist_data_train)/K

iid_dict = iid_partition(mnist_data_train, K)
TotalClients = []
Clients_all_idxs = [i for i in range(K)]
TotalEdges = []
ClientsNumForEdges = [0 for i in range(Edges)]
test_accuracies = []
BIASARRAY = [0.1, 0.2, 0.3, 0.4, 0, 0, 0, 0, 0, 0]
BIASDISTRIBUTE = [
    [0.1, 0.2, 0.3, 0.4, 0, 0, 0, 0, 0, 0],
    [0, 0.1, 0.2, 0.3, 0.4, 0, 0, 0, 0, 0],
    [0, 0, 0.1, 0.2, 0.3, 0.4, 0, 0, 0, 0],
    [0, 0, 0, 0.1, 0.2, 0.3, 0.4, 0, 0, 0],
    [0, 0, 0, 0, 0.1, 0.2, 0.3, 0.4, 0, 0],
    [0, 0, 0, 0, 0, 0.1, 0.2, 0.3, 0.4, 0],
    [0, 0, 0, 0, 0, 0, 0.1, 0.2, 0.3, 0.4],
    [0.4, 0, 0, 0, 0, 0, 0, 0.1, 0.2, 0.3],
    [0.3, 0.4, 0, 0, 0, 0, 0, 0, 0.1, 0.2],
    [0.2, 0.3, 0.4, 0, 0, 0, 0, 0, 0, 0.1],
]

Set_clients = set(Clients_all_idxs)

def Rescheduling(C=0.1, Max_clients_per_mediator=10):
    # 初始化客户端群体
    from random import randint
    print(f"Generating {K} clients and sampling random generation profile...")
    for i in range(K):
        # print(f"For {i}")
        samplingIdx = i % num_classes
        biasArrayTemp = BIASDISTRIBUTE[samplingIdx]
        clientDatasetIdxs = customizedIdxsGenerator(
            Bias_array=biasArrayTemp, local_datasize=local_datasize)
        clientTemp = Client(client_index=i, dataset=mnist_data_train, batchsize=batchSize,
                            learning_rate=lr, local_epochs=E, data_idxs=clientDatasetIdxs, bias_array=biasArrayTemp)
        if(i % 48 == 0):
            clientTemp.printProfile()
        TotalClients.append(clientTemp)
    print(f"Done, {K} clients are generated...")

    edge_index_temp = 0
    P_uniform = torch.Tensor([0.1 for i in range(10)])
    # 只要还有clients就重复最外层循环
    while(len(Set_clients) != 0):
        # 初始化一个mediator
        mediator_temp = Edge(edge_index=edge_index_temp,
                             region_clients_idxs=[], Cr=C)
        P_mediator = copy.deepcopy(mediator_temp.Bias_array)  # Tensor类型的
        curClientsCount = 0
        # 寻找一个P_mediator本次的全部从属client
        while(len(Set_clients) != 0 and len(mediator_temp.region_clients_idxs) < Max_clients_per_mediator):
            # 先定一个临时散度最大值
            kl_div_min = 9999999
            P_mediator = copy.deepcopy(mediator_temp.Bias_array)  # Tensor类型的
            # 遍历每一个Set_clients的client找到合适的k
            target_K = 1
            # 寻找一次mediator本轮中的一个client
            for client_idx in Set_clients.copy():
                P_client = copy.deepcopy(TotalClients[client_idx].bias_array)
                # 如果当前mediator还没有clients,就先给它分配当前的client
                if len(mediator_temp.region_clients_idxs) == 0:
                    print(
                        f"Current MediatorIndex is: {edge_index_temp} ,and first allocate client_index:{client_idx} ,and its bias is:{P_client}")
                    mediator_temp.addClient(client_idx)
                    Set_clients.remove(client_idx)
                    curClientsCount += 1
                    P_uniform += torch.Tensor([0.1 for i in range(10)])
                    P_mediator = copy.deepcopy(P_client)
                    continue
                else:
                    P_fusion_temp = P_client+P_mediator  # 论文中并没有提到具体怎么做的
                    # 想要计算D(p||q)，要写成kl_div（q.log（），p）,有点别扭
                    kl_div_temp = F.kl_div(
                        P_uniform.log(), P_fusion_temp, reduction='sum')/(curClientsCount+1)
                    if(kl_div_temp < kl_div_min):
                        kl_div_min = kl_div_temp
                        target_K = client_idx

            # 找到本轮的client k之后，更新当前mediator的各项参数
            mediator_temp.addClient(target_K)
            curClientsCount += 1
            P_client = TotalClients[target_K].bias_array
            P_fusion = P_mediator + P_client
            mediator_temp.Bias_array = copy.deepcopy(P_fusion)
            # 按次调整P_uniform
            P_uniform += torch.Tensor([0.1 for i in range(10)])
            # 从尚未匹配到的客户端Set中移除target_K
            Set_clients.remove(target_K)
        # 找完一个mediator的全部client之后
        mediator_temp.printProfile()
        edge_index_temp += 1
        TotalEdges.append(mediator_temp)

def main():
    # load model
    lenet = LeNet()
    if torch.cuda.is_available():
        lenet.cuda()
    Rescheduling(C=C, Max_clients_per_mediator=10)
# 后面先不管
    HyBridFL(model=lenet, rounds=100)
    print(test_accuracies)

main()


import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as npimg
from keras.models import Sequential
from keras.optimizers import Adam
from keras.layers import Conv2D,Convolution2D,MaxPooling2D,Dense,Flatten,Dropout
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from imgaug import augmenters as iaa
import cv2 as cv
import pandas as pd
import random
import ntpath

def path_leaf(path):
  head, tail = ntpath.split(path)
  return tail

def load_img_steering(datadir,df):
  image_path = []
  steering = []
  for i in range(len(data)):
    indexed_data = data.iloc[i]
    center , left , right = indexed_data[0] , indexed_data[1] , indexed_data[2]
    image_path.append(os.path.join(datadir,center.strip()))
    steering.append(float(indexed_data[3]))
  image_paths = np.asarray(image_path)
  steerings = np.asarray(steering)
  return image_paths ,steerings

def zoom(image):
  zoom = iaa.Affine(scale=(1,1.3))
  image = zoom.augment_image(image)
  return image

def pan(image):
  pan = iaa.Affine(translate_percent= {"x":(-.1,.1),"y":(-.1,.1)})
  image = pan.augment_image(image)
  return image

def image_brightness(image):
  brightness = iaa.Multiply((.2,1.2))
  image = brightness.augment_image(image)
  return image

def image_flip(image, steering_angle):
  image = cv.flip(image, 1)                       # 0 for vertical flipping, 1 for horizontal , -1 for both, but actually here we need horizontal flipping
  steering_angle = -steering_angle
  return image,steering_angle

def random_augment(image, steering_angle):
    image = npimg.imread(image)
    if np.random.rand() < 0.5:
        image = pan(image)
    if np.random.rand() < 0.5:
        image = zoom(image)
    if np.random.rand() < 0.5:
        image = image_brightness(image)
    if np.random.rand() < 0.5:
        image, steering_angle = image_flip(image, steering_angle)

    return image, steering_angle

def img_preprocess(image):
  image = image[60:130,:,:]
  image = cv.cvtColor(image,cv.COLOR_RGB2YUV)
  image = cv.GaussianBlur(image,(3,3), 0)
  image = cv.resize(image, (200,66))
  image = image/255
  return image

def batch_generator(image_paths, steering_ang, batch_size, istraining):
    while True:
        batch_img = []
        batch_steering = []

        for i in range(batch_size):
            random_index = random.randint(0, len(image_paths) - 1)

            if istraining:
                im, steering = random_augment(image_paths[random_index], steering_ang[random_index])

            else:
                im = npimg.imread(image_paths[random_index])
                steering = steering_ang[random_index]

            im = img_preprocess(im)
            batch_img.append(im)
            batch_steering.append(steering)
        yield (np.asarray(batch_img), np.asarray(batch_steering))

def nvidia_model():
    model = Sequential()
    model.add(Convolution2D(24, 5, 5, subsample=(2, 2), input_shape=(66, 200, 3), activation='relu'))
    model.add(Convolution2D(36, 5, 5, subsample=(2, 2), activation='elu'))
    model.add(Convolution2D(48, 5, 5, subsample=(2, 2), activation='elu'))
    model.add(Convolution2D(64, 3, 3, activation='elu'))
    model.add(Convolution2D(64, 3, 3, activation='elu'))

    # model.add(Dropout(.5))

    model.add(Flatten())
    model.add(Dense(100, activation='elu'))

    # model.add(Dropout(.5))

    model.add(Dense(15, activation='elu'))

    # model.add(Dropout(.5))

    model.add(Dense(10, activation='elu'))

    # model.add(Dropout(.5))

    model.add(Dense(1))
    model.compile(Adam(lr=.0001), loss='mse')
    return model

datadir = 'DATA'
columns = ['center','left','right','steering','throttle','reverse','speed']
data = pd.read_csv(os.path.join(datadir,'DATA/driving_log.csv'), names = columns)
pd.set_option('display.max_colwidth',-1)
data.head()

data['center'] = data['center'].apply(path_leaf)
data['left'] = data['left'].apply(path_leaf)
data['right'] = data['right'].apply(path_leaf)
data.head()

num_bins = 25
hist, bins = np.histogram(data['steering'],num_bins)
center = (bins[:-1]+bins[1:]) * .5
plt.bar(center,hist,width=.05)

print("total_data= ",len(data))
remove_list = []
for i in range (num_bins):
  List = []
  for j in range (len(data['steering'])):
    if data['steering'][j] >= bins[i] and data['steering'][j] <= bins[i+1]:
      List.append(j)
  List = shuffle(List)
  List = List[400:]
  remove_list.extend(List)

print("removed",len(remove_list))
data.drop(data.index[remove_list], inplace=True)
print("remaining",len(data))

hist, _ = np.histogram(data['steering'],num_bins)
plt.bar(center,hist,width=.05)

print(data.iloc[1])
image_paths, steerings = load_img_steering(datadir + '/DATA/IMG',data)

x_train, x_val, y_train, y_val = train_test_split(image_paths,steerings,test_size=.25,random_state = 6)
_, axs = plt.subplots(1, 2 ,figsize = (12,4))
axs[0].hist(y_train,bins=num_bins,width=.05,color='red')
axs[1].hist(y_val,bins=num_bins,width=.05,color='blue')

image = image_paths[random.randint(0,1000)]
original_image = npimg.imread(image)
zoomed_image = zoom(original_image)
fig,axs = plt.subplots(1,2,figsize=(15,10))
fig.tight_layout()
axs[0].imshow(original_image)
axs[0].set_title('original')
axs[1].imshow(zoomed_image)
axs[1].set_title('zoomed')

image = image_paths[random.randint(0,1000)]
original_image = npimg.imread(image)
panned_image = pan(original_image)
fig,axs = plt.subplots(1,2,figsize=(15,10))
fig.tight_layout()
axs[0].imshow(original_image)
axs[0].set_title('original')
axs[1].imshow(panned_image)
axs[1].set_title('panned')

image = image_paths[random.randint(0,1000)]
original_image = npimg.imread(image)
brightness_image = image_brightness(original_image)
fig,axs = plt.subplots(1,2,figsize=(15,10))
fig.tight_layout()
axs[0].imshow(original_image)
axs[0].set_title('original')
axs[1].imshow(brightness_image)
axs[1].set_title('brightness')

random_index = random.randint(0,1000)
image = image_paths[random_index]
steering_angle = steerings[random_index]
original_image = npimg.imread(image)
flipped_image, flipped_steering_angle = image_flip(original_image,steering_angle)
fig,axs = plt.subplots(1,2,figsize=(15,10))
fig.tight_layout()
axs[0].imshow(original_image)
axs[0].set_title('original')
axs[1].imshow(flipped_image)
axs[1].set_title('flipped')

image = image_paths[160]
original_image = npimg.imread(image)
preprocessed_image = img_preprocess(original_image)

fig, axs = plt.subplots(1,2,figsize=(15,10))
fig.tight_layout()
axs[0].imshow(original_image)
axs[0].set_title("original")
axs[1].imshow(preprocessed_image)
axs[1].set_title("preprocessed")

random_index = random.randint(0, len(image_paths) - 1)
image = image_paths[random_index]
image = npimg.imread(image)

x_train_gen, y_train_gen = next(batch_generator(x_train, y_train, 1, 1))
x_valid_gen, y_valid_gen = next(batch_generator(x_val, y_val, 1, 0))

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()

axs[0].imshow(x_train_gen[0])
axs[0].set_title('Training Image')

axs[1].imshow(x_valid_gen[0])
axs[1].set_title('Validation Image')

#x_train = np.array(list(map(img_preprocess,x_train)))
#x_val = np.array(list(map(img_preprocess,x_val)))

model = nvidia_model()
print(model.summary())

history = model.fit_generator(batch_generator(x_train,y_train,batch_size=100 ,istraining=1) ,steps_per_epoch=300 ,epochs=10,validation_data=batch_generator(x_val,y_val,100,0),validation_steps=200,verbose=1,shuffle=1)

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.legend(['training','validation'])
plt.title('Loss')
plt.xlabel('epochs')


from typing import Optional

import torch as th

def discounted_bwd_cumsum_(
    x: th.Tensor,
    discount: float,
    mask: Optional[th.Tensor] = None,
    dim: int = -1,
) -> th.Tensor:
    '''
    In-place computation of discounted cumulative sum, applied backwards (from n
    to 0) along a given dimension. If mask is provided, do not propagate values
    where it is 0; it is expected to be binary.
    '''
    if mask is not None and x.shape != mask.shape:
        raise ValueError(
            f'Input and mask are required to have the same shape ({x.shape} != {mask.shape})'
        )
    n = x.shape[dim]
    if mask is not None:
        for i in range(n - 2, -1, -1):
            x.select(dim, i).add_(
                mask.select(dim, i) * x.select(dim, i + 1), alpha=discount
            )
    else:
        for i in range(n - 2, -1, -1):
            x.select(dim, i).add_(x.select(dim, i + 1), alpha=discount)

    return x

def normalize(x: th.Tensor):
    '''
    Normalize x to zero mean and unit standard derivation.
    '''
    std, mu = th.std_mean(x)
    return (x - mu) / (std + 1e-6)

def normalize_masked(x: th.Tensor, mask: th.Tensor):
    '''
    Normalize x to zero mean and unit standard derivation.
    From rl-games
    '''
    sum_mask = mask.sum()
    x_mask = x * mask
    x_mean = x_mask.sum() / sum_mask
    min_sqr = (((x_mask) ** 2) / sum_mask).sum() - (
        (x_mask / sum_mask).sum()
    ) ** 2
    x_var = min_sqr * sum_mask / (sum_mask - 1)
    x_std = th.sqrt(x_var)
    return (x - x_mean) / (x_std + 1e-6)

_normalize_fn = normalize

def gae_advantage(
    reward: th.Tensor,
    value: th.Tensor,
    next_value: th.Tensor,
    gamma: float,
    lambd: float = 1,
    normalize: bool = True,
    mask: Optional[th.Tensor] = None,
    time_dim: int = 1,
):
    '''
    Generalized advantage estimation.  Expected shape of reward, value, mask is
    BxT -- unless time_dim is set to 0.
    '''
    assert reward.dim() == 2, f'Expected reward of BxT/TxB, got {reward.shape}'
    assert (
        value.shape == reward.shape
    ), f'Expected value shape of {reward.shape}, got {value.shape}'
    assert (
        next_value.shape == reward.shape
    ), f'Expected value shape of {reward.shape}, got {next_value.shape}'
    assert (
        mask is None or mask.shape == reward.shape
    ), f'Expected mask shape of {reward.shape}, got {mask.shape}'
    if mask is not None:
        deltas = reward + mask * gamma * next_value - value
        adv = discounted_bwd_cumsum_(
            deltas, gamma * lambd, mask=mask, dim=time_dim
        )
    else:
        deltas = reward + gamma * next_value - value
        adv = discounted_bwd_cumsum_(deltas, gamma * lambd, dim=time_dim)
    return _normalize_fn(adv) if normalize else adv

def batch_2to1(x: th.Tensor) -> th.Tensor:
    '''
    Flattens first two dimensions into a single one.
    '''
    s = list(x.shape)
    return x.reshape([s[0] * s[1]] + s[2:])

def clampt(x: th.Tensor, min: th.Tensor, max: th.Tensor) -> th.Tensor:
    return th.max(th.min(x, max), min)


import json
from tqdm import tqdm

"""
二次过滤：根据单个图片bbox最高score来对图像进行二次过滤, 判断该图像是否是正常样本
"""
# Need to set: Line 8, Line 11
test_json_raw = json.load(
    open("../../tcdata/tile_round1_testB_20210128/annotations/instances_test_crop_2000.json", "r"))
test_json = json.load(open('../../user_data/tmp_data/tile_result.bbox.json', "r"))
CROP_RES_PATH = '../../user_data/tmp_data/crop_result.json'  # 切块测试集的测试结果 --> json2submit

images_ids = {}
for img in test_json_raw["images"]:
    images_ids[img["id"]] = img["file_name"]  # image_ids['20210000001']='197_103_t20201119094043570_CAM1.jpg'

img_scores = dict()
for ann in tqdm(test_json):
    img_id = ann["image_id"]
    score = ann["score"]
    if img_id not in img_scores.keys():
        img_scores[img_id] = score
    else:
        if score > img_scores[img_id]:
            img_scores[img_id] = score

results = []
for ann in tqdm(test_json):
    img_id = ann["image_id"]
    if img_scores[img_id] > 0.1:
        label = ann["category_id"]
        bbox = ann["bbox"]
        filename = images_ids[img_id].split('/')[-1]
        w, h = bbox[2], bbox[3]
        xmin = bbox[0]
        ymin = bbox[1]
        xmax = bbox[0] + w
        ymax = bbox[1] + h
        score = ann["score"]
        results.append(
            {'name': filename, 'category': int(label), 'bbox': [xmin, ymin, xmax, ymax], 'score': float(score)})

with open(CROP_RES_PATH, 'w') as fp:
    json.dump(results, fp, indent=4, separators=(',', ': '))


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import json
import numpy as np
import tensorflow as tf
from tensorflow import feature_column as fc
# for python 2.x
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

flags = tf.app.flags
flags.DEFINE_string("model_dir", "./model_dir", "Base directory for the model.")
flags.DEFINE_string("output_model", "./model_output", "Path to the training data.")
flags.DEFINE_string("train_data", "data/samples", "Directory for storing mnist data")
flags.DEFINE_string("eval_data", "data/eval", "Path to the evaluation data.")
flags.DEFINE_string("input_format", "tfrecord", "the format(tfrecord/text) of the training & evaluation data.")
flags.DEFINE_integer("num_gpus", 1, "Total number of gpus for each machine."
                                    "If you don't use GPU, please set it to '0'")
flags.DEFINE_string("hidden_units", "512,128", "Comma-separated list of number of units in each hidden layer of the NN")
flags.DEFINE_integer("train_steps", 10000,
                     "Number of (global) training steps to perform")
flags.DEFINE_integer("batch_size", 512, "Training batch size")
flags.DEFINE_integer("shuffle_buffer_size", 10000, "dataset shuffle buffer size")
flags.DEFINE_float("learning_rate", 0.01, "Learning rate")
flags.DEFINE_integer("num_parallel_readers", 5, "number of parallel readers for training data")
flags.DEFINE_integer("save_checkpoints_steps", 10000, "Save checkpoints every this many steps")
flags.DEFINE_string("ps_hosts", "s-xiasha-10-2-176-43.hx:2222",
                    "Comma-separated list of hostname:port pairs")
flags.DEFINE_string("worker_hosts", "s-xiasha-10-2-176-42.hx:2223,s-xiasha-10-2-176-44.hx:2224",
                    "Comma-separated list of hostname:port pairs")
flags.DEFINE_string("job_name", None, "job name: worker or ps")
flags.DEFINE_integer("task_index", None,
                     "Worker task index, should be >= 0. task_index=0 is "
                     "the master worker task the performs the variable "
                     "initialization ")
flags.DEFINE_boolean("run_on_cluster", False, "Whether the cluster info need to be passed in as input")

FLAGS = flags.FLAGS

def set_tfconfig_environ():
  if "TF_CLUSTER_DEF" in os.environ:
    cluster = json.loads(os.environ["TF_CLUSTER_DEF"])
    task_index = int(os.environ["TF_INDEX"])
    task_type = os.environ["TF_ROLE"]

    tf_config = dict()
    worker_num = len(cluster["worker"])
    if task_type == "ps":
      tf_config["task"] = {"index": task_index, "type": task_type}
      FLAGS.job_name = "ps"
      FLAGS.task_index = task_index
    else:
      if task_index == 0:
        tf_config["task"] = {"index": 0, "type": "chief"}
      else:
        tf_config["task"] = {"index": task_index - 1, "type": task_type}
      FLAGS.job_name = "worker"
      FLAGS.task_index = task_index

    if worker_num == 1:
      cluster["chief"] = cluster["worker"]
      del cluster["worker"]
    else:
      cluster["chief"] = [cluster["worker"][0]]
      del cluster["worker"][0]

    tf_config["cluster"] = cluster
    os.environ["TF_CONFIG"] = json.dumps(tf_config)
    print("TF_CONFIG", json.loads(os.environ["TF_CONFIG"]))

  if "INPUT_FILE_LIST" in os.environ:
    INPUT_PATH = json.loads(os.environ["INPUT_FILE_LIST"])
    if INPUT_PATH:
      print("input path:", INPUT_PATH)
      FLAGS.train_data = INPUT_PATH.get(FLAGS.train_data)
      FLAGS.eval_data = INPUT_PATH.get(FLAGS.eval_data)
    else:  # for ps
      print("load input path failed.")
      FLAGS.train_data = None
      FLAGS.eval_data = None

def parse_argument():
  if FLAGS.job_name is None or FLAGS.job_name == "":
    raise ValueError("Must specify an explicit `job_name`")
  if FLAGS.task_index is None or FLAGS.task_index == "":
    raise ValueError("Must specify an explicit `task_index`")

  print("job name = %s" % FLAGS.job_name)
  print("task index = %d" % FLAGS.task_index)
  os.environ["TF_ROLE"] = FLAGS.job_name
  os.environ["TF_INDEX"] = str(FLAGS.task_index)

  # Construct the cluster and start the server
  ps_spec = FLAGS.ps_hosts.split(",")
  worker_spec = FLAGS.worker_hosts.split(",")
  cluster = {"worker": worker_spec, "ps": ps_spec}
  os.environ["TF_CLUSTER_DEF"] = json.dumps(cluster)

class DebugHook(tf.train.SessionRunHook):
  def begin(self):
    # You can add ops to the graph here.
    print('Before starting the session.')

  def after_create_session(self, session, coord):
    # When this is called, the graph is finalized and ops can no longer be added to the graph.
    print('Session created.')

  def before_run(self, run_context):
    # print('Before calling session.run().')
    return None  # SessionRunArgs(self.your_tensor)

  def after_run(self, run_context, run_values):
    # print('Done running one step. The value of my tensor: %s', run_values.results)
    # if you-need-to-stop-loop:
    #  run_context.request_stop()
    pass

  def end(self, session):
    print('Done with the session.')

def create_feature_columns():
  # user feature
  bids = fc.categorical_column_with_hash_bucket("behaviorBids", 10240, dtype=tf.int64)
  c1ids = fc.categorical_column_with_hash_bucket("behaviorC1ids", 100, dtype=tf.int64)
  cids = fc.categorical_column_with_hash_bucket("behaviorCids", 10240, dtype=tf.int64)
  sids = fc.categorical_column_with_hash_bucket("behaviorSids", 10240, dtype=tf.int64)
  pids = fc.categorical_column_with_hash_bucket("behaviorPids", 1000000, dtype=tf.int64)

  # item feature
  pid = fc.categorical_column_with_hash_bucket("productId", 1000000, dtype=tf.int64)
  sid = fc.categorical_column_with_hash_bucket("sellerId", 10240, dtype=tf.int64)
  bid = fc.categorical_column_with_hash_bucket("brandId", 10240, dtype=tf.int64)
  c1id = fc.categorical_column_with_hash_bucket("cate1Id", 100, dtype=tf.int64)
  cid = fc.categorical_column_with_hash_bucket("cateId", 10240, dtype=tf.int64)

  # context feature
  matchScore = fc.numeric_column("matchScore", default_value=0.0)
  popScore = fc.numeric_column("popScore", default_value=0.0)
  brandPrefer = fc.numeric_column("brandPrefer", default_value=0.0)
  cate2Prefer = fc.numeric_column("cate2Prefer", default_value=0.0)
  catePrefer = fc.numeric_column("catePrefer", default_value=0.0)
  sellerPrefer = fc.numeric_column("sellerPrefer", default_value=0.0)
  matchType = fc.indicator_column(fc.categorical_column_with_identity("matchType", 9, default_value=0))
  postition = fc.indicator_column(fc.categorical_column_with_identity("position", 201, default_value=200))
  triggerNum = fc.indicator_column(fc.categorical_column_with_identity("triggerNum", 51, default_value=50))
  triggerRank = fc.indicator_column(fc.categorical_column_with_identity("triggerRank", 51, default_value=50))
  sceneType = fc.indicator_column(fc.categorical_column_with_identity("type", 2, default_value=0))
  hour = fc.indicator_column(fc.categorical_column_with_identity("hour", 24, default_value=0))
  phoneBrand = fc.indicator_column(fc.categorical_column_with_hash_bucket("phoneBrand", 1000))
  phoneResolution = fc.indicator_column(fc.categorical_column_with_hash_bucket("phoneResolution", 500))
  phoneOs = fc.indicator_column(
    fc.categorical_column_with_vocabulary_list("phoneOs", ["android", "ios"], default_value=0))
  tab = fc.indicator_column(fc.categorical_column_with_vocabulary_list("tab",
                                                                       ["ALL", "TongZhuang", "XieBao", "MuYing",
                                                                        "NvZhuang", "MeiZhuang", "JuJia", "MeiShi"],
                                                                       default_value=0))

  pid_embed = fc.shared_embedding_columns([pids, pid], 32, combiner='sum', shared_embedding_collection_name="pid")
  bid_embed = fc.shared_embedding_columns([bids, bid], 16, combiner='sum', shared_embedding_collection_name="bid")
  cid_embed = fc.shared_embedding_columns([cids, cid], 16, combiner='sum', shared_embedding_collection_name="cid")
  c1id_embed = fc.shared_embedding_columns([c1ids, c1id], 8, combiner='sum', shared_embedding_collection_name="c1id")
  sid_embed = fc.shared_embedding_columns([sids, sid], 16, combiner='sum', shared_embedding_collection_name="sid")
  columns = [matchScore, matchType, postition, triggerNum, triggerRank, sceneType, hour, phoneBrand, phoneResolution,
             phoneOs, tab, popScore, sellerPrefer, brandPrefer, cate2Prefer, catePrefer]
  columns += pid_embed
  columns += sid_embed
  columns += bid_embed
  columns += cid_embed
  columns += c1id_embed
  print("feature columns:", columns)
  return columns

def parse_exmp(serial_exmp):
  features = {
    "click": tf.FixedLenFeature([], tf.int64),
    "behaviorBids": tf.FixedLenFeature([20], tf.int64),
    "behaviorCids": tf.FixedLenFeature([20], tf.int64),
    "behaviorC1ids": tf.FixedLenFeature([10], tf.int64),
    "behaviorSids": tf.FixedLenFeature([20], tf.int64),
    "behaviorPids": tf.FixedLenFeature([20], tf.int64),
    "productId": tf.FixedLenFeature([], tf.int64),
    "sellerId": tf.FixedLenFeature([], tf.int64),
    "brandId": tf.FixedLenFeature([], tf.int64),
    "cate1Id": tf.FixedLenFeature([], tf.int64),
    "cateId": tf.FixedLenFeature([], tf.int64),
    "tab": tf.FixedLenFeature([], tf.string),
    "matchType": tf.FixedLenFeature([], tf.int64),
    "position": tf.FixedLenFeature([], tf.int64),
    "type": tf.FixedLenFeature([], tf.int64),
    "triggerNum": tf.FixedLenFeature([], tf.int64),
    "triggerRank": tf.FixedLenFeature([], tf.int64),
    "hour": tf.FixedLenFeature([], tf.int64),
    "matchScore": tf.FixedLenFeature([], tf.float32),
    "popScore": tf.FixedLenFeature([], tf.float32),
    "brandPrefer": tf.FixedLenFeature([], tf.float32),
    "sellerPrefer": tf.FixedLenFeature([], tf.float32),
    "catePrefer": tf.FixedLenFeature([], tf.float32),
    "cate2Prefer": tf.FixedLenFeature([], tf.float32),
    "phoneBrand": tf.FixedLenFeature([], tf.string),
    "phoneOs": tf.FixedLenFeature([], tf.string),
    "phoneResolution": tf.FixedLenFeature([], tf.string)
  }
  feats = tf.parse_single_example(serial_exmp, features=features)
  labels = feats.pop('click')
  return feats, labels

def filter_func(line):
  fields = line.decode().split("\t")
  if len(fields) < 8:
    return False
  for field in fields:
    if not field:
      return False
  return True

def parse_line(line):
  _COLUMNS = ["sellerId", "brandId", "cate1Id", "cateId", "tab"]
  _INT_COLUMNS = ["click", "productId", "matchType", "position", "type", "triggerNum", "triggerRank", "hour"]
  _FLOAT_COLUMNS = ["matchScore", "popScore", "brandPrefer", "sellerPrefer", "catePrefer", "cate2Prefer"]
  _STRING_COLUMNS = ["phoneResolution", "phoneBrand", "phoneOs"]
  _SEQ_COLUMNS = ["behaviorC1ids", "behaviorBids", "behaviorCids", "behaviorSids", "behaviorPids"]

  def get_content(record):
    import datetime
    fields = record.decode().split("\t")
    if len(fields) < 8:
      raise ValueError("invalid record %s" % record)
    for field in fields:
      if not field:
        raise ValueError("invalid record %s" % record)
    fea = json.loads(fields[1])
    if fea["time"]:
      dt = datetime.datetime.fromtimestamp(fea["time"])
      fea["hour"] = dt.hour
    else:
      fea["hour"] = 0
    seq_len = 10
    for x in _SEQ_COLUMNS:
      sequence = fea.setdefault(x, [])
      n = len(sequence)
      if n < seq_len:
        sequence.extend([-1] * (seq_len - n))
      elif n > seq_len:
        fea[x] = sequence[:seq_len]
      seq_len = 20

    elems = [np.int64(fields[2]), np.int64(fields[3]), np.int64(fields[4]), np.int64(fields[6]), fields[7]]
    elems += [np.int64(fea.get(x, 0)) for x in _INT_COLUMNS]
    elems += [np.float32(fea.get(x, 0.0)) for x in _FLOAT_COLUMNS]
    elems += [fea.get(x, "") for x in _STRING_COLUMNS]
    elems += [np.int64(fea[x]) for x in _SEQ_COLUMNS]
    return elems

  out_type = [tf.int64] * 4 + [tf.string] + [tf.int64] * len(_INT_COLUMNS) + [tf.float32] * len(_FLOAT_COLUMNS) + [
    tf.string] * len(_STRING_COLUMNS) + [tf.int64] * len(_SEQ_COLUMNS)
  result = tf.py_func(get_content, [line], out_type)
  n = len(result) - len(_SEQ_COLUMNS)
  for i in range(n):
    result[i].set_shape([])
  result[n].set_shape([10])
  for i in range(n + 1, len(result)):
    result[i].set_shape([20])
  columns = _COLUMNS + _INT_COLUMNS + _FLOAT_COLUMNS + _STRING_COLUMNS + _SEQ_COLUMNS
  features = dict(zip(columns, result))
  labels = features.pop('click')
  return features, labels

def my_input_fn(filenames, batch_size, shuffle_buffer_size):
  print("start my_input_fn")
  dataset = tf.data.TextLineDataset(filenames)
  dataset = dataset.filter(lambda x: tf.py_func(filter_func, [x], tf.bool, False))
  dataset = dataset.map(parse_line, num_parallel_calls=8)
  # Shuffle, repeat, and batch the examples.
  if shuffle_buffer_size > 0:
    dataset = dataset.shuffle(shuffle_buffer_size)
  dataset = dataset.repeat().batch(batch_size)
  print(dataset.output_types)
  print(dataset.output_shapes)
  # Return the read end of the pipeline.
  return dataset

def train_input_fn(filenames, batch_size, shuffle_buffer_size):
  #dataset = tf.data.TFRecordDataset(filenames)
  files = tf.data.Dataset.list_files(filenames)
  dataset = files.apply(tf.contrib.data.parallel_interleave(tf.data.TFRecordDataset, cycle_length=FLAGS.num_parallel_readers))
  # Shuffle, repeat, and batch the examples.
  if shuffle_buffer_size > 0:
    dataset = dataset.shuffle(shuffle_buffer_size)
  #dataset = dataset.apply(tf.contrib.data.map_and_batch(map_func=parse_exmp, batch_size=batch_size))
  #dataset = dataset.repeat().prefetch(1)
  dataset = dataset.map(parse_exmp, num_parallel_calls=8)
  dataset = dataset.repeat().batch(batch_size).prefetch(1)
  print(dataset.output_types)
  print(dataset.output_shapes)
  # Return the read end of the pipeline.
  return dataset

def eval_input_fn(filename, batch_size):
  dataset = tf.data.TFRecordDataset(filename)
  dataset = dataset.map(parse_exmp, num_parallel_calls=8)
  # Shuffle, repeat, and batch the examples.
  dataset = dataset.batch(batch_size)
  # Return the read end of the pipeline.
  return dataset

def my_model(features, labels, mode, params):
  net = fc.input_layer(features, params['feature_columns'])
  # Build the hidden layers, sized according to the 'hidden_units' param.
  for units in params['hidden_units']:
    net = tf.layers.dense(net, units=units, activation=tf.nn.relu)
  my_head = tf.contrib.estimator.binary_classification_head(thresholds=[0.5])
  # Compute logits (1 per class).
  logits = tf.layers.dense(net, my_head.logits_dimension, activation=None, name="my_model_output_logits")
  optimizer = tf.train.AdagradOptimizer(learning_rate=params['learning_rate'])

  def _train_op_fn(loss):
    return optimizer.minimize(loss, global_step=tf.train.get_global_step())

  return my_head.create_estimator_spec(
    features=features,
    mode=mode,
    labels=labels,
    logits=logits,
    train_op_fn=_train_op_fn
  )

def main(unused_argv):
  set_tfconfig_environ()
  my_feature_columns = create_feature_columns()
  classifier = tf.estimator.Estimator(
    model_fn=my_model,
    params={
      'feature_columns': my_feature_columns,
      # Two hidden layers of 10 nodes each.
      'hidden_units': FLAGS.hidden_units.split(','),
      'learning_rate': FLAGS.learning_rate,
      # The model must choose between 2 classes.
      'n_classes': 2
    },
    config=tf.estimator.RunConfig(model_dir=FLAGS.model_dir, save_checkpoints_steps=FLAGS.save_checkpoints_steps)
  )
  batch_size = FLAGS.batch_size
  print("train steps:", FLAGS.train_steps, "batch_size:", batch_size)
  if isinstance(FLAGS.train_data, str) and os.path.isdir(FLAGS.train_data):
    train_files = [FLAGS.train_data + '/' + x for x in os.listdir(FLAGS.train_data)] if os.path.isdir(
      FLAGS.train_data) else FLAGS.train_data
  else:
    train_files = FLAGS.train_data
  if isinstance(FLAGS.eval_data, str) and os.path.isdir(FLAGS.eval_data):
    eval_files = [FLAGS.eval_data + '/' + x for x in os.listdir(FLAGS.eval_data)] if os.path.isdir(
      FLAGS.eval_data) else FLAGS.eval_data
  else:
    eval_files = FLAGS.eval_data
  shuffle_buffer_size = FLAGS.shuffle_buffer_size
  print("train_data:", train_files)
  print("eval_data:", eval_files)
  print("shuffle_buffer_size:", shuffle_buffer_size)
  hook = DebugHook()
  if FLAGS.input_format == "tfrecord":
    print("input_format: tfrecord")
    input_fn_for_training = lambda: train_input_fn(train_files, batch_size, shuffle_buffer_size)
    input_fn_for_eval = lambda: eval_input_fn(eval_files, batch_size)
  else:
    print("input_format: text")
    input_fn_for_training = lambda: my_input_fn(train_files, batch_size, shuffle_buffer_size)
    input_fn_for_eval = lambda: my_input_fn(eval_files, batch_size, 0)

  train_spec = tf.estimator.TrainSpec(
    input_fn=input_fn_for_training,
    max_steps=FLAGS.train_steps,
    hooks=[hook]
  )
  eval_spec = tf.estimator.EvalSpec(input_fn=input_fn_for_eval, throttle_secs=300)

  print("before train and evaluate")
  tf.estimator.train_and_evaluate(classifier, train_spec, eval_spec)
  print("after train and evaluate")

  # Evaluate accuracy.
  results = classifier.evaluate(input_fn=input_fn_for_eval)
  for key in sorted(results): print('%s: %s' % (key, results[key]))
  print("after evaluate")

  if FLAGS.job_name == "worker" and FLAGS.task_index == 0:
    print("exporting model ...")
    feature_spec = tf.feature_column.make_parse_example_spec(my_feature_columns)
    print(feature_spec)
    serving_input_receiver_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)
    classifier.export_savedmodel(FLAGS.output_model, serving_input_receiver_fn)
  print("quit main")

if __name__ == "__main__":
  if "CUDA_VISIBLE_DEVICES" in os.environ:
    print("CUDA_VISIBLE_DEVICES:", os.environ["CUDA_VISIBLE_DEVICES"])
  if FLAGS.run_on_cluster: parse_argument()
  tf.logging.set_verbosity(tf.logging.INFO)
  tf.app.run(main=main)


import unittest
import numpy as np
from .. import models

class TestAsset(unittest.TestCase):

    def test_constructor(self) :
        models.Asset()
        models.Asset( 1, 1 )
        models.Asset( constant_rate=0.2, constant_fee=0.1 )

    def test_process_period(self) :
        asset = models.Asset( constant_rate=0.02, constant_fee=0.01 )
        self.assertEqual( [], asset.values )

        self.assertEqual( asset.process_period( deposit=100, rate=0, fee=0 ), 100 )
        self.assertEqual( [100], asset.values )

        # test the sum of previous + new value
        self.assertEqual( asset.process_period( deposit=100, rate=0, fee=0 ), 200 )
        self.assertEqual( [100, 200], asset.values )

        # test interest rate
        self.assertEqual( asset.process_period( deposit=0, rate=0.02, fee=0 ), 204 )
        self.assertAlmostEqual( asset.process_period( deposit=0, rate=0, fee=0.01 ), 201.96 )

    def test_reset(self) :
        asset = models.Asset()
        asset.process_period()
        self.assertEqual( len(asset.values), 1 )

        asset.reset()
        self.assertEqual( len(asset.values), 0 )

class TestBrokerContract( unittest.TestCase ) :

    def test_constructor(self) :
        models.BrokerContract()
        models.BrokerContract( 1, 1, 1)
        models.BrokerContract( entry_fee=1, periodic_fee=1, exit_fee=1)

    def test_normalize_weights(self) :
        bc = models.BrokerContract( entry_fee=0.2 )

        # test no crash
        bc.normalize_weights()

        # test single entry
        bc.add_asset( models.Asset(), 0.5 )
        bc.normalize_weights()
        np.testing.assert_array_equal( bc.assets['weight'].values, [1])

        # test mutliple entries
        bc.add_asset( models.Asset(), 1 )
        bc.normalize_weights()
        np.testing.assert_array_equal( bc.assets['weight'].values, [0.5, 0.5])

    def test_add_asset( self ) :
        bc = models.BrokerContract( entry_fee=0.2 )

        bc.add_asset( models.Asset(), 0 )
        self.assertEqual( len(bc.assets), 0 )

        bc.add_asset( models.Asset(), -1 )
        self.assertEqual( len(bc.assets), 0 )

        bc.add_asset( models.Asset(), 0.2 )
        self.assertEqual( len(bc.assets), 1 )
        self.assertAlmostEqual( bc.assets['weight'].sum(), 0.2 )

        bc.add_asset( models.Asset(), 0.5 )
        self.assertEqual( len(bc.assets), 2 )
        self.assertAlmostEqual( bc.assets['weight'].sum(), 0.7 )


import os

from django.contrib.auth.decorators import login_required
from django.core.paginator import Paginator
from django.http import HttpResponse, HttpResponseRedirect, JsonResponse
from django.shortcuts import render, redirect, get_object_or_404
from django.urls import reverse
from article_app.models import Article, ExtraAuthor
from journal.forms import CreateJournalForm, UpdateJournalForm
from journal.models import Journal, JournalArticle
import PyPDF2

from user_app.decorators import allowed_users
from django.utils.translation import gettext_lazy as _

def is_ajax(request):
    return request.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest'

@login_required(login_url='login')
@allowed_users(role=['editor'])
def journal_dashboard(request):
    journals = Journal.objects.all()
    articles = Article.objects.filter(article_status_id=2, is_publish=True, is_publish_journal=False)
    data = {
        "journals": journals,
        "articles": articles
    }
    return render(request, "journal/journal_dashboard.html", context=data)

@login_required(login_url='login')
@allowed_users(role=['editor'])
def split_journal_pages(request, pk):
    journal = get_object_or_404(Journal, pk=pk)
    if request.method == "POST" and is_ajax(request):
        j_articles = JournalArticle.objects.filter(journal=journal)
        article_ids = request.POST.getlist('article')
        start_pages = request.POST.getlist('start_page')
        end_pages = request.POST.getlist('end_page')

        if j_articles.count() != len(article_ids):
            data = {"result": False, "message": _("Wrong Article Count")}
            return JsonResponse(data)
        full_file = journal.file_pdf.path
        pdf = PyPDF2.PdfFileReader(full_file)
        pages_count = pdf.getNumPages()
        last_article_end_page = int(end_pages[-1])
        if pages_count < last_article_end_page:
            data = {"result": False, "message": _("Journal page count is less than last article end page number! ")}
            return JsonResponse(data)
        for i in range(len(article_ids)):
            cwd = os.getcwd()
            article = get_object_or_404(Article, pk=int(article_ids[i]))
            start_page = int(start_pages[i])
            end_page = int(end_pages[i])
            if start_page == 0 or end_page == 0:
                data = {
                    "result": False,
                    "message": _(f"{i + 1}-article not entered!"),
                }
                return JsonResponse(data)
            if start_page >= end_page:
                data = {
                    "result": False,
                    "message": _(f"{i + 1}-article page must be {start_page} < {end_page}!"),
                }
                return JsonResponse(data)

            newpdf = PyPDF2.PdfFileWriter()
            for j in range(start_page, end_page + 1, 1):
                newpdf.addPage(pdf.getPage(j - 1))

            out_path = f"{cwd}\\media\\files\\split_article\\{article.id}.pdf"

            with open(out_path, "wb") as out:
                newpdf.write(out)

            table = get_object_or_404(JournalArticle, journal=journal, article=article)
            table.start_page = start_page
            table.end_page = end_page
            table.article_pdf = f"files\\split_article\\{article.id}.pdf"
            table.save()
            article.filePDF = table.article_pdf
            article.save()

        journal.is_split = True
        journal.save()
        data = {
            "result": True,
            "message": _("Splited Successfully"),
        }
        return JsonResponse(data)
    objects = JournalArticle.objects.filter(journal=journal)
    context = {
        'journal': journal,
        'objects': objects,
    }
    return render(request, 'journal/split_pages.html', context=context)

@login_required(login_url='login')
@allowed_users(role=['editor'])
def create_journal(request):
    if request.method == 'POST' and is_ajax(request):
        get_file = request.FILES.get('file_pdf', None)
        form = CreateJournalForm(request.POST, request.FILES)
        articles = Article.objects.filter(article_status_id=2, is_publish=True, is_publish_journal=False)
        if form.is_valid():
            journal = form.save(commit=False)
            journal.save()
            for article in articles:
                try:
                    JournalArticle.objects.create(
                        journal=journal,
                        article=article,
                    )
                    article.is_publish_journal = True
                    article.save()
                except:
                    print("Error Create Journal")

            data = {
                "result": True,
                "message": _("Created Journal Successfully!"),
            }
            return JsonResponse(data)
        else:
            data = {
                "result": False,
                "message": _("Form didn't valid!"),
            }
            return JsonResponse(data)
    context = {
        'form': CreateJournalForm(),
    }
    return render(request, "journal/create_journal.html", context=context)

def journal_detail(request, pk):
    journal = get_object_or_404(Journal, pk=pk)
    list_articles = JournalArticle.objects.filter(journal=journal)
    context = {
        'journal': journal,
        'list_articles': list_articles,
    }
    return render(request, "journal/journal_detail.html", context=context)

def journals_list(request):
    journals = Journal.objects.filter(is_publish=True, status=True).order_by('-id')
    context = {
        'journals': journals,
    }
    return render(request, "journal/journals.html", context=context)

@login_required(login_url='login')
@allowed_users(role=['editor'])
def edit_journal(request, pk):
    journal = get_object_or_404(Journal, pk=pk)
    if request.method == "POST" and is_ajax(request):
        form = UpdateJournalForm(request.POST, request.FILES, instance=journal)
        if not form.has_changed():
            data = {"message": "The information has not changed", "result": False}
            return JsonResponse(data)
        if form.is_valid():
            form.save()
            data = {
                "result": True,
                "message": "Succes"
            }
            return JsonResponse(data)
        else:
            data = {
                "result": False,
                "message": "Error"
            }
            return JsonResponse(data)
    context = {
        'form': UpdateJournalForm(instance=journal),
        'journal': journal,
    }
    return render(request, "journal/edit_journal.html", context=context)

@login_required(login_url='login')
@allowed_users(role=['editor'])
def delete_journal(request, pk):
    journal = get_object_or_404(Journal, pk=pk)
    if request.method == "POST" and is_ajax(request):
        journal_articles = JournalArticle.objects.filter(journal=journal)
        for item in journal_articles:
            article = item.article
            article.is_publish_journal = False
            article.save()
        journal.delete()
        return JsonResponse({"message": "Deleted Success"})

    data = {"journal": journal}
    return render(request, "journal/delete_journal.html", context=data)

def journal_article_view(request, pk):
    article = get_object_or_404(Article, pk=pk)
    authors = ExtraAuthor.objects.filter(article=article)
    return render(request, 'journal/journal_article_view.html', context={'article': article, "authors": authors})


from sklearn.linear_model import Perceptron

from konjac2.indicator.logistic_regression import prepare_indicators_data

def trend_perceptron(candles, predict_step=0):
    data_y, data_x = prepare_indicators_data(candles)
    if predict_step == 0:
        train_x = data_x[1:]
        train_y = data_y[1:]
        test_x = data_x[-1:]
    else:
        train_x = data_x.shift(predict_step)[predict_step:]
        train_y = data_y[predict_step:]
        test_x = data_x[-predict_step:]
    clf = Perceptron(tol=1e-3, random_state=0)
    clf.fit(train_x[100:], train_y[100:])
    score = clf.score(train_x[100:], train_y[100:])
    return clf.predict(test_x), score


def count(array, array_length):

    # Find the sum  & avg of elements
    sum_of = sum(array)

    average = sum_of / array_length

    # Lets insert the array elements in a hash map
    hashmap = {}
    for i in range(0, array_length):
        hashmap[array[i]] = hashmap.get(array[i], 0) + 1

    # Init a set to hold unique elements
    unique_set = set()

    # We insert elements and the given count
    for x in hashmap:
        unique_set.add((x, hashmap[x]))

    count = 0

    # We want to print count for given elements
    for x in sorted(unique_set):
        count += x[1]
        print(x[0], count)


# %%
import time
from pathlib import Path

import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from graspologic.match import graph_match

from neuropull import NetworkFrame

DATA_PATH = Path("./neuropull/data")

def load_networkframe(dataset):
    nodes = pd.read_csv(DATA_PATH / dataset / "nodes.csv", index_col=0)
    edges = pd.read_csv(DATA_PATH / dataset / "edgelist.csv")
    return NetworkFrame(nodes, edges)

summary_rows = []
for dataset in [
    "c_elegans_hermaphrodite",
    "c_elegans_male",
    "maggot_brain",
    "flywire_central",
    "flywire",
]:
    frame = load_networkframe(dataset)
    frame.query_nodes("side.isin(['left', 'right'])", inplace=True)

    sizes = frame.nodes.groupby("side").size()
    sizes.name = dataset

    if "pair" in frame.nodes.columns:
        paired_nodes = frame.nodes.groupby("pair").filter(
            lambda x: (len(x) == 2)
            and ("left" in x["side"].values)
            and ("right" in x["side"].values)
        )
        paired_frame = frame.loc[paired_nodes.index, paired_nodes.index]

    else:
        paired_frame = frame

    left_frame = paired_frame.query_nodes("side == 'left'")
    right_frame = paired_frame.query_nodes("side == 'right'")

    left_density = len(left_frame.edges) / (len(left_frame.nodes) ** 2)
    right_density = len(right_frame.edges) / (len(right_frame.nodes) ** 2)

    summary_rows.append(
        {
            "dataset": dataset,
            "side": "left",
            "n_nodes": len(left_frame.nodes),
            "density": left_density,
        }
    )
    summary_rows.append(
        {
            "dataset": dataset,
            "side": "right",
            "n_nodes": len(right_frame.nodes),
            "density": right_density,
        }
    )
# %%
summary = pd.DataFrame(summary_rows)

sns.set_context("talk")

fig, axs = plt.subplots(1, 2, figsize=(12, 8), constrained_layout=True)
ax = axs[0]
sns.barplot(data=summary, y="n_nodes", x="dataset", hue="side", ax=ax)
ax.set_yscale("log")
plt.setp(ax.get_xticklabels(), rotation=45, ha="right")
ax = axs[1]
sns.barplot(data=summary, y="density", x="dataset", hue="side", ax=ax)
plt.setp(ax.get_xticklabels(), rotation=45, ha="right")

# %%

n_init = 5

for dataset in [
    "c_elegans_hermaphrodite",
    "c_elegans_male",
    "maggot_brain",
]:
    frame = load_networkframe(dataset)
    frame.query_nodes("side.isin(['left', 'right'])", inplace=True)
    for i in range(n_init):
        frame.nodes = frame.nodes.sample(frac=1)

        paired_nodes = frame.nodes.groupby("pair").filter(
            lambda x: (len(x) == 2)
            and ("left" in x["side"].values)
            and ("right" in x["side"].values)
        )
        paired_frame = frame.loc[paired_nodes.index, paired_nodes.index]

        left_frame = paired_frame.query_nodes("side == 'left'")
        right_frame = paired_frame.query_nodes("side == 'right'")

        left_adjacency = left_frame.to_sparse_adjacency()
        right_adjacency = right_frame.to_sparse_adjacency()

        timer = time.time()
        left_indices, right_indices, score, misc = graph_match(
            left_adjacency, right_adjacency, shuffle_input=False
        )
        elapsed = time.time() - timer

        left_matched_nodes = left_frame.nodes.iloc[left_indices]
        right_matched_nodes = right_frame.nodes.iloc[right_indices]
        match_acc = (
            left_matched_nodes["pair"].values == right_matched_nodes["pair"].values
        ).mean()

        print(f"{dataset}: accuracy={match_acc:.2f}, elapsed={elapsed:.2f}s")

# %%


from BTrees.OOBTree import OOBTree
from collective.saml2auth.interfaces import IAuthNRequestStorage
from zope.annotation.interfaces import IAnnotations
from zope.interface import implements
import time

ANNOTATION_KEY = 'collective.saml2auth.requests'

class AuthNRequestStorage(object):
    implements(IAuthNRequestStorage)

    def __init__(self, context):
        self.context = context
        annotations = IAnnotations(context)
        self._id_to_url = annotations.setdefault(ANNOTATION_KEY, OOBTree())

    def add(self, id_, url):
        """Store the given id and url."""
        self._id_to_url[id_] = (url, int(time.time()))

    def pop(self, id_):
        """Remove the given id and return the associated url."""
        if id_ in self._id_to_url:
            url = self._id_to_url[id_][0]
            del self._id_to_url[id_]
            return url
        return None

    def cleanup(self):
        """Remove old entries."""
        now = int(time.time())
        for k in list(self._id_to_url.keys()):
            if now > self._id_to_url[k][1] + 120:
                del self._id_to_url[k]


#lalala
def isLess(i,j):
    if i < j:
        return True
    else:
        return False

def exch(a,i,j):
    temp = a[i]
    a[i] = a[j]
    a[j] = temp

def selectionSort(a):
    n = len(a)
    for i in range(n):
        min = i
        for j in range(i+1,n):
            if isLess(a[j],a[min]):
                min = j
                exch(a,i,j)

    return a


"""
Log Channel Module
"""

import discord
from discord import app_commands
from discord.ext import commands
import templates.embeds as embeds

from main import db

class SetLogChannelCog(commands.Cog):
    """Log Channel Cog

    Args:
        commands (Cog): the base class for all cogs
    """

    def __init__(self, bot):
        self.bot = bot

    @app_commands.command(
        name="set_log_channel",
        description="Sets the channel in which reported messages will go to",
    )
    @app_commands.describe(channel="The channel in which reported messages will go to")
    @app_commands.checks.has_permissions(administrator=True)
    async def set_log_channel(
        self, interaction: discord.Interaction, channel: discord.TextChannel
    ):
        """Sets the channel that reported messages will be sent in

        Args:
            interaction (discord.Interaction): Discord interaction. Occurs when user does notifiable action (e.g. slash commands)
            channel (discord.TextChannel): The Discord text channel
        """
        try:
            await interaction.response.defer(ephemeral=True)
            log_channel_set = db.fetchone_singlecolumn(
                0,
                "SELECT log_channel_id FROM bot_log_channel WHERE guild_id = ?",
                interaction.guild_id,
            )
            if log_channel_set is None:
                db.update_db(
                    "INSERT INTO bot_log_channel VALUES(?,?)",
                    interaction.guild_id,
                    channel.id,
                )
            else:
                db.update_db(
                    "UPDATE bot_log_channel SET log_channel_id = ? WHERE guild_id = ?",
                    channel.id,
                    interaction.guild_id,
                )
            await interaction.followup.send(
                embed=discord.Embed(
                    description="Log channel successfully set to <#"
                    + str(channel.id)
                    + ">!",
                    color=0x00AEFF,
                ),
            )
        except:
            await interaction.followup.send(
                embed=discord.Embed(
                    title="There was an error setting or updating the log channel. Please try again or contact the bot owner if you see this again",
                    color=0xFF0000,
                ),
            )
            raise

    @set_log_channel.error
    async def error(
        self, interaction: discord.Interaction, error: app_commands.AppCommandError
    ):
        """checks for missing permissions error

        Args:
            interaction (discord.Interaction): Discord interaction. Occurs when user does notifiable action (e.g. slash commands)
            error (app_commands.AppCommandError): The error that has been thrown
        """
        if isinstance(error, app_commands.MissingPermissions):
            await embeds.missing_permissions(interaction)

async def setup(bot: commands.Bot):
    """initializes the log channel cog

    Args:
        bot (commands.Bot): the discord bot
    """
    await bot.add_cog(SetLogChannelCog(bot))


import json
from pathlib import Path

import sc2

# Bots are created as classes and they need to have on_step method defined.
# Do not change the name of the class!
class TossBot(sc2.BotAI):
    with open(Path(__file__).parent / "../botinfo.json") as f:
        NAME = json.load(f)["name"]

    # On_step method is invoked each game-tick and should not take more than
    # 2 seconds to run, otherwise the bot will timeout and cannot receive new
    # orders.
    # It is important to note that on_step is asynchronous - meaning practices
    # for asynchronous programming should be followed.
    async def on_step(self, iteration):
        if iteration == 0:
            await self.chat_send("Tossbot activated.")
            for worker in self.workers:
                await self.do(worker.attack(self.enemy_start_locations[0]))


from torch.autograd import Variable
import numpy as np
import torch

def compute_gradient_penalty(
        D,
        real_samples,
        fake_samples,
        Tensor
    ):
    """Calculates the gradient penalty loss for WGAN GP"""
    alpha = Tensor(np.random.random((real_samples.size(0), 1)))

    interpolates = (alpha * real_samples + ((1 - alpha) * fake_samples)).requires_grad_(True)
    d_interpolates = D(interpolates)
    fake = Variable(Tensor(real_samples.shape[0], 1).fill_(1.0), requires_grad=False)
    # Get gradient w.r.t. interpolates
    gradients = torch.autograd.grad(
        outputs=d_interpolates,
        inputs=interpolates,
        grad_outputs=torch.ones_like(d_interpolates),
        create_graph=True,
        retain_graph=True,
        only_inputs=True,
    )[0]
    gradients = gradients.view(gradients.size(0), -1)
    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
    return gradient_penalty

def train(
        data,
        model,
        epochs,
        det_step,
        is_cuda=False
        ):

    G, D = model
    feature_dim = int(list(data.sampler.data_source[0][0].shape)[0])

    Tensor = torch.cuda.FloatTensor if is_cuda else torch.FloatTensor

    # Load previous model
    # DB = 'generator'
    # model_path = os.path.join('./model', )
    # if LOAD_MODEL:
    #     G.load_state_dict(torch.load(os.path.join(model_path, 'generator.pkl')))
    #     D.load_state_dict(torch.load(os.path.join(model_path, 'critic.pkl')))

    step = 0
    for epoch in range(epochs):
        for i, (batch, _) in enumerate(data):
            real_data = Variable(batch.type(Tensor))

            # ---------------------
            #  Train Discriminator
            # ---------------------

            # Sample noise as generator input
            z = Variable(Tensor(np.random.normal(0, 1, (batch.shape[0], feature_dim))))
            # Generate a batch of images
            fake_data = G(z)

            # Real and fake dataset
            real_validity = D(real_data)
            fake_validity = D(fake_data)
            # Gradient penalty
            gradient_penalty = compute_gradient_penalty(
                D,
                real_data.data,
                fake_data.data,
                Tensor
            )
            # Adversarial loss
            lambda_gp = 0.2

            d_loss = - torch.mean(real_validity) + torch.mean(fake_validity) + lambda_gp * gradient_penalty

            D.optimizer.zero_grad()
            d_loss.backward()
            D.optimizer.step()

            # -----------------
            #  Train Generator
            # -----------------
            if step % det_step == 0:

                fake_data = G(z)

                fake_validity = D(fake_data)
                g_loss = - torch.mean(fake_validity)

                G.optimizer.zero_grad()
                g_loss.backward()
                G.optimizer.step()

                print(
                    "[Epoch %d/%d] [Batch %d/%d] [D loss: %f (RV: %f)(FV: %f)] [G loss: %f] [Penalty: %f]"
                    % (epoch, epochs, i, len(data), d_loss.item(), torch.mean(real_validity), torch.mean(fake_validity), g_loss.item(), gradient_penalty.item())
                )

            step+=1

            # if batches_done % opt.sample_interval == 0:
            #     save_image(fake_imgs.dataset[:25], "images/%d.png" % batches_done, nrow=5, normalize=True)

            # batches_done += opt.n_critic


import numpy as np
import pandas as pd
from flask import Flask, request, jsonify, render_template
import pickle
import matplotlib.pyplot as plt
import os
plt.style.use('seaborn-bright')

app = Flask(__name__)
result = pickle.load(open('result.pkl','rb'))

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/predict',methods=['POST'])
def predict():
    '''
    For rendering results on HTML GUI
    '''
    date_initial = request.form['date_initial']
    date_final = request.form['date_final']
    prediction = result.predict(start=date_initial,end=date_final)
    #prediction = np.expm1(prediction)
    prediction = round(prediction,3)

    monthly = prediction.resample('m').sum()
    monthly = pd.DataFrame(monthly,columns=['Units'])
    monthly = round(monthly,3)
    monthly.index = pd.to_datetime(monthly.index).month_name()

    headings = ("Month","Consumption (Units)")
    tup = monthly.to_records(index=True).tolist()
    unit_list = [i[1] for i in tup]
    total = ('Total',int(sum(unit_list)))
    tup.append(total)
    tup = tuple(tup)

    monthly.plot(kind='bar',colormap='Set3',width=0.60)
    plt.xticks(rotation = 20)
    plt.xlabel('Months')
    plt.ylabel('Unit Consumption')
    plt.savefig('static/plot.png', transparent=True, bbox_inches='tight')

    return render_template('index.html',
    prediction_text= 'Monthly power consumption from {} to {}\n'.format(date_initial,date_final),
    filename='plot.png',headings= headings, tup=tup)

if __name__ == "__main__":
    app.run(debug=True)


# -*- coding:utf-8 -*-
# Created by Guozeping on 2019/10/2
import os
import glob
import zipfile

def unzip_file(dir_path):
    # 解压后文件存放的路径
    unzip_file_path = r"/Users/gzp/Desktop/解压文件"
    # 找到压缩文件夹
    dir_list = glob.glob(dir_path)
    if dir_list:
        # 循环zip文件夹
        for dir_zip in dir_list:
            # 以读的方式打开
            with zipfile.ZipFile(dir_zip, 'r') as f:
                for file in f.namelist():
                    f.extract(file,path=unzip_file_path)
            os.remove(dir_zip)
# 需要解压的文件路径
unzip_file(r"")

3-教程/压缩与解压/解压.py


import numpy as np
import matplotlib.pyplot as plt
from math import sqrt, floor, ceil
from scipy.interpolate import RegularGridInterpolator
import random
import warnings
import sys
import os
import csv

def is_perfect_cube(x):
    x = abs(x)
    return int(round(x ** (1. / 3))) ** 3 == x

def getRowColumnVectors(n):
    random.seed(0)

    while True:
        x = np.array([random.random(),random.random(),random.random()])
        x = x-np.dot(x,n)*n
        if np.dot(x,x)>0:
            break
    x = x/sqrt(np.dot(x,x))

    while True:
        y = np.array([random.random(),random.random(),random.random()])
        y = y-np.dot(y,n)*n
        y = y-np.dot(y,x)*x
        if np.dot(y,y)>0:
            break
    y = y/sqrt(np.dot(y,y))

    if np.dot(np.cross(x,y),n)<0:
        y=-y

    return (x[0],x[1],x[2],y[0],y[1],y[2])

def getPlaneBounds(center,x,y,size,s):
    (x0,y0,z0) = center
    (x1,x2,x3) = x
    (y1,y2,y3) = y
    (sizeX,sizeY,sizeZ) = size
    minX = max(floor(x0 -  s/2.0*abs(x1) -  s/2.0*abs(y1)),0)
    maxX = min(ceil( x0 +  s/2.0*abs(x1) +  s/2.0*abs(y1)),sizeX-1)
    minY = max(floor(y0 -  s/2.0*abs(x2) -  s/2.0*abs(y2)),0)
    maxY = min(ceil( y0 +  s/2.0*abs(x2) +  s/2.0*abs(y2)),sizeY-1)
    minZ = max(floor(z0 -  s/2.0*abs(x3) -  s/2.0*abs(y3)),0)
    maxZ = min(ceil( z0 +  s/2.0*abs(x3) +  s/2.0*abs(y3)),sizeZ-1)
    # print('Plane Bounds = ')
    # print(minX,maxX,minY,maxY,minZ,maxZ)
    return (minX,maxX,minY,maxY,minZ,maxZ)

def loadDat(fileName,size,minimum=None,maximum=None):
    img = np.fromfile(fileName, dtype=np.int16)
    img = img.reshape(size)
    if not (minimum is None):
        img = np.maximum(np.zeros(img.shape),img - minimum)
    if not (maximum is None):
        img = np.minimum(img,np.ones(img.shape)*maximum)
    # plt.hist(self.data.flatten().tolist(), bins=100)
    # plt.show()
    return np.transpose(img,axes=(2,1,0))

def reslicePlane(flowPath,csvPath):
    gridSize=64
    # print('Input flow path = {}'.format(flowPath))
    if not os.path.exists(flowPath):
        flowPath = '\\\\MPUFS7\\data_mrcv\\45_DATA_HUMANS\\CHEST\\STUDIES\\2017_Eldridge_NLP_MET\\002\\Normoxia\\CS_WAVELET_20_FRAMES_LOWRES\\dat\\CORRECTED'
        print('Flow Path not found, using default instead.')
    # print('4D flow path = {}'.format(flowPath))
    # print('Input CSV path = {}'.format(csvPath))
    if not os.path.exists(csvPath):
        csvPath = 'M:\\pcorrado\\CODE\\chestFlowSlicer\\testCSVSaveFile.csv'
        print('CSV Path not found, using default instead.')
    # print('CSV path = {}'.format(csvPath))

    headerFile = os.path.join(flowPath,'pcvipr_header.txt')
    for line in open(headerFile,'r'):
        (field,value) = line.split()
        if field=='matrixx': sizeX=int(float(value))
        if field=='matrixy': sizeY=int(float(value))
        if field=='matrixz': sizeZ=int(float(value))
        if field=='frames': sizeT=int(float(value))

    # print('Size = ({},{},{},{})'.format(sizeX,sizeY,sizeZ,sizeT))
    mag = loadDat(os.path.join(flowPath,'MAG.dat'),[sizeZ,sizeY,sizeX])
    cd = loadDat(os.path.join(flowPath,'CD.dat'),[sizeZ,sizeY,sizeX],minimum=0)
    vX = np.zeros([sizeX,sizeY,sizeZ,sizeT])
    vY = np.zeros([sizeX,sizeY,sizeZ,sizeT])
    vZ = np.zeros([sizeX,sizeY,sizeZ,sizeT])
    for ph in range(sizeT):
        vxFile = 'ph_{:0>3d}_vd_{}.dat'.format(ph,1)
        vyFile = 'ph_{:0>3d}_vd_{}.dat'.format(ph,2)
        vzFile = 'ph_{:0>3d}_vd_{}.dat'.format(ph,3)
        vX[:,:,:,ph] = -loadDat(os.path.join(flowPath,vxFile),[sizeZ,sizeY,sizeX])
        vY[:,:,:,ph] = -loadDat(os.path.join(flowPath,vyFile),[sizeZ,sizeY,sizeX])
        vZ[:,:,:,ph] = -loadDat(os.path.join(flowPath,vzFile),[sizeZ,sizeY,sizeX])

    with open(csvPath, mode='r') as csvFile:
        reader = csv.reader(csvFile, delimiter=',')
        for row in reader:
            if row and ((row[0]==flowPath) or (row[0].replace('\\\\MPUFS7','\\data').replace('\\','/')==flowPath)):
                print(row[0].replace('\\\\MPUFS7','\\data').replace('\\','/'))
                vessel = row[1]
                (x0,y0,z0) = (float(row[2]), float(row[3]), float(row[4]))
                (x0,y0,z0) = (x0*(sizeX/128.0),y0*(sizeY/128.0),z0*(sizeZ/128.0))
                (nx,ny,nz) = (float(row[5]), float(row[6]), float(row[7]))
                # (nx,ny,nz) = (nx*sizeX/128.0,ny*sizeY/128.0,nz*sizeZ/128.0)
                s = sqrt(nx**2+ny**2+nz**2)
                # s = float(row[5])
                (nx,ny,nz) = (nx/s,ny/s,nz/s)
                s = s*(max(sizeX,sizeY,sizeZ)/128.0)
                # print('Center = ({},{},{})'.format(x0,y0,z0))
                # print('Normal = ({},{},{})'.format(nx,ny,nz))
                # print('Side = {}'.format(s))

                (x1,x2,x3,y1,y2,y3) = getRowColumnVectors(np.array((nx,ny,nz)))
                (minX,maxX,minY,maxY,minZ,maxZ) = getPlaneBounds((x0,y0,z0),(x1,x2,x3),(y1,y2,y3),(sizeX,sizeY,sizeZ),s)

                # print('x=({},{},{})'.format(x1,x2,x3))
                # print('y=({},{},{})'.format(y1,y2,y3))
                (i, j) = np.meshgrid(np.linspace(start=-s/2.0, stop=s/2.0, num=gridSize), np.linspace(start=-s/2.0, stop=s/2.0, num=gridSize))

                # print('IJ Bounds')
                # print(min(i.flatten()),max(i.flatten()),min(j.flatten()),max(j.flatten()))
                # xp = np.minimum(np.maximum(x0 + i*x1 + j*y1,np.zeros(i.shape)),np.ones(i.shape)*(sizeX-1))
                # yp = np.minimum(np.maximum(y0 + i*x2 + j*y2,np.zeros(i.shape)),np.ones(i.shape)*(sizeY-1))
                # zp = np.minimum(np.maximum(z0 + i*x3 + j*y3,np.zeros(i.shape)),np.ones(i.shape)*(sizeZ-1))
                # print(x0,x1,y1)
                xp = x0 + x1*i + y1*j
                yp = y0 + x2*i + y2*j
                zp = z0 + x3*i + y3*j

                # print('Plane Query bounds:')
                # print(min(xp.flatten()),max(xp.flatten()),min(yp.flatten()),max(yp.flatten()),min(zp.flatten()),max(zp.flatten()))

                for (name,img) in [(vessel+'_MAG',mag),(vessel+'_CD',cd)]:
                    # print(img.shape)
                    rgi = RegularGridInterpolator((np.linspace(minX, maxX, maxX-minX+1),
                                                   np.linspace(minY, maxY, maxY-minY+1),
                                                   np.linspace(minZ, maxZ, maxZ-minZ+1)),
                                                   img[minX:(maxX+1), minY:(maxY+1), minZ:(maxZ+1)],
                                                   bounds_error=False, fill_value=0)

                    with warnings.catch_warnings():
                        warnings.filterwarnings('ignore', r'invalid value encountered in true_divide')
                        grid = np.transpose(np.reshape(rgi(np.transpose(np.array([xp.flatten(),yp.flatten(),zp.flatten()]))),(gridSize,gridSize)))
                    # plt.figure()
                    # plt.imshow(grid)
                    # plt.title(name)
                    np.save(os.path.join(flowPath,name),grid)

                grid =np.zeros((gridSize,gridSize,sizeT))
                for (v,n) in [(vX,nx),(vY,ny),(vZ,nz)]:
                    for ph in range(sizeT):

                        rgi = RegularGridInterpolator((np.linspace(minX, maxX, maxX-minX+1),
                                                       np.linspace(minY, maxY, maxY-minY+1),
                                                       np.linspace(minZ, maxZ, maxZ-minZ+1)),
                                                       v[minX:(maxX+1), minY:(maxY+1), minZ:(maxZ+1),ph],
                                                       bounds_error=False, fill_value=0)

                        with warnings.catch_warnings():
                            warnings.filterwarnings('ignore', r'invalid value encountered in true_divide')
                            # if ph==3:
                            #     plt.figure()
                            #     plt.imshow(np.transpose(np.reshape(rgi(np.transpose(np.array([xp.flatten(),yp.flatten(),zp.flatten()]))),(gridSize,gridSize))))
                            #     plt.title('{},{}'.format(n,ph))
                            #     plt.show()
                            grid[:,:,ph] += n*np.transpose(np.reshape(rgi(np.transpose(np.array([xp.flatten(),yp.flatten(),zp.flatten()]))),(gridSize,gridSize)))
                np.save(os.path.join(flowPath,vessel+'_velNormal'),grid)

if __name__ == "__main__":
    flowPath = sys.argv[1]
    csvPath = sys.argv[2]
    reslicePlane(flowPath,csvPath)


#!/usr/bin/env python3
import random
import re
from sys import stdin
import argparse

TOKEN_EXPRESSION = r'(\d+|[^\W\d_]+|\s+|\W)'

WORDS_EXPRESSION = r'([^\W\d_]+)'

def tokenize(line):
    tokens = [token for token in re.split(TOKEN_EXPRESSION, line) if token != '']
    return tokens

def words(line):
    return re.findall(WORDS_EXPRESSION, line)

class Chain:
    def __init__(self, text_tokens, depth):
        self.depth = depth
        self.chain = dict()
        self.text_tokens = text_tokens
        if depth == 0:
            self.make_zero_chain()
        else:
            self.make_chain()

    def make_zero_chain(self):
        dictionary = dict()
        number_of_words = 0
        for i in range(len(self.text_tokens)):
            for word in self.text_tokens[i]:
                if word not in dictionary.keys():
                    dictionary[word] = 0
                dictionary[word] += 1
                number_of_words += 1
        self.chain[''] = dict()
        for word in dictionary.keys():
            self.chain[''][word] = float(dictionary[word]) / number_of_words

    def make_chain(self):
        for line in self.text_tokens:
            for j in range(self.depth, len(line)):
                sub_string = ' '.join(line[j - self.depth:j])
                regular_expression = ' ' + sub_string + ' ([^\W\d_]+)'
                words_after_pattern = re.findall(regular_expression, ' ' + ' '.join(line))
                if len(words_after_pattern) != 0:
                    if sub_string not in self.chain.keys():
                        self.chain[sub_string] = dict()
                    for word in words_after_pattern:
                        if word not in self.chain[sub_string].keys():
                            self.chain[sub_string][word] = 0
                        self.chain[sub_string][word] += 1
            for pattern in self.chain.keys():
                for word in self.chain[pattern]:
                    self.chain[pattern][word] = (float(self.chain[pattern][word]) /
                                                 len(self.chain[pattern]))

    def __str__(self):
        result = list()
        pattern = '  %s: %.3f'
        for sub_string in sorted(self.chain.keys()):
            result.append(sub_string)
            for word in sorted(self.chain[sub_string]):
                probability = self.chain[sub_string][word]
                result.append(pattern % (word, probability))
        return '\n'.join(result)

class Generator:
    def __init__(self, text, max_depth):
        self.depth = 0
        self.max_depth = max_depth
        self.chains = list()
        self.text = text
        self.is_fitted = False

    def fit(self):
        self.is_fitted = True
        for i in range(self.depth, self.max_depth + 1):
            self.next_chain()

    def next_chain(self):
        assert self.is_fitted, "Not Fitted"
        self.chains.append(Chain(self.text, self.depth))
        self.depth += 1

    def __str__(self):
        assert self.is_fitted, "Not Fitted"
        concatenation = dict(self.chains[0].chain)
        for i in range(1, self.depth):
            concatenation.update(self.chains[i].chain)
        result = list()
        pattern = '  %s: %.2f'
        for sub_string in sorted(concatenation.keys()):
            result.append(sub_string)
            for word in sorted(concatenation[sub_string]):
                probability = concatenation[sub_string][word]
                result.append(pattern % (word, probability))
        return '\n'.join(result)

    def generate_next(self, prefix):
        assert self.is_fitted, "Not Fitted"
        depth = min(len(prefix), self.max_depth)
        previous_words = ' '.join(prefix[-depth:])
        if previous_words in self.chains[depth].chain.keys():
            after_words = self.chains[depth].chain[previous_words].keys()
            return self.choose_world(after_words, depth, previous_words)
        else:
            return None

    def choose_world(self, after_words, depth, previous_words):
        choice = random.uniform(0, 1)
        cumulative_summ = 0
        for word in after_words:
            cumulative_summ += self.chains[depth].chain[previous_words][word]
            if choice <= cumulative_summ:
                return word

    def generate(self, length):
        assert self.is_fitted, "Not Fitted"
        result = list()
        for i in range(length):
            begin = max(0, i - self.depth)
            next = self.generate_next(result[begin:])
            if next is None:
                break
            result.append(next)
        return ' '.join([result[0].title()].__add__(result[1:])) + '.'

class Test:
    def __init__(self):
        print(self.test_tokenize())
        print(self.test_generate())
        print(self.test_probabilities())

    def test_tokenize(self):
        tokenized = ['Hello', ',', ' ', 'world', '!']
        if tokenize('Hello, world!') != tokenized:
            print(tokenize('Hello, world!'))
            return False
        return True

    def test_probabilities(self):
        text = [['First', 'test', 'sentence'],
                ['Second', 'test', 'line']]
        result = '\n  First: 0.17\n' \
                 + '  Second: 0.17\n' \
                 + '  line: 0.17\n' \
                 + '  sentence: 0.17\n' \
                 + '  test: 0.33\n' \
                 + 'First\n' \
                 + '  test: 1.00\n' \
                 + 'Second\n' \
                 + '  test: 1.00\n' \
                 + 'test\n' \
                 + '  line: 0.50\n' \
                 + '  sentence: 0.50'
        chain = Generator(text, 1)
        chain.fit()
        if str(chain) != result:
            print(chain)
            print()
            print(result)
            return False
        return True

    def test_generate(self):
        text = [['First', 'test', 'sentence'],
                ['Second', 'test', 'line']]
        chain = Generator(text, 1)
        chain.fit()
        results = ['First test.', 'Test sentence.',
                   'Second test.', 'Test line.',
                   'Line.', 'Sentence.']
        generated = chain.generate(2)
        if generated not in results:
            print(generated)
            return False
        return True

def main():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers(dest='which')

    tokenize_parser = subparsers.add_parser('tokenize')

    test_parser = subparsers.add_parser('test')

    probabilities_parser = subparsers.add_parser('probabilities')
    probabilities_parser.add_argument('--depth', nargs='?', type=int, default=0)

    generate_parser = subparsers.add_parser('generate')
    generate_parser.add_argument('--depth', nargs='?', type=int, default=0)
    generate_parser.add_argument('--size', nargs='?', type=int, default=0)

    args = parser.parse_args(input().split())

    if args.which == 'tokenize':
        line = input()
        tokens = tokenize(line)
        print('\n'.join(tokens))
    elif args.which == 'probabilities':
        text = list()
        for line in stdin:
            text.append(words(line))
        chain = Generator(text, args.depth)
        chain.fit()
        print(chain)
    elif args.which == 'generate':
        text = list()
        for line in stdin:
            text.append(words(line))
        chain = Generator(text, args.depth)
        chain.fit()
        print(chain.generate(args.size))
    elif args.which == 'test':
        Test()

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
"""Show a text-mode spectrogram using live microphone data."""

#Importe todas as bibliotecas

import numpy as np
import sounddevice as sd
import matplotlib.pyplot as plt
from scipy.fftpack import fft
from scipy import signal as window
from suaBibSignal import signalMeu
import time
import peakutils
from peakutils.plot import plot as pplot

#funcao para transformar intensidade acustica em dB
def todB(s):
    sdB = 10*np.log10(s)
    return(sdB)

def main():

    #declare um objeto da classe da sua biblioteca de apoio (cedida)
    #declare uma variavel com a frequencia de amostragem, sendo 44100
    signal = signalMeu()
    freqDeAmostragem = 44100

    #voce importou a bilioteca sounddevice como, por exemplo, sd. entao
    # os seguintes parametros devem ser setados:

    sd.default.samplerate = freqDeAmostragem  #taxa de amostragem
    sd.default.channels = 1  #voce pode ter que alterar isso dependendo da sua placa
    duration = 2 #tempo em segundos que ira aquisitar o sinal acustico captado pelo mic

    # faca um print na tela dizendo que a captacao comecará em n segundos. e entao
    n = 1
    print("A captaçao comecara em {} segundos ".format(n))
    time.sleep(n)
    #use um time.sleep para a espera
    #faca um print informando que a gravacao foi inicializada
    print("A gravaçao foi inicializada")

    #declare uma variavel "duracao" com a duracao em segundos da gravacao. poucos segundos ...
    numAmostras =  freqDeAmostragem * duration
    #calcule o numero de amostras "numAmostras" que serao feitas (numero de aquisicoes)

    audio = sd.rec(int(numAmostras), freqDeAmostragem, channels=1)
    sd.wait()
    print("...     FIM")

    #analise sua variavel "audio". pode ser um vetor com 1 ou 2 colunas, lista ...
    #grave uma variavel com apenas a parte que interessa (dados)
    dados = []
    for e in audio[:,0]:
        dados.append(e)

    # use a funcao linspace e crie o vetor tempo. Um instante correspondente a cada amostra!

    t = np.linspace(0,duration,int(numAmostras))

    #plot do gravico  áudio vs tempo!
    plt.plot(dados,t)
    plt.grid()
    plt.title('audio vs tempo')
    plt.show()

    ## Calcula e exibe o Fourier do sinal audio. como saida tem-se a amplitude e as frequencias

    xf, yf = signal.calcFFT(dados, freqDeAmostragem)
    plt.figure("F(y)")
    plt.plot(xf,yf)
    plt.grid()
    plt.title('Fourier audio')
    plt.show()

    #esta funcao analisa o fourier e encontra os picos
    #voce deve aprender a usa-la. ha como ajustar a sensibilidade, ou seja, o que é um pico?
    #voce deve tambem evitar que dois picos proximos sejam identificados, pois pequenas variacoes na
    #frequencia do sinal podem gerar mais de um pico, e na verdade tempos apenas 1.

    #index = peakutils.indexes(,,)
    #printe os picos encontrados!
    index = peakutils.indexes(yf, thres=0.1, min_dist=50)
    picos = [xf[i] for i in index if xf[i] > 600 and xf[i] < 1800]
    maxi = max(picos, key=int)
    minn = min(picos, key=int)
    delta = 10
    print("Picos : {}  ".format(picos))

    tabela_DTMF = {"1":[1209, 697], "2":[1336, 697], "3":[1477, 697], "A":[1633, 697],
         "4":[1209, 770], "5":[1336, 770], "6":[1477, 770], "B":[1633, 770],
         "7":[1209, 852], "8":[1336, 852], "9":[1477, 852], "C":[1633, 852],
         "X":[1209, 941], "0":[1336, 941], "#":[1477, 941], "D":[1633, 941]}

    if 1209 + delta >= maxi >= 1209 - delta :
        if 697 + delta >= minn >= 697 - delta:
            print("A tecla encontrada foi a : 1  ")

    if 1336 + delta >= maxi >= 1336 - delta:
        if 697 + delta >= minn >= 697 - delta:
            print("A tecla encontrada foi a : 2  ")

    if 1477 + delta >=  maxi >= 1477 - delta:
        if 697 + delta >= minn >= 697 - delta :
            print("A tecla encontrada foi a : 3  ")

    if 1633 + delta >= maxi >= 1633 - delta:
        if 697 + delta >= minn >= 697 - delta :
            print("A tecla encontrada foi a : A  ")

    if 1209 + delta >= maxi >= 1209 - delta :
        if 770 + delta >= minn >= 770 - delta :
            print("A tecla encontrada foi a : 4  ")

    if 1336 + delta >= maxi >= 1336 - delta :
        if 770 + delta >= minn >= 770 - delta:
            print("A tecla encontrada foi a : 5  ")

    if 1477 + delta >= maxi >= 1477 - delta:
        if 770 + delta >= minn >= 770 - delta:
            print("A tecla encontrada foi a : 6  ")

    if 1633 + delta >= maxi >= 1633 - delta:
        if 770 + delta >= minn >= 770 - delta :
            print("A tecla encontrada foi a : B  ")

    if 1209 + delta >= maxi >= 1209 - delta:
        if 852 + delta >= minn >= 852 - delta :
            print("A tecla encontrada foi a : 7  ")

    if 1336 + delta >= maxi >= 1336 - delta:
        if 852 + delta >= minn >= 852 - delta :
            print("A tecla encontrada foi a : 8  ")

    if 1477 + delta >= maxi >= 1477 - delta :
        if 852 + delta >= minn >= 852 - delta :
            print("A tecla encontrada foi a : 9  ")
    if 1633 + delta >= maxi >= 1633 - delta :
        if 852 + delta >= minn >= 852 - delta :
            print("A tecla encontrada foi a : C  ")

    if 1209 + delta  >= maxi >= 1209 - delta :
        if 941 + delta >= minn >= 941 - delta  :
            print("A tecla encontrada foi a : X  ")
    if 1336 + delta  >= maxi >= 1336 - delta:
        if 941 + delta >= minn >= 941 - delta :
            print("A tecla encontrada foi a : 0  ")
    if 1477 + delta  >= maxi >= 1477 - delta:
        if 941 + delta >= minn >= 941 - delta  :
            print("A tecla encontrada foi a : #  ")
    if 1633 + delta >= maxi >= 1633 - delta:
        if 941 + delta >= minn >= 941 - delta  :
            print("A tecla encontrada foi a : D  ")

    #encontre na tabela duas frequencias proximas às frequencias de pico encontradas e descubra qual foi a tecla
    #print a tecla.

    ## Exibe gráficos
        #plt.show()

if __name__ == "__main__":
    main()


from discord.ext.pages import Paginator
import discord
from config import emotes, colors

e = emotes.error
ce = colors.error
embed = discord.Embed(
    description=f"{e} This paginator is not for you!", color=ce)

class CustomPaginator(Paginator):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    async def interaction_check(self, interaction: discord.Interaction) -> bool:
        if self.usercheck:
            if self.user != interaction.user:
                await interaction.response.send_message(embed=embed, ephemeral=True)
                return False
            return self.user == interaction.user
        return True


'''
In a given grid, each cell can have one of three values:

the value 0 representing an empty cell;
the value 1 representing a fresh orange;
the value 2 representing a rotten orange.
Every minute, any fresh orange that is adjacent (4-directionally) to a rotten orange becomes rotten.

Return the minimum number of minutes that must elapse until no cell has a fresh orange.  If this is impossible, return -1 instead.

Example 1:

Input: [[2,1,1],[1,1,0],[0,1,1]]
Output: 4
Example 2:

Input: [[2,1,1],[0,1,1],[1,0,1]]
Output: -1
Explanation:  The orange in the bottom left corner (row 2, column 0) is never rotten, because rotting only happens 4-directionally.
Example 3:

Input: [[0,2]]
Output: 0
Explanation:  Since there are already no fresh oranges at minute 0, the answer is just 0.

Note:

1 <= grid.length <= 10
1 <= grid[0].length <= 10
grid[i][j] is only 0, 1, or 2.

'''

from collections import deque

class Solution:
    def is_cell_valid(self, cell_i, cell_j):
        return (cell_i >= 0 and cell_i <= self.n - 1) and (cell_j >= 0 and cell_j <= self.m -1)

    def orangesRotting(self, grid: List[List[int]]) -> int:
        q = deque()
        n, m = len(grid), len(grid[0])
        seen_fresh = 0
        longest_time = 0
        self.n, self.m = n,m

        # Seeding queue and fresh counter
        for i in range(n):
            for j in range(m):
                if grid[i][j] == 2:
                    q.append((i, j, 0))

                elif grid[i][j] == 1:
                    seen_fresh += 1

        neighbors = [(1, 0), (-1, 0), (0, 1), (0, -1)]

        while q:
            cell_i, cell_j, time = q.popleft()

            if grid[cell_i][cell_j] == 1:
                seen_fresh -= 1
                grid[cell_i][cell_j] = 2
                longest_time = max(longest_time, time)

            for nei_i, nei_j in neighbors:
                new_i, new_j = cell_i + nei_i, cell_j + nei_j

                if self.is_cell_valid(new_i, new_j) and grid[new_i][new_j] == 1:
                    q.append((new_i, new_j, time + 1))

        return longest_time if not seen_fresh else -1


from django.shortcuts import render
from .models import News

# Create your views here.
def home_page(request):
    articles_list = News.objects.all()
    context = {
        'articles' : articles_list
    }
    return render(request, 'articles.html', context)


import numpy as np
import matplotlib.pyplot as plt
from glob import glob
import librosa as lr

audio='arabic6'
y, sr = lr.load('./{}.wav'.format(audio))
y_fast = lr.effects.time_stretch(y, 2.0)
time = np.arange(0,len(y_fast))/sr
fig, ax = plt.subplots()
ax.plot(time,y_fast)
ax.set(xlabel='Time(s)',ylabel='sound amplitude')
plt.show()#compress to be twice as fast

y_slow = lr.effects.time_stretch(y, 0.5)
time = np.arange(0,len(y_slow))/sr
fig, ax = plt.subplots()
ax.plot(time,y_slow)
ax.set(xlabel='Time(s)',ylabel='sound amplitude')
plt.show()#half the original speed