from copy import copy
import numpy as np
import os
import pickle as pkl
import networkx as nx
import scipy.sparse as sp
from scipy.sparse.linalg.eigen.arpack import eigsh
from sklearn import preprocessing
import sys

def parse_index_file(filename):
    """Parse index file."""
    index = []
    for line in open(filename):
        index.append(int(line.strip()))
    return index

def sample_mask(idx, l):
    """Create mask."""
    mask = np.zeros(l)
    mask[idx] = 1
    return np.array(mask, dtype=np.bool)

'''
def load_data(dataset_str):
    """
    Loads input data from gcn/data directory

    ind.dataset_str.x => the feature vectors of the training instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.tx => the feature vectors of the test instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.allx => the feature vectors of both labeled and unlabeled training instances
        (a superset of ind.dataset_str.x) as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.y => the one-hot labels of the labeled training instances as numpy.ndarray object;
    ind.dataset_str.ty => the one-hot labels of the test instances as numpy.ndarray object;
    ind.dataset_str.ally => the labels for instances in ind.dataset_str.allx as numpy.ndarray object;
    ind.dataset_str.graph => a dict in the format {index: [index_of_neighbor_nodes]} as collections.defaultdict
        object;
    ind.dataset_str.test.index => the indices of test instances in graph, for the inductive setting as list object.

    All objects above must be saved using python pickle module.

    :param dataset_str: Dataset name
    :return: All data input files loaded (as well the training/test data).
    """
    names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph']
    objects = []
    for i in range(len(names)):
        with open("data/ind.{}.{}".format(dataset_str, names[i]), 'rb') as f:
            if sys.version_info > (3, 0):
                objects.append(pkl.load(f, encoding='latin1'))
            else:
                objects.append(pkl.load(f))

    x, y, tx, ty, allx, ally, graph = tuple(objects)
    test_idx_reorder = parse_index_file("data/ind.{}.test.index".format(dataset_str))
    test_idx_range = np.sort(test_idx_reorder)

    if dataset_str == 'citeseer':
        # Fix citeseer dataset (there are some isolated nodes in the graph)
        # Find isolated nodes, add them as zero-vecs into the right position
        test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder)+1)
        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
        tx_extended[test_idx_range-min(test_idx_range), :] = tx
        tx = tx_extended
        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
        ty_extended[test_idx_range-min(test_idx_range), :] = ty
        ty = ty_extended

    features = sp.vstack((allx, tx)).tolil()
    features[test_idx_reorder, :] = features[test_idx_range, :]
    adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph))

    labels = np.vstack((ally, ty))
    labels[test_idx_reorder, :] = labels[test_idx_range, :]

    idx_test = test_idx_range.tolist()
    idx_train = range(len(y))
    idx_val = range(len(y), len(y)+500)

    train_mask = sample_mask(idx_train, labels.shape[0])
    val_mask = sample_mask(idx_val, labels.shape[0])
    test_mask = sample_mask(idx_test, labels.shape[0])

    y_train = np.zeros(labels.shape)
    y_val = np.zeros(labels.shape)
    y_test = np.zeros(labels.shape)
    y_train[train_mask, :] = labels[train_mask, :]
    y_val[val_mask, :] = labels[val_mask, :]
    y_test[test_mask, :] = labels[test_mask, :]

    return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask
'''
def save_sparse_csr(filename,array):
    np.savez(filename, data=array.data, indices=array.indices,
             indptr=array.indptr, shape=array.shape)

def load_sparse_csr(filename):
    loader = np.load(filename)
    return sp.csr_matrix(
        (loader['data'], loader['indices'], loader['indptr']),
        shape=loader['shape']
    )

def load_data(dataset_str):
    """
    Loads input data from gcn/data directory

    ind.dataset_str.x => the feature vectors of the training instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.tx => the feature vectors of the test instances as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.allx => the feature vectors of both labeled and unlabeled training instances
        (a superset of ind.dataset_str.x) as scipy.sparse.csr.csr_matrix object;
    ind.dataset_str.y => the one-hot labels of the labeled training instances as numpy.ndarray object;
    ind.dataset_str.ty => the one-hot labels of the test instances as numpy.ndarray object;
    ind.dataset_str.ally => the labels for instances in ind.dataset_str.allx as numpy.ndarray object;
    ind.dataset_str.graph => a dict in the format {index: [index_of_neighbor_nodes]} as collections.defaultdict
        object;
    ind.dataset_str.test.index => the indices of test instances in graph, for the inductive setting as list object.

    All objects above must be saved using python pickle module.

    :param dataset_str: Dataset name
    :return: All data input files loaded (as well the training/test data).
    """
    names = ['x', 'y', 'tx', 'ty', 'allx', 'ally', 'graph']
    objects = []
    for i in range(len(names)):
        with open("data/ind.{}.{}".format(dataset_str, names[i]), 'rb') as f:
            if sys.version_info > (3, 0):
                objects.append(pkl.load(f, encoding='latin1'))
            else:
                objects.append(pkl.load(f))

    x, y, tx, ty, allx, ally, graph = tuple(objects)
    test_idx_reorder = parse_index_file("data/ind.{}.test.index".format(dataset_str))
    test_idx_range = np.sort(test_idx_reorder)

    if dataset_str == 'citeseer':
        # Fix citeseer dataset (there are some isolated nodes in the graph)
        # Find isolated nodes, add them as zero-vecs into the right position
        test_idx_range_full = range(min(test_idx_reorder), max(test_idx_reorder)+1)
        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
        tx_extended[test_idx_range-min(test_idx_range), :] = tx
        tx = tx_extended
        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
        ty_extended[test_idx_range-min(test_idx_range), :] = ty
        ty = ty_extended

    if 'nell.0' in dataset_str:
        # Find relation nodes, add them as zero-vecs into the right position
        test_idx_range_full = range(allx.shape[0], len(graph))
        isolated_node_idx = np.setdiff1d(test_idx_range_full, test_idx_reorder)
        tx_extended = sp.lil_matrix((len(test_idx_range_full), x.shape[1]))
        tx_extended[test_idx_range - allx.shape[0], :] = tx
        tx = tx_extended
        ty_extended = np.zeros((len(test_idx_range_full), y.shape[1]))
        ty_extended[test_idx_range - allx.shape[0], :] = ty
        ty = ty_extended

        features = sp.vstack((allx, tx)).tolil()
        features[test_idx_reorder, :] = features[test_idx_range, :]

        idx_all = np.setdiff1d(range(len(graph)), isolated_node_idx)

        if not os.path.isfile("data/{}.features.npz".format(dataset_str)):
            print("Creating feature vectors for relations - this might take a while...")
            features_extended = sp.hstack(
                (features, sp.lil_matrix(
                    (features.shape[0], len(isolated_node_idx))
                )), dtype=np.float32).todense()
            features_extended[isolated_node_idx, features.shape[1]:] = np.eye(
                len(isolated_node_idx), dtype=np.float32)
            features = sp.csr_matrix(features_extended)
            print("Done!")
            save_sparse_csr("data/{}.features".format(dataset_str),
                            features)
        else:
            features = load_sparse_csr(
                "data/{}.features.npz".format(dataset_str))

        adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph))
    else:
        features = sp.vstack((allx, tx)).tolil()
        features[test_idx_reorder, :] = features[test_idx_range, :]
        adj = nx.adjacency_matrix(nx.from_dict_of_lists(graph))

    labels = np.vstack((ally, ty))
    labels[test_idx_reorder, :] = labels[test_idx_range, :]

    idx_test = test_idx_range.tolist()
    idx_train = range(len(y))
    idx_val = range(len(y), len(y)+500)

    train_mask = sample_mask(idx_train, labels.shape[0])
    val_mask = sample_mask(idx_val, labels.shape[0])
    test_mask = sample_mask(idx_test, labels.shape[0])

    y_train = np.zeros(labels.shape)
    y_val = np.zeros(labels.shape)
    y_test = np.zeros(labels.shape)
    y_train[train_mask, :] = labels[train_mask, :]
    y_val[val_mask, :] = labels[val_mask, :]
    y_test[test_mask, :] = labels[test_mask, :]

    return adj, features, y_train, y_val, y_test, train_mask, val_mask, test_mask

def load_embedding(model_type, dataset_str, all_embedding=False):
    fname = "data/{}.{}.emb".format(model_type, dataset_str)
    if all_embedding:
        fname += 'all'
    print(fname)
    with open(fname, 'rb') as f:
        if sys.version_info > (3, 0):
            saved_embeddings = pkl.load(f, encoding='latin1')
        else:
            saved_embeddings = pkl.load(f)
    print(saved_embeddings.shape)
    return preprocessing.normalize(saved_embeddings, norm='l2')

def sparse_to_tuple(sparse_mx):
    """Convert sparse matrix to tuple representation."""
    def to_tuple(mx):
        if not sp.isspmatrix_coo(mx):
            mx = mx.tocoo()
        coords = np.vstack((mx.row, mx.col)).transpose()
        values = mx.data
        shape = mx.shape
        return coords, values, shape

    if isinstance(sparse_mx, list):
        for i in range(len(sparse_mx)):
            sparse_mx[i] = to_tuple(sparse_mx[i])
    else:
        sparse_mx = to_tuple(sparse_mx)

    return sparse_mx

def preprocess_features(features, sparse=False):
    """Row-normalize feature matrix and convert to tuple representation"""
    rowsum = np.array(features.sum(1))
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = sp.diags(r_inv)
    features = r_mat_inv.dot(features)
    if sparse:
        return sparse_to_tuple(features)
    else:
        return sp.spmatrix.toarray(features)

def normalize_adj(adj):
    """Symmetrically normalize adjacency matrix."""
    adj = sp.coo_matrix(adj)
    rowsum = np.array(adj.sum(1))
    d_inv_sqrt = np.power(rowsum, -0.5).flatten()
    d_inv_sqrt[np.isinf(d_inv_sqrt)] = 0.
    d_mat_inv_sqrt = sp.diags(d_inv_sqrt)
    return adj.dot(d_mat_inv_sqrt).transpose().dot(d_mat_inv_sqrt).tocoo()

def preprocess_adj(adj):
    """Preprocessing of adjacency matrix for simple GCN model and conversion to tuple representation."""
    adj_normalized = normalize_adj(adj + sp.eye(adj.shape[0]))
    return sparse_to_tuple(adj_normalized)

def preprocess_adj_dense(adj):
    """Preprocessing of adjacency matrix for simple GCN model and conversion to tuple representation."""
    adj_normalized = normalize_adj(adj + sp.eye(adj.shape[0]))
    # return sparse_to_tuple(adj_normalized)
    return sp.spmatrix.toarray(adj_normalized)

def construct_feed_dict(features, support, labels, labels_mask, placeholders,
                        neighbor_features=None, neighbor_ids=None):
    """Construct feed dictionary."""
    feed_dict = dict()
    feed_dict.update({placeholders['labels']: labels})
    feed_dict.update({placeholders['labels_mask']: labels_mask})
    feed_dict.update({placeholders['features']: features})
    if 'neighbor_features' in placeholders.keys():
        feed_dict.update(
            {placeholders['neighbor_features'][i]: \
                 neighbor_features[i] for i in range(len(neighbor_features))}
        )
    if 'neighbor_ids' in placeholders.keys():
        feed_dict.update(
            {placeholders['neighbor_ids'][i]: \
                 neighbor_ids[i] for i in range(len(neighbor_ids))}
        )
    feed_dict.update({placeholders['support']: support})
    feed_dict.update({placeholders['num_features_nonzero']: features[1].shape})
    return feed_dict

def chebyshev_polynomials(adj, k):
    """Calculate Chebyshev polynomials up to order k. Return a list of sparse matrices (tuple representation)."""
    print("Calculating Chebyshev polynomials up to order {}...".format(k))

    adj_normalized = normalize_adj(adj)
    laplacian = sp.eye(adj.shape[0]) - adj_normalized
    largest_eigval, _ = eigsh(laplacian, 1, which='LM')
    scaled_laplacian = (2. / largest_eigval[0]) * laplacian - sp.eye(adj.shape[0])

    t_k = list()
    t_k.append(sp.eye(adj.shape[0]))
    t_k.append(scaled_laplacian)

    def chebyshev_recurrence(t_k_minus_one, t_k_minus_two, scaled_lap):
        s_lap = sp.csr_matrix(scaled_lap, copy=True)
        return 2 * s_lap.dot(t_k_minus_one) - t_k_minus_two

    for i in range(2, k+1):
        t_k.append(chebyshev_recurrence(t_k[-1], t_k[-2], scaled_laplacian))

    return sparse_to_tuple(t_k)

'''
adj is a csr_matrix
'''
def get_col_indices(adj):
    print(adj.shape)
    col_indices = []
    coo_adj = adj.tocoo()
    row_starts = []
    pre_row = -1
    for i in range(len(coo_adj.row)):
        if not coo_adj.row[i] == pre_row:
            row_starts.append(i)
            pre_row = coo_adj.row[i]
    row_starts.append(len(coo_adj.row))
    assert len(row_starts) == adj.shape[0] + 1
    for i in range(adj.shape[0]):
        col_indices.append(coo_adj.col[row_starts[i]: row_starts[i + 1]])
    return col_indices

def sample_neighbors(col_indices, features, count=1):
    neighbor_features = []
    for i in range(count):
        one_neighbor_features = np.zeros(features.shape, dtype=features.dtype)
        for j in range(features.shape[0]):
            offset = np.random.randint(len(col_indices[j]))
            nei_col = col_indices[j][offset]
            one_neighbor_features[j, :] = copy(features[nei_col, :])
        neighbor_features.append(one_neighbor_features)
    return neighbor_features

def sample_neighbors_id(col_indices, count=1, neighbor_probs=None):
    neighbor_features_id = []
    for i in range(count):
        one_neighbor_features_id = np.zeros(len(col_indices), dtype=np.int32)
        for j in range(len(col_indices)):
            if neighbor_probs is None:
                offset = np.random.randint(len(col_indices[j]))
                one_neighbor_features_id[j] = col_indices[j][offset]
            else:
                one_neighbor_features_id[j] = np.random.choice(
                    col_indices[j], p=neighbor_probs[j]
                )
        neighbor_features_id.append(one_neighbor_features_id)
    return neighbor_features_id

def generate_random_splits(y_train, y_val, y_test, train_mask, val_mask,
                           test_mask, dataset, repeats=5):
    labels = y_train + y_val + y_test

def get_sampling_probability(col_indices, sampling_strategy, dataset):
    if sampling_strategy == 'uniform':
        return None
    elif sampling_strategy == 'degree' or sampling_strategy == 'degree_reverse':
        n = len(col_indices)

        # calculate degree of each node
        degrees = np.zeros(n, dtype=np.float32)
        for i in range(n):
            degrees[i] = len(col_indices[i])

        # generate probability of each node
        sampling_probabilities = []
        for i in range(n):
            neighbor_degrees = copy(degrees[col_indices[i]])
            if sampling_strategy == 'degree_reverse':
                neighbor_degrees = np.reciprocal(neighbor_degrees)
            neighbor_degrees = neighbor_degrees / np.sum(neighbor_degrees)
            sampling_probabilities.append(neighbor_degrees)
        return sampling_probabilities
    elif sampling_strategy == 'pagerank':
        pr_scores = np.load("data/" + dataset + '.prs.npy')
        n = len(col_indices)
        assert n == len(pr_scores), \
            'mismatch lengths of page rank score and graph nodes' + \
            str(n) + ':::' + len(pr_scores)

        # generate probability of each node
        sampling_probabilities = []
        for i in range(n):
            neighbor_scores = copy(pr_scores[col_indices[i]])
            sum_scores = np.sum(neighbor_scores)
            if abs(sum_scores) < 1e-9:
                neighbor_scores = np.ones(neighbor_scores.shape,
                                          dtype=np.float32) / neighbor_scores.shape[0]
            else:
                neighbor_scores = neighbor_scores / np.sum(neighbor_scores)
            sampling_probabilities.append(neighbor_scores)
        return sampling_probabilities
    else:
        raise ValueError('unexpected sampling strategy:' + sampling_strategy)


from collections import deque

def BFS():
    while location:
        x,y = location.popleft()
        for i in range(4):
            pre_x = x + dx[i]
            pre_y = y + dy[i]
            if 0 <= pre_x < M and 0 <= pre_y < N and maze[pre_y][pre_x] == 1:
                maze[pre_y][pre_x] = maze[y][x] + 1
                location.append((pre_x,pre_y))
    return

N,M = map(int,input().split())

maze = [list(map(int,input())) for i in range(N)]

location = deque()
location.append((0,0))
dx = [0,0,-1,1] #상하좌우
dy = [-1,1,0,0]

BFS()
print(maze[N-1][M-1])


# 9.4 Write a program to read through the mbox-short.txt
# and figure out who has sent the greatest number of mail messages.
# The program looks for 'From ' lines
# and takes the second word of those lines as the person who sent the mail.
# The program creates a Python dictionary that maps the sender's mail
# address to a count of the number of times they appear in the file.
# After the dictionary is produced, the program reads through the
# dictionary using a maximum loop to find the most prolific committer.

# a program to read through the mbox-short.txt
name = input("Enter file:")
if len(name) < 1:
    name = "mbox-short.txt"
handle = open(name, 'r')
counts = dict()
list2 = list()

# The program looks for 'From ' lines
for line in handle:
    line = line.rstrip()
    if line.startswith('From '):
        line = (line.rstrip()).split(" ")

# and takes the second word of those lines as the person who sent the mail.
        list2.append(line[1])

# The program creates a Python dictionary that maps
# the sender's mail address to a count of the number of
# times they appear in the file.
for word in list2:
    counts[word] = counts.get(word, 0) + 1

# After the dictionary is produced, the program
# reads through the dictionary using a maximum
# loop to find the most prolific committer.
maximum = None
for sender, number in counts.items():
    if maximum is None:
        maximum = number
    else:
        if number > maximum:
            maximum = number
for sender, number in counts.items():
    if maximum == number:
        print(sender, number)

Coursera_Python for everybody/Cours2_Python Data Structures/Cours2_Week5.py


import numpy as np
import matplotlib.pyplot as plt
import pickle
import random as rd
import compute_grad_num as num_grad
import pandas as pd

# Set random seed to get reproducable results
np.random.seed(400)

"Loads the CIFAR-10 data"
# returns:
#	X = image pixel data (d x N)
#	Y = one-hot class representation for each image (K x N)
# 	y = label for each image 0-9 (1 x N)
def read_data(file_name):
	path = 'Datasets/cifar-10/'
	file = path + file_name
	with open(file, 'rb') as fo:
		dict = pickle.load(fo, encoding='bytes')

	X = dict[b'data']
	#transpose X to get d x N matrix
	X = X.T
	# normalize to get pixel values in [0,1]
	X = X / 255

	# zero center the data
	x_mean = np.mean(X, axis=1)
	x_mean = x_mean.reshape((len(x_mean), 1))
	X = X - x_mean

	y = np.array(dict[b'labels'])

	#get one-hot encoding of labels
	Y = np.zeros((10, len(X[0])))
	for i, label in enumerate(y):
		Y[label][i] = 1

	return np.array(X), np.array(Y), np.array(y)

"Init model parameters"
# initialize weights and biases to have Gaussian random values
# with mean = mu and standard deviation = std
# returns:
#	theta = [W_1, b_1, W_2, b_2]
#	W_1 = weight matrices of size m x d
#	W_2 = weight matrices of size K x m
#	b_1 = bias vectors of length m
#	b_2 = bias vectors of length K
def init_model_params(m, d, K, mu = 0, std = 1e-3, xavier = True):

	if(xavier):
		W_1 = np.random.normal(mu, 2/np.sqrt(d), (m,d))
		b_1 = np.random.normal(mu, 2/np.sqrt(m), m)
		W_2 = np.random.normal(mu, 2/np.sqrt(m), (K, m))
		b_2 = np.random.normal(mu, 2/np.sqrt(K), K)
	else:
		W_1 = np.random.normal(mu, std, (m,d))
		b_1 = np.zeros(m)
		W_2 = np.random.normal(mu, std, (K, m))
		b_2 = np.zeros(K)

	theta = [W_1, b_1, W_2, b_2]

	return theta

"Calculate softmax"
# returns:
#	normalized exponential values of vector s
def softmax(s):
	return np.exp(s)/np.sum(np.exp(s), axis=0)

"Perform a forward pass"
# evaluates the network function,
# X = the dataset
# theta = [W_1, b_1, W_2, b_2]
# ----------------------------
# s_1 = W_1X + b_1 (m x N)
# h = max(0, s_1) (ReLu activation function)
# s = W_2h + b2 = ([K x m] x [m x N] => K x N)
# i.e softmax(W_2 x max(0, [WX + b]))
# returns:
#	P - (K x N) matrix of the probabilities for each class
# 	s_1, h for gradient computations
def forward_pass(X, theta):
	b_1 = theta[1].reshape((len(theta[1]), 1))
	b_2 = theta[3].reshape((len(theta[3]), 1))

	s_1 = np.matmul(theta[0], X)
	s_1 = np.add(s_1, b_1)
	h = np.maximum(0, s_1)
	s = np.matmul(theta[2], h)
	s = np.add(s, b_2)
	P = softmax(s)

	return P, h

"Compute the cost function, J (cross entropy loss)"
# returns:
#	J = sum of cross entropy loss for the network
#	+ a l_2-regulizer
# returns:
#	The cost of the model, J
def compute_cost(X, Y, theta, lambda_reg):
	D = len(X[0])
	P, h = forward_pass(X, theta)
	l_cross = 0
	for data_point in range(D):
		l_cross -= np.log(np.dot(Y[:,data_point], P[:,data_point]))

	reg_term_1 = (np.square(theta[0])).sum()
	reg_term_2 = (np.square(theta[2])).sum()
	reg_term = reg_term_1 + reg_term_2

	J = (l_cross/D) + (lambda_reg * reg_term)

	return J

"Compute accuracy"
# calc ratio between correct predictions and total
# number of predictions
# returns:
#	The accuracy of the model, acc (correctly classified/samples)
def compute_accuracy(X, y, theta):
	p, h = forward_pass(X, theta)
	# get columnwise argmax
	p_star = np.argmax(p, axis=0)
	correct = np.sum(p_star == y)
	acc = correct/len(p_star)

	return acc

"Computes the weight and bias gradients"
# returns:
#	The gradients, [grad_W_1, grad_W_2], [grad_b_1, grad_b_2]
def compute_gradients(X, Y, P, theta, h, lambda_reg):

	D = len(X[0])

	grad_W_1 = np.zeros(theta[0].shape)
	grad_b_1 = np.zeros(theta[1].shape)
	grad_W_2 = np.zeros(theta[2].shape)
	grad_b_2 = np.zeros(theta[3].shape)

	for data_point in range(D):
		g = -(Y[:, data_point]-P[:, data_point]).T
		h_i = h[:, data_point]
		h_i = h_i.reshape((1, len(h_i)))
		x_i = X[:, data_point]
		x_i = x_i.reshape((len(x_i), 1))

		grad_b_2 += g

		g = g.reshape((1, len(g)))

		grad_W_2 += np.matmul(g.T, h_i)

		g = np.matmul(g, theta[2])

		h_i[h_i > 0] = 1
		g = np.matmul(g, np.diag(h_i[0]))[0,:]

		grad_b_1 += g
		g = g.reshape((len(g), 1))
		grad_W_1 += np.matmul(g, x_i.T)

	grad_b_1 = (1/D) * grad_b_1
	grad_b_2 = (1/D) * grad_b_2
	grad_W_1 = (1/D) * grad_W_1 + 2*lambda_reg*theta[0]
	grad_W_2 = (1/D) * grad_W_2 + 2*lambda_reg*theta[2]

	return  [grad_W_1, grad_W_2], [grad_b_1, grad_b_2]

"Performs mini-batch gradient decent"
# GD_params = [eta, n_batch, n_epochs]
# X = Y = [train, validation]
# W, b = the initialized weight matrix and bias vector
# lambda_reg = the panalizing factor for l2-regularization
# i = which parameter setting (used for the plotting)
# plot = boolean for plotting
# returns:
#	W_star, b_star = the updated weight matrix and bias vector
def mini_batch_GD(X, Y, GD_params, theta, lambda_reg, i = 1, rho = 0.9, plot = True, momentum = True):
	batches_X, batches_Y = generate_batches(X[0], Y[0], GD_params[1])
	W_star_1 = theta[0]
	b_star_1 = theta[1]
	W_star_2 = theta[2]
	b_star_2 = theta[3]

	decay_rate = 0.95
	eta = GD_params[0]

	mom_W_1 = np.zeros((W_star_1.shape))
	mom_b_1 = np.zeros((b_star_1.shape))
	mom_W_2 = np.zeros((W_star_2.shape))
	mom_b_2 = np.zeros((b_star_2.shape))

	theta_star = [W_star_1, b_star_1, W_star_2, b_star_2]
	if(plot):
		train_cost = np.zeros(GD_params[2] + 1)
		val_cost = np.zeros(GD_params[2] + 1)

	best_theta = theta_star
	min_val = 1000
	check = 0

	for epoch in range(GD_params[2]):
		if(epoch % 10 == 0):
			print("epoch: ", epoch)

		if(plot):
			train_cost[epoch] = compute_cost(X[0], Y[0], theta_star, lambda_reg)
			val_cost[epoch] = compute_cost(X[1], Y[1], theta_star, lambda_reg)

		for batch in range(GD_params[1]):
			X_batch = batches_X[:,:,batch].T
			Y_batch = batches_Y[:,:,batch].T

			P, h = forward_pass(X_batch, theta_star)

			grad_W, grad_b = compute_gradients(X_batch, Y_batch, P, theta_star, h, lambda_reg)

			if(momentum):
				mom_W_1 = (rho * mom_W_1) + (GD_params[0] * grad_W[0])
				mom_b_1 = (rho * mom_b_1) + (GD_params[0] * grad_b[0])
				mom_W_2 = (rho * mom_W_2) + (GD_params[0] * grad_W[1])
				mom_b_2 = (rho * mom_b_2) + (GD_params[0] * grad_b[1])

				W_star_1 = W_star_1 - mom_W_1
				W_star_2 = W_star_2 - mom_W_2
				b_star_1 = b_star_1 - mom_b_1
				b_star_2 = b_star_2 - mom_b_2
			else:
				W_star_1 = W_star_1 - (GD_params[0] * grad_W[0])
				W_star_2 = W_star_2 - (GD_params[0] * grad_W[1])
				b_star_1 = b_star_1 - (GD_params[0] * grad_b[0])
				b_star_2 = b_star_2 - (GD_params[0] * grad_b[1])

			theta_star = [W_star_1, b_star_1, W_star_2, b_star_2]

		v_cost = compute_cost(X[1], Y[1], theta_star, lambda_reg)

		if(v_cost < min_val):
			min_val = v_cost
			best_theta = theta_star
			check = 0
		else:
			check += 1

		if(check == 5):
			eta = eta/2
		elif(check > 10):
			return best_theta

		if(plot):
			t_cost = compute_cost(X[0], Y[0], theta_star, lambda_reg)
			#v_cost = compute_cost(X[1], Y[1], theta_star, lambda_reg)

			# If weights are very small => log(0) in compute cost => inf/NaN
			if(np.isnan(t_cost) or np.isinf(t_cost)):
				t_cost = train_cost[epoch - 1]
			if(np.isnan(v_cost) or np.isinf(v_cost)):
				v_cost = val_cost[epoch - 1]

			train_cost[epoch + 1] = t_cost
			val_cost[epoch + 1] = v_cost

		if(momentum):
			eta = eta * decay_rate

	if(plot):
		plot_cost(train_cost, val_cost, i, GD_params[0])

	return best_theta

"Generates the batches to use for mini-batch GD"
# X, Y = the data and labels (one-hot encoded)
# n_batch = how many batches to use
# returns:
#	batches_X,  batches_Y = arrays containging the batches
def generate_batches(X, Y, n_batch):
	batch_size = int(len(X[0])/n_batch)

	batches_X = np.zeros((batch_size, len(X), n_batch))
	batches_Y = np.zeros((batch_size, len(Y), n_batch))

	for i in range(batch_size):
		start = i*n_batch
		end = (i+1)*n_batch
		batches_X[i] = X[:,start:end]
		batches_Y[i] = Y[:,start:end]

	return batches_X, batches_Y

"Compares the analytical and numerical gradients"
# X, Y = the data and labels (one-hot encoded)
# lambda_reg = the panalizing factor for l2-regularization
# h = the small shift used for numerical gradients
# slow = boolean for using the slow numerical gradient
# check_size = number of data points used for computing the gradients
def check_grad(X, Y, lambda_reg, h_diff = 1e-5, check_size = 10, dim_size = 3072):
	X = X[:dim_size,:check_size]
	Y = Y[:dim_size,:check_size]

	K = len(Y)
	d = len(X)
	m = 50

	theta = init_model_params(m, d, K)

	num_grad_W, num_grad_b = grad_slow(X, Y, theta, lambda_reg, h_diff)

	P, h = forward_pass(X, theta)
	grad_W, grad_b = compute_gradients(X, Y, P, theta, h, lambda_reg)
	epsilon = 1e-10

	comp_W_1 = np.zeros(theta[0].shape)
	comp_b_1 = np.zeros(theta[1].shape)
	comp_W_2 = np.zeros(theta[2].shape)
	comp_b_2 = np.zeros(theta[3].shape)

	abs_W_1 = np.zeros(theta[0].shape)
	abs_b_1 = np.zeros(theta[1].shape)
	abs_W_2 = np.zeros(theta[2].shape)
	abs_b_2 = np.zeros(theta[3].shape)

	comp_W = [comp_W_1, comp_W_2]
	comp_b = [comp_b_1, comp_b_2]

	abs_W = [abs_W_1, abs_W_2]
	abs_b = [abs_b_1, abs_b_2]

	errors = np.zeros((6,2))

	for layer in range(2):
		for i in range(len(comp_W[layer])):
			for j in range(len(comp_W[layer][0])):
				comp_W[layer][i][j] = abs(grad_W[layer][i][j] - \
					num_grad_W[layer][i][j])/max(epsilon, abs(grad_W[layer][i][j]) \
					 + abs(num_grad_W[layer][i][j]))
				abs_W[layer][i][j] = abs(num_grad_W[layer][i][j]) - abs(grad_W[layer][i][j])

		for i in range(len(comp_b[layer])):
			comp_b[layer][i] = abs(num_grad_b[layer][i] - \
				grad_b[layer][i])/max(epsilon, abs(num_grad_b[layer][i]) + abs(grad_b[layer][i]))
			abs_b[layer][i] = abs(num_grad_b[layer][i]) - abs(grad_b[layer][i])

		print("layer: ", layer + 1)
		for i in range(len(num_grad_b[layer])):
			print("n: ", num_grad_b[layer][i], "a: ", grad_b[layer][i])

		tol_error = 1e-6 if layer == 0 else 1e-6

		print("---------------", layer + 1, "---------------")
		print("max relError W", layer + 1, ":", np.max(comp_W[layer]))
		print("min relError W", layer + 1, ":", np.min(comp_W[layer]))
		print("max relError b", layer + 1, ":", np.max(comp_b[layer]))
		print("min relError b", layer + 1, ":", np.min(comp_b[layer]))
		print("Tollerated error: ", tol_error)
		print("# wrong W", layer + 1, ":", np.sum(comp_W[layer] > tol_error), "/", comp_W[layer].size)
		print("# wrong b", layer + 1, ":", np.sum(comp_b[layer] > tol_error), "/", comp_b[layer].size)

"Compute numerical gradients (SLOW)"
def grad_slow(X, Y, theta, lambda_reg, h=1e-6):

	grad_W_1 = np.zeros(theta[0].shape)
	grad_b_1 = np.zeros((len(theta[1]), 1))
	grad_W_2 = np.zeros(theta[2].shape)
	grad_b_2 = np.zeros((len(theta[3]), 1))

	for i in range(len(theta[1])):
		b_try = np.copy(theta[1])
		b_try[i] = b_try[i] - h
		c1 = compute_cost(X, Y, [theta[0], b_try, theta[2], theta[3]], lambda_reg)

		b_try = np.copy(theta[1])
		b_try[i] = b_try[i] + h
		c2 = compute_cost(X, Y, [theta[0], b_try, theta[2], theta[3]], lambda_reg)

		grad_b_1[i] = (c2-c1) / (2*h)

	for i in range(len(theta[3])):
		b_try = np.copy(theta[3])
		b_try[i] = b_try[i] - h
		c1 = compute_cost(X, Y, [theta[0], theta[1], theta[2], b_try], lambda_reg)

		b_try = np.copy(theta[3])
		b_try[i] = b_try[i] + h
		c2 = compute_cost(X, Y, [theta[0], theta[1], theta[2], b_try], lambda_reg)

		grad_b_2[i] = (c2-c1) / (2*h)

	for i in range(len(theta[0])):
		for j in range(len(theta[0][0])):
			W_try = np.copy(theta[0])
			W_try[i][j] = W_try[i][j] - h
			c1 = compute_cost(X, Y, [W_try, theta[1], theta[2], theta[3]], lambda_reg)

			W_try = np.copy(theta[0])
			W_try[i][j] = W_try[i][j] + h;
			c2 = compute_cost(X, Y, [W_try, theta[1], theta[2], theta[3]], lambda_reg);

			grad_W_1[i][j] = (c2-c1) / (2*h);

	for i in range(len(theta[2])):
		for j in range(len(theta[2][0])):
			W_try = np.copy(theta[2])
			W_try[i][j] = W_try[i][j] - h
			c1 = compute_cost(X, Y, [theta[0], theta[1], W_try, theta[3]], lambda_reg)

			W_try = np.copy(theta[2])
			W_try[i][j] = W_try[i][j] + h;
			c2 = compute_cost(X, Y, [theta[0], theta[1], W_try, theta[3]], lambda_reg);

			grad_W_2[i][j] = (c2-c1) / (2*h);

	return [grad_W_1, grad_W_2], [grad_b_1, grad_b_2]

"Plots the training and validation cost as a function of epochs"
def plot_cost(train_cost, val_cost, eta, ind = 1):
	colors = ["green", "red", "yellow", "blue", "black"]
	plt.xlabel("Epochs")
	plt.ylabel("Cost")
	epochs = len(train_cost)
	X = np.linspace(0,epochs,epochs)
	#plt.axis([0, epochs, 1, 3])
	plt.plot(X, train_cost, color = "green", label="Training")
	plt.plot(X, val_cost, color = "red", label="Validation")
	plt.legend()
	plt.savefig("cost_plot_ALL_30E" + str(ind) + ".png")
	plt.close()

"Visualizes the final weight representations"
def plot_weights(W, ind):
	plt.figure(figsize=(8.4, 2))
	for i, w in enumerate(W):
		plt.subplot(1, 10, i + 1)
		im = w.reshape((3, 32, 32))
		im = im.transpose((1,2,0))
		im = ((im - np.min(im))/(np.max(im) - np.min(im)))
		plt.imshow(im)
		plt.xticks(())
		plt.yticks(())
	plt.suptitle("Visualization of the weight matrix", fontsize=16)
	plt.subplots_adjust(0.08, 0.02, 0.92, 0.85, 0.08, 0.23)
	plt.savefig("weight_plot_" + str(ind) + ".png")
	plt.close()
	print("Weight visualization", ind, "completed")

def grad_check():
	X, Y, train_y = read_data('data_batch_1')
	lambda_reg = 0
	check_grad(X, Y, lambda_reg, check_size = 10, dim_size=100)

def gen_rand_etas(e_min, e_max, samples):
	e = e_min + (e_max - e_min) * np.random.uniform(0,1,samples)
	etas = np.power(10, e)

	return np.sort(etas)

def gen_rand_lambdas(e_min, e_max, samples):
	e = e_min + (e_max - e_min) * np.random.uniform(0,1,samples)
	lambdas = np.power(10, e)

	return np.sort(lambdas)

def main():
	# train_X, train_Y, train_y = read_data('data_batch_1')
	# val_X, val_Y, val_y = read_data('data_batch_2')
	# test_X, test_Y, test_y = read_data('test_batch')

	#Use all batches of data
	X_1, Y_1, y_1 = read_data('data_batch_1')
	X_2, Y_2, y_2 = read_data('data_batch_2')
	X_3, Y_3, y_3 = read_data('data_batch_3')
	X_4, Y_4, y_4 = read_data('data_batch_4')
	X_5, Y_5, y_5 = read_data('data_batch_5')
	#val_X, val_Y, val_y = read_data('data_batch_2')
	test_X, test_Y, test_y = read_data('test_batch')

	val_for_train = 0

	train_X = np.array([np.concatenate((X_1, X_2, X_3, X_4, X_5[:,:val_for_train]), axis = 1)]).reshape((3072, (40000 + val_for_train)))
	train_Y = np.array([np.concatenate((Y_1, Y_2, Y_3, Y_4, Y_5[:,:val_for_train]), axis = 1)]).reshape((10, (40000 + val_for_train)))
	val_X = X_5[:,val_for_train:]
	val_Y = Y_5[:,val_for_train:]

	K = len(train_Y)
	d = len(train_X)
	m = 50

	# theta = [W_1, b_1, W_2, b_2]
	theta = init_model_params(m, d, K, xavier = False)

	# Use only a subset of batch_1
	# subset_size = 100
	# train_X = train_X[:, :subset_size]
	# train_Y = train_Y[:, :subset_size]
	# val_X = val_X[:, :subset_size]
	# val_Y = val_Y[:, :subset_size]

	X = [train_X, val_X]
	Y = [train_Y, val_Y]

	n_epochs = 80
	n_batch = 100

	# lambdas = gen_rand_lambdas(-6, -4, 10)
	# etas = gen_rand_etas(-1.7, -1.15, 10)

	lambdas = [1e-5]
	etas = [0.03]

	accuracy = np.zeros((len(lambdas), len(etas)))

	for lamb in range(len(lambdas)):
		print("lambda: ", lamb + 1, "/", len(lambdas))
		for e in range(len(etas)):
			print("eta: ", e + 1, "/", len(etas))
			lambda_reg = lambdas[lamb]
			eta = etas[e]
			GD_params = [eta, n_batch, n_epochs]

			P, h = forward_pass(train_X, theta)

			theta_star = mini_batch_GD(X, Y, GD_params, theta, lambda_reg)
			v_acc = compute_accuracy(test_X, test_y, theta_star)
			print("acc:", v_acc)
			accuracy[lamb][e] = v_acc

	print("Accuracy:")
	for i, lamb in enumerate(lambdas):
		print("------ lambda = ", lamb, "-----------")
		for j, eta in enumerate(etas):
			print("eta = ", eta, " accuracy = ", accuracy[i][j])

	# df_acc = pd.DataFrame(accuracy, index=lambdas, columns=etas)
	# df_acc.to_csv("grid_search.csv", sep = ',')

main()
#grad_check()


# Notes v1.0:
# Tested with Python 3.6.5 on Windows 10.

import tkinter
from tkinter import ttk, scrolledtext, filedialog, messagebox
from tkinter import *

# initialise a window
window = Tk()
window.title('Notes')
window.geometry('960x540')

# use the ScrolledText widget
scr = scrolledtext.ScrolledText(window, wrap = tkinter.WORD)
scr.pack(side = LEFT, fill = BOTH, expand = YES)

class menu_items():
    def __init__(self, scr):
        self.scr = scr
        self.filename = None
        self.is_saved = False

    def _new(self):
        scr.delete('1.0', END)
        self.filename = None

    def _open(self):
        filename =  filedialog.askopenfilename(title = 'Select file', filetypes = (('Text Documents','*.txt'),('All Files','*.*')))
        if filename:
            f = open(filename, 'r+')
            text = f.read()
            f.close()
            scr.delete('1.0', END)
            scr.insert(tkinter.INSERT, text)
            self.filename = filename

    def _save(self):
        if self.filename:
            try:
                f = open(self.filename, 'w')
            except FileNotFoundError as error:
                print('The file name was not specified')
                self.is_saved = False
            else:
                contents = scr.get(1.0, END)
                f.write(contents)
                f.close
                self.is_saved = True
        else:
            filename =  filedialog.asksaveasfilename(title = 'Select file',filetypes = (('Text Documents','*.txt'),('All Files','*.*')), defaultextension = '.txt')
            try:
                f = open(filename, 'w')
                self.filename = filename
            except FileNotFoundError as error:
                print('The file name was not specified')
                self.is_saved = False
            else:
                contents = scr.get(1.0, END)
                f.write(contents)
                f.close
                self.is_saved = True

    def _saveas(self):
        filename =  filedialog.asksaveasfilename(title = 'Select file',filetypes = (('Text Documents','*.txt'),('All Files','*.*')), defaultextension = '.txt')
        if filename:
            f = open(filename, 'w')
            contents = scr.get(1.0, END)
            f.write(contents)
            f.close

    def _quit(self):
# Save confirm from https://www.programcreek.com/python/example/57788/tkMessageBox.askyesnocancel
        if scr.compare("end-1c", "!=", "1.0"):
            message = "Do you want to save %s before closing?" % (
            self.filename or "this untitled document")
            confirm = messagebox.askyesnocancel(
            title="Save On Close",
            message=message,
            default=messagebox.YES,
            parent=self.scr)
            if confirm:
                self._save()
                if not self.is_saved:
                   return
            elif confirm is None:
                return
            else:
                reply = "no"
        window.quit()
        window.destroy()
        exit()

    def _about(self):
        messagebox.showinfo('Notes v1.0', 'Tested with Python 3.6.5 on Windows 10. \nAuthor: Madis Võrklaev')

# create a menu bar
t = menu_items(scr)
menuBar = tkinter.Menu(window)
window.config(menu = menuBar)
fileMenu = tkinter.Menu(menuBar, tearoff = 0)
fileMenu.add_command(label = 'New', command = t._new)
fileMenu.add_command(label = 'Open', command = t._open)
fileMenu.add_command(label = 'Save', command = t._save)
fileMenu.add_command(label = 'Save as', command = t._saveas)
menuBar.add_cascade(label = 'File', menu = fileMenu)
fileMenu.add_separator()
fileMenu.add_command(label = 'Exit', command = t._quit)

helpMenu = tkinter.Menu(menuBar, tearoff = 0)
helpMenu.add_command(label ='About', command = t._about)
menuBar.add_cascade(label = 'Help', menu = helpMenu)

# Start the main window (root window)
window.mainloop()


from pyspark.ml import Pipeline
from pyspark.ml.classification import RandomForestClassifier as RF
from pyspark.ml.feature import StringIndexer, VectorIndexer, VectorAssembler, SQLTransformer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator, BinaryClassificationEvaluator
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
import numpy as np
import functools
from pyspark.ml.feature import OneHotEncoder
from pyspark.ml.clustering import KMeans
from numpy.linalg import norm

def getOutliers(df, opt_k, dist_cutoff):

	#Getting categorical columns
	column_vec_in = []
	for (a,b) in df.dtypes:
		if b=="string":
				column_vec_in.append(a)

	column_vec_out = [x + '_catVec' for x in column_vec_in]
	cols_now = [x for x in df.columns if x not in column_vec_in]
	cols_now += column_vec_out

	indexers = [StringIndexer(inputCol=x, outputCol=x+'_tmp') for x in column_vec_in ]
	encoders = [OneHotEncoder(dropLast=False, inputCol=x+"_tmp", outputCol=y) for x,y in zip(column_vec_in, column_vec_out)]
	tmp = [[i,j] for i,j in zip(indexers, encoders)]
	tmp = [i for sublist in tmp for i in sublist]

	assembler_features = VectorAssembler(inputCols=cols_now, outputCol='features')
	tmp += [assembler_features]

	pipeline = Pipeline(stages=tmp)
	df_t = pipeline.fit(df).transform(df)
	df_t.cache()

	kmeans=KMeans().setK(opt_k).setSeed(0)
	model=kmeans.fit(df_t)

	cluster_centres = model.clusterCenters()
	df_t=model.transform(df_t)

	cluster_nos=df_t.select('prediction').rdd.flatMap(lambda x:x).collect()
	outliers = []

	row_points = df_t.select('features').rdd.flatMap(lambda x:x).collect()

	#Calculating distances to cluster centres
	for i in range(df.count()):
		a = np.array(row_points[i])
		b = np.array(cluster_centres[cluster_nos[i]])
		if norm(a-b)>dist_cutoff:
			outliers.append(i)

	return outliers


#!/usr/bin/python3
from daemon import DaemonContext
from lockfile import FileLock
import time
import logging
from mqtt2serial import main

class MqttApp(object):
    def __init__(self):
        self.pidfile_path = '/var/run/mqtt2seriald.pid'
        self.pidfile_timeout = 10

    def run(self):
        logging.basicConfig(format="%(asctime)s [ARDUINO/%(processName)s] %(levelname)s %(message)s",
                            filename='/var/log/nagf2rpi/mqtt2seriald.log',
                            level=logging.DEBUG)
        logging.info("Starting")
        try:
            logging.info("before main")
            main(logging)
        except(SystemExit, KeyboardInterrupt):
            logging.info("Exiting normally...")
        except:
            logging.exception("some exception")

app = MqttApp()
with DaemonContext(pidfile=FileLock(app.pidfile_path, app.pidfile_timeout)):
    app.run()


# -*- coding: utf-8 -*-
""" Script to crawl Article from thejointblog.com

the-joint-blog
"""
try:
    from crawler import BaseCrawler
except:
    from __init__ import BaseCrawler

class TheJointBlogCrawler(BaseCrawler):
    start_urls = [
        'https://thejointblog.com/category/news/'
    ]

    # source = 'The Joint Blog'
    source_id = 'the-joint-blog'

    config_selectors = {
        'POST_URLS': '.entry-title > a::attr(href)',
        'ARTICLE_CONTENT': '.entry-content > *:not(.first-child):not(.last-child)',
        'ARTICLE_AUTHOR': '.author [rel=author]::text',
        'ARTICLE_TITLE': 'h1.post-title::text',
    }

    # override parent's class method
    def extract_title(self, selector):
        css = self.config_selectors.get('ARTICLE_TITLE')
        return selector.css(css).extract_first('').strip()

if __name__ == "__main__":
    crawler = TheJointBlogCrawler()
    crawler.run()


#!/usr/bin/python
#-*- coding: utf-8 -*-

from flask import request, jsonify
import base, api

@base.web_app.app.route('/api/transceiver/<key>/', methods=['GET', 'PUT'])
@base.web_app.auth.login_required
def transceiver_mgr_CURD(key):
    if request.method == 'GET':
        resp = api.transceiver.get(key)
    elif request.method == 'PUT':
        resp = api.transceiver.modify(base.parse.parse_json(request.data))
    return jsonify(resp)


import os
import numpy as np
from matplotlib import pyplot as plt

from scipy.stats import gamma

import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
from matplotlib import pyplot as plt

disttype = 'general-gaussian'

model = 'consistency-cifar-1530-0.50.pth'
k = '1530'

# model = 'cohen-cifar-1530-0.50.pth'
# k = '1530'

# model = 'cohen-mnist-380-1.00.pth'
# k = '380'

# model = 'consistency-mnist-380-1.00.pth'
# k = '380'

# model = 'cohen-imagenet-75260-0.50.pth'
# k = '75260'

sigma = '0.5'
N = '50000'
alpha = '0.0005'

params = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.75, 0.8, 0.85, 0.9]
# params = [0.5, 0.6, 0.75, 0.9, 0.95]
# params = [0.5, 0.6]

pAs = dict()

old_radius = dict()

new_radius = dict()

max_radius = dict()

left_param = dict()

right_param = dict()

x = list()
y1 = list()
y2 = list()
y3 = list()

if __name__ == '__main__':
    with open(os.path.join('data/sampling', model, f'{disttype}-{k}-{sigma}-{N}-{alpha}.txt'), 'r') as f:
        for r in f.readlines():
            if r.startswith('o'):
                things = r.split(' ')
                pAs[int(things[1])] = float(things[2])

    with open(os.path.join('data/orig-radius', model, f'orig-rad-{disttype}-{k}-{sigma}-{N}-{alpha}.txt'), 'r') as f:
        for r in f.readlines():
            things = r.split(' ')
            old_radius[int(things[0])] = float(things[1])

    for param in params:
        new_radius[param] = dict()
        with open(os.path.join('data/new-radius', model, f'new-rad-{disttype}-th-{k}-{sigma}-{param}-{N}-{alpha}.txt'), 'r') as f:
            for r in f.readlines():
                things = r.split(' ')
                new_radius[param][int(things[0])] = float(things[1])

    for no in pAs:
        now_max = -1.
        for param in params:
            if new_radius[param][no] > now_max:
                now_max = new_radius[param][no]
        max_radius[no] = now_max

    for no in pAs:
        if max_radius[no] > 1e-5 and max_radius[no] - old_radius[no] > 1e-5:
            for param in params:
                t = new_radius[param][no]
                if max_radius[no] - t < 1e-5:
                    if no not in left_param: left_param[no] = param
                    if no not in right_param: right_param[no] = param
                    left_param[no] = min(left_param[no], param)
                    right_param[no] = max(right_param[no], param)

    for no in pAs:
        if max_radius[no] > 1e-5 and max_radius[no] - old_radius[no] > 1e-5:
            x.append(-np.log(1.0-pAs[no]))
            y1.append(left_param[no])
            y2.append(right_param[no])
            y3.append((left_param[no] + right_param[no]) / 2.)

    plt.figure(figsize=(8, 4), dpi=80)
    # plt.scatter(x, y1)
    # plt.scatter(x, y2)
    plt.errorbar(x, y3, np.array(y2) - np.array(y3), fmt='o')

    plt.xlabel('$-\\ln(1 - P_A)$')

    plt.ylabel('Best Hyperparameter $p$')

    y4 = [max(0.08 * _x + 0.2, 0.5) for _x in sorted(x)]
    # y4 = [1. - np.exp(-_x) - 0.1 for _x in sorted(x)]
    plt.plot(sorted(x), y4)

    plt.savefig('ablation/t_heuristic.pdf')
    # plt.show()


import os
import sys

#
# Complete the gradingStudents function below.
#
def gradingStudents(grades):
    finalgrades = []
    for grade in grades:
        if (grade < 38):
            #print("First if ",grade)
            finalgrades.append(grade)
        else:
            #print("First else ", grade)
            if (grade % 5 <= 2):
                finalgrades.append(grade)
            else:
                finalgrades.append(grade + (5 - (grade % 5)))

    return (finalgrades)

if __name__ == '__main__':
    f = open(os.environ['OUTPUT_PATH'], 'w')

    n = int(input())

    grades = []

    for _ in range(n):
        grades_item = int(input())
        grades.append(grades_item)

    result = gradingStudents(grades)

    f.write('\n'.join(map(str, result)))
    f.write('\n')

    f.close()


from stagesepx.cutter import VideoCutter
from stagesepx.classifier import SVMClassifier
from stagesepx.reporter import Reporter

video_list = [
    '../demo.mp4',
    # 把别的视频也配置在这里即可
]

for each_video_path in video_list:
    cutter = VideoCutter()
    res = cutter.cut(each_video_path)
    stable = res.get_stable_range()
    data_home = res.pick_and_save(stable, 3)
    print(stable)

    # classify
    cl = SVMClassifier()
    cl.load(data_home)
    cl.train()

    # 注意，如果在classify方法指定了范围
    # 那么分析时只会分析处于范围内的帧！
    # 例如，这里只传入了stable的范围，那么非stable范围内的帧都会被忽略掉，标记为 -1
    res = cl.classify(
        each_video_path,
        stable,
        # 步长，可以自行设置用于平衡效率与颗粒度
        # 默认为1，即每帧都检测
        step=1
    )

    # 为了更方便的可读性，stagesepx已经内置了图表绘制功能
    # 你可以直接把分析结果绘制成图表
    report = Reporter()
    # 你可以将把一些文件夹路径插入到报告中
    # 这样你可以很方便地从报告中查看各项相关内容
    # 当然，你需要想好这些路径与报告最后所在位置之间的相对位置，以确保他们能够被访问到
    report.add_dir_link(data_home)

    report.draw(res)


"""
1 make board
2 display board
3 play game
4 check win
    check rows
    check columns
    check diagonals
5 check tie
6 flip player
"""

board = ['-', '-', '-',
         '-', '-', '-',
         '-', '-', '-', ]
# ---------------GLOBAL VARIABLES-----------------

# if game is still going
game_is_going = True

# who's the winner
winner = None

# who's turn is now
current_player = 'X'

def display_board():
    print('    ' + board[0] + ' | ' + board[1] + ' | ' + board[2])
    print('    ' + board[3] + ' | ' + board[4] + ' | ' + board[5])
    print('    ' + board[6] + ' | ' + board[7] + ' | ' + board[8])
    print()

def play_game():
    display_board()

    while game_is_going:

        handle_turn(current_player)

        check_if_game_over()

        flip_player()
        # declare the result
        if winner == 'X' or winner == 'O':
            print(winner, 'won the match.')
        elif winner == 'No one':
            print('OOPS! Tie.')

def handle_turn(player):
    print(player + 's turn.')

    position = int(input('Enter a number from 1-9: '))
    # chek for valid input
    valid = False
    while not valid:
        while position not in [1, 2, 3, 4, 5, 6, 7, 8, 9]:
            position = int(input('INVELID INPUT.Enter a number from 1-9: '))

        position = position - 1

        if board[position] == '-':
            valid = True
        else:
            print('you can not write here !')

    board[position] = player
    display_board()

def check_if_game_over():
    check_for_win()
    check_if_tie()

def check_for_win():
    global winner
    # check rows
    row_winner = check_rows()
    # check columns
    column_winner = check_columns()
    # check diagonals
    diagonal_winner = check_diagonals()
    if row_winner:
        winner = row_winner
    elif column_winner:
        winner = column_winner
    elif diagonal_winner:
        winner = diagonal_winner
    return

def check_rows():
    global game_is_going
    # check if any row has same value
    row_1 = board[0] == board[1] == board[2] != '-'
    row_2 = board[3] == board[4] == board[5] != '-'
    row_3 = board[6] == board[7] == board[8] != '-'
    # if any row has same value, end the game and declare winner
    if row_1 or row_2 or row_3:
        game_is_going = False

    if row_1:
        return board[0]
    elif row_2:
        return board[3]
    elif row_3:
        return board[6]
    return

def check_columns():
    global game_is_going
    # check if any column has same value
    col_1 = board[0] == board[3] == board[6] != '-'
    col_2 = board[1] == board[4] == board[7] != '-'
    col_3 = board[2] == board[5] == board[8] != '-'
    # if any row has same value, end the game and declare winner
    if col_1 or col_2 or col_3:
        game_is_going = False

    if col_1:
        return board[0]
    elif col_2:
        return board[1]
    elif col_3:
        return board[2]
    return

def check_diagonals():
    global game_is_going
    # check if any row has same value
    dia_1 = board[0] == board[4] == board[8] != '-'
    dia_2 = board[2] == board[4] == board[6] != '-'
    # if any row has same value, end the game and declare winner
    if dia_1 or dia_2:
        game_is_going = False

    if dia_1:
        return board[0]
    elif dia_2:
        return board[2]
    return

def check_if_tie():
    global game_is_going
    global winner
    if '-' not in board:
        game_is_going = False
        winner = 'No one'
    return

def flip_player():
    global current_player
    if current_player == 'X':
        current_player = 'O'
    elif current_player == 'O':
        current_player = 'X'
    return

play_game()


#group: 120, name: Eivind Kjosbakken

from Product import Product

class CustomerOrder():
    def __init__(self, customerName : str):
        self.customerName = customerName
        self.order = dict()

#getters and setters
    def getCustomerName(self):
        return self.customerName
    def getOrder(self):
        return self.order
    def setCustomerName(self, name : str):
        self.name = name
    def setOrder(self, order : list):
        self.order = order

    def getMax40FromOrder(self, warehouse):
        """returns a (product, amount) with product and the amount, where total weight <= 40 also avoids filling up customerorder twice (by getting same products as other robots are already getting) """
        productToCarry = None
        amountToCarry = 0
        totalWeight = 0
        currentlyGettingPickedUp = warehouse.getAllProductsGettingPickedUp()
        for product, amount in self.order.items():
            if (product in currentlyGettingPickedUp.keys()):
                amount -= currentlyGettingPickedUp[product]
            if (amount<=0): #if the rest of the product is already getting picked up by other robots
                continue

            productToCarry = product
            productWeight = product.getWeight()
            for i in range(amount):
                if (totalWeight + productWeight > 40):
                    return (productToCarry, amountToCarry)
                totalWeight += productWeight
                amountToCarry+=1
            return (productToCarry, amountToCarry) #only want to run 1 iteration since robot can only carry one type of product at a time
        return None

    def calculateTotalWeight(self):
        """returns total weight of all products in customerOrder"""
        totalWeight = 0
        for product, amount in self.order.items():
            totalWeight += (product.getWeight() * amount)
        return totalWeight

    def addToOrder(self, product : Product):
        """adds a single product to customer order (amount = 1)"""
        if (product in self.order.keys()):
            currentAmount = self.order[product]
            currentAmount+=1
            self.order[product] = currentAmount
            return True
        self.order[product] = 1

    def removeFromOrder(self, product : Product):
        """remove 1 of a product from the CustomerOrder, raises exception if product was not in the order"""
        if (product in self.order.keys()):
            currentAmount = self.order[product]
            if (currentAmount<=0):
                raise Exception("Cant remove product there is nothing from in removeFromOrder")
            currentAmount-=1
            self.order[product] = currentAmount
            if (currentAmount == 0):
                del self.order[product]
        else:
            productName = product
            if (product != None):
                productName = product.getName()
            raise Exception(f"product: {productName} was not in dict")

    def removeFromOrderByName(self, productName : str):
        """if creating new customerOrder, new product objects are created, so have to remove by name (not by object). Raises exception if product was not in the customerOrder"""
        for product in self.order.keys():
            if (str(product.getName()) == productName):
                currentAmount = self.order[product]
                if (currentAmount<=0):
                    raise Exception("Cant remove product there is nothing from in removeFromOrderByName")
                currentAmount -=1
                self.order[product] = currentAmount
                if (currentAmount == 0):
                    del self.order[product]
                return True
        raise Exception(f"product: {productName} was not in dict in removeFromOrderByName")

    def hasOrder(self, warehouseStock : dict):
        """checks if warehouse has the stock to fill the order, returns True if so"""
        for product in self.order.keys():
            if (product not in warehouseStock.keys()):
                return False
            elif (warehouseStock[product] < self.order[product]): #if there is not enough of product in warehouse
                return False
        return True


# Exer-11-Managing-a-context-using-the-with-statement

import csv
import pathlib

some_source = [[2,3,5], [7,11,13], [17,19,23]]
target_path = pathlib.Path('code/test.csv')
with target_path.open('w', newline='') as target_file:
    writer = csv.writer(target_file)
    writer.writerow(['column', 'data', 'headings'])
    for data in some_source:
        writer.writerow(data)

print('finished writing', target_path)

###

try:
    target_path = pathlib.Path('code/test.csv')
    with target_path.open('w', newline='') as target_file:
        writer = csv.writer(target_file)
        writer.writerow(['column', 'headings'])
        for data in some_source:
            writer.writerow(data)
            raise Exception("Just Testing")
except Exception as exc:
    print(target_file.closed)
    print(exc)
print('finished writing', target_path)


""" Defines 'types' that various fields can be """

from typing import Tuple, Union

# Literal only available in python 3.8 + so try import otherwise use extensions
try:
    from typing import Literal
except ImportError:
    from typing_extensions import Literal
from typing_extensions import Annotated

import pydantic
import numpy as np
# from matplotlib.axes import Axes
Axes = None
from shapely.geometry.base import BaseGeometry
from ..log import ValidationError

""" Numpy Arrays """

# type tag default name
TYPE_TAG_STR = "type"

def annotate_type(UnionType):  # pylint:disable=invalid-name
    """Annotated union type using TYPE_TAG_STR as discriminator."""
    return Annotated[UnionType, pydantic.Field(discriminator=TYPE_TAG_STR)]

# generic numpy array
Numpy = np.ndarray

class TypedArrayLike(np.ndarray):
    """A numpy array with a type given by cls.inner_type"""

    @classmethod
    def make_tuple(cls, v):
        """Converts a nested list of lists into a list of tuples."""
        return (
            tuple(cls.make_tuple(x) for x in v)
            if isinstance(v, list)
            else cls.inner_type(v)  # pylint:disable=no-member
        )

    @classmethod
    def __get_validators__(cls):
        """boilerplate"""
        yield cls.validate_type

    @classmethod
    def validate_type(cls, val):
        """validator"""
        # need to fix, doesnt work for simulationdata_export and load?

        if isinstance(val, np.ndarray):
            val_ndims = len(val.shape)
            cls_ndims = cls.ndims  # pylint:disable=no-member

            if (cls_ndims is not None) and (cls_ndims != val_ndims):
                raise ValidationError(
                    "wrong number of dimensions given. " f"Given {val_ndims}, expected {cls_ndims}."
                )
            return cls.make_tuple(val.tolist())

        return tuple(val)

    @classmethod
    def __modify_schema__(cls, field_schema):
        """Sets the schema of ArrayLike."""

        field_schema.update(
            dict(
                title="Array Like",
                description="Accepts sequence (tuple, list, numpy array) and converts to tuple.",
                type="tuple",
                properties={},
                required=[],
            )
        )

class ArrayLikeMeta(type):
    """metclass for Array, enables Array[type] -> TypedArray"""

    def __getitem__(cls, type_ndims):
        """Array[type, ndims] -> TypedArrayLike"""
        desired_type, ndims = type_ndims
        return type("Array", (TypedArrayLike,), {"inner_type": desired_type, "ndims": ndims})

class ArrayLike(np.ndarray, metaclass=ArrayLikeMeta):
    """type of numpy array with annotated type (Array[float], Array[complex])"""

""" Complex Values """

class ComplexNumber(pydantic.BaseModel):
    """Complex number with a well defined schema."""

    real: float
    imag: float

    @property
    def as_complex(self):
        """return complex representation of ComplexNumber."""
        return self.real + 1j * self.imag

class tidycomplex(complex):  # pylint: disable=invalid-name
    """complex type that we can use in our models."""

    @classmethod
    def __get_validators__(cls):
        """Defines which validator function to use for ComplexNumber."""
        yield cls.validate

    @classmethod
    def validate(cls, value):
        """What gets called when you construct a tidycomplex."""

        if isinstance(value, ComplexNumber):
            return value.as_complex
        if isinstance(value, dict):
            c = ComplexNumber(**value)
            return c.as_complex
        return cls(value)

    @classmethod
    def __modify_schema__(cls, field_schema):
        """Sets the schema of ComplexNumber."""
        field_schema.update(ComplexNumber.schema())

""" symmetry """

Symmetry = Literal[0, -1, 1]

""" geometric """

Size1D = pydantic.NonNegativeFloat
Size = Tuple[Size1D, Size1D, Size1D]
Coordinate = Tuple[float, float, float]
Coordinate2D = Tuple[float, float]
Bound = Tuple[Coordinate, Coordinate]
GridSize = Union[pydantic.PositiveFloat, Tuple[pydantic.PositiveFloat, ...]]
Axis = Literal[0, 1, 2]
Axis2D = Literal[0, 1]
Shapely = BaseGeometry
Vertices = Union[Tuple[Coordinate2D, ...], ArrayLike[float, 2]]
PlanePosition = Literal["bottom", "middle", "top"]

""" medium """

# custom medium
InterpMethod = Literal["nearest", "linear"]

# Complex = Union[complex, ComplexNumber]
Complex = Union[tidycomplex, ComplexNumber]
PoleAndResidue = Tuple[Complex, Complex]

# PoleAndResidue = Tuple[Tuple[float, float], Tuple[float, float]]
FreqBoundMax = float
FreqBoundMin = float
FreqBound = Tuple[FreqBoundMin, FreqBoundMax]

""" sources """

Polarization = Literal["Ex", "Ey", "Ez", "Hx", "Hy", "Hz"]
Direction = Literal["+", "-"]

""" monitors """

EMField = Literal["Ex", "Ey", "Ez", "Hx", "Hy", "Hz"]
FieldType = Literal["Ex", "Ey", "Ez", "Hx", "Hy", "Hz"]
FreqArray = Union[Tuple[float, ...], ArrayLike[float, 1]]
ObsGridArray = Union[Tuple[float, ...], ArrayLike[float, 1]]

""" plotting """

#Ax = Axes
Ax = None

""" mode tracking """

TrackFreq = Literal["central", "lowest", "highest"]


"""Draw a square with dashed lines"""

from turtle import Turtle, Screen

# creating turtle
timmy = Turtle()
timmy.shape("turtle")

# moving the turtle
for _ in range(4):
    for _ in range(10):
        timmy.forward(10)
        timmy.penup()
        timmy.forward(10)
        timmy.pendown()
    timmy.right(90)

# creating screen
screen = Screen()
screen.exitonclick()

Basic_Projects/Day 18 (Hirst Painting)/Turtle Challanges/Challenge 2.py


import csv

NUM_JUDGES = 1
PROJECTS_CSV = 'hackillinois.csv'
csv_encoding = 'iso-8859-1'

mappings = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z']
NUM_LETTERS = 14

f = open(PROJECTS_CSV, encoding=csv_encoding, newline='')
c = csv.reader(f)
NUM_PROJECTS = 0
for row in c:
    NUM_PROJECTS += 1
NUM_PROJECTS -= 1

PROJECTS_PER_JUDGE = 117

f.seek(0,0)
next(c)
print(NUM_PROJECTS)
ij = 0

def convert_number_to_table(number):
    letter = int(number / 6)
    let_num = number % 6

    letter = mappings[letter]
    return str(letter) + str(let_num+1)

for i in range(NUM_JUDGES):
    j = open('{}.txt'.format(i), mode='w+')
    for k in range(ij, ij + PROJECTS_PER_JUDGE):
        try:
            proj_name = next(c)[0]
        except StopIteration:
            f.seek(0,0)
            next(c)
            proj_name = next(c)[0]
        table_map = convert_number_to_table(k%117)
        print('{}'.format(table_map), file=j)
    ij += PROJECTS_PER_JUDGE

j.truncate()


import json
import logging
import os
import re
from typing import Any, Dict, List, Optional, Tuple, Union

from neural_compressor.ux.utils.exceptions import InternalException
from neural_compressor.ux.utils.json_serializer import JsonSerializer
from neural_compressor.ux.utils.logger import log
from neural_compressor.ux.utils.templates.metric import Metric
from neural_compressor.ux.utils.utils import get_size

logging.basicConfig(level=logging.INFO)

class WorkloadInfo(JsonSerializer):
    """Create template for workload_list entity."""

    def __init__(
        self,
        request_id: str,
        project_name: Optional[str],
        workload_path: Optional[str],
        model_path: Optional[str],
        input_precision: Optional[str],
        model_output_path: Optional[str],
        output_precision: Optional[str],
        mode: Optional[str],
        metric: Optional[Union[Metric, dict]],
        status: Optional[str],
        code_template_path: Optional[str],
        created_at: Optional[str],
        supports_profiling: Optional[bool],
        execution_details: Optional[Dict[str, dict]] = None,
    ) -> None:
        """Initialize configuration WorkloadInfo class."""
        super().__init__()
        self._id = request_id
        self._project_name = project_name
        self._model_path = model_path
        self._input_precision = input_precision
        self._model_output_path = model_output_path
        self._output_precision = output_precision
        self._mode = mode
        self._workload_path = workload_path
        self._status = status
        self._metric = metric
        self._code_template_path = code_template_path
        self._config_path: Optional[str] = None
        self._log_path: Optional[str] = None
        self._execution_details = execution_details
        self._created_at = created_at
        self._supports_profiling = supports_profiling
        if self._workload_path:
            self._config_path = os.path.join(
                self._workload_path,
                "config.yaml",
            )
            self._log_path = os.path.join(self._workload_path, "output.txt")
            if not os.path.isfile(self._log_path):
                os.makedirs(os.path.dirname(self._log_path), exist_ok=True)
                with open(self._log_path, "w") as log_file:
                    log_file.write("Configuration created.\n")
        if self._model_path and self._metric:
            if isinstance(self._metric, dict) and not self._metric.get("size_input_model"):
                self._metric["size_input_model"] = get_size(self._model_path)
            if isinstance(self._metric, Metric) and not self._metric.size_input_model:
                self._metric.insert_data("size_input_model", str(get_size(self._model_path)))

    def insert_data(self, data: dict) -> None:
        """
        Set all available properties from workload_info dict.

        param: data
        type: dict
        """
        for key, value in data:
            attribute = "_" + key
            if attribute in self.__dict__:
                self.__setattr__(attribute, data[key])

    def serialize(
        self,
        serialization_type: str = "default",
    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
        """
        Serialize class to dict.

        :param serialization_type: serialization type, defaults to "default"
        :type serialization_type: str, optional
        :return: serialized class
        :rtype: Union[dict, List[dict]]
        """
        result = {}
        for key, value in self.__dict__.items():
            variable_name = re.sub(r"^_", "", key)
            if key in self._skip:
                continue
            elif issubclass(type(value), JsonSerializer):
                # pylint: disable=maybe-no-member
                result[variable_name] = value.serialize(serialization_type)
            else:
                result[variable_name] = self.serialize_item(value)

        if result.get("metric", None):
            result.update(result["metric"])
        return result

class WorkloadsListMigrator:
    """Workloads list migrator."""

    def __init__(self) -> None:
        """Initialize workloads list migrator."""
        self.workloads_json = os.path.join(
            os.environ.get("HOME", ""),
            ".neural_compressor",
            "workloads_list.json",
        )
        self.workloads_data: dict = {}
        self.version_migrators: Dict[int, Any] = {
            2: self._migrate_to_v2,
            3: self._migrate_to_v3,
            4: self._migrate_to_v4,
        }

    @property
    def current_version(self) -> int:
        """Get version of current workloads list."""
        self.ensure_workloads_loaded()
        return int(float(self.workloads_data.get("version", 1)))

    @property
    def require_migration(self) -> bool:
        """Check if workloads list require migration."""
        if not os.path.isfile(self.workloads_json):
            log.debug("Workloads list does not exits.")
            return False
        if self.current_version >= max(self.version_migrators.keys()):
            log.debug("Workloads list already up to date.")
            return False
        return True

    def load_workloads_data(self) -> None:
        """Load workloads data from json."""
        with open(self.workloads_json, encoding="utf-8") as workloads_list:
            self.workloads_data = json.load(workloads_list)

    def ensure_workloads_loaded(self) -> None:
        """Make sure that workloads list is loaded."""
        if not self.workloads_data and os.path.isfile(self.workloads_json):
            self.load_workloads_data()

    def dump(self) -> None:
        """Dump workloads information to json."""
        with open(self.workloads_json, "w") as f:
            json.dump(self.workloads_data, f, indent=4)

    def migrate(self) -> None:
        """Migrate workloads list to latest version."""
        self.ensure_workloads_loaded()
        if not self.require_migration:
            return

        migration_steps = range(self.current_version, max(self.version_migrators.keys()))
        for step in migration_steps:
            migration_version = step + 1
            self._migrate_workloads_list(migration_version)

    def _migrate_workloads_list(self, migration_version: int) -> None:
        """Migrate workloads list one version up."""
        migrate_workloads = self.version_migrators.get(migration_version, None)
        if migrate_workloads is None:
            raise InternalException(
                f"Could not migrate workloads list to version {migration_version}",
            )
        migrate_workloads()

    def _migrate_to_v2(self) -> None:
        """Migrate workloads list from v1 to v2."""
        parsed_workload_data = {
            "active_workspace_path": "",
            "workloads": "",
            "version": 2,
        }
        parsed_workload_data.update(
            {"active_workspace_path": self.workloads_data["active_workspace_path"]},
        )
        parsed_workloads = {}
        for workload_id, workload_data in self.workloads_data["workloads"].items():
            parsed_workloads.update({workload_id: self._parse_workload_to_v2(workload_data)})
        parsed_workload_data.update({"workloads": parsed_workloads})
        self.workloads_data = parsed_workload_data

    def _parse_workload_to_v2(self, workload_data: dict) -> dict:
        """Parse workload from v1 to v2."""
        parsed_workload = {
            "input_precision": "fp32",
            "output_precision": "int8",
            "mode": "tuning",
        }
        for key, value in workload_data.items():
            if key == "metric":
                parsed_key, parsed_value = self._parse_metric_to_v2(value)
            elif key == "execution_details":
                parsed_key, parsed_value = self._parse_exec_details_to_v2(value)
            else:
                parsed_key, parsed_value = self._parse_regular_key_to_v2(key, value)
            parsed_workload.update({parsed_key: parsed_value})

        return parsed_workload

    @staticmethod
    def _parse_metric_to_v2(metric_data: dict) -> Tuple:
        """Parse metric from v1 to v2."""
        metric_mappings = {
            "acc_fp32": "acc_input_model",
            "acc_int8": "acc_optimized_model",
            "tuning_time": "optimization_time",
            "size_fp32": "size_input_model",
            "size_int8": "size_optimized_model",
            "perf_throughput_fp32": "perf_throughput_input_model",
            "perf_throughput_int8": "perf_throughput_optimized_model",
        }
        parsed_metric = {}
        for key, value in metric_data.items():
            parsed_key = metric_mappings.get(key, key)
            parsed_metric.update({parsed_key: value})
        return "metric", parsed_metric

    def _parse_exec_details_to_v2(self, execution_details: Optional[dict]) -> Tuple:
        """Parse metric from v1 to v2."""
        parsed_exec_details = {}
        if execution_details is None:
            return "execution_details", None
        if "tuning" in execution_details:
            parsed_exec_details.update(
                {
                    "optimization": {
                        "input_graph": execution_details["tuning"]["model_path"],
                        "input_precision": "fp32",
                        "output_graph": execution_details["tuning"]["model_output_path"],
                        "output_precision": "int8",
                        "framework": execution_details["tuning"]["framework"],
                        "input_nodes": "N/A",
                        "output_nodes": "N/A",
                        "tune": True,
                        "config_path": execution_details["tuning"]["config_path"],
                        "instances": execution_details["tuning"]["instances"],
                        "cores_per_instance": execution_details["tuning"]["cores_per_instance"],
                        "command": execution_details["tuning"]["command"],
                    },
                },
            )
        if "fp32_benchmark" in execution_details:
            parsed_exec_details.update(
                {
                    "input_model_benchmark": {
                        "performance": self._parse_perf_data_to_v2(
                            execution_details["fp32_benchmark"],
                        ),
                    },
                },
            )
        if "int8_benchmark" in execution_details:
            parsed_exec_details.update(
                {
                    "optimized_model_benchmark": {
                        "performance": self._parse_perf_data_to_v2(
                            execution_details["int8_benchmark"],
                        ),
                    },
                },
            )

        return "execution_details", parsed_exec_details

    @staticmethod
    def _parse_perf_data_to_v2(perf_data: dict) -> dict:
        """Perse performance data from v1 to v2."""
        key_mapping = {"datatype": "precision"}
        parsed_perf_data = {}
        for key, value in perf_data.items():
            parsed_key = key
            if key in key_mapping:
                parsed_key = key_mapping[key]
            parsed_perf_data.update({parsed_key: value})
        return parsed_perf_data

    @staticmethod
    def _parse_regular_key_to_v2(key: str, value: Any) -> Tuple:
        """Parse metric from v1 to v2."""
        key_mapping = {
            "fp32": "input_model",
            "int8": "optimized_model",
            "tuning": "optimization",
            "datatype": "precision",
        }
        for old_key, new_key in key_mapping.items():
            if old_key in key:
                key = key.replace(old_key, new_key)
        return key, value

    def _migrate_to_v3(self) -> None:
        """Migrate workloads list from v2 to v3."""
        for workload_id, workload_data in self.workloads_data["workloads"].items():
            workload_data.update(
                {
                    "project_name": os.path.basename(workload_data.get("model_path", "")),
                    "created_at": "2021-07-15T14:19:18.860579",
                },
            )
        self.workloads_data.update(
            {
                "version": 3,
            },
        )

    def _migrate_to_v4(self) -> None:
        """Migrate workloads list from v3 to v4."""
        for workload_id, workload_data in self.workloads_data["workloads"].items():
            workload_data.update(
                {
                    "supports_profiling": False,
                },
            )
        self.workloads_data.update(
            {
                "version": 4,
            },
        )


#!/usr/bin/env python3
# coding=UTF-8
import sys
import os

from wrapt.wrappers import transient_function_wrapper

import pyagxrobots.UGVConfigMsg as UGVBaseMsg
import pyagxrobots.agxrobots as agxrobots
from pyagxrobots.agxbase import ActuatorStateMessageV1, GetRobotStae
from pyagxrobots.agxbase import ActuatorStateMessageV2

class BunkerBase(GetRobotStae):
    def __init__(self) :
        self.basetype='bunker'
        self.motronum= 2
        self.bunkerbase=agxrobots.UGV()
        self.baseversion=self.bunkerbase.base_version
        self.max_linear=3.0
        self.max_angular=2.5235
        self.max_lateral=0.0
        self.min_linear=-self.max_linear
        self.min_angular=-self.max_angular
        self.min_lateral=-self.max_lateral
        if self.baseversion==1:
            self.BunkerActuator1=ActuatorStateMessageV1(motro_id=1)
            self.BunkerActuator2=ActuatorStateMessageV1(motro_id=2)
        elif self.baseversion==2:
            self.BunkerActuator1=ActuatorStateMessageV2(motro_id=1)
            self.BunkerActuator2=ActuatorStateMessageV2(motro_id=2)

    def  SetMotionCommand(self,linear_vel,angular_vel):

        if linear_vel>self.max_linear:
            linear_vel=self.max_linear
        elif linear_vel<self.min_linear:
            linear_vel=self.min_linear
        if angular_vel>self.max_angular:
            angular_vel=self.max_angular
        elif angular_vel<self.min_angular:
            angular_vel=self.min_angular
        self.bunkerbase.SetMotionCommand(linear_vel,angular_vel,0,0)

        # self.version=
    def EnableCAN(self):
        if self.baseversion==2:
            self.bunkerbase.EnableCANCtrl()

class HunterBase(GetRobotStae):
    def __init__(self) :
        self.basetype='hunter'
        self.motronum= 2
        self.bunterbase=agxrobots.UGV()
        self.baseversion=self.bunterbase.base_version
        self.max_linear=1.5
        self.max_angular=0.444
        self.max_lateral=0.0
        self.min_linear=-self.max_linear
        self.min_angular=-self.max_angular
        self.min_lateral=-self.max_lateral
        if self.baseversion==1:
            self.HunterActuator1=ActuatorStateMessageV1(motro_id=1)
            self.HunterActuator2=ActuatorStateMessageV1(motro_id=2)
        elif self.baseversion==2:
            self.HunterActuator1=ActuatorStateMessageV2(motro_id=1)
            self.HunterActuator2=ActuatorStateMessageV2(motro_id=2)
    def  SetMotionCommand(self,linear_vel,angular_vel):
        if self.baseversion==1:
            if linear_vel>self.max_linear:
                linear_vel=self.max_linear
            elif linear_vel<self.min_linear:
                linear_vel=self.min_linear
            if angular_vel>self.max_angular:
                angular_vel=self.max_angular
            elif angular_vel<self.min_angular:
                angular_vel=self.min_angular
        self.bunterbase.SetMotionCommand(linear_vel,0,0,angular_vel)

    def EnableCAN(self):
        if self.baseversion==2:
            self.bunterbase.EnableCANCtrl()

class RangerBase(GetRobotStae):
    def __init__(self) :
        self.basetype='ranger'
        self.motronum= 4
        self.rangerbase=agxrobots.UGV()
        self.baseversion=self.rangerbase.base_version
        if self.baseversion==1:
            self.RangerActuator1=ActuatorStateMessageV1(motro_id=1)
            self.RangerActuator2=ActuatorStateMessageV1(motro_id=2)
            self.RangerActuator3=ActuatorStateMessageV1(motro_id=3)
            self.RangerActuator4=ActuatorStateMessageV2(motro_id=4)
        elif self.baseversion==2:
            self.RangerActuator1=ActuatorStateMessageV2(motro_id=1)
            self.RangerActuator2=ActuatorStateMessageV2(motro_id=2)
            self.RangerActuator3=ActuatorStateMessageV2(motro_id=3)
            self.RangerActuator4=ActuatorStateMessageV2(motro_id=4)

    def  SetMotionCommand(self,linear_vel=0.0,lateral_vel=0.0,angular_vel=0.0,steer_angle=0.0):

        self.rangerbase.SetMotionCommand(linear_vel,angular_vel,lateral_vel,steer_angle)
    def SetLightCommand(self,f_mode,f_value,r_mode,r_value):
        self.rangerbase.SetLightCommand(f_mode,f_value,r_mode,r_value)

       # self.version=
    def EnableCAN(self):
        if self.baseversion==2:
            self.rangerbase.EnableCANCtrl()

class ScoutBase(GetRobotStae):
    def __init__(self):
        self.basetype='scout'
        self.motronum= 4
        self.scoutbase=agxrobots.UGV()
        self.baseversion=self.scoutbase.base_version
        self.max_linear=1.5
        self.max_angular=0.5235
        self.max_lateral=0.0
        self.min_linear=-self.max_linear
        self.min_angular=-self.max_angular
        self.min_lateral=-self.max_lateral
        if self.baseversion==1:
            self.ScoutActuator1=ActuatorStateMessageV1(motro_id=1)
            self.ScoutActuator2=ActuatorStateMessageV1(motro_id=2)
            self.ScoutActuator3=ActuatorStateMessageV1(motro_id=3)
            self.ScoutActuator4=ActuatorStateMessageV2(motro_id=4)
        elif self.baseversion==2:
            self.ScoutActuator1=ActuatorStateMessageV2(motro_id=1)
            self.ScoutActuator2=ActuatorStateMessageV2(motro_id=2)
            self.ScoutActuator3=ActuatorStateMessageV2(motro_id=3)
            self.ScoutActuator4=ActuatorStateMessageV2(motro_id=4)

    def  SetMotionCommand(self,linear_vel=0.0,angular_vel=0.0):
        if self.baseversion==2:
            if linear_vel>self.max_linear:
                linear_vel=self.max_linear
            elif linear_vel<self.min_linear:
                linear_vel=self.min_linear
            if angular_vel>self.max_angular:
                angular_vel=self.max_angular
            elif angular_vel<self.min_angular:
                angular_vel=self.min_angular
        self.scoutbase.SetMotionCommand(linear_vel,angular_vel,0,0)
    def SetLightCommand(self,f_mode,f_value,r_mode,r_value):
        self.scoutbase.SetLightCommand(f_mode,f_value,r_mode,r_value)
        # self.version=

    def EnableCAN(self):
        if self.baseversion==2:
            self.scoutbase.EnableCANCtrl()

class ScoutMiniBase(GetRobotStae):
    def __init__(self):
        self.basetype='scoutMini'
        self.motronum= 4
        self.scoutminibase=agxrobots.UGV()
        self.baseversion=self.scoutminibase.base_version
        self.max_linear=3.0
        self.max_angular=2.5235
        self.max_lateral=2.0
        self.min_linear=-self.max_linear
        self.min_angular=-self.max_angular
        self.min_lateral=-self.max_lateral
        if self.baseversion==1:
            self.ScoutMiniActuator1=ActuatorStateMessageV1(motro_id=1)
            self.ScoutMiniActuator2=ActuatorStateMessageV1(motro_id=2)
            self.ScoutMiniActuator3=ActuatorStateMessageV1(motro_id=3)
            self.ScoutMiniActuator4=ActuatorStateMessageV2(motro_id=4)
        elif self.baseversion==2:
            self.ScoutMiniActuator1=ActuatorStateMessageV2(motro_id=1)
            self.ScoutMiniActuator2=ActuatorStateMessageV2(motro_id=2)
            self.ScoutMiniActuator3=ActuatorStateMessageV2(motro_id=3)
            self.ScoutMiniActuator4=ActuatorStateMessageV2(motro_id=4)

    def  SetMotionCommand(self,linear_vel=0.0,angular_vel=0.0,lateral_velocity=0.0):
        if self.baseversion==1:
            if linear_vel<0:
                linear_vel=0
            elif linear_vel>100:
                linear_vel=100
        elif self.baseversion==2:
            if linear_vel>self.max_linear:
                linear_vel=self.max_linear
            elif linear_vel<self.min_linear:
                linear_vel=self.min_linear
            if angular_vel>self.max_angular:
                angular_vel=self.max_angular
            elif angular_vel<self.min_angular:
                angular_vel=self.min_angular
            if lateral_velocity>self.max_lateral:
                lateral_velocity=self.max_lateral
            elif lateral_velocity<self.min_lateral:
                lateral_velocity=self.min_lateral
        self.scoutminibase.SetMotionCommand(linear_vel,angular_vel,lateral_velocity,0)
    def SetLightCommand(self,f_mode,f_value,r_mode,r_value):
        self.scoutminibase.SetLightCommand(f_mode,f_value,r_mode,r_value)
    def EnableCAN(self):
        if self.baseversion==2:
            self.scoutminibase.EnableCANCtrl()
      # self.version=

class TracerBase(GetRobotStae):
    def __init__(self):
        self.basetype='tracer'
        self.motronum= 2
        self.tracerbase=agxrobots.UGV()
        self.baseversion=self.tracerbase.base_version
        if self.baseversion==1:
            self.TracerActuator1=ActuatorStateMessageV1(motro_id=1)
            self.TracerActuator2=ActuatorStateMessageV1(motro_id=2)
        elif self.baseversion==2:
            self.TracerActuator1=ActuatorStateMessageV2(motro_id=1)
            self.TracerActuator2=ActuatorStateMessageV2(motro_id=2)

    def  SetMotionCommand(self,linear_vel=0.0,angular_vel=0.0):
        self.tracerbase.SetMotionCommand(linear_vel,angular_vel,0,0)
    def SetLightCommand(self,f_mode,f_value):
        self.tracerbase.SetLightCommand(f_mode,f_value)

    def EnableCAN(self):
        if self.baseversion==2:
            self.tracerbase.EnableCANCtrl()
        # self.version=
    # TracerAuator1=ActuatorStateMessageV2(motro_id=1)
    # TracerAuator2=ActuatorStateMessageV2(motro_id=2)


"""Difference"""
def diff(num_n, num_m):
    """Difference"""
    set_a, set_b = set(), set()
    #SET A
    for _ in range(num_n):
        a_member = int(input())
        set_a.add(a_member)
    # SET B
    for _ in range(num_m):
        b_member = int(input())
        set_b.add(b_member)
    #final
    total = sorted(set_a - set_b)
    print(*total)
diff(int(input()), int(input()))


import json
from random import shuffle, randrange

from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect

# Create your views here.
from django.shortcuts import render_to_response, get_object_or_404
from django.views.generic import UpdateView, DetailView
import requests
from board.models import UserProfile, BoardState
from board.tasks import board_iteration

@login_required()
def secret_page(request, *args, **kwargs):
    return HttpResponse('Secret contents!', status=200)

@login_required()
def start_game(request, *args, **kwargs):
    if not request.user.is_superuser:
        return HttpResponse('You should be admin to display this page')

    BoardState.objects.all().delete()

    empty_board_state = {
        "the_end": False,
        "message": "",
        #"last_player": 0,
        "reset": True,
        "players": []
    }

    players = []
    for user_profile in UserProfile.active_gamers.all():
        players.append({
            "name": user_profile.user.username,
            "url": user_profile.url,
            "avatar": "",
            "dice": [],
            "bid": [],
            "active": True,
        })

    shuffle(players)
    for i, gamer in enumerate(players):
        gamer['id'] = i
    empty_board_state['players'] = players

    empy_state_data = {
        'gameplay': [],
        'players': []
    }
    for user in players:
        empy_state_data['players'].append({'id': user['id'], 'name': user['name'], 'dice': 1})

    empty_board_state['last_player'] = len(empy_state_data['players']) - 1

    BoardState.objects.create(iteration=0, board_data=empty_board_state, state_data=empy_state_data)
    board_iteration.delay(iteration=1)
    return HttpResponseRedirect(reverse('board'))

@login_required()
def board(request, *args, **kwargs):
    return render_to_response('board/board.html')

# @login_required()
def board_json(request, pk):
    object = get_object_or_404(BoardState.objects.filter(iteration=pk))
    return HttpResponse(json.dumps(object.board_data))

class UserProfileUpdate(UpdateView):
    model = UserProfile
    fields = ['url']

    def get_object(self, queryset=None):
        if queryset is None:
            queryset = self.get_queryset()
        return queryset.filter(user=self.request.user)[0]

    def form_valid(self, form):
        url = form.cleaned_data['url']
        url = url.rstrip('/')
        try:
            r = requests.get(url + '/ping', timeout=1)
        except:
            messages.add_message(self.request, messages.ERROR,
                                 'Your URL does not reply "pong" on %s/ping [HTTP %d]' % (url, r.status_code))
            return self.form_invalid(form)

        if r.status_code == 200 and r.text.strip() == 'pong':
            messages.add_message(self.request, messages.SUCCESS,
                                 'Ping-Pong OK! URL successfully saved!')
            return super(UserProfileUpdate, self).form_valid(form)
        else:
            messages.add_message(self.request, messages.ERROR,
                                 'Your URL does not reply "pong" on %s/ping [HTTP %d]' % (url, r.status_code))
            return self.form_invalid(form)

    def get_success_url(self):
        return reverse('profile')


#class memorySetting():

global packet_db
packet_db = {}
global lan_hosts
lan_hosts = {}
global destination_hosts
destination_hosts = {}
global tor_nodes
tor_nodes = []
global possible_tor_traffic
possible_tor_traffic = []
global malicious_traffic
possible_mal_traffic = []


"""The LinkedList code from before is provided below.
Add three functions to the LinkedList.
"get_position" returns the element at a certain position.
The "insert" function will add an element to a particular
spot in the list.
"delete" will delete the first element with that
particular value."""

class Element(object):
    def __init__(self, value):
        self.value = value
        self.next = None                # type is Element

class LinkedList(object):
    def __init__(self, head=None):      # head should be Element type
        self.head = head                # first element is head

    def append(self, new_element):      # O(n)
        current = self.head
        if self.head:                   # only if linked list has elements
            while current.next:
                current = current.next
            current.next = new_element
        else:                           # if linked list inisialized without elements
            self.head = new_element

    def get_position(self, position):
        """Get an element from a particular position.
        Assume the first position is "1".
        Return "None" if position is not in the list."""
        current = self.head
        if self.head:                               # only if linked list has elements
            for i in range(1, position):
                if current.next:
                    current = current.next
                else:
                    return None
        return current

    def insert(self, new_element, position):        # O(n)
        """Insert a new node at the given position.
        Assume the first position is "1".
        Inserting at position 3 means between
        the 2nd and 3rd elements."""
        current = self.head
        if self.head:                               # only if linked list has elements
            for i in range(1, position-1):
                if current.next:
                    current = current.next
                else:
                    pass
            else:
                new_element.next = current.next     # firstly map pointer of new_element to next element
                current.next = new_element          # secondly map pointer of current to new_element

    def delete(self, value):                        # O(n)
        """Delete the first node with a given value."""
        current = self.head
        if self.head:                               # only if linked list has elements
            if self.head.value == value:            # check if the head neads to be removed
                self.head = current.next
                return None
            while current:                          # check if any successive element to be removed
                if current.next.value == value:
                    current.next = current.next.next
                current = current.next

# Test cases
if __name__ == '__main__':
    # Set up some Elements
    e1 = Element(1)
    e2 = Element(2)
    e3 = Element(3)
    e4 = Element(4)
    e5 = Element(5)
    e6 = Element(6)
    e7 = Element(7)
    e8 = Element(8)
    e9 = Element(9)
    e10 = Element(10)

    # Start setting up a LinkedList
    ll = LinkedList()
    ll.append(e1)
    ll.append(e2)
    ll.append(e3)
    ll.append(e5)
    ll.append(e6)
    ll.append(e7)

    # Test get_position
    print('Test get_position():')
    print(ll.head.value)                # 1
    print(ll.get_position(1).value)     # also 1
    print(ll.get_position(2).value)     # 2
    print(ll.head.next.next.value)      # 3
    print(ll.get_position(3).value)     # also 3
    print(ll.get_position(4).value)     # 5
    print(ll.get_position(5).value)     # 6
    print(ll.get_position(6).value)     # 7

    # Test insert
    print('Test insert():')
    ll.insert(e4,3)
    print(ll.head.value)                # 1
    print(ll.head.next.value)           # 2
    print(ll.head.next.next.value)      # 4 now
    print(ll.head.next.next.next.value) # 3 now
    print(ll.get_position(5).value)     # 5
    ll.insert(e10,8)
    ll.insert(e8,8)
    ll.insert(e9,9)
    print(ll.get_position(6).value)     # 6
    print(ll.get_position(7).value)     # 7
    print(ll.get_position(8).value)     # 8
    print(ll.get_position(9).value)     # 9
    print(ll.get_position(10).value)    # 10

    # Test delete
    print('Test delete():')
    ll.delete(1)
    ll.delete(10)
    print(ll.get_position(1).value)     # 2 now
    print(ll.get_position(2).value)     # 4 now
    print(ll.get_position(3).value)     # 3 now
    print(ll.get_position(8).value)     # 9 now


from tkinter import *
from tkinter import ttk

class AboutView(Toplevel):

    def __init__(self, parent):
        Toplevel.__init__(self, parent)
        project_desc = "Project done as a part of Bachelor's Thesis."
        labels = ["Thesis title:", "Thesis supervisor:", "Author:", "College:", "Faculty:", "Term:"]

        thesis_title = "<title>"
        thesis_supervisor = "dr inz. Izabela Zoltowska"
        author = "Filip Mazur"
        college = "Warsaw University of Technology"
        faculty = "The Faculty of Electronics and Information Technology"
        term = "2021Z"

        desc_labels = [thesis_title, thesis_supervisor, author, college, faculty, term]

        print("About creating")
        self.title("About")
        self.geometry("450x250")
        self.resizable(False, False)

        self.create_label(project_desc, 0, 0, 2, 1, (N), (10, 10), (20))
        for i in range(0, 6):
            self.create_label(labels[i], 0, i+1, 1, 1, (NW), (10, 10), (10, 0))
            self.create_label(desc_labels[i], 1, i+1, 1, 1, (NW), (10, 10), (10, 0))

    def create_label(self, text, col, row, colspan, rowspan, st, padx, pady, wraptext=0):
        label = ttk.Label(self, text=text)
        label.grid(column=col, row=row, columnspan=colspan, rowspan=rowspan, padx=padx, pady=pady, sticky=st)
        label.configure(wraplength=wraptext)


from keras.layers import Conv2D
from keras.layers import Input
from keras.layers import MaxPooling2D, UpSampling2D
from keras.models import Model
from keras.datasets.mnist import load_data
from numpy import reshape
import matplotlib.pyplot as plt
from keras.callbacks import ModelCheckpoint
import numpy as np

(xtrain0, ytrain0), (xtest0, ytest0) = load_data()

def wheredex(x,y,d=7):
  assert len(x)==len(y),f"{len(x)} {len(y)}"
  i=np.where(y==d)
  k=x[i]
  if len(k)>5000:k=k[:5000]
  return reshape(k.astype('float32') / 255, (len(k), 28, 28, 1))#[:5000]
def wherenotdex(x,y,d=7):
  assert len(x)==len(y),f"{len(x)} {len(y)}"
  i=np.where(y!=d)
  k=x[i]
  if len(k)>5000:k=k[:5000]
  return reshape(k.astype('float32') / 255, (len(k), 28, 28, 1))#[:5000]

xtrain=wheredex(xtrain0,ytrain0,7)
xatrain=wherenotdex(xtrain0,ytrain0,7)
xtest=wheredex(xtest0,ytest0,7)
xatest=wherenotdex(xtest0,ytest0,7)

np.savez_compressed("data",x=xtrain,xa=xatrain,xval=xtest,xaval=xatest)

print(xtrain.shape, xtest.shape)

input_img = Input(shape=(28, 28, 1))

enc_conv1 = Conv2D(12, (3, 3), activation='relu', padding='same')(input_img)
enc_pool1 = MaxPooling2D((2, 2), padding='same')(enc_conv1)
enc_conv2 = Conv2D(8, (4, 4), activation='relu', padding='same')(enc_pool1)
enc_ouput = MaxPooling2D((4, 4), padding='same')(enc_conv2)

dec_conv2 = Conv2D(8, (4, 4), activation='relu', padding='same')(enc_ouput)
dec_upsample2 = UpSampling2D((4, 4))(dec_conv2)
dec_conv3 = Conv2D(12, (3, 3), activation='relu')(dec_upsample2)
dec_upsample3 = UpSampling2D((2, 2))(dec_conv3)
dec_output = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(dec_upsample3)

autoencoder = Model(input_img, dec_output)
autoencoder.compile(optimizer='rmsprop', loss='binary_crossentropy')
autoencoder.summary()

checkpoint = ModelCheckpoint("model.hdf5", monitor='loss', verbose=1,
    save_best_only=True, mode='auto', period=1)

autoencoder.fit(xtrain, xtrain, epochs=20, batch_size=128, shuffle=True,callbacks=[checkpoint])

decoded_imgs = autoencoder.predict(x_test)

n = 10
plt.figure(figsize=(20, 4))
for i in range(n):
  plt.gray()
  ax = plt.subplot(2, n, i+1)
  plt.imshow(x_test[i].reshape(28, 28))
  ax.get_xaxis().set_visible(False)
  ax.get_yaxis().set_visible(False)

  ax = plt.subplot(2, n, i +1+n)
  plt.imshow(decoded_imgs[i].reshape(28, 28))
  ax.get_xaxis().set_visible(False)
  ax.get_yaxis().set_visible(False)
plt.show()


from sklearn.decomposition import PCA
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os

plot = False

df_fname = r"E:\nanopores\2023_02_20\ch1\pore2\data3\f1.pkl"
root = os.path.dirname(df_fname)
name,ext = os.path.splitext(os.path.basename(df_fname))

df = pd.read_pickle(df_fname)

dfInfToNan = df.replace([np.inf, -np.inf], np.nan, inplace=False)
noNan = dfInfToNan.copy().dropna()
conditionedDf = noNan.iloc[:,1:]
array = conditionedDf.to_numpy()

print(f"All values finite: {np.isfinite(array).all()}")

pca = PCA(n_components = 2)
pca.fit(array)
new = pca.transform(array)

if plot:
    fig, ax = plt.subplots()
    ax.scatter(new[:,0],new[:,1], alpha = 0.1)
    plt.show()

noNan['pc_1'] = new[:,0]
noNan['pc_2'] = new[:,1]

noNan.to_pickle(os.path.join(root,f"{name}_PCA.pkl"))

names = list(conditionedDf.columns)
pc1 = dict(zip(names,pca.components_[0]))
pc2 = dict(zip(names,pca.components_[1]))
print(f"PC1: {pc1}")
print(f"PC2: {pc2}")