import unittest, pytest
from utils import Utils
from pathlib import Path
import os
import shutil
import nltk
nltk.download('wordnet')
from nltk.corpus import wordnet as wn

class TestUtils(unittest.TestCase):

    maxDiff = None

    # == TEST GET RANDOM WORD == #

    def test_get_random_word_returns_a_string_found_in_wordnet_word_list(self):
        # given - an instance of the utils class and wordnet words list
        undertest_class = Utils()
        wordnet = wn.words()

        # when - we call the get random word method of the utils class
        actual_word = undertest_class.get_random_word()

        # then - the returned value is type string, with value found in the wordnet
        self.assertTrue(type(actual_word) == str)
        self.assertTrue(actual_word in wordnet)

    def test_get_random_word_returns_two_different_words_when_called_twice(self):
        # given - an instance of the utils class
        undertest_class = Utils()

        # when - we call the get random word method of the utils class twice
        actual_word1 = undertest_class.get_random_word()
        actual_word2 = undertest_class.get_random_word()

        # then - the words returned are not the same
        self.assertNotEqual(actual_word1, actual_word2)

    # == TEST GET SYNSETS == #
    def test_get_synsets_returns_list_of_synsets_and_number_of_synsets(self):
        # given - an instance of the utils class
        undertest_class = Utils()

        # when - we call the get_synsets method, passing in a common word that can be found in wordnet with multiple synsets
        actual_synsets, actual_num_synsets = undertest_class.get_synsets("class")

        # then - returned values are of the expected types
        self.assertTrue(type(actual_synsets) == list)
        self.assertTrue(type(actual_num_synsets) == int)

    # == TEST GET WORDS FROM SYNSETS == #
    def test_get_words_from_synsets_returns_a_list_of_strings(self):
        # given - an instance of the utils class
        undertest_class = Utils()

        # when - we call the get words from synsets method passing in a list of synsets
        actual_synsets, _ = undertest_class.get_synsets("class")
        actual_words = undertest_class.get_words_from_synsets(actual_synsets)

        # then - a list of strings is returned
        self.assertTrue(type(actual_words) == list)
        self.assertTrue(all([type(i) == str for i in actual_words]))

    # == TEST GET LEMMAS FROM SYNSET == #
    def test_get_lemmas_from_synset_returns_list_of_lemmas(self):
        # given - an instance of the utils class
        undertest_class = Utils()

        # when - we call the get lemmas from synsets method passing in a synset
        actual_synsets, _ = undertest_class.get_synsets("class")
        actual_words = undertest_class.get_lemmas_from_synset(actual_synsets[0])

        # then - a list of strings is returned
        self.assertTrue(type(actual_words) == list)
        self.assertTrue(all([type(i) == nltk.corpus.reader.wordnet.Lemma for i in actual_words]))

    # == TEST VALIDATE RELATED WORDS == #
    def test_validate_related_words_returns_same_words_if_all_valid(self):
        # given - an instance of the undertest utils class, a word and a list of related words that are all considered valid
        undertest_class = Utils()
        word = "example"
        original_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)

        # then - the same list of related words is returned
        self.assertEqual(actual_list, original_list)

    def test_validate_related_words_removes_multiple_words_with_underscore_between(self):
        # given - an instance of the undertest utils class, a word and a list of related words with some invalid due to underscore
        undertest_class = Utils()
        word = "example"
        original_list = ["specimen", "sample", "illustration", "case_in_point", "role_model", "guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)
        expected_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # then - the list of related words is returned without the invalid words
        self.assertEqual(actual_list, expected_list)

    def test_validate_related_words_removes_words_with_numbers(self):
        # given - an instance of the undertest utils class, a word and a list of related words with some invalid due to numbers
        undertest_class = Utils()
        word = "example"
        original_list = ["specimen", "100", "sample", "illustration", "eg1","guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)
        expected_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # then - the list of related words is returned without the invalid words
        self.assertEqual(actual_list, expected_list)

    def test_validate_related_words_removes_words_that_are_very_similar(self):
        # given - an instance of the utils class, a word and a list of related words with some invalid due to extreme similarity to original word
        undertest_class = Utils()
        word = "example"
        original_list = ["examples", "specimen", "sample", "exampled", "illustration", "guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)
        expected_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # then - the list of related words is returned without the invalid words
        self.assertEqual(actual_list, expected_list)

    def test_validate_related_words_removes_words_that_are_the_same_with_different_caps(self):
        # given - an instance of the utils class, a word and a list of related words with some invalid - differing by caps to original word
        undertest_class = Utils()
        word = "example"
        original_list = [ "Example", "specimen", "sample", "EXAMPLE", "illustration", "guide", "blueprint", "ideal"]

        # when - we call the validate related words method passing in the word and list of words
        actual_list = undertest_class._validate_related_words(word, original_list)
        expected_list = ["specimen", "sample", "illustration", "guide", "blueprint", "ideal"]

        # then - the list of related words is returned without the invalid words
        self.assertEqual(actual_list, expected_list)

if __name__ == "__main__":
    unittest.main()


"""Deep Residual Network with Stochastic Depth."""
import cPickle as pickle
import datetime as dt
import gzip
import lasagne as nn
import numpy as np
import sys
import theano
import theano.tensor as T
import time

from Deep_Residual_Learning_CIFAR10 import load_data
from Deep_Residual_Learning_CIFAR10 import iterate_minibatches
from helpers import report
from helpers import ResBlockLayer

# Basic settings
sys.setrecursionlimit(2 ** 20)

TITLE = 'stochastic_depth'
OUTPUT_FILE = './output/{}.txt'.format(TITLE)

if __name__ == '__main__':
    report('\n\nSTART: {}'.format(dt.datetime.now()), OUTPUT_FILE)
    report('>>> {}'.format(OUTPUT_FILE), OUTPUT_FILE)

    # Model parameters
    try:
        N = int(sys.argv[1])
    except:
        N = 9 # default depth uses N = 9.

    epochs, batchsize = 500, 128
    report('Depth: N = {}, #Layers = {}.'.format(N, 6*N+2), OUTPUT_FILE)
    delta = 0.5 / (3 * N - 1)
    decay = [1.0 - i * delta for i in xrange(3*N)] # linear decay rule
    switches = [theano.shared(nn.utils.floatX(1.)) for i in xrange(3*N)]
    on, off = 1.0, 0.0 # nn.utils.floatX(1.), nn.utils.floatX(0.)

    # Use to turn on all blocks (during testing)
    def full_depth():
        global switches, on
        for c in switches:
            c.set_value(on)

    # Use to randomly on/off blocks based on decay rule (during training)
    def stochastic_depth():
        global switches, on, off, decay
        assert len(switches) == len(decay)
        drop = np.random.uniform(low=0, high=1, size=len(switches)) > decay
        for c, d in zip(switches, drop):
            c.set_value(off) if d else c.set_value(on)

    # Symbolic variables
    X, Y = T.tensor4('inputs'), T.ivector('targets')

    # Model architecture
    net = nn.layers.InputLayer(shape=(None, 3, 32, 32), input_var=X)

    # 16 x 32 x 32
    net = nn.layers.Conv2DLayer(
        incoming=net, num_filters=16, filter_size=(3,3), stride=(1,1),
        nonlinearity=nn.nonlinearities.rectify, pad='same',
        W=nn.init.HeNormal(gain='relu'), flip_filters=False)
    net = nn.layers.batch_norm(net)
    i = 0  # add i-th switch to i-th block

    for _ in range(N):
        net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=False)
        i += 1

    # 32 x 16 x 16
    net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=True)
    i += 1

    for _ in range(N-1):
        net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=False)
        i += 1

    # 64 x 8 x 8
    net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=True)
    i += 1

    for _ in range(N-1):
        net = ResBlockLayer(incoming=net,C=switches[i],increase_channels=False)
        i += 1

    net = nn.layers.GlobalPoolLayer(net)
    net = nn.layers.DenseLayer(
        incoming=net, num_units=10, W=nn.init.HeNormal(),
        nonlinearity=nn.nonlinearities.softmax)
    report('Model OK...', OUTPUT_FILE)

    num_params = nn.layers.count_params(net, trainable=True)
    report("Number of parameters: {}".format(num_params), OUTPUT_FILE)

    # Training function
    output = nn.layers.get_output(net, deterministic=False)
    loss = nn.objectives.categorical_crossentropy(output, Y).mean()
    accuracy = T.mean(
        T.eq(T.argmax(output, axis=1), Y), dtype=theano.config.floatX)
    all_layers = nn.layers.get_all_layers(net)
    regularization = nn.regularization.regularize_layer_params(
        layer=all_layers, penalty=nn.regularization.l2)
    loss = loss + 1e-4 * regularization
    params = nn.layers.get_all_params(net, trainable=True)
    learning_rate = theano.shared(nn.utils.floatX(0.1))
    updates = nn.updates.momentum(
        loss_or_grads=loss, params=params,
        learning_rate=learning_rate, momentum=0.9)
    training_function = theano.function([X,Y],[loss,accuracy],updates=updates)
    report('Training function OK...', OUTPUT_FILE)

    # Test/validation function
    test_output = nn.layers.get_output(net, deterministic=True)
    test_loss = nn.objectives.categorical_crossentropy(test_output, Y).mean()
    test_accuracy = T.mean(
        T.eq(T.argmax(test_output, axis=1), Y), dtype=theano.config.floatX)
    test_function = theano.function([X,Y], [test_loss,test_accuracy])
    report('Validation function OK...', OUTPUT_FILE)

    # Load data
    data = load_data()
    X_train, Y_train = data['X_train'], data['Y_train']
    X_test, Y_test = data['X_test'], data['Y_test']
    report('Data OK...', OUTPUT_FILE)

    # Start training
    # `training loss, training accuracy, validation loss, validation accuracy`
    TL, TA, VL, VA = [], [], [], []
    report('Starting training...', OUTPUT_FILE)
    header = ['Epoch', 'TL', 'TA', 'VL', 'VA', 'Time']
    report('{:<10}{:<20}{:<20}{:<20}{:<20}{:<20}'.format(*header), OUTPUT_FILE)
    learning_rate_schedule = {250:0.01, 375:0.001}

    for e in xrange(epochs):
        if (e + 1) in learning_rate_schedule:
            learning_rate.set_value(learning_rate_schedule[e + 1])

        start_time = time.time()
        t_batches, v_batches = 0, 0
        tl, ta, vl, va = 0., 0., 0., 0.

        # Train with stochastic depth
        stochastic_depth()
        minibatches = iterate_minibatches(X_train, Y_train, batchsize,
            shuffle=True, augment=True)

        for data, target in minibatches:
            l, a = training_function(data, target)
            tl += l
            ta += a
            t_batches += 1

        tl /= t_batches
        ta /= t_batches
        TL.append(tl)
        TA.append(ta)

        # Test with full depth
        full_depth()
        minibatches = iterate_minibatches(X_test, Y_test, 500, shuffle=False,
            augment=False)

        for data, target in minibatches:
            l, a = test_function(data, target)
            vl += l
            va += a
            v_batches += 1

        vl /= v_batches
        va /= v_batches
        VL.append(vl)
        VA.append(va)

        row = [e + 1, tl, ta, vl, va, time.time() - start_time]
        report('{:<10}{:<20}{:<20}{:<20}{:<20}{:<20}'.format(*row),OUTPUT_FILE)

    report('Finished training...', OUTPUT_FILE)

    # Save training information
    f = gzip.open('./output/{}_info.pkl.gz'.format(TITLE), 'wb')
    info = {
        'training loss': TL,
        'training accuracy': TA,
        'validation loss': VL,
        'validation accuracy': VA
    }
    pickle.dump(info, f)
    f.close()
    report('Saved training information...', OUTPUT_FILE)

    # Save weights
    weights = nn.layers.get_all_params(net)
    weights = [np.array(w.get_value()) for w in weights]
    f = gzip.open('./output/{}_weights.pkl.gz'.format(TITLE), 'wb')
    pickle.dump(weights, f)
    f.close()

    report('Saved weights...', OUTPUT_FILE)
    report('END: {}'.format(dt.datetime.now()), OUTPUT_FILE)


import yaml
import pandas as pd
import numpy as np

def build_cpu_and_mem_distribution_dataframes(trace_df : pd.DataFrame,
                            col_flavor_cpu : str = 'vmcorecount', col_flavor_mem : str = 'vmmemory',
                            col_vm_created : str = 'vmcreated', col_vm_deleted : str = 'vmdeleted',
                            timestamp_begin : int = None, timestamp_end : int = None, timestamp_step : int = 3600 ):

    if timestamp_begin is None: timestamp_begin = trace_df[col_vm_created].min()
    if timestamp_end is None: timestamp_end = trace_df[col_vm_created].max()

    keys_core, core_values_per_key = __init_values_per_key(trace_df, col_flavor_cpu)
    keys_mem, mem_values_per_key = __init_values_per_key(trace_df, col_flavor_mem)

    considered_timestamps = list()
    for timestamp in range(timestamp_begin, timestamp_end, timestamp_step):

        # Dataset containing all VM alive in current range (deleted after min, created before max)
        filtered_by_aliveness = trace_df.loc[(trace_df[col_vm_deleted] >= (timestamp)) & (trace_df[col_vm_created] <= (timestamp+timestamp_step) )]

        __add_to_result_dict_observed_freq(alive_df=filtered_by_aliveness, metric=col_flavor_cpu, metric_keys=keys_core, result_dict=core_values_per_key)
        __add_to_result_dict_observed_freq(alive_df=filtered_by_aliveness, metric=col_flavor_mem, metric_keys=keys_mem, result_dict=mem_values_per_key)
        considered_timestamps.append(timestamp)

    core_values_per_key['timestamp'] = considered_timestamps
    mem_values_per_key['timestamp'] = considered_timestamps

    return pd.DataFrame(core_values_per_key), pd.DataFrame(mem_values_per_key)

def get_cpu_and_mem_average_distribution(trace_df : pd.DataFrame,
                            col_flavor_cpu : str = 'vmcorecount', col_flavor_mem : str = 'vmmemory',
                            col_vm_created : str = 'vmcreated', col_vm_deleted : str = 'vmdeleted',
                            timestamp_begin : int = None, timestamp_end : int = None, timestamp_step : int = 3600 ):

    cpu_timestamped_df, mem_timestamped_df = build_cpu_and_mem_distribution_dataframes(trace_df=trace_df,
                                        col_flavor_cpu=col_flavor_cpu, col_flavor_mem=col_flavor_mem,
                                        col_vm_created=col_vm_created, col_vm_deleted=col_vm_deleted,
                                        timestamp_begin=timestamp_begin, timestamp_end=timestamp_end, timestamp_step=timestamp_step)

    cpu_grouped = cpu_timestamped_df.drop('timestamp', axis=1)
    cpu_grouped = cpu_grouped.mean().to_frame()

    cpu_keys = cpu_grouped.index.values
    cpu_vals = cpu_grouped.values.tolist()
    res_cpu = pd.DataFrame({col_flavor_cpu: [int(key) for key in cpu_keys], 'freq': [round(np.mean(value),2) for value in cpu_vals]})

    mem_grouped = mem_timestamped_df.drop('timestamp', axis=1)
    mem_grouped = mem_grouped.mean().to_frame()

    mem_keys = mem_grouped.index.values
    mem_vals = mem_grouped.values.tolist()
    res_mem = pd.DataFrame({col_flavor_mem: [float(key) for key in mem_keys], 'freq': [round(np.mean(value),2) for value in mem_vals]})

    return res_cpu.loc[res_cpu['freq'] > 0.01].reset_index(drop=True), res_mem.loc[res_mem['freq'] > 0.01].reset_index(drop=True)

def __init_values_per_key(trace_df : pd.DataFrame, column_name : str):
    keys = list(trace_df[column_name].unique())
    keys.sort()
    values_per_keys = dict()
    for key in keys: values_per_keys[str(key)] = list()
    return keys, values_per_keys

def __add_to_result_dict_observed_freq(alive_df : pd.DataFrame, metric : str, metric_keys : list, result_dict : dict):

    distribution_df = pd.DataFrame(alive_df.groupby(metric).size().rename('count')).reset_index()
    total = distribution_df['count'].sum()
    distribution_df['freq'] = round((distribution_df['count']/total),2)

    observed_metric = list(distribution_df[metric] )
    for key in metric_keys:
        if key not in observed_metric:
            result_dict[str(key)].append(0)
        else:
            result_dict[str(key)].append(float(distribution_df.loc[distribution_df[metric] == key]['freq'].iloc[0]))

def convert_distribution_to_scenario(cpu_distribution : pd.DataFrame,  mem_distribution : pd.DataFrame,
                                     col_freq_cpu : str = 'freq', col_flavor_cpu = 'vmcorecount',
                                     col_freq_mem : str = 'freq', col_flavor_mem = 'vmmemory',
                                     output_file : str = 'scenario-vm-distribution.yml'
                                    ):

    ordered_cpu_distribution = cpu_distribution.sort_values(by=[col_flavor_cpu]).set_index(col_flavor_cpu)
    ordered_mem_distribution = mem_distribution.sort_values(by=[col_flavor_mem]).set_index(col_flavor_mem)

    yml_dict = dict()
    yml_dict["vm_distribution"] = dict()
    yml_dict["vm_distribution"]["config_cpu"] = ordered_cpu_distribution.apply(
        lambda row : float(row[col_freq_cpu]), axis=1).to_dict()

    yml_dict["vm_distribution"]["config_mem"] = ordered_mem_distribution.apply(
        lambda row :float(row[col_freq_mem]), axis=1).to_dict()

    with open(output_file, 'w') as outfile:
        yaml.dump(yml_dict, outfile, default_flow_style=False)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author            : desertsniper87 <torshobuet@gmail.com>
# Date              : 27.01.2018
# Last Modified Date: 28.01.2018
# Completed         : 27.01.2018

# https://en.wikipedia.org/wiki/Reverse_Polish_notatifor each token in the postfix expression:
# for each token in the postfix expression:
# if token is an operator:
    # operand_2 ← pop from the stack
    # operand_1 ← pop from the stack
    # result ← evaluate token with operand_1 and operand_2
    # push result back onto the stack
# else if token is an operand:
    # push token onto the stack
    # result ← pop from the stackon

def operation(s, o1, o2):
    if s=='+':
        return o1+o2
    elif s=='-':
        return o1-o2
    elif s=='*':
        return o1*o2
    elif s=='/':
        return o1/o2

def calc(expr):
    print(expr)
    if (expr==""): return '0'
    notation = expr.split(" ")
    stack = []
    result = []
    for token in notation:
        if token.replace('.', '1').isdigit()==False:
            o2 = stack.pop()
            o1 = stack.pop()
            result = operation(token, int(o1), int(o2))
            stack.append(result)
        else:
            stack.append(token)
    print(stack[-1])
    a = stack[-1]
    try:
        return float(a)
    except:
        return int(a)

if __name__ == "__main__":
    calc("")
    calc("1 2 3")
    calc("1 2 3.5")
    calc("1 3 +")
    calc("1 3 *")
    calc("1 3 -")
    calc("4 2 /")


# solution: 맵의 테두리를 이동 할 수 없는 상수(1)로 대체한다.
#   0: 이동가능, 1: 바다, 2: 이미 가 본 칸
#
# time-complexity: O(N^2)...? 맵의 칸 갯수만큼의 시간복잡도를 갖는다. 라고 생각한다....

# [U, R, D, L]
move_x = [0, 1, 0, -1]
move_y = [-1, 0, 1, 0]

# start_input
R, C = map(int, input().split())
x_pos, y_pos, dir = map(int, input().split())
_map = []
for i in range(R):
    _map.append(list(map(int,input().split())))
# end_input

# Map 테두리 보정
for row in _map:
    row.insert(0, 1)
    row.append(1)
_map.insert(0, [1]*(C+2))
_map.append([1]*(C+2))
x_pos += 1
y_pos += 1

cnt = 1 if _map[y_pos][x_pos] == 0 else 0
_map[y_pos][x_pos] = 2

is_possible = True
while is_possible:
    is_possible = False

    for i in range(4):
        dir = (dir + 3) % 4
        x_moved = x_pos + move_x[dir]
        y_moved = y_pos + move_y[dir]

        if _map[y_moved][x_moved] == 0:
            x_pos = x_moved
            y_pos = y_moved
            _map[y_pos][x_pos] = 2
            cnt += 1
            is_possible = True
            break

    if is_possible == False:
        # Move back
        backward = (dir + 2) % 4
        x_moved = x_pos + move_x[backward]
        y_moved = y_pos + move_y[backward]

        if _map[y_moved][x_moved] != 1:
            x_pos = x_moved
            y_pos = y_moved
            is_possible = True

# start_print
print(cnt)
# end_print


#coding=utf-8
# File      :   cut_wav.py
# Time      :   2021/12/15 14:39:17
# Author    :   Jinghan Peng
# Desciption:   切割无效音频为小的chunk

import os, sys
from re import L
import wave
from pydub import AudioSegment
from pydub.utils import make_chunks
from tqdm import tqdm

from multiprocessing import Pool
import collections

def main():
    wav_dir = "/data1/pengjinghan/tsvad/invalid_sound/sum"

    out_wav_dir  = "/data1/pengjinghan/tsvad/invalid_sound/wav_chunk/wav"
    out_data_dir = "/data1/pengjinghan/tsvad/invalid_sound/wav_chunk/data"
    os.makedirs(out_wav_dir, exist_ok=True)
    os.makedirs(out_data_dir, exist_ok=True)

    wav_length = 16.02 # 单位: 秒，=16020秒
    chunk_length_ms    = int(wav_length*1000)
    chunk_length_frame = int(wav_length*100)

    """对音频进行切割chunk"""
    with open(os.path.join(out_data_dir, 'target_chunk.lst'), 'w') as wf:
        for file in os.listdir(wav_dir):
            wav_path = os.path.join(wav_dir, file) # 输入音频路径
            utt      = file.split(".wav")[0]
            audio    = AudioSegment.from_file(wav_path)    # 输入音频
            duration = audio.duration_seconds
            print(f"Processing {file}")
            print(f"{duration}s audio is going to be cut to {int(duration//wav_length)} chunk with chunk_size={wav_length}s")
            chunks   = make_chunks(audio, chunk_length_ms) # 切片

            """"对每个chunk"""
            for index, chunk in tqdm(enumerate(chunks)):
                if chunk.duration_seconds != wav_length: # 最后一个块可能长度不够而丢弃
                    continue

                """保存chunk音频"""
                chunk_name = utt+"-{0:08d}_{1:08d}.wav".format(index*chunk_length_ms, (index+1)*chunk_length_ms)
                chunk_path = os.path.join(out_wav_dir, chunk_name)
                chunk.export(chunk_path, format="wav") # 保存剪切后的样本音频
                print("Exporting", chunk_path)

                """记录chunk的target"""
                chunk_utt = chunk_name.split('.wav')[0]
                target_chunk_length_frame = chunk_length_frame - 2 # 特征要比音频时长小2秒
                target_line = ' '.join(['0' for i in range(target_chunk_length_frame)])

                wf.write(f"{chunk_utt} {target_line}\n")

if __name__ == "__main__":
    main()


import cv2
import numpy as np
import easyocr
import PIL.Image as pil_img

def Book_Image(Snap):

    pic = pil_img.open(Snap)
    img = np.array(pic).astype("uint8")
    picture = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    image = cv2.rotate(picture, cv2.cv2.ROTATE_90_COUNTERCLOCKWISE)
    reader = easyocr.Reader(['en'])
    result = reader.readtext(image, detail=0)

    Words = [re for re in result]
    text = " ".join(Words)
    return text


import sqlite3

fname = input('Enter file name: ')
if(len(fname) < 1):
    fname = 'mbox.txt'

fh = open(fname)
emailList = list()
dic = dict()

for line in fh:
    if( line.startswith('From: ') ):
        pieces = line.split()
        email = pieces[1]

        secPc =  email.split('@')
        orgN = secPc[1]

        emailList.append(orgN)
    else:
        continue

# print(emailList)

#count email sending
for email in emailList:
    dic[email] = dic.get(email, 0) + 1

# print(dic)

conn = sqlite3.connect('assignmentdb.sqlite')
cur = conn.cursor()

cur.execute('''
DROP TABLE IF EXISTS Counts''')

cur.execute('''
    CREATE TABLE Counts (org TEXT, count INTEGER)
''')

## email insert
for email, count in dic.items():
    # print(email, count)
    cur.execute('''INSERT into Counts (org, count) values (?, ?)''', (email, count))
    conn.commit()

sqlstr = 'SELECT org, count FROM Counts ORDER BY count DESC'

print(cur.execute(sqlstr))
for row in cur.execute(sqlstr):
    print(str(row[0]), row[1])

#     email = pieces[1]
#     cur.execute('SELECT count from Counts where email = ?', (email,))

#     row = cur.fetchone()  # grab the first one
#     if row is None:
#         cur.execute('''INSERT into Counts (email, count)
#         values (?, 1)''', (email,))
#     else:
#         cur.execute(
#             'update Counts set count = count + 1 where email = ?', (email,))
#     conn.commit()

# sqlstr = 'SELECT email, count FROM Counts ORDER BY count DESC LIMIT 10'

# for row in cur.execute(sqlstr):
#     print(str(row[0]), row[1])


# @Time : 2022-09-20 21:09
# @Author : Phalange
# @File : 128. 最长连续序列.py
# @Software: PyCharm
# C'est la vie,enjoy it! :D
from typing import List

class Solution:

    def __init__(self):
        self.nodes = dict()
        self.parents = dict()
        self.sizeMap = dict()

    def longestConsecutive(self, nums: List[int]) -> int:
        if nums == []:
            return 0
        n = len(nums)
        for i in range(n):
            self.nodes[nums[i]] = nums[i]
            self.parents[nums[i]] = nums[i]
            self.sizeMap[nums[i]] = 1

        # 连通起来
        for i in range(n):
            if nums[i]-1 in self.nodes:
                self.union(nums[i]-1,nums[i])

        return max(list(self.sizeMap.values()))

    def isSameSet(self,a,b):
        if a not in self.nodes or b not in self.nodes:
            return
        return self.findFather(a) == self.findFather(b)

    def findFather(self,cur):
        path = []
        while cur !=self.parents[cur]:
            path.append(cur)
            cur = self.parents[cur]

        while path:
            self.parents[path.pop()] = cur
        return cur

    def union(self,a,b):
        if a not in self.nodes or b not in self.nodes:
            return

        aHead = self.findFather(a)
        bHead = self.findFather(b)
        if aHead != bHead:
            aSize = self.sizeMap[aHead]
            bSize = self.sizeMap[bHead]
            big = aHead if aSize >=bSize else bHead
            small = bHead if aHead == big else aHead
            self.parents[small] = big
            self.sizeMap[big] = aSize + bSize
            self.sizeMap.pop(small)

print(Solution().longestConsecutive([1,3,2]))

哈希表/128. 最长连续序列.py


import pytest

from ephios.core.models import Qualification, QualificationCategory, QualificationGrant
from ephios.plugins.qualification_management.importing import (
    DeserializedQualification,
    QualificationChangeManager,
)

SANH_UUID = "b1faab38-2e7c-4507-b753-06d1e653412d"
NOTSAN_UUID = "d114125b-7cf4-49e2-8908-f93e2f95dfb8"

@pytest.fixture
def deserialized_qualifications():
    b = DeserializedQualification(
        {
            "uuid": "247fab6a-8784-4976-a406-985fe47dc683",
            "title": "Deutsches Rettungsschwimmabzeichen Bronze",
            "abbreviation": "DRSA Bronze",
            "includes": [],
            "included_by": ["ef95a854-2eeb-431c-a795-bc291b341d49"],
            "category": {
                "uuid": "cd10e68f-41fe-4ca0-a624-3ab3eb85bd08",
                "title": "Wasserrettung Allgemein",
            },
        }
    )
    s = DeserializedQualification(
        {
            "uuid": "ef95a854-2eeb-431c-a795-bc291b341d49",
            "title": "Deutsches Rettungsschwimmabzeichen Silber",
            "abbreviation": "DRSA Silber",
            "includes": ["247fab6a-8784-4976-a406-985fe47dc683"],
            "included_by": ["b601a18b-cee8-4037-af33-dd7aabeac295"],
            "category": {
                "uuid": "cd10e68f-41fe-4ca0-a624-3ab3eb85bd08",
                "title": "Wasserrettung Allgemein",
            },
        }
    )
    g = DeserializedQualification(
        {
            "uuid": "b601a18b-cee8-4037-af33-dd7aabeac295",
            "title": "Deutsches Rettungsschwimmabzeichen Gold",
            "abbreviation": "DRSA Gold",
            "includes": ["ef95a854-2eeb-431c-a795-bc291b341d49"],
            "included_by": [],
            "category": {
                "uuid": "cd10e68f-41fe-4ca0-a624-3ab3eb85bd08",
                "title": "Wasserrettung Allgemein",
            },
        }
    )
    return b, s, g

@pytest.fixture
def saved_deserialized_qualifications(deserialized_qualifications):
    QualificationChangeManager().add_deserialized_qualifications_to_db(
        *deserialized_qualifications
    ).commit()
    return deserialized_qualifications

@pytest.fixture
def qualification_grant(saved_deserialized_qualifications, volunteer):
    return QualificationGrant.objects.create(
        user=volunteer,
        qualification=Qualification.objects.create(
            title="custom",
            is_imported=False,
            abbreviation="cstm",
            uuid="5543ce30-5593-48b7-aa01-78d4cc54bf22",
            category=QualificationCategory.objects.get(),
        ),
    )


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import plotly.graph_objs as go
import pandas as pd

df = pd.read_csv('./data/Stapleton.csv')

app = dash.Dash()

# https://dash.plot.ly/dash-core-components/dropdown
# We need to construct a dictionary of dropdown values for the years
year_options = []
for year in df['YEAR'].unique():
    year_options.append({'label':str(year),'value':year})

app.layout = html.Div([
    html.H3('Denver Max Daily Temp'),
    dcc.Graph(id='graph'),
    dcc.Dropdown(id='year-picker',options=year_options,value=df['YEAR'].min())
])

@app.callback(Output('graph', 'figure'),
              [Input('year-picker', 'value')])
def update_figure(selected_year):
    filtered_df = df[df['YEAR'] == selected_year]
    traces = []
    for month_num in filtered_df['MONTH'].unique():
        df_by_month = filtered_df[filtered_df['MONTH'] == month_num]
        traces.append(go.Scatter(
            x=df_by_month['DAY'],
            y=df_by_month['TMAX'],
            text=df_by_month['NAME'],
            mode='markers + lines',
            opacity=0.7,
            marker={'size': 5},
            name=month_num
        ))

    return {
        'data': traces,
        'layout': go.Layout(
            xaxis={'title': 'TIME'},
            yaxis={'title': 'TMAX'},
            hovermode='closest'
        )
    }

if __name__ == '__main__':
    app.run_server()


import tweaks as _
import numpy as np
from numpy import *

# ------------------------------------------------------ Random distributions --

def p_logp(p):
    if p < 1e-8:
        return 0
    return p * log(p)

def entropy(probabilities):
    return sum(-p_logp(p) for p in probabilities)

def entropy_uniform(N):
    p = 1.0 / N
    return -p_logp(p) * N

def random_distribution(N, min_entropy):
    if N == 1:
        return array([1.0])

    distribution = np.random.random_sample(N)
    distribution /= sum(distribution)

    while entropy_uniform(N) / entropy(distribution) < min_entropy:
        distribution[np.random.randint(N)] *= 1.1
        distribution /= sum(distribution)

    return distribution

def random_distribution_nonzeros(N, N_nonzeros, min_entropy):
    result = zeros(N)
    non_zeros = np.random.choice(N, min(N, N_nonzeros), replace=False)
    result[non_zeros] = random_distribution(N, min_entropy)
    return result

def random(probabilities):
    N = len(probabilities)
    bins = add.accumulate(probabilities)
    values = range(N)
    return values[digitize([np.random.rand()], bins)]

# ----------------------------------------------------- Transition & emission --

def transition():
    result = zeros([_.N_STATES, _.N_STATES])

    # Honest state
    result[0, 0] = _.TRANSITION_HONEST_INERTIA
    result[0, 1:] = \
        ones(_.N_STATES - 1) * (1 - _.TRANSITION_HONEST_INERTIA) / (_.N_STATES - 1)

    # Dishonest states, return to honest
    result[1:, 0] = 1 - _.TRANSITION_DISHONEST_INERTIA

    # Dishonest states, rest
    for s in range(1, _.N_STATES):
        # Dishonest self-inertia
        result[s, s] = _.N_STATES > 2 and _.TRANSITION_DISHONEST_SELF_INERTIA or 1.0

        # Transition to other dishonest states
        if _.N_STATES > 2:
            indices = range(1, s) + range(s + 1, _.N_STATES)

            distribution = random_distribution_nonzeros(
                _.N_STATES - 2,
                _.TRANSITION_DISHONEST_MAX,
                _.TRANSITION_MIN_ENTROPY
            )

            result[s, indices] = \
                distribution * (1 - _.TRANSITION_DISHONEST_SELF_INERTIA)

        # Multiply by dishonest inertia (rest is transition to honest state)
        result[s, 1:] *= _.TRANSITION_DISHONEST_INERTIA

    return result

def emission():
    result = zeros([_.N_STATES, _.N_VALUES])

    # Honest emission
    result[0] = ones(_.N_VALUES) / _.N_VALUES

    # Dishonest emissions
    for state in range(1, _.N_STATES):
        result[state] = random_distribution_nonzeros(
            _.N_VALUES,
            _.EMISSION_DISHONEST_MAX,
            _.EMISSION_MIN_ENTROPY
        )

    return result

# ------------------------------------------------------------------ Modeling --

def states(transition_matrix, sample_length):
    state = 0
    result = [0] * sample_length

    for i in range(sample_length):
        result[i] = state
        state = random(transition_matrix[state])

    return array(result, dtype=int)

def values(states, emission_matrix):
    return array([random(emission_matrix[state]) for state in states], dtype=int)

# -------------------------------------------------- Reproducible experiments --

def experiment_seeds():
    return np.random.randint(0, 1000, _.K)


import matplotlib.pyplot as plt
import time
import Functions
import numpy as np

#---------------------------------------------------------------
# LOADING OF THE DIFFERENT TEST IMAGES
#---------------------------------------------------------------

img1 = Functions.extraction_image("../data/Test1.png")
img1=img1[:,:,:3]
img2 = Functions.extraction_image("../data/landscape.jpg")
img3 = Functions.extraction_image("../data/falaise.jpg")
img4 = Functions.small_matrix_test()
img5 = Functions.extraction_image("../data/desert.jpg")
img6 = Functions.extraction_image("../data/test_50x50.jpg")
img7 = Functions.extraction_image("../data/depth_sea.jpg")
img8 = Functions.extraction_image("../data/forest.jpg")
img9 = Functions.extraction_image("../data/city.jpg")
img10 = Functions.extraction_image("../data/canyon.jpg")
img11 = Functions.extraction_image("../data/mountains.jpg")
img7_cut = Functions.extraction_image("../data/depth_sea_cut.jpg")
img8_cut = Functions.extraction_image("../data/forest_cut.jpg")
img9_cut = Functions.extraction_image("../data/city_cut.jpg")
img10_cut = Functions.extraction_image("../data/canyon_cut.jpg")
img11_cut = Functions.extraction_image("../data/mountains_cut.jpg")

"""
Parameters:
- epsilon: float, default = 0.0001
    The threshold for the convergence of the algorithm
- iter: int, default = 3
    The number of iterations
- alpha: float or list of float, default = 1.8, can be a list if iterations (ex: for 3 iterations, in a list, alpha = [1, 2, 1.6])
    The different alpha values for the different iterations
- lbda: float or list of float, default = 0.35, can be a list if iterations (ex: for 3 iterations, in a list, lambda = [1, 2, 1.6])
    The lambda value for the algorithm
- weight: float, list of float or list of list of float, default = 1.2 (can be a list if there are iterations, like: weight = [2, 3, 5] for 3 iterations, and weight each details of iteration)
    Linked to the display parameter
    The weight for the details in iterations
- display: str, default = "smoothed detailed full smoothed full final 3"
    The different images to display
    "smoothed": the smoothed image
    "detailed": the detailed image
    "full": the full option show all iterations of detail or smoothed add with a space after detailed or smoothed
    "final": the final image, with weights and X final images if add a int after. For ex: affichage = "final 3", means we want 3 final ones. We can change weights of each one by using weight = [[3, 2], [1.2, 0], [0.2, 5]] for example if we want 3 images, in 2 iterations.
    for "final 2" with iter = 3 for ex can also be weights = [1.2, 3] eqivlent to [[1.2, 1.2, 1.2], [3, 3, 3]] or [2, [2, 3, 0.1]] eqivalent to [[2, 2, 2], [2, 3, 0.1]]
- comparison: str, default = "None"
    The filtered images to compare with the comapared one, but currently only the original image is available, need to add the comparison to gaussian or BLF filters

Also possible to use the class ParamClassFilter

Here are examples below:
"""

#---------------------------------------------------------------
# TEST 1
#---------------------------------------------------------------
epsilon1 = 0.0001
alpha1 = [1.8, 1.2, 1.6]
iter1 = 3
lbda1 = 0.35
display1 = "smoothed detailed full final 3"
comparaison1 = "original"
weight1 = [[1.2, 0, 1], [0, 5, 10], [5, 5, 4]]
curr_img1 = img1
paramFilter1 = Functions.ParamClassFilter(iter1, alpha1, lbda1, weight1, display1, comparaison1, epsilon1)

#---------------------------------------------------------------
# TEST 2
#---------------------------------------------------------------
epsilon2 = 0.0001
alpha2 = 1.5
iter2 = 5
lbda2 = 0.3
display2 = "final 4"
comparaison2 = None
weight2 = [1.2, [0, 5, 10, 0, 0], [3, 2, 5, 5, 4], 5]
curr_img2 = Functions.limit_size(img9_cut, 300)
paramFilter2 = Functions.ParamClassFilter(iter2, alpha2, lbda2, weight2, display2, comparaison2, epsilon2)

#---------------------------------------------------------------
# TEST 3
#---------------------------------------------------------------

epsilon3 = 0.0001
alpha3 = 1.5
iter3 = 1
lbda3 = 0.3
display3 = "final"
comparaison3 = None
weight3 = 1.3
curr_img3 = Functions.limit_size(img10_cut, 300)
paramFilter3 = Functions.ParamClassFilter(iter3, alpha3, lbda3, weight3, display3, comparaison3, epsilon3)
#---------------------------------------------------------------
# TEST 4
#---------------------------------------------------------------
epsilon4 = 0.0001
alpha4 = 2
iter4 = 3
lbda4 = 0.3
display4 = "smoothed detailed full"
comparaison4 = None
weight4 = 1.5
curr_img4 = Functions.limit_size(img8_cut, 600)
paramFilter4 = Functions.ParamClassFilter(iter4, alpha4, lbda4, weight4, display4, comparaison4, epsilon4)

#---------------------------------------------------------------
# MAIN LOOP
#---------------------------------------------------------------

if __name__ == "__main__":
    # Functions.WLS_full(curr_img1, iter = iter1, alpha = alpha1, lbda = lbda1, weight = weight1, display = display1, comparaison = comparaison1, epsilon = epsilon1)
    Functions.WLS_full(curr_img1, param = paramFilter1)


def sum_pairs(lists, sum):
    answer_list=[]
    dictionary={}
    for x in lists:
        dictionary[x]=x
    print(dictionary)
    for key in dictionary:
        one_value=dictionary[key]
        print(one_value)
        for secondkey in dictionary:
            if key==secondkey:
                continue
            if isinstance(dictionary[key], str)or isinstance(dictionary[secondkey], str):
                continue
            elif dictionary[key]+dictionary[secondkey]==sum:
                answer_list.append([key, secondkey])
                dictionary[key]="used"
                dictionary[secondkey]="used"
            print(dictionary)
    if answer_list==[]:
        return 'unable to find pairs'
    print(answer_list)
    return answer_list

# sum_pairs([1,2,3,4,5], 9)
# sum_pairs([1,2,3,4,5], 7)
# sum_pairs([3, 1, 5, 8, 2], 27)

# Before writing your code, don't forget to write your tests!!

# Write a method that takes an integer array and a desired sum. The output will be pairs of numbers from the inputed integer array that equal that desired sum. If there are no pairs that work, return 'unable to find pairs'

# Sample output:

# sum_pairs([1,2,3,4,5], 9) # [[4,5]]
# sum_pairs([1,2,3,4,5], 7) # [[2,5], [3,4]]
# sum_pairs([3, 1, 5, 8, 2], 27) # 'unable to find pairs'


import sys
from pyLingo import *
def book(s1Discount1,s1Discount2,s2Discount,s3Discount,s4Discount):
    #create Lingo enviroment object
    pEnv = lingo.pyLScreateEnvLng()
    if pEnv is None:
        print("cannot create LINGO environment!")
        exit(1)
    #open LINGO's log file
    name='_'+str(s1Discount1)+'_'+str(s1Discount2)+'_'+str(s2Discount)+'_'+str(s3Discount)+'_'+str(s4Discount)
    cur_name='book'+name+'.log'
    errorcode = lingo.pyLSopenLogFileLng(pEnv,cur_name)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    #pass memory transfer pointers to LINGO
    #define pnPointersNow
    pnPointersNow = N.array([0],dtype=N.int32)
    #@POINTER(1)
    s1Discount1_cur = N.array([s1Discount1],dtype=N.double)
    errorcode = lingo.pyLSsetDouPointerLng(pEnv, s1Discount1_cur, pnPointersNow)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    #@POINTER(2)
    s1Discount2_cur = N.array([s1Discount2],dtype=N.double)
    errorcode = lingo.pyLSsetDouPointerLng(pEnv, s1Discount2_cur, pnPointersNow)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    #@POINTER(3)
    s2Discount_cur = N.array([s2Discount],dtype=N.double)
    errorcode = lingo.pyLSsetDouPointerLng(pEnv, s2Discount_cur, pnPointersNow)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    #@POINTER(4)
    s3Discount_cur = N.array([s3Discount],dtype=N.double)
    errorcode = lingo.pyLSsetDouPointerLng(pEnv, s3Discount_cur, pnPointersNow)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    #@POINTER(5)
    s4Discount_cur = N.array([s4Discount],dtype=N.double)
    errorcode = lingo.pyLSsetDouPointerLng(pEnv, s4Discount_cur, pnPointersNow)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    #@POINTER(6)
    Status = N.array([-1.0],dtype=N.double)
    errorcode = lingo.pyLSsetDouPointerLng(pEnv, Status, pnPointersNow)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    #Run the script
    cScript = "SET ECHOIN 1 \n TAKE book.lng \n GO \n QUIT \n"
    errorcode = lingo.pyLSexecuteScriptLng(pEnv, cScript)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    #Close the log file
    errorcode = lingo.pyLScloseLogFileLng(pEnv)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        print("errorcode = ", errorcode)
        exit(1)
    if Status[0] == const.LS_STATUS_GLOBAL_LNG:
        print("\nGlobal optimum found!")
    elif Status[0] == const.LS_STATUS_LOCAL_LNG:
        print("\nLocal optimum found!")
    else:
        print("\nSolution is non-optimal\n")
    #delete Lingo enviroment object
    errorcode = lingo.pyLSdeleteEnvLng(pEnv)
    if errorcode != const.LSERR_NO_ERROR_LNG:
        exit(1)

if __name__ == '__main__':
    rate=[0.1,0.2,0.3,0.4,0.5]
    discount_float_2=[18,21,24] #s2店铺浮动折扣
    discount_float_3=[0.88,0.85,0.82,0.79,0.76,0.73]  #s3店铺浮动折扣
    boundary_4=[125,118,113,105] #s4边界浮动
    for i in discount_float_2:
        for j in discount_float_3:
            for k in boundary_4:
                book(16,40,i,j,k)
                sys.stdin.read(1)


'''
Created on Mar 31, 2014

@author: anuvrat

Use python 2.7 for this one
'''

from ortools.constraint_solver import pywrapcp

def read_puzzle(lines):
    puzzle_name = lines[0]
    puzzle = []

    for line in lines[1:]:
        puzzle.extend(map(int, list(line.strip())))

    return puzzle_name.strip(), puzzle

def solve_puzzle(lines):
    puzzle_name, puzzle = read_puzzle(lines)
    print(puzzle_name)

    solver = pywrapcp.Solver(puzzle_name)
    tiles = [solver.IntVar(1, 9, 'Tile-%i' % i) for i in range(81)]

    # rows all different condition
    for row in range(9):
        solver.Add(solver.AllDifferent(tiles[row * 9 : (row + 1) * 9]))

    # columns all different condition
    for column in range(9):
        solver.Add(solver.AllDifferent([tiles[column + 9 * i] for i in range(9)]))

    # blocks all different condition
    block_start = 0
    for block in range(9):
        block_tiles = []
        for a, b in ((i, j) for i in range(3) for j in range(3)):
            block_tiles.append(tiles[block_start + 9 * a + b])
        solver.Add(solver.AllDifferent(block_tiles))
        block_start = block_start + 21 if block % 3 == 2 else block_start + 3

    # add known tile values
    for idx in range(81):
        tile_value = puzzle[idx]
        if tile_value != 0:
            solver.Add(tiles[idx] == tile_value)

    solution = solver.Assignment()
    solution.Add(tiles)

    collector = solver.FirstSolutionCollector(solution)
    solver.Solve(solver.Phase(tiles, solver.CHOOSE_LOWEST_MIN, solver.ASSIGN_MIN_VALUE), [collector])

    solved_puzzle = map(str, [collector.Value(0, val) for val in tiles])

    # print solved puzzle
    for row in range(9):
        print(' '.join(solved_puzzle[row * 9 : (row + 1) * 9]))

    return int(solved_puzzle[0]) * 100 + int(solved_puzzle[1]) * 10 + int(solved_puzzle[2])

if __name__ == '__main__':
    input_data_file = open('/Users/anuvrat/git/project-euler-python/resource/problem_96_input.txt', 'r')
    lines = input_data_file.readlines()
    input_data_file.close()

    total = 0
    for idx in range(len(lines) / 10):
        total += solve_puzzle(lines[idx * 10 : (idx + 1) * 10])

    print(total)


from tensorflow.keras.models import Sequential, load_model as keras_load_model
from tensorflow.keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D, BatchNormalization
from tensorflow.keras.optimizers import Adam, SGD
from tensorflow.keras.metrics import categorical_crossentropy
from tensorflow.keras.regularizers import l2
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, CSVLogger, Callback
from sklearn.metrics import confusion_matrix, accuracy_score, precision_recall_fscore_support, classification_report
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from modules.m2_preprocessing import load_train_valid, load_testing, load_train_valid_tuner, load_testlines
from modules.utils import CLASSES, IMAGE_SIZE, model_metadata, save_model_metadata, \
	update_model_retrained, update_model_status, get_project_name, _load_json, _save_json, show_classification_metrics

from collections import Counter
from scipy.stats import shapiro
from numpy import random

import keras_tuner, json, os, codecs
import tensorflow_addons as tfa
import tensorflow as tf
import numpy as np
import seaborn as sb
import pandas as pd
import scipy.stats as stats

# dictionary for epochs
EPOCHS = {
	'max_dummy': 3,
	'patience_dummy': 3,
	'max': 500,
	'patience': 20,
	# 'patience': 10,
	# observation: 15 epochs is the max interval,
	# gender_2L_nocanny stops learning
}

# dictionary for tuner constants
TUNER = {
	'epochs_dummy': 3,
	'patience_dummy': 3,
	'max_trials_dummy': 3,
	'epochs': 200,
	'patience': 10,
	'max_trials': 10,
	'objective': 'val_accuracy',
	'overwrite': False,
}

class TrainingProcess():
	def __init__(
		self,
		save_path,
		dataset_path,
		nlayer,
		canny,
		category,
		dummy=False,
	):

		# --------- model metadata ---------
		self.dataset_path = dataset_path
		self.project_name = get_project_name(nlayer, canny, category)
		self.save_path = self.init_save_path(save_path)
		self.nlayer = nlayer
		self.canny = canny
		self.category = category
		self.binary = category != 'gender-handedness'
		self.morera = not canny and nlayer == 2
		metadata = model_metadata( # from utils.py
			path=self.save_path,
			nlayer=nlayer,
			canny=canny,
			category=category,
			morera=self.morera)

		# --------- if already done ---------
		if metadata['max_epoch'] > -1:
			raise Exception('Training process for this model is already completed. If you want to redo, delete or rename the project folder first.\n-dev')

		# ----------- set paths ------------
		rootprefix = f'{self.save_path}/{self.project_name}'
		self.metadata = rootprefix + '_metadata.json'
		self.checkpointfile = rootprefix + '_model.h5'
		self.csvloggerfile = rootprefix + '_model.csv'
		self.historyfile = rootprefix + '_history.json'

		# -------- if dummy test run --------
		self.dummy = dummy
		self.dummy_suffix = '_dummy' if self.dummy else ''

		# ------- set number of epochs ------
		self.epochs = EPOCHS['max'+self.dummy_suffix]
		self.patience = EPOCHS['patience'+self.dummy_suffix]

		# ------ if not morera's archi ------
		if not self.morera:
			self.save_path_tuner = self.init_save_path_tuner(self.save_path)
			self.checkpointtuner = self.save_path_tuner + '/latest_trial_model.h5'
			self.istuned = metadata['best_hp'] != 'None'
			self.epochs_tuner = TUNER['epochs'+self.dummy_suffix]
			self.patience_tuner = TUNER['patience'+self.dummy_suffix]
			self.tunername = self.project_name + '_tuner'

		# ----------- load model ------------
		self.model = self.load_model()  # from this class

		# -------- done initializing ----------
		print(f'\nMODEL: {self.project_name}\n(project_name)\n\nInitialized!')

	# creates and returns a save path for the project
	# if already a dir, it just returns the path
	def init_save_path(
		self,
		save_path
	):
		save_path = save_path + '/' + self.project_name

		if not os.path.isdir(save_path):
			os.makedirs(save_path)

		return save_path

	# creates and returns a save path for the tuner
	# if already a dir, it just returns the path
	def init_save_path_tuner(
		self,
		save_path
	):
		# save_path is already output/project_name
		save_path = save_path + '/' + self.project_name + '_tuner_output'

		# save_path will be output/project_name/project_name_tuner_output
		if not os.path.isdir(save_path):
			os.makedirs(save_path)

		return save_path

	# returns the number of trials
	# raises exception if didnt find anything
	def get_current_trials(self):
		max_trials = TUNER['max_trials'+self.dummy_suffix]
		trials = 0
		for trials in range(max_trials):
			trial_dir = 'trial_'
			if trials < 10 and max_trials >= 10:
				trial_dir += '0'

			if not os.path.isdir(f'{self.save_path}/{self.tunername}/{trial_dir}{trials}'):
				return trials
		return max_trials
		raise Exception('Bug. Did not return anything from function get_current_trials. -dev')

	# returns the number of epochs
	# returns 0 if no csv logger
	def get_max_epoch(
		self,
		csvlogger,
	):
		try: # get max epoch from the logger file path
			logs = pd.read_csv(csvlogger) # get logs
			return logs.shape[0]
		# if log not found, must be new
		except FileNotFoundError:
			return 0

	# if existing, returns the h5 file model
	# else, returns the base model, compiled if morera's
	def load_model(self):
		if os.path.isfile(self.checkpointfile):
			return keras_load_model(self.checkpointfile, compile=True) # from keras
		else:
			if self.morera:
				return self.build_model(None)
			else:
				csvloggerfile_tuner = f'{self.save_path_tuner}/latest_trial_log.csv'
				if os.path.isfile(csvloggerfile_tuner) and os.path.isfile(self.checkpointtuner):
					return keras_load_model(self.checkpointtuner, compile=True) # from keras
				return None

	# uses the entire function and compiles the base model if morera's
	# takes the entire function and is passed to the tuner, if tuner
	def build_model(
		self,
		hp,
	):
		input_shape = (IMAGE_SIZE['height'], IMAGE_SIZE['width'], 1)

		# ========== morera's ==========
		learning_rate = 0.001
		kernel_size = (5,5)
		maxpooling_size = (2,2)
		padding = 'same' # zero-padding
		dropout_layer = 0.25
		dropout_dense = 0.5
		units = 512 # last dense layer
		weight_decay = 1E-7

		# constant
		activation_layer = 'relu'
		activation_dense = 'softmax'

		if self.category == 'gender':
			filters = 128
		elif self.category == 'handedness':
			filters = 64
		else:
			filters = 32

		# ========== start of bayesian optimization ==========
		if hp:
			filters = hp.Int('filters', min_value=32, max_value=128, step=32)

			learning_rate = hp.Float(
				'learning_rate',
				min_value=1e-4,
				max_value=1e-2,
				sampling='log',
			)

			dropout_layer = hp.Float(
				'dropout_layer',
				min_value=0.0001,
				max_value=0.5,
				sampling='log',
			)

			dropout_dense = hp.Float(
				'dropout_dense',
				min_value=0.0001,
				max_value=0.5,
				sampling='log',
			)

			weight_decay = hp.Float(
				'weight_decay',
				min_value=1e-8,
				max_value=1e-6,
				sampling='log',
			)

		# Model
		model = Sequential()

		# First block
		model.add(Conv2D(filters, # 32
			kernel_size,
			input_shape=input_shape,
			padding=padding,
		))
		model.add(Activation(activation_layer)) # relu: activates the input of block to block
		model.add(Dropout(dropout_layer)) # reduces number of neurons randomly
		model.add(MaxPooling2D(maxpooling_size)) # summarizes the pixel values which makes the conv smaller essentially

		# Second block
		model.add(Conv2D(filters * 2, # 64
			kernel_size,
			padding=padding,
		))
		model.add(Activation(activation_layer))
		model.add(Dropout(dropout_layer))
		model.add(MaxPooling2D(maxpooling_size))

		# Third block
		# filters *2 is somewhat a default / norm in setting filters
		if self.nlayer == 3:
			model.add(Conv2D(filters * 4, # 128
				kernel_size,
				padding=padding,
			))
			model.add(Activation(activation_layer))
			model.add(Dropout(dropout_layer))
			model.add(MaxPooling2D(maxpooling_size))

		# Fully connected blocks
		model.add(Flatten()) # into 1 dimension for dense layer; prep for output

		# Output Layer
		model.add(Dense(units)) # summarizes the flattened
		model.add(Activation(activation_layer))
		model.add(Dropout(dropout_dense))
		model.add(Dense(2 if self.binary else 4)) # actual output
		model.add(Activation(activation_dense)) # softmax: activates the input of blocks to prediction

		# Instantiate a optimizer
		if self.binary:
			optimizer = tfa.optimizers.SGDW(
				learning_rate=learning_rate,
				weight_decay=weight_decay)
		else:
			optimizer = tfa.optimizers.AdamW(
				learning_rate=learning_rate,
				weight_decay=weight_decay)

		# Instantiate a logistic loss function that expects integer targets
		loss = 'categorical_crossentropy'

		# Instantiate an accuracy metric
		accuracy = 'accuracy'

		model.compile(optimizer=optimizer, loss=loss, metrics=[accuracy])

		return model

	# callback for search and train
	# makes a folder for logs and tensorboard
	def create_callbacks(
		self,
		csvlogger,
		checkpoint,
		patience,
		monitor,
	):
		return [
			CSVLogger(csvlogger, separator=',', append=True),
			ModelCheckpoint(checkpoint, monitor=monitor, verbose=1, save_best_only=True),
			EarlyStopping(monitor=monitor, patience=patience, verbose=1, baseline=None),
		]

	# search best hyperparameters using bayesian optimization tuner
	# will be skipped if morera, or if tuner is done searching
	def search(
		self,
		training_dataset_only
	):
		if self.morera:
			print('\nSearching with Tuner skipped: Morera\'s architecture is being used.')
			return

		# get current number of trials
		trial_count = max(self.get_current_trials(), 1)

		# if theres continuation epoch
		csvloggerfile = f'{self.save_path_tuner}/latest_trial_log.csv'
		cont_epoch = self.get_max_epoch(csvloggerfile)

		# instantiate custom model to tune
		model = MyHyperModel(
			save_path=self.save_path_tuner,
			build_model=self.build_model,
			category=self.category,
			nlayer=self.nlayer,
			binary=self.binary,
			epochs=self.epochs_tuner,
			cont_epoch=cont_epoch,
			model=self.model,
			trial_count=trial_count,
		)

		tuner =  keras_tuner.BayesianOptimization(
			hypermodel=model,
			objective=TUNER['objective'],
			max_trials=TUNER['max_trials'+self.dummy_suffix],
			overwrite=TUNER['overwrite'],
			directory=self.save_path,
			project_name=self.tunername,
		)

		if not self.istuned:
			tuner_train_batch, tuner_valid_batch = load_train_valid_tuner( # from module 2
				path=self.dataset_path,
				category=self.category,
				canny=self.canny,
				dummy=self.dummy,
				training_dataset_only=training_dataset_only,
			)

			tuner.search(
				tuner_train_batch,
				validation_data = tuner_valid_batch,
				verbose = 1,
				callbacks=self.create_callbacks(
					csvlogger=csvloggerfile,
					checkpoint=self.checkpointtuner,
					patience=self.patience_tuner,
					monitor='val_accuracy',
				),
			)

			update_model_status( # from utils.py
				project_path = self.metadata,
				best_hp = tuner.get_best_hyperparameters()[0].values,
			)

		print(f'\nSearching with Tuner finished: {TUNER["max_trials"+self.dummy_suffix]} "max_trial" for the tuner has been reached,')

		# get the top 1 hyperparameters.
		best_hp = tuner.get_best_hyperparameters()[0]

		print('With best hyperparameters:', end=' ')
		print(best_hp.values, end='\n')

		if self.istuned:
			return

		# build the model with the best hp.
		self.model = self.build_model(best_hp)

	# train with best hyperparameters
	# returns the trained model
	def train(
		self,
		training_dataset_only,
	):

		# get last epoch if stopped, then print message
		cont_epoch = self.get_max_epoch(csvlogger=self.csvloggerfile)
		if cont_epoch > 0:
			self.epochs -= cont_epoch
			message = f'Project stopped training at {cont_epoch} epoch/s... '
			message += f'Continuing with {self.epochs} epoch/s left.'
			print(message, end='\n')

		# load dataset
		train_batch, valid_batch = load_train_valid( # from module 2
			path=self.dataset_path,
			category=self.category,
			canny=self.canny,
			dummy=self.dummy,
			training_dataset_only=training_dataset_only,
		)

		# start/continue to train the model
		self.model.fit(
			train_batch,
			validation_data=valid_batch,
			epochs=self.epochs,
			verbose=1,
			callbacks=self.create_callbacks(
				csvlogger=self.csvloggerfile,
				checkpoint=self.checkpointfile,
				patience=self.patience,
				monitor='val_accuracy',
			),
		)

		update_model_status( # from utils.py
			project_path = self.metadata,
			max_epoch = self.get_max_epoch(csvlogger=self.csvloggerfile),
		)

	# function that will start the training process
	# calls search and train function
	def start(
		self,
		training_dataset_only=False,
	):
		self.search(training_dataset_only)
		self.train(training_dataset_only)

class MyHyperModel(keras_tuner.HyperModel):
	def __init__(
		self,
		save_path,
		build_model, 		# function
		category, 			# metadata
		nlayer,
		binary,
		epochs,				# epochs of tuner
		cont_epoch,			# continutation epoch
		model,				# h5 file of trial model
		trial_count,		# current number of trials
	):
		super(MyHyperModel, self).__init__()
		self.save_path = save_path
		self.build_model = build_model
		self.category = category
		self.nlayer = nlayer
		self.binary = binary
		self.epochs = epochs
		self.cont_epoch = cont_epoch
		self.model = model
		self.trial_count = trial_count

	# uses the build of TrainingProcess
	# returns the compiled model
	def build(self, hp):
		model = self.build_model(hp)
		return model

	# custom fit function for this model
	# only used when tuning
	def fit(self, hp, model, *args, **kwargs):
		if self.model is not None:
			model = self.model

		# get last epoch if stopped, then print message
		epochs = self.epochs
		if self.cont_epoch > 0:
			epochs -= self.cont_epoch
			message = f'Project stopped tuning at {self.cont_epoch} epoch/s... '
			message += f'Continuing with {epochs} epoch/s left.'
			print(message, end='\n')

		# fit for one trial
		history = model.fit(
			*args,
			epochs=epochs,
			**kwargs,
		)

		max_trials = TUNER['max_trials']
		trial_suffix = ''
		if self.trial_count < 10 and max_trials >= 10:
			trial_suffix += '0'

		# rename the csv logger to not overwrite it
		os.rename(
			f'{self.save_path}/latest_trial_log.csv',
			f'{self.save_path}/trial_{trial_suffix}{self.trial_count-1}.csv',

		)

		# increase trial count
		self.trial_count += 1

		# reset cont_epoch, since trial is done
		self.cont_epoch = 0

		return history

class TestingProcess():
	def __init__(self,
		save_path,
		dataset_path=None,
		dummy=False,
		show_skipped=False,
		show_results=False,
		return_skipped=False,
	):
		self.save_path_root = save_path
		self.dataset_path = dataset_path
		self.dummy = dummy
		self.show_skipped = show_skipped
		self.show_results = show_results
		self.return_skipped = return_skipped

	def load_model(
		self,
		checkpointfile,
	):
		if os.path.isfile(checkpointfile):
			return keras_load_model(checkpointfile, compile=True) # from keras
		else:
			return None

	def model_summary(self, project_name):
		project_path = f'{self.save_path_root}/{project_name}'
		model = self.load_model(f'{project_path}/{project_name}_model.h5')
		print(model.summary())

	def test(self, project_name):
		if not self.dataset_path:
			raise Exception('Attempting to test without dataset_path. -dev')

		if type(project_name) is not list:
			project_name = [project_name]

		skipped = {}

		for project in project_name:
			# set path
			project_path = f'{self.save_path_root}/{project}'

			# if project does not exist
			if not os.path.isdir(project_path):
				skipped[project] = 'Project directory not found.'
				continue

			# get metadata
			metadata = model_metadata( # from utils.py
				path=project_path,
				project_name=project,
			)
			canny = metadata['canny'] == 'True'
			category = metadata['category']
			binary = category != 'gender-handedness'

			# if already done with results
			if os.path.isfile(f'{project_path}/{project}_results.json'):
				skipped[project] = 'Already been tested. Please check the \"results\" json.'
				continue

			# if not done training
			if metadata['max_epoch'] == -1:
				skipped[project] = 'This model has not finished training yet.'
				continue

			# load model and dataset
			model = self.load_model(f'{project_path}/{project}_model.h5')

			# if model is missing
			if model is None:
				skipped[project] = 'The "_model.h5" file is missing.'
				continue

			# prompt project name
			print(f'\nMODEL: {project}\n(project_name)\n\n')

			# dataset path
			if self.dummy:
				dataset_path = 'data/sample'
			else:
				dataset_path = self.dataset_path

			# load the test lines
			test_x, test_y = load_testlines( # from module 2
				path=dataset_path,
				category=category,
				canny=canny,
				batch_size=1,
				dummy=self.dummy,
			)

			# test the model
			results_data = self.test_category_model(
				model=model,
				test_x=test_x,
				test_y=test_y,
				project_name=project,
				project_path=project_path,
			)

			if self.show_results:
				show_classification_metrics(
					classification_metrics=results_data['classification_metrics'],
					project_name=project,
					category=category,
				)

		if self.show_skipped:
			if skipped != {}:
				print('Skipped:')
				for key, val in skipped.items():
					print(f'{key}: {val}')

		if self.return_skipped:
			return skipped

	def majority_voting_scheme(
		self,
		pred_words_y,
	):
		vote_pred = []
		vote_class = []

		for i, y in enumerate(pred_words_y):
			pred_words_y[i] = y[0]

		#get the highest prediction and its class
		for y in pred_words_y:
			pred_conf = max(y)
			pred_class = y.index(pred_conf)
			# print(f'y: {y}, pred_conf: {pred_conf}, pred_class: {pred_class}')
			vote_pred.append(pred_conf)
			vote_class.append(pred_class)

		# convert array into dictionary
		count = Counter(vote_class)

		# traverse dictionary and check majority element
		size = len(vote_class)

		highest = {
			'class': [],
			'votes': 0,
		}
		for (perclass, votes_perclass) in count.items():
			# print(f'class: {perclass}, votes: {votes_perclass}')
			if votes_perclass > highest['votes']: # if found higher
				highest['class'] = [perclass]
				highest['votes'] = votes_perclass
			elif votes_perclass == highest['votes']: # if tie
				highest['class'].append(perclass)

		if len(highest['class']) == 1:
			return perclass, {'votes/confidence': f'{highest["votes"]}/{size}'}
		else: # if tie
			highest_conf_pred = 0
			highest_conf_class = 0
			for i in range(len(vote_class)):
				if vote_class[i] in highest['class'] and vote_pred[i] > highest_conf_pred:
					highest_conf_pred = vote_pred[i]
					highest_conf_class = vote_class[i]
			return highest_conf_class, {'votes/confidence': highest_conf_pred}

	def predict(
		self,
		model,
		word_imgs,
	):
		pred_words_y = []
		for word in word_imgs:
			pred_words_y.append(model.predict(word).tolist())
		pred_line, pred_metadata = self.majority_voting_scheme(pred_words_y)
		return pred_line, pred_metadata

	def test_category_model(
		self,
		model,
		test_x,
		test_y=None,
		project_name=None,
		project_path=None,
	):
		line_counter =  0
		pred_y_perword = []
		pred_y = []
		pred_y_md = []
		for data_line, data_label in zip(test_x, test_y):
			pred_words_y = []
			# print(f'LABEL: {data_label}')
			for word in data_line:
				pred_words_y.append(model.predict(word).tolist())
			pred_y_perword.append(pred_words_y)
			pred_line, pred_metadata = self.majority_voting_scheme(pred_words_y)
			pred_y.append(pred_line)
			pred_y_md.append(pred_metadata)
			print(f'{line_counter+1} lines predicted.', end='\r')
			line_counter += 1
		# print()
		print('\nDone predicting\n')
		# print('\npred_y:')
		# print(pred_y)

		# performance metrics
		classification_metrics = self.compute_classification_metrics(
			ground=test_y,
			predicted=pred_y,
		)

		# save all as json
		return self.save_model_results(
			test_y=test_y,
			pred_y=pred_y,
			pred_y_md=pred_y_md,
			pred_y_perword=pred_y_perword,
			classification_metrics=classification_metrics,
			project_name=project_name,
			path=project_path,
		)

	def compute_classification_metrics(
		self,
		ground,
		predicted,
	):
		# accuracy
		acc = accuracy_score(ground, predicted)

		# precision, recall, fscore, support
		p, r, f, s = precision_recall_fscore_support( # from sklearn
			ground,
			predicted,
			labels=list(set(ground)),
		)

		# confusion matrix
		conf_matrix = confusion_matrix(ground, predicted)

		# classification report
		report = classification_report(ground, predicted)

		# return metrics
		metrics = {
			'matrix': conf_matrix,
			'accuracy': acc,
			'precision': p,
			'recall': r,
			'fscore': f,
			'support': s,
			'report': report,
		}

		return metrics

	def save_model_results(
		self,
		test_y,
		pred_y,
		pred_y_md,
		pred_y_perword,
		classification_metrics,
		project_name,
		path,
	):
		# PERFORMANCE METRICS
		matrix = []
		for row in classification_metrics['matrix']:
			matrix.append(row.tolist())
		classification_metrics['matrix'] = matrix
		classification_metrics['precision'] = classification_metrics['precision'].tolist()
		classification_metrics['recall'] = classification_metrics['recall'].tolist()
		classification_metrics['fscore'] = classification_metrics['fscore'].tolist()
		classification_metrics['support'] = classification_metrics['support'].tolist()

		# DATA
		data = {
			'name': project_name,
			'test_y': test_y,
			'pred_y': pred_y,
			'pred_y_md': pred_y_md,
			'pred_y_perword': pred_y_perword,
			'classification_metrics': classification_metrics,
		}

		_save_json(f'{path}/{project_name}_results.json', data)

		return data


####################################################
# CS 31, Prof. Muldrow
# Name: Javier Rodriguez
# Assignment: Chapter 13 Homework
# Due Date: 23 MAY 2021
####################################################

import tkinter
import tkinter.messagebox

########## Problem 7, Long-Distance Calls ##########
class LongDistanceGUI:
    def __init__(self):
        self.main_window = tkinter.Tk()
        self.minutes_frame = tkinter.Frame(self.main_window)
        self.op_frame = tkinter.Frame(self.main_window)
        self.buttons_frame = tkinter.Frame(self.main_window)

        # widgets for minutes_frame
        self.minutes_label = tkinter.Label(self.minutes_frame, text = 'Enter number of minutes:')
        self.minutes_entry = tkinter.Entry(self.minutes_frame, width = 10)
        self.minutes_label.pack(side = 'left')
        self.minutes_entry.pack(side = 'left')

        # widgets for op_frame
        self.rb_var = tkinter.IntVar()
        self.rb_var.set(0)
        self.daytime_rb = tkinter.Radiobutton(self.op_frame, text = 'Daytime (6:00 am through 5:59 pm)', variable = self.rb_var, value = 1)
        self.evening_rb = tkinter.Radiobutton(self.op_frame, text = 'Evening (6:00 pm through 11:59 pm)', variable = self.rb_var, value = 2)
        self.offpeak_rb = tkinter.Radiobutton(self.op_frame, text = 'Off-Peak (midnight through 5:59 am)', variable = self.rb_var, value = 3)
        self.daytime_rb.pack(side = 'left')
        self.evening_rb.pack(side = 'left')
        self.offpeak_rb.pack(side = 'left')

        # widgets for buttons_frame
        self.calc_button = tkinter.Button(self.buttons_frame, text = 'Calculate', command = self.rate)
        self.quit_button = tkinter.Button(self.buttons_frame, text = 'Quit', command = self.main_window.destroy)
        self.calc_button.pack(side = 'left')
        self.quit_button.pack(side = 'left')

        # pack the frames and go into main loop
        self.minutes_frame.pack()
        self.op_frame.pack()
        self.buttons_frame.pack()

        tkinter.mainloop()

    def rate(self):
        self.mins = int(self.minutes_entry.get())
        if self.rb_var.get() == 1:
            self.cost = float(self.mins * .07)
        elif self.rb_var.get() == 2:
            self.cost = float(self.mins * .12)
        elif self.rb_var.get() == 3:
            self.cost = float(self.mins * .05)

        tkinter.messagebox.showinfo('Long-Distance Calls', f'Cost of the call: ${self.cost:.2f}')

callGUI = LongDistanceGUI()


import numpy as np
from sklearn.neighbors import NearestNeighbors
import numpy as np

def calc_SL(W,n_objs=3):
    num = min(n_objs,len(W))

    neigh = NearestNeighbors(num)
    neigh.fit(W)

    neighbors = neigh.kneighbors(W, num)[0].tolist()
    SL = [np.prod(item[1:num]) for item in neighbors]
    return SL

def delete_vector_random(X, Y, W, z, Archive, delete_ratio = 0.01,lower_pop=10):
    arg_edges = (W == 1).any(axis=1)
    W_edges = W[arg_edges]
    X_edges = X[arg_edges]
    Y_edges = Y[arg_edges]
    if len(W)<=lower_pop:
        return X, Y, W, max(2, len(W)//10)

    W_ = W[~arg_edges]
    X_ = X[~arg_edges]
    Y_ = Y[~arg_edges]

    num  = int(len(W_) - len(W_)*delete_ratio)
    arg_remain = np.random.choice(range(len(W_)),num,replace=False)
    W_ = W_[arg_remain]
    X_ = X_[arg_remain]
    Y_ = Y_[arg_remain]

    W_ = np.append(W_, W_edges,axis=0)
    X_ = np.append(X_, X_edges,axis=0)
    Y_ = np.append(Y_, Y_edges,axis=0)
    T = max(2, len(W_)//10)

    return X_, Y_, W_, T

# def delete_vector(X, Y, W, z, Archive, delete_ratio = 0.01,lower_pop=10):
#     arg_edges = (W == 1).any(axis=1)
#     W_edges = W[arg_edges]
#     X_edges = X[arg_edges]
#     Y_edges = Y[arg_edges]
#     if len(W)<=lower_pop:
#         return X, Y, W, max(2, len(W)//10)

#     W_ = W[~arg_edges]
#     X_ = X[~arg_edges]
#     Y_ = Y[~arg_edges]

#     SL = calc_SL(W_)
#     num  = int(len(W_) - len(W_)*delete_ratio)
#     L = np.argsort(SL)[::-1][-num:]

#     arg_remain = np.array([False if i in L else True for i in range(len(W_))])

#     W_ = W_[arg_remain]
#     X_ = X_[arg_remain]
#     Y_ = Y_[arg_remain]

#     W_ = np.append(W_, W_edges,axis=0)
#     X_ = np.append(X_, X_edges,axis=0)
#     Y_ = np.append(Y_, Y_edges,axis=0)
#     T = max(2, len(W_)//10)

#     return X_, Y_, W_, T

def delete_vector_AWA(X, Y, W, z, Archive, update_ratio = 0.05,first_pop=105):
    arg_edges = (W == 1).any(axis=1)
    W_edges = W[arg_edges]
    X_edges = X[arg_edges]
    Y_edges = Y[arg_edges]

    W_ = W[~arg_edges]
    X_ = X[~arg_edges]
    Y_ = Y[~arg_edges]

    nus = max(int(first_pop*update_ratio),1)
    L = []
    for i in range(nus):
        SL = calc_SL(W_)
        arg = np.argmin(SL)
        index = np.ones(len(W_),dtype=bool)
        index[arg] = False
        W_ = W_[index]
        X_ = X_[index]
        Y_ = Y_[index]

    W_ = np.append(W_, W_edges,axis=0)
    X_ = np.append(X_, X_edges,axis=0)
    Y_ = np.append(Y_, Y_edges,axis=0)
    T = max(2, len(W_)//10)

    return X_, Y_, W_, T


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 31 17:35:22 2022

@author: Gabriel
"""

from osgeo import gdal
import numpy as np
import cv2
import tkinter as tk
from tkinter import filedialog
import os

root = tk.Tk()
img_new = filedialog.askopenfilename(filetypes=(("tif files","*.tif"),("All files","*.*")))
img_old = filedialog.askopenfilename(filetypes=(("tif files","*.tif"),("All files","*.*")))
root.destroy()

saida = 'D:/TCC/IMAGENS/Classificadas/Reprocessada/Change_' + os.path.basename(img_old)

img_new = gdal.Open(img_new,gdal.GA_ReadOnly)
img_old = gdal.Open(img_old,gdal.GA_ReadOnly)

l,c = img_new.RasterYSize,img_new.RasterXSize

array_new = img_new.ReadAsArray()

array_old = img_old.ReadAsArray()

geo_transform = img_new.GetGeoTransform()
projection = img_new.GetProjectionRef()

#re_cls = np.zeros([l,c])

for i in range(0,l):
    for j in range(0,c):
        if array_new[i,j] == 2 and array_old[i,j] != 2:
            array_old[i,j] = 2
        if array_new[i,j] == 3 and array_old[i,j] != 3:
            array_old[i,j] = 3

array_old = array_old.astype('uint8')
kernel = np.ones([3,3])
array_old = cv2.morphologyEx(array_old, cv2.MORPH_CLOSE, kernel)

driver = gdal.GetDriverByName('GTiff')
rows, cols = array_old.shape
rasterDS = driver.Create(saida, cols, rows, 1, gdal.GDT_Int32)
rasterDS.SetProjection(projection)
rasterDS.SetGeoTransform(geo_transform)
band = rasterDS.GetRasterBand(1)
band.WriteArray(array_old)
rasterDS = None


# Book Index
# ------------------------------------------------------------------------
# Write a function that given a sorted array of page numbers, return a string representing a book index. Combine consecutive pages to create ranges. Given
# [1, 3, 4, 5, 7, 8, 10], return the string "1, 3-5, 7-8, 10".
def bookIndex(num_string):
    var = ""
    string_x = ""
    for x in range (0, len(num_string)-1):
        if (num_string[x]-(num_string[x+1])) != 1:
            string_x += str(num_string[x])
            print(num_string[x+1])
            print(string_x + ',')
            # print(num_string[x]+1)
        # elif num_string[x+1] == (num_string[x]+1):
        #     print(num_string[x+1])
        #     print(num_string[x]+1)
        #     var = num_string[x+1]
        # else:
        #     string_x += str(num_string[x])
        #     string_x += str(var)

    return string_x

print(bookIndex([1, 3, 4, 5, 7, 8, 10]))

# Common Suffix
# ------------------------------------------------------------------------
# When given an array of words, returns the largest suffix (word-end) that is common between all words. For example, for input ["ovation", "notation", "action"], return "tion". Given ["nice", "ice", "sic"], return "".


# Definition for singly-linked list.
class ListNode:

    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution:

    def reverseList(self, head: ListNode) -> ListNode:
        '''反转链表'''
        ret = None
        while head:
            tmp = head.next
            head.next = ret
            ret = head
            head = tmp
        return ret


# 객체지향 프로그래밍( OOP )를 지원함.
# "class 클래스명" 을 사용하여 정의

# Python은 접근 제한자(public, protected, private)를 갖지 않는다. = 기본적으로 public
# 특정 변수명이나 메서드를 private로 만들려면 두개의 밑줄(__)을 이름 앞에 붙이자!

class MyClass:
    pass    # 빈 클래스

# 클래스 멤버
    # 데이터 표현 ( 속성 )
    # 클래스의 행위를 표현 ( 메서드 ) = 클래스 내의 함수
    # 프로퍼티, 클래스 변수, 인스턴스 변수, 초기자 등등...

# 메서드
    # 인스턴스, 클래스, 정적 메서드
    # 인스턴스 메서드 : 인스턴스 변수에 엑서스 할 수 있도록
    #                  메서드의 첫번째 파라미터에 항상 객체 자신을 의미하는
    #                   "self"라는 파라미터를 갖는다.
class Rectangle:
    count = 0   # 클래스 변수

    # 초기자
    def __init__(self, width, height):
        # self.* : 인스턴스 변수
        self.width = width
        self.height = height
        Rectangle.count += 1

    # 메서드
    def calcArea(self):
        area = self.width * self.height
        return area

# 클래스 변수
    # 메서드 밖에 존재하는 변수
    # 클래스 내외부에서 "클래스명.변수명" 으로 엑세스 할 수 있다.
    # 하나의 클래스에 하나만 존재

# 인스턴스 변수
    # 하나의 클래스로부터 여러 객체 인스턴스를 생성하여 사용
    # 메서드 안에서 사용되면서 "self.변수명" 처럼 사용

def __init__(self, width, height):
    self.width = width
    self.height = height

    # private 변수 __area           # private 사용시 이름 앞에 두개의 밑줄(__) 사용
    self.__area = width * height

    # private 메서드                # private 사용시 이름 앞에 두개의 밑줄(__) 사용
def __internalRun(self):
    pass

# 초기자
    # 클래스로부터 새 객체를 생성할 때마다 실행되는
    # __init__() 이라는 메서드가 있는데, 이를 흔히 클래스 Initializer 라 부른다

# 정적 메서드와 클래스 메서드
    # 정적 메서드
        # 메서드 앞에 @staticmethod 표시

    # 클래스 메서드
        # 메서드 앞에 @classmethod 표시

class Rectangle:
    count = 0  # 클래스 변수

    def __init__(self, width, height):
        self.width = width
        self.height = height
        Rectangle.count += 1

    # 인스턴스 메서드
    def calcArea(self):
        area = self.width * self.height
        return area

    # 정적 메서드
    @staticmethod
    def isSquare(rectWidth, rectHeight):
        return rectWidth == rectHeight

    # 클래스 메서드
    @classmethod
    def printCount(cls):
        print(cls.count)

# 테스트
square = Rectangle.isSquare(5, 5)
print(square)   # True

rect1 = Rectangle(5, 5)
rect2 = Rectangle(2, 5)
rect1.printCount()  # 2

# Special Method(Magic Method)
    # 객체가 소멸될때 (__del__) 메서드
    # 두 개의 객체를 더하는 + (__add__) 메서드
    # 두 개의 객체를 빼는 - (__sub__) 메서드
    # 두 개의 객체를 비교 (__cmp__) 메서드
    # 문자열로 객체를 표현 (__str__) 메서드

def __add__(self, other):
    obj = Rectangle(self.width + other.width, self.height + other.height)
    return obj

# 사용 예
r1 = Rectangle(10, 5)
r2 = Rectangle(20, 15)
r3 = r1 + r2  # __add__()가 호출됨

# 클래스 인스턴스 생성과 사용
    # 클래스 생성전 인스턴스(객체)를 생성해야 함
    # "객체변수명 = 클래스명()"과 같이 클래스명을 함수 호출처럼 사용

# 인스턴스 생성
r = Rectangle(2, 3)

# 메서드 호출
area = r.calcArea()
print("area = ", area)

# 인스턴스 변수 엑세스
r.width = 10
print("width = ", r.width)

# 클래스 변수 엑세스
print(Rectangle.count)
print(r.count)

r = Rectangle(2, 3)

Rectangle.count = 50
r.count = 10   # count 인스턴스 변수가 새로 생성됨

print(r.count, Rectangle.count)  # 10  50 출력

# 클래스 상속과 다형성
    # 상속
        # 자식클래스에서 클래스명 뒤에 부모클래스 이름을 괄호와 함께 넣어주면 된다.

class Animal:
    def __init__(self, name):
        self.name = name
    def move(self):
        print("move")
    def speak(self):
        pass

class Dog (Animal):
    def speak(self):
        print("bark")

class Duck (Animal):
    def speak(self):
        print("quack")

# 자식클래스는 부모클래스의 멤버들을 호출하거나 사용, 자신의 멤버들을 사용가능
dog = Dog("doggy") # 부모클래스의 생성자
n = dog.name # 부모클래스의 인스턴스변수
dog.move()   # 부모클래스의 메서드
dog.speak()  # 파생클래스의 멤버

    # 다형성
        # animals 리스트에 Dog, Duck 객체를 넣고 이들의 speak() 메서드 호출
        # 객체의 타입에 따라 서로 다른 speak() 메서드가 호출됨
animals = [Dog('doggy'), Duck('duck')]

for a in animals:
    a.speak()


from django.shortcuts import render
from django.http import JsonResponse
from firstApp.models import Employee
from rest_framework.permissions import IsAuthenticated

# Create your views here.
def employeeView(request):
    emp = {
    'id':123,
    'name':'Pavan',
    'sal': 100000
    }

    data = Employee.objects.all();
    response = {'employees':list(data.values('name','sal'))}

    return JsonResponse(response)

    permission_classes = (IsAuthenticated,)