import numpy as np
from layers import Linear, LinearAndReLU, Softmax
import pickle

"""
Define the neural network class
"""
class NeuralNetwork:
    def __init__(self, input_dim, hidden_dims, output_dim, regularization, learning_rate):
        """
        Initialize the layers
        """
        # Initialize linear and relu layer
        dimensions = [input_dim] + hidden_dims
        self.layers = []
        for i in range(1, len(dimensions)):
            self.layers.append(LinearAndReLU(dimensions[i-1], dimensions[i], regularization, learning_rate))

        # Initialize the final layer
        self.final_layer = Linear(output_dim, dimensions[-1], regularization, learning_rate)

        # Initialize Softmax loss
        self.softmax = Softmax()

    def update_learning_rate(self, learning_rate):
        """
        Update the learning rates of the layers
        """
        for layer in self.layers:
            layer.linear.lr = learning_rate
        self.final_layer.lr = learning_rate

    def predict(self, X):
        """
        Perform the prediction of the Neural Network
        """
        # Initialize the current value
        hidden = X

        # Pass through the layers
        for layer in self.layers:
            hidden = layer.forward(hidden)

        # Pass through the final layer
        scores = self.final_layer.forward(hidden)

        return np.argmax(scores, axis=1)

    def forward(self, X, y, training=True):
        """
        Perform the forward pass of the neural network
        """
        # Initialize the current value
        hidden = X

        # Pass through the layers
        for layer in self.layers:
            hidden = layer.forward(hidden)

        # Pass through the final layer
        scores = self.final_layer.forward(hidden)

        # Calculate the cost
        cost = self.softmax.forward(scores, y)

        if not training:
            return cost

        # Perform backward pass on final layers
        dLoss = self.softmax.backward(y)
        dLoss, _, _ = self.final_layer.backward(dLoss)

        # Perform backward pass on the layers
        for i in range(len(self.layers) - 1, -1, -1):
            dLoss, _, _ = self.layers[i].backward(dLoss)

        # Return the cost
        return cost

    @staticmethod
    def save(model):
        """
        Save the neural network into a pickle file
        """
        with open("model/model.pickle", "wb") as f:
            pickle.dump(model, f, protocol=pickle.HIGHEST_PROTOCOL)

    @staticmethod
    def load():
        """
        Load the model from a pickle file
        """
        with open("model/model.pickle", "rb") as f:
            model = pickle.load(f)
        return model


import itertools
import random
import numpy as np
import matplotlib.pyplot as plt
import torch
from torch import nn, optim
from torchvision.utils import save_image
from DualGan.net.Generator import Generator
from DualGan.net.Discriminator import Discriminator
from DualGan.util.parseArgs import parseArgs
from DualGan.util.loader import loadData
from DualGan.util.logger import my_log

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

## 训练判别器A ##
def D_A_train(D_A: Discriminator, G_B: Generator, X, Y, BCELoss, optimizer_D):
    """
    训练判别器A
    :param BCELoss: 二分交叉熵损失函数
    :param optimizer_D: 判别器优化器
    :return: 判别器的损失值
    """
    x = X.to(device)
    y = Y.to(device)
    # 梯度初始化为0
    yx = torch.cat([y, x], dim=1)
    D_A.zero_grad()
    # 在真数据上
    D_output_r = D_A(yx).squeeze()
    # 在假数据上
    G_B_output = G_B(y)
    yx_fake = torch.cat([y, G_B_output], dim=1)
    D_output_f = D_A(yx_fake).squeeze()
    if random.random() < 0.1:
        D_real_loss = BCELoss(D_output_r, torch.zeros(D_output_r.size()).to(device))
        D_fake_loss = BCELoss(D_output_f, torch.ones(D_output_f.size()).to(device))
    else:
        D_real_loss = BCELoss(D_output_r, torch.ones(D_output_r.size()).to(device))
        D_fake_loss = BCELoss(D_output_f, torch.zeros(D_output_f.size()).to(device))
    # 反向传播并优化
    D_loss = (D_real_loss + D_fake_loss) * 0.5
    D_loss.backward()
    optimizer_D.step()

    return D_loss.data.item()

## 训练判别器B ##
def D_B_train(D_B: Discriminator, G_A: Generator, X, Y, BCELoss, optimizer_D):
    """
    训练判别器B
    :param BCELoss: 二分交叉熵损失函数
    :param optimizer_D: 判别器优化器
    :return: 判别器的损失值
    """
    # 标签转实物（右转左）
    x = X.to(device)
    y = Y.to(device)
    # 梯度初始化为0
    D_B.zero_grad()
    # 在真数据上
    xy = torch.cat([x, y], dim=1)
    D_output_r = D_B(xy).squeeze()
    # 在假数据上
    G_A_output = G_A(x)
    xy_fake = torch.cat([x, G_A_output], dim=1)
    D_output_f = D_B(xy_fake).squeeze()
    if random.random() < 0.1:
        D_real_loss = BCELoss(D_output_r, torch.zeros(D_output_r.size()).to(device))
        D_fake_loss = BCELoss(D_output_f, torch.ones(D_output_f.size()).to(device))
    else:
        D_real_loss = BCELoss(D_output_r, torch.ones(D_output_r.size()).to(device))
        D_fake_loss = BCELoss(D_output_f, torch.zeros(D_output_f.size()).to(device))

    # 反向传播并优化
    D_loss = (D_real_loss + D_fake_loss) * 0.5
    D_loss.backward()
    optimizer_D.step()

    return D_loss.data.item()

## 训练生成器 ##
def G_train(D_A: Discriminator, D_B: Discriminator, G_A: Generator, G_B: Generator, X, Y, BCELoss, L1, optimizer_G,
            lamb=100):
    """
    训练生成器
    :param BCELoss: 二分交叉熵损失函数
    :param L1: L1正则化函数
    :param optimizer_G: 生成器优化器
    :param lamb: L1正则化的权重
    :return: 生成器的损失值
    """

    x = X.to(device)
    y = Y.to(device)

    # 梯度初始化为0
    G_A.zero_grad()
    G_B.zero_grad()
    # 在假数据上
    G_A_output = G_A(x)
    xy_fake = torch.cat([x, G_A_output], dim=1)
    D_B_output_f = D_B(xy_fake).squeeze()
    G_A_BCE_loss = BCELoss(D_B_output_f, torch.ones(D_B_output_f.size()).to(device))
    G_A_L1_Loss = L1(G_A_output, y)
    # 反向传播并优化
    G_A_loss = G_A_BCE_loss + lamb * G_A_L1_Loss
    # 在假数据上
    G_B_output = G_B(y)
    yx_fake = torch.cat([y, G_B_output], dim=1)
    D_A_output_f = D_A(yx_fake).squeeze()
    G_B_BCE_loss = BCELoss(D_A_output_f, torch.ones(D_A_output_f.size()).to(device))
    G_B_L1_Loss = L1(G_B_output, x)
    # 反向传播并优化
    G_B_loss = G_B_BCE_loss + lamb * G_B_L1_Loss
    G_loss = G_A_loss + G_B_loss
    G_loss.backward(retain_graph=True)
    optimizer_G.step()

    return G_loss.data.item()

#主函数：训练DualGan
def main():
    # 加载训练数据
    args = parseArgs()
    logger = my_log()
    save_path = args.save_path
    data_path = args.data_path
    GT_path = args.origin
    hazy_path = args.hazy
    batch_size = args.batch_size
    image_size = args.image_size
    GT_loader = loadData(data_path, GT_path, image_size=image_size, batch_size=batch_size)
    hazy_loader = loadData(data_path, hazy_path, image_size=image_size, batch_size=batch_size)

    # 定义结构参数
    in_ch, out_ch = 3, 3  # 输入输出图片通道数
    ngf, ndf = 64, 64  # 生成数、判别器第一层卷积通道数

    # 定义训练参数
    lr_G, lr_D = 0.0005, 0.0001  # G、D的学习速率
    lamb = 100  # 在生成器的目标函数中L1正则化的权重
    epochs = 200  # 训练迭代次数

    # 声明生成器、判别器
    G_A = Generator(in_ch, out_ch, ngf).to(device)  # 生成无雾
    G_B = Generator(in_ch, out_ch, ngf).to(device)  # 生成有雾
    D_A = Discriminator(in_ch + out_ch, ndf).to(device)  # 鉴别有雾
    D_B = Discriminator(in_ch + out_ch, ndf).to(device)  # 鉴别无雾

    # 目标函数 & 优化器
    BCELoss = nn.BCELoss().to(device)
    L1 = nn.L1Loss().to(device)
    optimizer_G = optim.RMSprop(itertools.chain(G_A.parameters(), G_B.parameters()), lr=lr_G, alpha=0.99, eps=1e-08)
    optimizer_D_A = optim.RMSprop(D_A.parameters(), lr=lr_D, alpha=0.99, eps=1e-08)
    optimizer_D_B = optim.RMSprop(D_B.parameters(), lr=lr_D, alpha=0.99, eps=1e-08)
    # 输入数据
    X, labelx = next(iter(hazy_loader))
    Y, labely = next(iter(GT_loader))

    g_a = G_A(X.to(device))  # 有雾->无雾
    g_b = G_B(Y.to(device))  # 无雾->有雾
    logger.info(g_a.size())
    save_image(X, save_path + labelx[0].split('.')[0] + '_input.png')
    save_image(Y, save_path + labely[0].split('.')[0] + '_ground-truth.png')
    save_image(g_a, save_path + labelx[0].split('.')[0] + '_sample_GT_0.png')
    save_image(g_b, save_path + labelx[0].split('.')[0] + '_sample_hazy_0.png')

    # 开始训练
    G_A.train()
    G_B.train()
    D_A.train()
    D_B.train()
    D_A_Loss, D_B_Loss, G_Loss, Epochs = [], [], [], range(1, epochs + 1)
    for epoch in range(epochs):
        D_A_losses, D_B_losses, G_losses, batch, d_a_l, d_b_l, g_l = [], [], [], 0, 0, 0, 0
        GT_iter = iter(GT_loader)
        hazy_iter = iter(hazy_loader)
        for i in range(0, len(GT_loader)):
            X, _ = next(hazy_iter)
            Y, _ = next(GT_iter)
            batch += 1
            # 训练Discriminator并保存loss
            D_A_losses.append(D_A_train(D_A, G_B, X, Y, BCELoss, optimizer_D_A))
            D_B_losses.append(D_B_train(D_B, G_A, X, Y, BCELoss, optimizer_D_B))
            # 训练Generator
            G_losses.append(G_train(D_A, D_B, G_A, G_B, X, Y, BCELoss, L1, optimizer_G, lamb))
            if batch % 10 == 1:
                # 打印每十次batch的平均loss
                d_a_l, d_b_l, g_l = np.array(D_A_losses).mean(), np.array(D_B_losses).mean(), np.array(G_losses).mean()
                print('[%d / %d]: batch#%d loss_d_a= %.3f  loss_d_b= %.3f  loss_g= %.3f' %
                      (epoch + 1, epochs, batch, d_a_l, d_b_l, g_l))
        # 测试每十次epoch的生成效果
        if (epoch + 1) % 10 == 0:
            X, labelx = next(iter(hazy_loader))
            Y, labely = next(iter(GT_loader))
            g_a = G_A(X.to(device))
            g_b = G_B(Y.to(device))
            save_image(g_a, save_path + labelx[0].split('.')[0] + '_sample_GT_' + str(epoch + 1) + '.jpg')
            save_image(g_b, save_path + labelx[0].split('.')[0] + '_sample_hazy_' + str(epoch + 1) + '.jpg')
        # 保存每次epoch的loss
        D_A_Loss.append(d_a_l)
        D_B_Loss.append(d_b_l)
        G_Loss.append(g_l)
    print("Done!")

    # 保存训练结果
    torch.save(G_A, 'generator_a.pkl')
    torch.save(G_B, 'generator_b.pkl')
    torch.save(D_A, 'discriminator_a.pkl')
    torch.save(D_B, 'discriminator_b.pkl')
    # 画出loss图
    # G的loss因为包含L1 相比D的loss太大了 画图效果不好 所以除以100
    plt.plot(Epochs, D_A_Loss, label='Discriminator_A Losses')
    plt.plot(Epochs, D_B_Loss, label='Discriminator_B Losses')
    plt.plot(Epochs, np.array(G_Loss) / 100, label='Generator_A Losses / 100')
    plt.legend()
    plt.savefig(save_path + 'loss.png')
    plt.show()

# 运行
if __name__ == '__main__':
    main()


# -*- coding:utf-8 -*-
#    author    :   丁雪峰
#    time      :   2019-08-22 11:54:00
#    email     :   fengidri@yeah.net
#    version   :   1.0.1

import random
import json

class g:
    status = 0
    level_n = None
    level_c = None
    color = None

def send(peer, tp, msg, clear = False):
    msg['type'] = tp
    msg['status'] = peer.status
    msg['level_n'] = g.level_n
    msg['level_c'] = g.level_c

    if clear:
        msg['clear'] = True

    peer.ws.write_message(json.dumps(msg))

class Peer(object):
    def __init__(self):
        self.ID = 0
        self.ws = None
        self.status = 0

    def init(self):
        self.pokers = []
        self.draft = []

peers = [Peer(), Peer(), Peer(), Peer()]
peers[0].ID = 0
peers[1].ID = 1
peers[2].ID = 2
peers[3].ID = 3

def gen_pokers():
    a = []
    for i in range(1, 14):
        a.append('R%d' % i)
        a.append('B%d' % i)
        a.append('S%d' % i)
        a.append('P%d' % i)

    for i in range(1, 14):
        a.append('R%d' % i)
        a.append('B%d' % i)
        a.append('S%d' % i)
        a.append('P%d' % i)

    a.append('J19')
    a.append('J20')
    a.append('J19')
    a.append('J20')
    return a

def shuffle():
    p = gen_pokers()
    random.shuffle(p)

    for peer in peers:
        peer.init()

    while p:
        i = p.pop()
        peers[0].pokers.append(i)

        i = p.pop()
        peers[1].pokers.append(i)

        i = p.pop()
        peers[2].pokers.append(i)

        i = p.pop()
        peers[3].pokers.append(i)

def turn_done():
    if peers[0].draft and \
        peers[1].draft and \
        peers[2].draft and \
        peers[3].draft:
            return True
    else:
        return False

def peer_next(peer):
    i = peers.index(peer)

    i += 1
    if i == 4:
        i = 0

    return peers[i]

def post(peer, msg):
    flag_clear = turn_done()
    if flag_clear:
        for p in peers:
            p.draft = []

    peer.draft = msg['data']

    if turn_done():
        for p in peers:
            p.status = 0
    else:
        for p in peers:
            p.status = None
        peer_next(peer).status = 0
        peer.status = 1

    for d in peer.draft:
        peer.pokers.remove(d)

    for p in peers:
        if p == peer:
            continue

        if not p.ws:
            continue

        off =  peer.ID - p.ID

        if off < 0:
            off = off + 4

        msg = {}
        msg['type'] = 'peer_post'
        msg['list_post%d' % off] = peer.draft

        send(p, "peer_post",  msg, clear = flag_clear)

    send(peer, "post_ack",  {}, clear = flag_clear)

def init(peer, msg):
    num = msg['data'][0][1:]
    num = int(num)
    g.status = 1
    g.level_n = num

    shuffle()

    for p in peers:
        if p.ws:
            sync(p)

def sync(peer):
    if g.status == 0:
        send(peer, "init",  {})
        return

    msg = {}
    msg['list_mine'] = peer.pokers
    msg['list_post0'] = peer.draft

    send(peer, "sync",  msg, clear = True)

    for p in peers:
        if p == peer:
            continue

        off =  p.ID - peer.ID

        if off < 0:
            off = off + 4

        msg['list_post%d' % off] = p.draft

    peer.ws.write_message(json.dumps(msg))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov  4 19:57:13 2020

@author: camila
"""

#%% Import

import tensorflow as tf
#tf.enable_eager_execution()

import igraph
import numpy as np
import matplotlib.pyplot as plt
import math
import scipy.sparse.linalg
import random

from pathlib import Path
import sys
path = str(Path(Path(__file__).parent.absolute()).parent.absolute())
sys.path.insert(0, path)
sys.path.insert(0, path + '/source_community_detection/')
sys.path.insert(0, path + '/source_community_embedding/')
sys.path.insert(0, path + '/source_graph_util/')

import SpecOV_partitioning_v2 as spec_ov
#import threading,
import copy

import spectral_functions as sf

import os

#from multiprocessing import Process
import multiprocessing
import queue

import time

#%% Defines

bool_verbose=False

bool_verbose_thread =False

#%% Classes with variables

class Pop:
    part=None
    Q=None

    RGroup_pos = None
    RGroup_neg = None

    Sov = None

# class Offs:
#     RGroup_pos = None
#     RGroup_neg = None

#     part=None
#     Q=None

class Params:
    num_pareto = None
    n_gen = None
    n_pop = None
    n_pop_off = None
    p_offs = None
    #it_max_ls_gen = 4 #2 #5
    it_max_ls_ini = 2#290921 1 #2 #5 #260721
    it_max_ls_gen = 3#290921 1 #2 #5 #260721
    it_max_ls_final = 5#290921 #4  #10
    ov_thr=0.1

class Ensemble:
    pareto_parts = None
    p_thr = None
    DEF_SEL_PARETO=0.8
    consensus = None

class Graph:
    g = None
    KNumber = None

    ri_vert_pos = None
    ri_vert_neg = None

class Eig:
    val = None
    vec = None
    pos_ind = None
    neg_ind = None

#%% Initialize

def initialize_params(g, num_pareto, ensemble_thr, n_gen, n_pop, p_offs,ov_thr=0):
    params = Params()

    params.num_pareto = num_pareto

    ensemble = Ensemble()
    ensemble.pareto_parts = np.zeros(( num_pareto, g.vcount()))
    ensemble.p_thr = ensemble_thr

    params.n_gen = n_gen
    params.n_pop = n_pop
    # params.n_pop_off = n_pop + 1
    params.n_pop_off = n_pop
    params.p_offs = p_offs

    params.ov_thr=ov_thr

    #return pareto_parts, n_pop_off, pop_part, pop_Q, offs_part, offs_Q
    return params, ensemble

def initialize_pop_arrays(g, n_pop_off, KNumber, eig_pos_ind, eig_neg_ind ):
    pop = Pop()
    pop.part = np.zeros((n_pop_off, g.vcount()),dtype=int)
    pop.Q = np.zeros(n_pop_off)

    pop.RGroup_pos = np.zeros(( n_pop_off, KNumber,len(eig_pos_ind)))
    pop.RGroup_neg = np.zeros(( n_pop_off, KNumber,len(eig_neg_ind)))

    pop.Sov = np.zeros(( n_pop_off, g.vcount(), KNumber)) #201220
    #print(pop.Sov)

    offs = Pop()
    offs.part = np.zeros((n_pop_off, g.vcount()),dtype=int)
    offs.Q = np.zeros(n_pop_off)

    offs.Sov = np.zeros(( n_pop_off, g.vcount(), KNumber)) #201220

    offs.RGroup_pos =  np.zeros((n_pop_off, KNumber,len(eig_pos_ind)))
    offs.RGroup_neg =  np.zeros((n_pop_off, KNumber,len(eig_neg_ind)))

    return pop, offs

#%% print file

def print_com_part_Sov(file_out, Sov):
    f = open(file_out, "w")

    k_num = np.shape(Sov)[1]
    v_num = np.shape(Sov)[0]

    for v in range(0, v_num):
        #returns only cols, since Sov[:,c] is one dim -> use #[0]
        vert_com = np.where(Sov[v,:] > 0)[0] + 1 #vertices start at 0 in python

        #print(Sov[com_c_vert,c])
        if len(vert_com)>0:
             np.savetxt(f, vert_com.reshape(1, vert_com.shape[0]), newline='\n', fmt='%i')
            #print(vert_com) #, newline='\n'
        else:
            print("Vertex ", v , " is empty")
    f.close()

def print_com_part(file_com_out, part):
    f = open(file_com_out, "w")

    # v_num = len(part)
    # k_num = len(np.unique(part))

    #for v in range(0, v_num):
    np.savetxt(f, part, newline='\n', fmt='%i')

    f.close()
#%% Aux

def print_ind(graph, pop_off, ind):
    print("Ind: "+ str(ind))

    conf_Q = sf.calc_Q_Rgroup(pop_off.RGroup_pos[ind], pop_off.RGroup_neg[ind], 2* np.sum(graph.g.strength(weights='weight'))/2) #121021
    conf_Q_vec = graph.g.modularity(pop_off.part[ind],  graph.g.es['weight'])
    print("Q:"+ str(conf_Q) + ", Q_pop=" + str(pop_off.Q[ind])+", Q_igraph="+str(conf_Q_vec))

    #print("pop_part:")

def move_vert(graph, eig, offs, ind_offs, v, ld, lold):
    # print("v="+ str(v))

    #print("offs_part:")
    #print(offs.part[ind_offs])

    offs.part[ind_offs,v]=ld

    #201220 TODO: c
    #always move everything?
    #print("move ", offs.Sov[ind_offs,v,lold],"\n")

    #300321
    #if offs.Sov[ind_offs,v,lold] == 0:
    #    print("move 0 offs")

    deltaS = offs.Sov[ind_offs,v,lold]
    offs.Sov[ind_offs,v,ld] += deltaS
    offs.Sov[ind_offs,v,lold]=0

    #offs.RGroup_pos[ind_offs], offs.RGroup_neg[ind_offs] = sf.move_vertex_R_group(offs.RGroup_pos[ind_offs], offs.RGroup_neg[ind_offs], ld, lold, graph.ri_vert_pos,  graph.ri_vert_neg, v) #todo: return new
    offs.RGroup_pos[ind_offs], offs.RGroup_neg[ind_offs] = sf.move_vertex_ov_R_group(offs.RGroup_pos[ind_offs], offs.RGroup_neg[ind_offs], ld, lold, deltaS, graph.ri_vert_pos,  graph.ri_vert_neg, v) #231220

    # offs.RGroup_pos[ind_offs][ls] -= eig.ri_vert_pos[v]
    # offs.RGroup_neg[ind_offs][ls] -= eig.ri_vert_neg[v]

    # offs.RGroup_pos[ind_offs][ld] += eig.ri_vert_pos[v]
    # offs.RGroup_neg[ind_offs][ld] += eig.ri_vert_neg[v]

    #deltaQ = sf.calc_gainQ_Rgroup_move(offs.RGroup_pos[ind_offs], offs.RGroup_neg[ind_offs], ld, lold, graph.ri_vert_pos, graph.ri_vert_neg, v)
    deltaQ = sf.calc_gainQ_ov_Rgroup_update(offs.RGroup_pos[ind_offs], offs.RGroup_neg[ind_offs], ld, lold, deltaS, graph.ri_vert_pos, graph.ri_vert_neg, v)

    # print("deltaQ="+str(deltaQ))
    return deltaQ

#%% Threads aux

def th_get_res_queue_pop(res_queue, ind, pop):

    res_t = res_queue.get() #False? not wait

    th_ind = res_t[0]
    pop.RGroup_pos[ind] = res_t[1]
    pop.RGroup_neg[ind] = res_t[2]
    pop.Sov[ind] = res_t[3] #201220
    pop.part[ind] = res_t[4]
    pop.Q[ind] = res_t[5]

    return pop, th_ind, res_queue

#%% Crossover

def print_details_crossover(pop, offs, pSourceQ2, pDestQ1, ld, ls, ind_offs):
    print("TODO: print detail in files")
    print("crossover; " + str(ind_offs) + "\tls; " + str(ls) + "\tld; " + str(ld))
    print("crossover; " + str(ind_offs) + ";\tsource ;" + str(pSourceQ2)+  ";\tQ=" + str(pop.Q[pSourceQ2]) + ";");
    print(pop.part[pSourceQ2])
    print("crossover; " + str(ind_offs) + ";\tdest ;" + str(pDestQ1)+  ";\tQ=" + str(pop.Q[pDestQ1]) + ";");
    print(pop.part[pDestQ1])
    print("crossover; " + str(ind_offs) + ";\toffspring ;" + str(ind_offs)+  ";\tQ=" + str(offs.Q[ind_offs]) + ";");
    print(offs.part[ind_offs])

def copy_rgroup_pop(offs, ind_offs, pop, pDestQ1):
    # print("ind_offs: "+str(ind_offs))
    # print("pDestQ1:" + str(pDestQ1))
    offs.Q[ind_offs] = pop.Q[pDestQ1]
    offs.part[ind_offs] = pop.part[pDestQ1].copy()

    offs.RGroup_pos[ind_offs] = pop.RGroup_pos[pDestQ1].copy()
    offs.RGroup_neg[ind_offs] = pop.RGroup_neg[pDestQ1].copy()

    offs.Sov[ind_offs] = pop.Sov[pDestQ1].copy() #201220

    return offs

def select_crossover_dest_cluster(graph, pop, pSourceQ2, pDestQ1, ls):
    val_best = -2
    s_best = -1

    for s in np.arange(0, graph.KNumber,1):
        val_pos = np.inner(pop.RGroup_pos[pSourceQ2,ls], pop.RGroup_pos[pDestQ1,s])
        val_neg = np.inner(pop.RGroup_neg[pSourceQ2,ls], pop.RGroup_neg[pDestQ1,s])
        val = val_pos + val_neg

        if s_best == -1 or val > val_best:
            val_best = val
            s_best = s
       #print("s="+str(s)+ ", cos_pos")
    return s_best

def crossover_offspring(graph, eig, params, pop, offs,g): #not used
    sumQ=0
    deltaQ=0

    #ind_prob = np.zeros(params.n_pop)
    ind_prob = pop.Q + 1

    for ind_offs in np.arange(0, params.n_pop,1):
        #roulete whell
        #sumQ = sum(pop.Q) + params.n_pop
        sumQ = sum(ind_prob)
        ind_prob = ind_prob/sumQ

        # print("ind_prob")
        # print(ind_prob)
        pSourceQ2,pDestQ1 = np.random.choice(np.arange(0,params.n_pop,1), 2,p=ind_prob, replace=False)

        # print("offs_part b")
        # print(offs.part)
        # #offs[indoffs] receives a copy of pDestQ1
        offs = copy_rgroup_pop(offs, ind_offs, pop, pDestQ1)

        # print("offs_part a")
        # print(offs.part)

        #one-way crossover
		#source:pSourceQ2, dest:pDestQ1
		#update only if new dest is better than source
        vi = np.random.choice( np.arange(0,graph.g.vcount() ), 1 , replace=False )[0]
        #C++ was implemented using the sampling strategy: it did not work.. try to implement coarsening first

        ls = pop.part[pSourceQ2,vi]
        #TODO: select from Sov...

        ld = select_crossover_dest_cluster(graph, pop, pSourceQ2, pDestQ1, ls)  #TODO

        if bool_verbose == True:
            print("... vi="+str(vi) + ", ls="+str(ls)+ ", ld="+str(ld))

        #v in community ls and not in community ld
        v_ls = np.where(pop.part[pSourceQ2] == ls)
        v_not_ld = np.where(pop.part[pDestQ1] != ld)
        v_move = np.intersect1d(v_ls, v_not_ld)

        for i in v_move:
            lold = pop.part[pDestQ1,i]
            # print("offs_part:")
            # print(offs.part)
            # print("pop_part:")
            # print(pop.part)
            #deltaQ += move_vert(graph, eig, offs, ind_offs, i, ld, lold)
            offs.Q[ind_offs] = move_vert(graph, eig, offs, ind_offs, i, ld, lold) #181120
            #print(deltaQ)
            offs.Q[ind_offs]

        if bool_verbose == True:
            print_ind(graph, offs, ind_offs)

    return offs

def crossover_offspring_ind_th(res_queue, graph, eig, params, pop, offs, ind_offs, ind_prob, g):
    np.random.seed()
    #roulete whell
    #sumQ = sum(pop.Q) + params.n_pop
    sumQ = sum(ind_prob)
    ind_prob = ind_prob/sumQ

    # print("ind_prob")
    # print(ind_prob)
    pSourceQ2,pDestQ1 = np.random.choice(np.arange(0,params.n_pop,1), 2,p=ind_prob, replace=False)
    offs = copy_rgroup_pop(offs, ind_offs, pop, pDestQ1)

    # print("offs_part a")
    # print(offs.part)

    #one-way crossover
		#source:pSourceQ2, dest:pDestQ1
		#update only if new dest is better than source
    vi = np.random.choice( np.arange(0,graph.g.vcount() ), 1 , replace=False )[0]
    #C++ was implemented using the sampling strategy: it did not work.. try to implement coarsening first

    ls = pop.part[pSourceQ2,vi]
    ld = select_crossover_dest_cluster(graph, pop, pSourceQ2, pDestQ1, ls)  #TODO

    if bool_verbose == True:
        print("... vi="+str(vi) + ", ls="+str(ls)+ ", ld="+str(ld))

    #v in community ls and not in community ld
    v_ls = np.where(pop.part[pSourceQ2] == ls)
    v_not_ld = np.where(pop.part[pDestQ1] != ld)
    v_move = np.intersect1d(v_ls, v_not_ld)

    for i in v_move:
        lold = pop.part[pDestQ1,i]
        # print("offs_part:")
        # print(offs.part)
        # print("pop_part:")
        # print(pop.part)
        offs.Q[ind_offs] = move_vert(graph, eig, offs, ind_offs, i, ld, lold) #181120
        #print(deltaQ)

    if bool_verbose == True:
        print_ind(graph, offs, ind_offs)

    #res_queue.put([ind_offs, offs])
    res_queue.put([ind_offs,pop.RGroup_pos[ind_offs], pop.RGroup_neg[ind_offs],  pop.Sov[ind_offs], pop.part[ind_offs], pop.Q[ind_offs]])

def crossover_offspring_offs_call(graph, eig, params, pop, offs,g):
    sumQ=0
    deltaQ=0

    #ind_prob = np.zeros(params.n_pop)
    ind_prob = pop.Q + 1

    threads = []
    manager = multiprocessing.Manager()
    res_queue = manager.Queue()

    for ind_offs in np.arange(0, params.n_pop,1):
        t = multiprocessing.Process(target=crossover_offspring_ind_th,args=[res_queue, graph, eig, params, pop, offs, ind_offs, ind_prob, g])
        threads.append(t)
        t.start()
        if bool_verbose_thread == True:
            print("Crossover: thread started..."+ str(ind_offs))

    for ind_offs in np.arange(0, params.n_pop,1):
        threads[ind_offs].join()

    for ind_offs in np.arange(0, params.n_pop,1):
        offs, th_ind, res_queue = th_get_res_queue_pop(res_queue, ind_offs, offs)
        if bool_verbose_thread == True:
            print("Crossover: thread finished...exiting" + ": " + str(th_ind) + ", Q=" + str(offs.Q[ind_offs]))

    return offs

#%% Mutation

def print_details_mutation(offs, ind,lold, lnew,g):
    extra="mutation_" + str(g)
    print("mutation; " + str(ind) + ";\t" + str(extra) + ";\tQ="+ str(offs.Q[ind]))
    print(offs.part[ind])

def mutation(graph, eig, params, pop, offs,g):

    num_mutation= int(np.random.choice( np.arange(1,np.floor(graph.g.vcount()/2)+1,1) ,1, replace=False))
    #print("num_mutation",num_mutation)
    ind = np.random.choice( np.arange(0, params.n_pop),1, replace=False )[0]
    v_shuffle = np.random.choice(np.arange(0,graph.g.vcount(),1) , num_mutation, replace=False )

    for vi in v_shuffle:
        #np.arange(0,num_mutation,1):
        lnew = np.random.choice( np.arange(0,graph.KNumber), 1, replace=False )[0]
        lold = offs.part[ind,vi]

        # print("ind="+str(ind)+ ", vi="+str(vi))
        # print("lnew = " + str(lnew) + ", lold="+str(lold))
        if lnew != lold:
            offs.Q[ind] += move_vert(graph, eig, offs,ind, vi, lnew, lold)

    if bool_verbose == True:
        print("Ind " + str(ind) + " after mutation ")
        print_ind(graph, offs, ind)
        print_details_mutation(offs, ind,lold, lnew,g)

    return offs
    #destroy shuffle

#%% Update population

def update_population_offspring(pop, offs, params, graph):
    nhighest = int(np.floor( params.p_offs*params.n_pop ))
    #find the highest nreplace individuals from offspring
    if bool_verbose == True:
        print("nhighest="+str(nhighest))
    #print("nhighest="+str(nhighest))

    ind_offs = np.argsort(offs.Q)[-nhighest:]
    ind_pop = np.argsort(pop.Q)[:nhighest]
    #print("ind_offs")
    #print(ind_offs)
    #print("ind_pop")
    #print(ind_pop)

    for a in np.arange(0,nhighest):

        #copy_rgroup_pop(offs, ind_offs[a], pop, ind_pop[a])
        #260721
        pop = copy_rgroup_pop( pop, ind_pop[a], offs, ind_offs[a])
        #print(a)
        #print_ind(graph, offs, ind_pop[a])

    #print("print offs")
    ind_best = np.argsort(pop.Q)[-1:][0]
    #print(ind_best)
    mod_best=graph.g.modularity(pop.part[ind_best],  graph.g.es['weight'])
    #print("Best Q=" ,mod_best)

    return pop

#%% Initial partition

def initial_partition_ind_th(res_queue, graph, pop, ind, eig, it_max, bool_verbose):
    np.random.seed()
    pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov[ind], pop.part[ind], pop.Q[ind] = sf.spec_initial_partition(graph.g, pop.RGroup_pos[ind], pop.RGroup_neg[ind], graph.ri_vert_pos, graph.ri_vert_neg, eig.pos_ind, eig.neg_ind, graph.KNumber, bool_verbose)

    #res_queue.put([pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov, pop.part[ind], pop.Q[ind]])
    res_queue.put([ind, pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov[ind], pop.part[ind], pop.Q[ind]])

def initial_partition_ov_ind_th(res_queue, graph, pop, ind, eig, params, it_max, bool_verbose):
    np.random.seed()
    pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov[ind],  pop.part[ind], pop.Q[ind] = sf.spec_initial_partition(graph.g, pop.RGroup_pos[ind], pop.RGroup_neg[ind], graph.ri_vert_pos, graph.ri_vert_neg, eig.pos_ind, eig.neg_ind, graph.KNumber, bool_verbose)
    #print("ini Q=", pop.Q[ind])

    #TEST: local search on initial partition?

    pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.part[ind], pop.Sov[ind], pop.Q[ind] = sf.spec_local_search(graph.g, graph.ri_vert_pos, graph.ri_vert_neg, eig.pos_ind, eig.neg_ind, graph.KNumber, pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov[ind], pop.part[ind], it_max= it_max, bool_verbose=bool_verbose)
    pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.part[ind], pop.Sov[ind], pop.Q[ind]  = spec_ov.spec_overlapping(graph.g, graph.ri_vert_pos, graph.ri_vert_neg, eig.pos_ind, eig.neg_ind,  graph.KNumber,pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov[ind],  pop.part[ind], it_max= it_max, bool_verbose=False)
    #print("ls Q=", pop.Q[ind])
    res_queue.put([ind, pop.RGroup_pos[ind], pop.RGroup_neg[ind],  pop.Sov[ind], pop.part[ind], pop.Q[ind]])

def initial_partition_pop_call(params, graph, pop, eig, it_max, bool_ov):
    threads = []
    #res_queue = queue.Queue()
    manager = multiprocessing.Manager()
    res_queue = manager.Queue()
    #qOut = manager.Queue()

    #res_queue = multiprocessing.Queue()
    if bool_verbose_thread == True:
        print("Initial partitions thread")

    for ind in np.arange(0, params.n_pop, 1):
        # if bool_verbose == True:
        #     print("\n## ind="+ str(ind))

        #pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov, pop.part[ind], pop.Q[ind] = sf.spec_initial_partition(graph.g, pop.RGroup_pos[ind], pop.RGroup_neg[ind], graph.ri_vert_pos, graph.ri_vert_neg, eig.pos_ind, eig.neg_ind, graph.KNumber, bool_verbose)
        #t = threading.Thread(target=spec_th_initial_partition,args=[res_queue, graph, pop, ind, eig, bool_verbose])
        if bool_ov == False:
            t = multiprocessing.Process(target= initial_partition_ind_th ,args=[res_queue, graph, pop, ind, eig, it_max, bool_verbose])
        else:
            if bool_verbose_thread == True:
                print("initial ov")

            t = multiprocessing.Process(target= initial_partition_ov_ind_th ,args=[res_queue, graph, pop, ind, eig, params, it_max, bool_verbose])

        threads.append(t)
        threads[ind].start()
        if bool_verbose_thread == True:
            print("thread started..."+ str(ind))

    for ind in np.arange(0, params.n_pop, 1):
        threads[ind].join()
        if bool_verbose_thread == True:
            print("Initial: thread finished...exiting:", ind)

    for ind in np.arange(0, params.n_pop, 1):

        pop, th_ind, res_queue = th_get_res_queue_pop(res_queue, ind, pop)
        if bool_verbose_thread == True:
            print("Initial: thread finished...exiting" + ": " + str(th_ind) + ", Q=" + str( pop.Q[ind]))
        #print("thread finished...exiting" + ": " + str(res_t[0]) + ", Q=" + str(res_t[1]))

    #print("Initial: CalcQ="+str(valQ) + ", (" + str(th_num) + ")")
    #if bool_verbose==True:
        #print(pcurr)

    return pop

#%% Local search

def local_search_ind_th(res_queue, graph, pop, ind, eig, it_max, bool_verbose):
    np.random.seed()
    pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.part[ind], pop.Sov[ind], pop.Q[ind] = sf.spec_local_search(graph.g, graph.ri_vert_pos, graph.ri_vert_neg, eig.pos_ind, eig.neg_ind, graph.KNumber, pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov[ind], pop.part[ind], it_max=it_max, bool_verbose=bool_verbose)

    #res_queue.put([pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov, pop.part[ind], pop.Q[ind]])
    res_queue.put([ind, pop.RGroup_pos[ind], pop.RGroup_neg[ind],  pop.Sov[ind], pop.part[ind], pop.Q[ind]])

def local_search_ov_ind_th(res_queue, graph, pop, ind, eig, params, it_max, bool_verbose):
    np.random.seed()

    pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.part[ind], pop.Sov[ind], pop.Q[ind]  = spec_ov.spec_overlapping(graph.g, graph.ri_vert_pos, graph.ri_vert_neg, eig.pos_ind, eig.neg_ind,  graph.KNumber,pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov[ind],  pop.part[ind], it_max=it_max, bool_verbose=False)

    #Post processing
    # print("TEST LS POST.")
    # print(pop.Sov[ind]) #OK

    pop.Sov[ind] = spec_ov.spec_post_processing(pop.Sov[ind], params.ov_thr)

    res_queue.put([ind, pop.RGroup_pos[ind], pop.RGroup_neg[ind],  pop.Sov[ind], pop.part[ind], pop.Q[ind]])

#TODO: try to send function as parameter
def local_search_pop_call(params, graph, pop, eig, it_max, bool_ov, bool_verbose):
    threads = []
    manager = multiprocessing.Manager()
    res_queue = manager.Queue()
    #print("Local search thread")

    for ind in np.arange(0, params.n_pop, 1):
        # if bool_verbose == True:
        #     print("\n## ind="+ str(ind))

        #multiprocessing.Process
        if bool_ov == False:
            t = multiprocessing.Process(target=local_search_ind_th,args=[res_queue, graph, pop, ind, eig, it_max, bool_verbose])
        else:
            t = multiprocessing.Process(target=local_search_ov_ind_th,args=[res_queue, graph, pop, ind, eig, params, it_max, bool_verbose])

        threads.append(t)

    for ind in np.arange(0, params.n_pop, 1):
        threads[ind].start()
        if bool_verbose_thread == True:
            print("Local Search: thread started..."+ str(ind))

    for ind in np.arange(0, params.n_pop, 1):
        # print("wait to join:",ind)
        threads[ind].join()
        # print("join ok", ind)

    for ind in np.arange(0, params.n_pop, 1):
        pop, th_ind, res_queue = th_get_res_queue_pop(res_queue, ind, pop)
        if bool_verbose_thread == True:
            print("Local Search: thread finished...exiting" + ": " + str(th_ind) + ", Q=" + str( pop.Q[ind]))
        #print("thread finished...exiting" + ": " + str(res_t[0]) + ", Q=" + str(res_t[1]))

    return pop

#%% Genetic algorithm

def genetic_algorithm(graph, eig, params, bool_ov):

    #nconv, eig_val, eig_vec, eig_pos_ind, eig_neg_ind, n_pop, g, ri_vert_pos, ri_vert_neg, KNumber, pop_part, pop_Q, pop_RGroup_pos, pop_RGroup_neg, offs_RGroup_pos, offs_RGroup_neg ):
    nconv= len(eig.val)

    pop, offs = initialize_pop_arrays(graph.g, params.n_pop_off, graph.KNumber, eig.pos_ind, eig.neg_ind )

    list_gen_Q = np.zeros(params.n_gen)

    #initial solution
    #TODO THREAD
    if bool_verbose == True:
        print("\n*** Initial")

    pop = initial_partition_pop_call(params, graph, pop, eig, it_max=params.it_max_ls_ini, bool_ov=bool_ov)
    #print("\n*** Initial")
    #print_ind(graph, pop, 0)
    #print("\n")

    #print("TEST-- error")
    #print_ind(graph, pop, 0)

    for g in np.arange(0,params.n_gen):

        if bool_verbose == True:
            print("\n*** Crossover")

        #offs = crossover_offspring(graph, eig, params, pop, offs,g)
        offs = crossover_offspring_offs_call(graph, eig, params, pop, offs,g)

        if bool_verbose == True:
            print("\n*** Mutation")

        offs = mutation(graph, eig, params, pop, offs,g)
    #local search - already implemented
        if bool_verbose == True:
            print("\n*** Local search")

        if bool_verbose == True:
            print("part to local search")
            #print(pop.part[ind])

        #TODO threads..
        #pop.RGroup_pos[ind], pop.RGroup_neg[ind],  pop.part[ind], pop.Sov, pop.Q[ind] = sf.spec_local_search(graph.g, graph.ri_vert_pos, graph.ri_vert_neg, eig.pos_ind, eig.neg_ind, graph.KNumber, pop.RGroup_pos[ind], pop.RGroup_neg[ind], pop.Sov, pop.part[ind], bool_verbose)

        #pop = local_search_pop_call(params, graph, pop, eig, it_max=params.it_max_ls_gen, bool_ov=False, bool_verbose=True)
        #pop = local_search_pop_call(params, graph, pop, eig, it_max=params.it_max_ls_gen, bool_ov=bool_ov, bool_verbose=True)
        #260721
        pop = local_search_pop_call(params, graph, offs, eig, it_max=params.it_max_ls_gen, bool_ov=bool_ov, bool_verbose=bool_verbose)

        #260721

        pop = update_population_offspring(pop, offs, params, graph);

        #260721

        #print(ind_best)
        #list_gen_Q[g] =
        #print(pop.Q[ind_best])
        #print(pop.part[ind_best])
        ind_best = np.argsort(pop.Q)[-1:][0]
        #mod_best=graph.g.modularity(pop.part[ind_best],  graph.g.es['weight'])
        print("Best Q=" ,pop.Q[ind_best])
        #list_gen_Q[g] = mod_best
        list_gen_Q[g] = pop.Q[ind_best]

    if bool_ov == True:
        #print("---------------")
        #print("final local search")
        #Local search with overlapping only after
        #pop = local_search_pop_call(params, graph, pop, eig, it_max=2, bool_ov=False, bool_verbose=True)
        print("---------------")
        print("OV final local search 2")
        pop = local_search_pop_call(params, graph, pop, eig, it_max=params.it_max_ls_final, bool_ov=bool_ov, bool_verbose=True)

    ind_best = np.argsort(pop.Q)[-1:]
    print("Best Q=" + str(pop.Q[ind_best]))
    #print(pop.RGroup_pos)
    if bool_verbose == True:
        for ind in np.arange(0,params.n_pop):
            print_ind(graph, pop, ind)

        print("Best part=")
        print(pop.part[ind_best,:])

    #post processing Sov
    if bool_ov == True:
        #TODO: ov_thr must be an input parameter
        pop.Sov[ind_best] = spec_ov.spec_post_processing(pop.Sov[ind_best], ov_thr=0.1) #290321

    #print( pop.Sov[ind_best])

    #return pop.part[ind_best], pop.Sov[ind_best], pop, offs''
    return pop.part[ind_best], pop.Sov[ind_best], pop, offs, pop.RGroup_pos[ind_best], pop.RGroup_neg[ind_best], list_gen_Q #010421

#%% Ensemble
def ensemble_threshold(ensemble, graph, params):

    # print(ensemble.consensus)

    avg_thr = np.mean( ensemble.consensus )

    print("Avg threshold before="+str(avg_thr))
    # print(ensemble.p_thr)

    nnz=0
    for i in np.arange(0,graph.g.vcount()):
        j_larg = np.argmax( ensemble.consensus[i,:])
        val_larg = ensemble.consensus[i, j_larg]

        # np.where: returns rows[a], cols[a] indicate the a-th element that satisfies the condition
        #returns only cols, since ensemble[i,:] is one dim -> use #consensus_where_zero[0]

        consensus_col_where_zero = np.where( ensemble.consensus[i,:] < ensemble.p_thr )[0]
        #only count in rows, otherwise, it would count twice
        nnz_i = len( consensus_col_where_zero)
        #print(ensemble.consensus[i,:])
        #print("nnz_"+str(i)+" =" + str(nnz_i))
        nnz+= nnz_i

        ensemble.consensus[ i, consensus_col_where_zero ] = 0

        #check if i is disconnected and link to the vert whose consensus value if the highest
        if nnz_i==0:
            ensemble.consensus[i,j_larg] = val_larg
            nnz+=1

    avg_thr = np.mean( ensemble.consensus )

    print("Avg threshold after="+str(avg_thr))
    return ensemble, nnz

def ensemble_consensus_number(ensemble, graph, params):
    k_sum = 0
    for ind in np.arange(0,params.num_pareto+1):
        #number of different communities
        k_num = len(np.unique(ensemble.pareto_parts[ind,:]))
        k_sum += k_num

    k_sum = k_sum/params.num_pareto
    return k_sum
        #len( ensemble.pareto_parts[ind,:] )

def ensemble_consensus_parts(graph, eig, params, ensemble, DEF_SEL_PARETO):

    consensus = np.zeros((graph.g.vcount(), graph.g.vcount()), dtype=np.double )
    ind_ini = int(np.floor( np.double(1-DEF_SEL_PARETO)*np.double(params.num_pareto/2) ) )
    ind_fim = int(params.num_pareto -1 -ind_ini)

    if bool_verbose == True:
        print("ind_ini="+str(ind_ini) + ", ind_fim="+str(ind_fim))

    sel_pareto = ind_fim - ind_ini + 1

    for i in np.arange(0,graph.g.vcount()):
        for j in np.arange(0,graph.g.vcount()):#IMPROVE: SYMMETRIC
            #consensus[i,j]=0
            for ind in np.arange(ind_ini,ind_fim+1):
                if ensemble.pareto_parts[ind,i] == ensemble.pareto_parts[ind,j]:
                    consensus[i,j] += 1

    consensus = consensus/sel_pareto

    # print(consensus)

    return consensus

def ensemble_construct_graph_ensemble(graph, ensemble):
    adj = np.matrix(graph.g.get_adjacency(attribute="weight").data)
    adj_ens = np.add(adj, ensemble.consensus)
    #graph_ens = graph.copy() #TEST
    #shallow copy.. make sure not to use original graph again
    graph_ens = copy.copy(graph)
    #ERROR: SHOULD BE UNDIRECTED
    graph_ens.g = igraph.Graph.Weighted_Adjacency(adj_ens.tolist(), attr="weight", mode=igraph.ADJ_UNDIRECTED) #replace graph with the adjusted for ensemble

    return graph_ens

#%% Eigendecomposition

def spec_eigen_dec(graph, mod_mat, p_leading,pos_neg=True ):
    eig = Eig()

    if pos_neg==True:
        eig.val, eig.vec, eig.pos_ind, eig.neg_ind = sf.calc_eigen(graph.g, mod_mat, p_leading, which="LM")
    else:
        eig.val, eig.vec, eig.pos_ind, eig.neg_ind = sf.calc_eigen(graph.g, mod_mat, p_leading, which="LR")

    #print(eig_val)
    graph.KNumber = sf.est_number_communities(eig.val)
    print("Estimation k: " + str( graph.KNumber))
    #TODO: benson algorithm to estimate the number of communities

    print("** Calculate vertex vectors ri... **")

    graph.ri_vert_pos = sf.calc_ri_vert_pos(graph.g, eig.val, eig.vec, eig.pos_ind)
    graph.ri_vert_neg = sf.calc_ri_vert_neg(graph.g, eig.val, eig.vec, eig.neg_ind)

    return eig, graph

#%% MOSpecG - main

def MO_main(MO, IT, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, mod_orig,bool_ov,T):
    graph = Graph()
    graph.g = sf.read_construct_g(file_graph)

    #initalize the pop arrays that are dependent on the eigenvalues
    #pareto_parts, n_pop_off, pop_part, pop_Q, offs_part, offs_Q = construct_modularity_cosine_param
    params, ensemble = initialize_params( graph.g, num_pareto, ensemble_thr, n_gen, n_pop, p_offs)

    step = 1/(params.num_pareto-1)

    print("step="+str(step))
    for it in np.arange(0,num_pareto,1):
        print("\n-----------------------\n**** it="+str(it))
    #it=1 #TEST
    #it=5
        mo_weight_Qin=it*step
        mo_weight_Qnull = 1-mo_weight_Qin

        print("mo_Qin="+str(mo_weight_Qin)+ ", mo_Qnull="+str(mo_weight_Qnull))

        #mod_mat = sf.construct_modularity_generalized(graph.g, mo_weight_Qin, mo_weight_Qnull).transpose()

        if mod_orig==False:
            S= sf.construct_cosine(graph.g)
            mod_mat = sf.construct_modularity_cosine_param(graph.g,S, mo_weight_Qin, mo_weight_Qnull, T).transpose()
        else:
            mod_mat = sf.construct_modularity_generalized(graph.g, mo_weight_Qin, mo_weight_Qnull).transpose()

        eig, graph = spec_eigen_dec(graph, mod_mat, p_leading  )

        print("** Start GA... **")
        #aloc pop arrays
        p_best, Sov_best, pop, offs, Rp, Rn, list_gen_Q = genetic_algorithm(graph, eig, params, bool_ov)
        ensemble.pareto_parts[it,:] = p_best.copy()

    return ensemble, graph, params, eig, pop, offs, list_gen_Q

def SpecG_single_main(MO, IT, p_leading, file_graph, file_com_out, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, ov_thr=0, mo_weight_Qin=1 , mo_weight_Qnull =0, gamma3=0, mod_orig=True,
                      KNumber_pre = -1, pos_neg=True, bool_link_pred=False, T=None):
    graph = Graph()
    graph.g = sf.read_construct_g(file_graph,bool_isolated=bool_link_pred)

    print(graph.g.vs.attribute_names())
    print("out:",file_com_out)
    #initalize the pop arrays that are dependent on the eigenvalues
    #pareto_parts, n_pop_off, pop_part, pop_Q, offs_part, offs_Q =
    params, ensemble = initialize_params( graph.g, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, ov_thr)

    print("gamma1=",mo_weight_Qin,", gamma2=",mo_weight_Qnull)
    #
    #
    print("* mo_Qin="+str(mo_weight_Qin)+ ", mo_Qnull="+str(mo_weight_Qnull))

    if mod_orig==False:
        #print("encoder")
        graph.g, mod_mat, adj, null_model = sf.construct_modularity_cosine(graph.g, mo_weight_Qin, mo_weight_Qnull, gamma3, T=T)
        #why transpose?
        mod_mat = mod_mat.transpose()

        #mod_mat = sf.construct_modularity_encoder(graph.g, mo_weight_Qin, mo_weight_Qnull).transpose()
    else:
        mod_mat, adj, null_model = sf.construct_modularity_generalized(graph.g, mo_weight_Qin, mo_weight_Qnull)
        mod_mat = mod_mat.transpose()
    #TODO sparsify mod_mat - STOP SPARSIFY!!
    #mod_mat[mod_mat < 0] =0 #260721

    eig, graph = spec_eigen_dec(graph, mod_mat, p_leading,pos_neg  )

    if KNumber_pre > 0:
        graph.KNumber = KNumber_pre

    print("** Start GA... **")
    #aloc pop arrays

    p_best, Sov_best, pop, offs, Rp , Rn, list_gen_Q = genetic_algorithm(graph, eig, params, bool_ov)
    #END PUT INSIDE ANOTHER FUNCTION!
    mod0 = graph.g.modularity(p_best[0], graph.g.es['weight'])
    print("# mod =",mod0)

    return p_best, graph, params, eig, pop, offs, Sov_best, Rp, Rn, list_gen_Q, mod_mat, adj, null_model

def SpecG_rp_only(MO, IT, p_leading, file_graph, file_com_out, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, ov_thr=0, mo_weight_Qin=1 , mo_weight_Qnull =1, mod_orig=True,
                      KNumber_pre = -1, pos_neg=True, bool_link_pred=False, T=None):
    graph = Graph()
    graph.g = sf.read_construct_g(file_graph,bool_isolated=bool_link_pred)

    mod_mat, adj, null_model = sf.construct_modularity_generalized(graph.g, mo_weight_Qin, mo_weight_Qnull)
    mod_mat = mod_mat.transpose()
    #TODO sparsify mod_mat - STOP SPARSIFY!!
    #mod_mat[mod_mat < 0] =0 #260721

    eig, graph = spec_eigen_dec(graph, mod_mat, p_leading,pos_neg  )

    p_best=[1]
    Sov_best=[1]
    return p_best, graph, -1, eig,  -1,  -1, Sov_best,  -1,  -1, -1, mod_mat, adj, null_model

#%% Ensemble - main

def ensemble_MOSpecG_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs,  bool_ov, ov_thr,  mod_orig=True):
    DEF_SEL_PARETO = 0.8
    DEF_MULT_CONSENSUS = 2

    ensemble, graph, params, eig, pop, offs,list_gen_Q = MO_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs,mod_orig, bool_ov=False)

    print("\n-----------------\n ***** ENSEMBLE \n")
    if bool_verbose == True:
        print("\n ensemble_consensus_parts ")

    ensemble.consensus = ensemble_consensus_parts(graph, eig, params, ensemble,DEF_SEL_PARETO)
    #k_consensus = ensemble_consensus_number( ensemble, graph, params) #not used
    #print("Consensus number="+str(k_consensus)) #not used

    ensemble, nnz = ensemble_threshold(ensemble, graph, params)

    #already done - params, ensemble = initialize_params( graph.g, num_pareto, ensemble_thr, n_gen, n_pop, p_offs)

    mo_weight_Qin=1
    mo_weight_Qnull =1
    print("\n-----------------\n ***** MOSpecG \n")
    print("mo_Qin=",mo_weight_Qin, ", mo_Qnull=",mo_weight_Qnull)

    #graph_ens is a shallow copy.. make sure not to use original graph again
    graph_ens= ensemble_construct_graph_ensemble(graph, ensemble)
    mod_mat_ens = sf.construct_modularity_generalized(graph_ens.g, mo_weight_Qin, mo_weight_Qnull).transpose()

    eig_ens, graph_ens = spec_eigen_dec(graph_ens, mod_mat_ens, p_leading  )

    print("** Start GA... **")
    p_best, Sov_best, pop, offs, Rp,Rn,list_gen_Q = genetic_algorithm(graph_ens, eig_ens, params, bool_ov)

    #0.55 for karate.. ???

    return p_best, graph_ens, params, eig,pop, offs, Sov_best, Rp,list_gen_Q

#%% Call main - example

def call_MOSpecG_example():
    #base_folder="/home/camila/not_sync/Lancichinetti2017/Binary_2017_Lancichinetti2011_S/paj_Binary_2017_S/"
    #file_graph=base_folder+"net_Binary_2017_S1000_1_1.net"

    #base_folder="/home/camila/not_sync/Lancichinetti2017/Binary_DirNormal2_OV_2017_Lancichinetti2011_S/paj_Binary_DirNormal2_OV_2017_S/"
    #file_graph=base_folder+"net_Binary_DirNormal2_OV_2017_S1000_1_1.net"

    base_folder="grafos_nao_direcionados/"
    file_graph=base_folder+"karate.net"
    #file_graph=base_folder+"net_Binary_2017_S1000_1_1.net"

    results_folder=""
    time_folder=""
    pareto_file=""
    #file_expected="karate_expected.txt"

    num_pareto=11
    ensemble_thr=0.5

    num_it=10
    n_gen=10 #10
    n_pop=2 #5  #3 #ERROR!!
    p_offs=0.6

    # output_time="time_aux.txt"
    # net_type="real"
    p_leading=1
    #p_leading=0.3

    #TODO:
    bool_ov=False
    ov_thr=0.5
    bool_verbose=False
    out_verbose=[]

    MO_bool=False

    #start_time = time.time()
    start_time = time.perf_counter()
    print(file_graph)
    #p_best, graph, params, eig, pop, offs, Sov_best = SpecG_single_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov)

    #p_best, graph, params, eig, pop, offs, Sov_best, Rp = ensemble_MOSpecG_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, ov_thr)
    p_best, graph, params, eig, pop, offs, Sov_best, Rp,Rn,list_gen_Q = SpecG_single_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, ov_thr)
    print(list_gen_Q)

    #print("--- %s seconds ---" % (time.time() - start_time))
    print("--- %s seconds ---" , (time.perf_counter() - start_time))

    return graph, params, eig, pop, offs

if __name__ == "__main__":
    graph, params, eig, pop, offs=call_MOSpecG_example()

#%% Call main - LFR

def call_spec_ov_LFR(nome_alg="SpecG_OV",bool_ov=True,bool_ensemble=True,lfr_set = "S"):
    #Default parameters:
    output_time="time_aux.txt"
    net_type="real"

    gamma1=1
    gamma2=1
    #bool_ensemble=True
    #bool_ov=False
    #ov_thr=0.5
    ov_thr=0.1

    bool_verbose=False
    out_verbose=[]

    folder_res="results_Tese_2021_3"
    #nome_alg = "SpecG_OV"
    if bool_ensemble == True:
        nome_alg=nome_alg+"_ens"
    #LFR

    # OV:
    if bool_ov==False:
        lfr_type="Binary_2017"
    else:
        lfr_type="Binary_DirNormal2_OV_2017" #Binary2017_DirNormal2_OV_2017

    #lfr_type="Binary_2017"

    #lfr_type="Binary_Sparse6_OV2_2017"

    tam = 1000

    results_folder=""
    time_folder=""
    pareto_file=""

    num_pareto=11
    ensemble_thr=0.5

    num_it=1
    n_gen=10 #10
    n_pop=5  #3 #ERROR!!
    p_offs=0.6

    #p_leading=0.95
    p_leading=0.3

    # for mu in range(1,9):
    #     for n2 in range(1,6):

    print("\n-----Alg=", nome_alg)

    ex=1
    for mu in range(1,8+1): #7
        for n2 in range(1,5+1):
            inst_base =  lfr_type +"_Lancichinetti2011_"+ lfr_set
            base_folder_in="/home/camila/not_sync/Lancichinetti2017/" + inst_base + "/paj_" + lfr_type + "_" + lfr_set + "/"
            inst_name =  lfr_type + "_" + lfr_set + str(tam) + "_" + str(mu) + "_" + str(n2)
            file_graph=base_folder_in + "net_" + inst_name + ".net"

            #base_folder_exp="/home/camila/not_sync/Lancichinetti2017/" + lfr_type +"_Lancichinetti2011_"+ lfr_set + "/com_" + lfr_type + "_" + lfr_set + "/"
            #file_exp=base_folder_in+"net_" + lfr_type + "_" + lfr_set + str(tam) + "_" + str(mu) + "_" + str(n2) + ".net"

            print("Graph: ", file_graph)
            if bool_ov == True:
                base_folder_out = folder_res+"/"+nome_alg + "/" + inst_base  +"/lines_"+ inst_base + "_" + str(ex)+ "/"
                file_com_out = base_folder_out+"lines_net_" + inst_name + "_ov.com"

                file_time_out = base_folder_out+"time_net_" + inst_name + "_ov.txt"
                file_k_num_out = base_folder_out+"num_net_" + inst_name + "_ov.txt"
                file_gen_out = base_folder_out+"gen_net_" + inst_name + "_ov.txt"
                file_gen_plot_out = base_folder_out+"plot_gen_net_" + inst_name + "_ov.png"

            else:
                base_folder_out = folder_res+"/"+nome_alg + "/" + inst_base  +"/part_"+ inst_base + "_" + str(ex)+ "/"
                file_com_out = base_folder_out+"part_net_" + inst_name + ".com"

                file_time_out = base_folder_out+"time_net_" + inst_name + ".txt"
                file_k_num_out = base_folder_out+"num_net_" + inst_name + ".txt"

                file_gen_out = base_folder_out+"gen_net_" + inst_name + ".txt"
                file_gen_plot_out = base_folder_out+"plot_gen_net_" + inst_name + ".png"
            print("Output: ", file_com_out)

            os.makedirs(base_folder_out, exist_ok=True)

            #start_time = time.perf_counter()
            #q_best, Sov,g, eig_val, ri_vert_pos, ri_vert_neg, Rp, Rn = spec_ov_main(file_graph, 4, plarg, gamma1, gamma2, ov_thr, bool_ov, file_com_out, bool_verbose, out_verbose)

            MO_bool=False
            #print("out 1:", file_com_out)

            start_time = time.perf_counter()

            if bool_ensemble == True:
                p_best, graph, params, eig, pop, offs, Sov_best, Rp,list_gen_Q = ensemble_MOSpecG_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs,  bool_ov, ov_thr)
            else:
                print("=======================\n\n")
                p_best, graph, params, eig, pop, offs, Sov_best, Rp,Rn,list_gen_Q = SpecG_single_main(MO_bool, num_it, p_leading, file_graph, file_com_out, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov, ov_thr)
                print(list_gen_Q)

                f = open(file_gen_out, "w")
                print(list_gen_Q.reshape(1, list_gen_Q.shape[0]))
                np.savetxt(f, list_gen_Q.reshape(1, list_gen_Q.shape[0]), newline='\n')
                f.close()

                #print chart

                fig, ax = plt.subplots()
                ax.plot(list_gen_Q);
                fig.savefig(file_gen_plot_out,dpi=100)

            if bool_ov == True:
                print("ok")
                #------ write communities
                spec_ov.print_com_lines_file(file_com_out, Sov_best[0]) #231220

                #------ write k number of communities
                Sov_best=Sov_best[0]
                idx= np.where( np.all(Sov_best[..., :] == 0, axis=0) == False )[0]
                Sov_nonzero = Sov_best[:,idx]
                k_num_best = Sov_nonzero.shape[1]
                f = open(file_k_num_out, "w")
                f.write(str(k_num_best ))
                f.close()

            else:
                #write partition

                base_folder_expected="/home/camila/not_sync/Lancichinetti2017/" + inst_base + "/com_" + lfr_type + "_" + lfr_set + "/"
                file_expected=base_folder_expected+"com_"+inst_name+".dat"

                #------ write partition
                print_com_part(file_com_out, p_best[0])
                #------ calc NMI
                print("file expected: ", file_expected)
                p_expected = sf.read_partition_expected(file_expected)
                nmi = sf.calc_nmi(p_best[0], p_expected)
                print("nmi=",nmi)

                #------ write k number of communities
                f = open(file_k_num_out, "w")
                f.write(str(len( np.unique(p_best[0]))))
                f.close()

            #------ write time

            #q_best, Sov = spec_ov_main_single(file_graph, 4, plarg, gamma1, gamma2, ov_thr, bool_ov, bool_verbose, out_verbose)
            time_tot = time.perf_counter() - start_time
            print("--- %s seconds ---" , time_tot)
            f = open(file_time_out, "w")
            f.write(str(time_tot))
            f.close()

            #TODO: save time

    #return q_best, Sov,g, eig_val, ri_vert_pos, ri_vert_neg, Rp, Rn
    return p_best,Sov_best

#if __name__ == "__main__":
    #Sov_best = call_spec_ov_LFR(nome_alg="SpecG_OV",bool_ov=True,bool_ensemble=True,lfr_set = "S")
    #Sov_best = call_spec_ov_LFR(nome_alg="SpecG_OV",bool_ov=True,bool_ensemble=True,lfr_set = "L")
    #Sov_best = call_spec_ov_LFR(nome_alg="SpecG",bool_ov=False,bool_ensemble=False)

#%% parameters

#  Parameters
# 		1. MO
# 			MO=10 to run MOSpecG for maximizing the classical modularity (MOSpecG-mod).
# 		    MO=11 to run MOSpecG to find Pareto sets and obtain a final consensus partition with SpecG algorithm.

# 		2. IT: maximum number of iterations for the local search multiplied by 10*****.

# 		3. p: consider only the largest p*n eigenvalues in absolute value, where n is the number of vertices.

# 		4. <instance>: full path to the input graph in pajek format.

# 		5. <res_base>: base name of the file to output the partitions in between "".
# 			The algorithm will automatically generate output files.

# 		6. <time_base>: path and base name of the file to output the execution time in seconds in between "".
# 			The algorithm will automatically generate files containing the execution times in between "".

# 		7. <pareto_file>: full path to file to save the objective values

# 		8. NF: number of solutions in the Pareto set --  When M0=10 this input can be set to any value.

# 		9. tau: threshold parameter of SpecG - When M0=10 this input can be set to any value.

# 		10. NG: number of generations of the memetic algorithm.

# 		11. NP: size of the population of the memetic algorithm.

# 		12. NO: the NO% fittest individuals from the offspring replace the NO% least fit individuals from a current population in the memetic algorithm.

#%% test

# a

#%%% test 0

    # nex = 5 #10
    # valQ_max = -100
    # nmi_max = -2

    # threads = []
    # res_queue = queue.Queue()

    # for it in range(0,nex):
    #   #print("---------------" + str(it))
    #   t = threading.Thread(target=spec_ov_th,args=[it, res_queue, g, comms_est, ri_vert_pos, ri_vert_neg, eig_pos_ind, eig_neg_ind, bool_ov, ov_thr, bool_verbose, out_verbose])
    #   threads.append(t)
    #   t.start()

    # for it in range(0,nex):
    #   threads[it].join()
    #   res_t = res_queue.get()
    #   print("thread finished...exiting" + ": " + str(res_t[0]) + ", Q=" + str(res_t[1]))

    #pareto_parts = genetic_algorithm()

    #ensemble_clustering()

#%%% test 1

#pop_RGroup_pos, pop_RGroup_neg, offs_RGroup_pos, offs_RGroup_neg

#R_group_pos, R_group_neg, Sov, pcurr, valQ = sf.spec_initial_partition(g, ri_vert_pos, ri_vert_neg, eig_pos_ind, eig_neg_ind, KNumber, bool_verbose)

# print(R_group_pos)
#print(KNumber)
#pop_RGroup_pos[0]= R_group_pos
#print(pop_RGroup_pos)
#pop_RGroup_pos[3] = R_group_pos.transpose() ##CORRIGIR NO MODULO

#print(pop_RGroup_pos)

#print(len(pop_RGroup_pos))

#qual parte vai ser paralelizada? cada etapa do crossover, solução inicial, etc
#def MOSpecG_th(MO, it, plarg, file_graph, results_folder, time_folder, pareto_file, num_pareto, threshold, n_gen, n_pop, p_offs, bool_verbose=True, out_verbose=True):
#%%% test 2

# a = np.array([9, 4, 4, 3, 3, 9, 0, 4, 6, 0])
# #ret = np.argpartition(a, 4)[-4:]
# #ret = np.argpartition(a, 9)
# ret = np.argsort(a)[-7:]
# print(ret)
# print(a[ret])

# ind_offs = np.sort(offs.Q)[-2:]
# ind_pop = np.sort(pop.Q)[2:]

# print(offs.Q[ind_offs])
# print(pop.Q[ind_pop])

#print(offs.Q)
# ind
# array([1, 5, 8, 0])
# a[ind]
# array([4, 9, 6, 9])

#%%% test 3
#ls = 0
# #ld = 1

# v_ls = np.where(pop.part[0] == 0)
# v_not_ld = np.where(pop.part[1] != 1)

# print(v_ls)
# print(v_not_ld)

# #v in ls and in v_not_ld
# print( np.intersect1d(v_ls, v_not_ld) )
#print(v_ls.intersection(v_not_ld ))
# print(v_ls[v_ls in v_not_ld]        )

#%%% test 4

# a = [1,2,4,5]
# p=[0.2,0.1,0.5,0.2]
# p1,p2 = np.random.choice(a, 2,p=p, replace=False)
# print(p1)
# print(p2)

# print(np.inner(a,a))

#%%% test 5

# print(pop.Q)
#print(pop.part[0].copy)

# params.n_pop_off

#%%% test 6

#ERROR:
# Process Process-6176:
# Process Process-6178:
# Process Process-6177:
# Traceback (most recent call last):
# Traceback (most recent call last):
# Process Process-6179:
# Process Process-6180:
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
#     self._target(*self._args, **self._kwargs)
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
#     self.run()
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
#     self.run()
#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 340, in crossover_offspring_ind_th
#     pSourceQ2,pDestQ1 = np.random.choice(np.arange(0,params.n_pop,1), 2,p=ind_prob, replace=False)
# Traceback (most recent call last):
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
#     self.run()
# Traceback (most recent call last):
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
#     self._target(*self._args, **self._kwargs)
#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 340, in crossover_offspring_ind_th
#     pSourceQ2,pDestQ1 = np.random.choice(np.arange(0,params.n_pop,1), 2,p=ind_prob, replace=False)
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
#     self.run()
# Traceback (most recent call last):
#   File "mtrand.pyx", line 930, in numpy.random.mtrand.RandomState.choice
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 258, in _bootstrap
#     self.run()
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
#     self._target(*self._args, **self._kwargs)
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
#     self._target(*self._args, **self._kwargs)
# ValueError: probabilities are not non-negative
#   File "mtrand.pyx", line 930, in numpy.random.mtrand.RandomState.choice
#   File "/usr/lib/python3.6/multiprocessing/process.py", line 93, in run
#     self._target(*self._args, **self._kwargs)
#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 340, in crossover_offspring_ind_th
#     pSourceQ2,pDestQ1 = np.random.choice(np.arange(0,params.n_pop,1), 2,p=ind_prob, replace=False)
#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 340, in crossover_offspring_ind_th
#     pSourceQ2,pDestQ1 = np.random.choice(np.arange(0,params.n_pop,1), 2,p=ind_prob, replace=False)
# ValueError: probabilities are not non-negative
#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 340, in crossover_offspring_ind_th
#     pSourceQ2,pDestQ1 = np.random.choice(np.arange(0,params.n_pop,1), 2,p=ind_prob, replace=False)
#   File "mtrand.pyx", line 930, in numpy.random.mtrand.RandomState.choice
#   File "mtrand.pyx", line 930, in numpy.random.mtrand.RandomState.choice
#   File "mtrand.pyx", line 930, in numpy.random.mtrand.RandomState.choice
# ValueError: probabilities are not non-negative
# ValueError: probabilities are not non-negative
# ValueError: probabilities are not non-negative
# Traceback (most recent call last):

#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 1152, in <module>
#     Sov_best = call_spec_ov_LFR(nome_alg="SpecG_OV",bool_ov=True,bool_ensemble=True)

#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 1085, in call_spec_ov_LFR
#     p_best, graph, params, eig, pop, offs, Sov_best, Rp,list_gen_Q = ensemble_MOSpecG_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs,  bool_ov, ov_thr)

#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 891, in ensemble_MOSpecG_main
#     ensemble, graph, params, eig, pop, offs,list_gen_Q = MO_main(MO_bool, num_it, p_leading, file_graph, results_folder, time_folder, pareto_file, num_pareto, ensemble_thr, n_gen, n_pop, p_offs, bool_ov=False)

#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 847, in MO_main
#     p_best, Sov_best, pop, offs, Rp, list_gen_Q = genetic_algorithm(graph, eig, params, bool_ov)

#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 645, in genetic_algorithm
#     offs = crossover_offspring_offs_call(graph, eig, params, pop, offs,g)

#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 402, in crossover_offspring_offs_call
#     offs, th_ind, res_queue = th_get_res_queue_pop(res_queue, ind_offs, offs)

#   File "/home/camila/sync_files/spectral_python/MOSpecG_partitioning_OV.py", line 217, in th_get_res_queue_pop
#     res_t = res_queue.get() #False? not wait

#   File "<string>", line 2, in get

#   File "/usr/lib/python3.6/multiprocessing/managers.py", line 757, in _callmethod
#     kind, result = conn.recv()

#   File "/usr/lib/python3.6/multiprocessing/connection.py", line 250, in recv
#     buf = self._recv_bytes()

#   File "/usr/lib/python3.6/multiprocessing/connection.py", line 407, in _recv_bytes
#     buf = self._recv(4)

#   File "/usr/lib/python3.6/multiprocessing/connection.py", line 379, in _recv
#     chunk = read(handle, remaining)

# KeyboardInterrupt


'''
Created on 17/10/2016

@author: joshbigdaddy
'''
#encoding: utf-8

import urllib2, re
import os.path
from bs4 import BeautifulSoup
import sqlite3
import tkMessageBox
import requests
def extraer_datos():
    url="https://www.rottentomatoes.com/top/bestofrt/?year=2016"
    soup = BeautifulSoup(requests.get(url).content,'html.parser')
    return soup

def almacena_peliculas():
    conn = sqlite3.connect('test.db')
    conn.text_factory = str  # para evitar problemas con el conjunto de caracteres que maneja la BD
    conn.execute("DROP TABLE IF EXISTS PELICULAS")
    conn.execute('''CREATE TABLE PELICULAS
       (ID INTEGER PRIMARY KEY  AUTOINCREMENT,
       YEAR            INTEGER    NOT NULL,
       TITULO          TEXT    NOT NULL,
       AVERAGECRITICS          TEXT    NOT NULL,
       AVERAGEUSERS          TEXT    NOT NULL,
       DESCRIPTION        TEXT NOT NULL,
       PERCENTLIKEUSERS        TEXT NOT NULL,
       PERCENTLIKECRITICS        TEXT NOT NULL);''')

    l = extraer_datos()
    fa=l.findAll("table", class_="table")
    #fa1=fa.findAll("href",class_="unstyled articleLink")

    for p in fa:
        if(p.find("a",class_="unstyled articleLink") is not None):
            link=p.find("a",class_="unstyled articleLink")

            if(link is not None):
                fb=link['href']
                url="https://www.rottentomatoes.com"+fb
                data = BeautifulSoup(requests.get(url).content,'html.parser')
                description=data.find("meta",property="og:description")['content']
                title=data.find("h1",class_="title hidden-xs")
                year=title.find("span").getText()
                finaltitle=title.getText()[:-len(year)]
                ratings=data.find("div",class_="score_panel col-sm-17 col-xs-15")

                conn.execute("""INSERT INTO PELICULAS (YEAR,TITULO, AVERAGECRITICS, AVERAGEUSERS, DESCRIPTION, PERCENTLIKEUSERS,PERCENTLIKECRITICS) VALUES (?,?,?,?.?,?)""",(year,finaltitle,avgcritics,avgusers,description,percentusers,percentcritics))

    conn.commit()
    conn.close()

if __name__ == "__main__":
    almacena_peliculas()


from __future__ import print_function

import argparse
import json
import os
import sys

from docutils import nodes
from docutils.statemachine import ViewList
from sphinx.util.compat import Directive
from sphinx.util.nodes import nested_parse_with_titles

from vsts.cli.vsts_cli_help import VstsCLIHelp
from vsts.cli.vsts_commands_loader import VstsCommandsLoader

from knack import CLI
from knack import help as _help

class VstsHelpGenDirective(Directive):
    def make_rst(self):
        INDENT = '   '
        DOUBLEINDENT = INDENT * 2
        parser_keys = []
        parser_values = []
        sub_parser_keys = []
        sub_parser_values = []

        # similar to what the Application object provides for in "az"
        cli_name = "vsts"
        vstscli = CLI(cli_name=cli_name,
                config_dir=os.path.join('~', '.{}'.format(cli_name)),
                config_env_var_prefix=cli_name,
                commands_loader_cls=VstsCommandsLoader,
                help_cls=VstsCLIHelp)

        loader = vstscli.commands_loader_cls()
        loader.__init__(vstscli)
        loader.load_command_table([])
        for command in loader.command_table:
            loader.load_arguments(command)

        vstsclihelp = vstscli.help_cls(cli_ctx=vstscli)

        global_parser = vstscli.parser_cls.create_global_parser(cli_ctx=vstscli)
        parser = vstscli.parser_cls(cli_ctx=vstscli, prog=vstscli.name, parents=[global_parser])
        parser.load_command_table(loader.command_table)

        _store_parsers(parser, parser_keys, parser_values, sub_parser_keys, sub_parser_values)
        for cmd, parser in zip(parser_keys, parser_values):
            if cmd not in sub_parser_keys:
                sub_parser_keys.append(cmd)
                sub_parser_values.append(parser)
        doc_source_map = _load_doc_source_map()

        help_files = []
        for cmd, parser in zip(sub_parser_keys, sub_parser_values):
            try:
                help_file = _help.GroupHelpFile(cmd, parser) if _is_group(parser) else _help.CommandHelpFile(cmd, parser)
                help_file.load(parser)
                help_files.append(help_file)
            except Exception as ex:
                print("Skipped '{}' due to '{}'".format(cmd, ex))
        help_files = sorted(help_files, key=lambda x: x.command)

        for help_file in help_files:
            is_command = isinstance(help_file, _help.CommandHelpFile)
            yield '.. cli{}:: {}'.format('command' if is_command else 'group', help_file.command if help_file.command else 'vsts') #it is top level group vsts if command is empty
            yield ''
            yield '{}:summary: {}'.format(INDENT, help_file.short_summary)
            yield '{}:description: {}'.format(INDENT, help_file.long_summary)
            if not is_command:
                top_group_name = help_file.command.split()[0] if help_file.command else 'vsts'
                yield '{}:docsource: {}'.format(INDENT, doc_source_map[top_group_name] if top_group_name in doc_source_map else '')
            else:
                top_command_name = help_file.command.split()[0] if help_file.command else ''
                if top_command_name in doc_source_map:
                    yield '{}:docsource: {}'.format(INDENT, doc_source_map[top_command_name])
            yield ''

            if is_command and help_file.parameters:
               group_registry = _help.ArgumentGroupRegistry(
                  [p.group_name for p in help_file.parameters if p.group_name])

               for arg in sorted(help_file.parameters,
                                key=lambda p: group_registry.get_group_priority(p.group_name)
                                + str(not p.required) + p.name):
                  yield '{}.. cliarg:: {}'.format(INDENT, arg.name)
                  yield ''
                  yield '{}:required: {}'.format(DOUBLEINDENT, arg.required)
                  short_summary = arg.short_summary or ''
                  possible_values_index = short_summary.find(' Possible values include')
                  short_summary = short_summary[0:possible_values_index
                                                if possible_values_index >= 0 else len(short_summary)]
                  short_summary = short_summary.strip()
                  yield '{}:summary: {}'.format(DOUBLEINDENT, short_summary)
                  yield '{}:description: {}'.format(DOUBLEINDENT, arg.long_summary)
                  if arg.choices:
                     yield '{}:values: {}'.format(DOUBLEINDENT, ', '.join(sorted([str(x) for x in arg.choices])))
                  if arg.default and arg.default != argparse.SUPPRESS:
                     yield '{}:default: {}'.format(DOUBLEINDENT, arg.default)
                  if arg.value_sources:
                     yield '{}:source: {}'.format(DOUBLEINDENT, ', '.join(arg.value_sources))
                  yield ''
            yield ''
            if len(help_file.examples) > 0:
               for e in help_file.examples:
                  yield '{}.. cliexample:: {}'.format(INDENT, e.name)
                  yield ''
                  yield DOUBLEINDENT + e.text
                  yield ''

    def run(self):
        node = nodes.section()
        node.document = self.state.document
        result = ViewList()
        for line in self.make_rst():
            result.append(line, '<vsts>')

        nested_parse_with_titles(self.state, result, node)
        return node.children

def setup(app):
    app.add_directive('vsts', VstsHelpGenDirective)

def _store_parsers(parser, parser_keys, parser_values, sub_parser_keys, sub_parser_values):
    for s in parser.subparsers.values():
        parser_keys.append(_get_parser_name(s))
        parser_values.append(s)
        if _is_group(s):
            for c in s.choices.values():
                sub_parser_keys.append(_get_parser_name(c))
                sub_parser_values.append(c)
                _store_parsers(c, parser_keys, parser_values, sub_parser_keys, sub_parser_values)

def _load_doc_source_map():
    with open('doc_source_map.json') as open_file:
        return json.load(open_file)

def _is_group(parser):
    return getattr(parser, '_subparsers', None) is not None \
        or getattr(parser, 'choices', None) is not None

def _get_parser_name(s):
    return (s._prog_prefix if hasattr(s, '_prog_prefix') else s.prog)[5:]


import reportlab
import printing

from reportlab.graphics import barcode
from reportlab.graphics.barcode import eanbc, widgets
from reportlab.graphics.shapes import Group, String, Rect
from reportlab.graphics.widgetbase import Widget
from reportlab.lib import colors
from reportlab.pdfbase.pdfmetrics import stringWidth
from reportlab.lib.validators import isNumber, isColor, isString, Validator, isBoolean
from reportlab.lib.attrmap import *
from reportlab.graphics.charts.areas import PlotArea
from reportlab.lib.units import mm, inch

class Ean13Ext5BarcodeWidget(PlotArea):
    codeName = "EAN13EXT5"

    _attrMap = AttrMap(
        BASE=PlotArea,
        value=AttrMapValue(eanbc.nDigits(18), desc="the number"),
        fontName=AttrMapValue(isString, desc="fontName"),
        fontSize=AttrMapValue(isNumber, desc="font size"),
        x=AttrMapValue(isNumber, desc="x-coord"),
        y=AttrMapValue(isNumber, desc="y-coord"),
        barFillColor=AttrMapValue(isColor, desc="bar color"),
        barHeight=AttrMapValue(isNumber, desc="Height of bars."),
        barWidth=AttrMapValue(isNumber, desc="Width of bars."),
        barStrokeWidth=AttrMapValue(isNumber, desc="Width of bar borders."),
        barStrokeColor=AttrMapValue(isColor, desc="Color of bar borders."),
        textColor=AttrMapValue(isColor, desc="human readable text color"),
        humanReadable=AttrMapValue(isBoolean, desc="if human readable"),
        quiet=AttrMapValue(isBoolean, desc="if quiet zone to be used"),
        lquiet=AttrMapValue(isBoolean, desc="left quiet zone length"),
        rquiet=AttrMapValue(isBoolean, desc="right quiet zone length"),
        shouldValidate=AttrMapValue(isBoolean, desc="validate or not"),
        barcode13=AttrMapValue(isAnything, desc="isbn barcode"),
        barcode5=AttrMapValue(isAnything, desc="ean5 barcode"),
    )

    _digits = 18
    quiet = 1
    rquiet = lquiet = None
    fontSize = 8  # millimeters
    fontName = "Helvetica"
    textColor = barFillColor = colors.black
    barStrokeColor = None
    barStrokeWidth = 0
    x = 0
    y = 0

    def __init__(self, value="123456789012345678", **kw):
        for k, v in list(kw.items()):
            setattr(self, k, v)
        ean13 = value[0:12]

        self.barcode13 = barcode.createBarcodeDrawing(
            "EAN13",
            value=str(ean13),
            validate=True,
            width=1.44 * inch,
            height=1.4 * inch,
            humanReadable=self.humanReadable,
            fontName=self.fontName,
        )
        ean5 = value[13:18]
        self.barcode5 = barcode.createBarcodeDrawing(
            "EAN5",
            value=str(ean5),
            validate=True,
            width=0.72 * inch,
            height=1.4 * inch,
            barHeight=self.barcode13.height * 0.66,
            humanReadable=self.humanReadable,
            fontName=self.fontName,
        )
        self.barcode5.contents[0].barHeight = (
            self.barcode13.contents[0].barHeight - self.fontSize
        )
        self.barcode5.translate(2.8 * inch, 0)

    def draw(self):
        """Just return a group"""
        return Group(self.barcode13, self.barcode5)

def getCodes():
    """Returns a dict mapping code names to widgets"""

    from reportlab.graphics.barcode.widgets import (
        BarcodeI2of5,
        BarcodeCode128,
        BarcodeStandard93,
        BarcodeExtended93,
        BarcodeStandard39,
        BarcodeExtended39,
        BarcodeMSI,
        BarcodeCodabar,
        BarcodeCode11,
        BarcodeFIM,
        BarcodePOSTNET,
        BarcodeUSPS_4State,
    )

    # newer codes will typically get their own module
    from reportlab.graphics.barcode.eanbc import (
        Ean13BarcodeWidget,
        Ean8BarcodeWidget,
        Ean5BarcodeWidget,
        UPCA,
    )
    from reportlab.graphics.barcode.qr import QrCodeWidget

    # the module exports a dictionary of names to widgets, to make it easy for
    # apps and doc tools to display information about them.
    codes = {}
    for widget in (
        BarcodeI2of5,
        BarcodeCode128,
        BarcodeStandard93,
        BarcodeExtended93,
        BarcodeStandard39,
        BarcodeExtended39,
        BarcodeMSI,
        BarcodeCodabar,
        BarcodeCode11,
        BarcodeFIM,
        BarcodePOSTNET,
        BarcodeUSPS_4State,
        Ean13BarcodeWidget,
        Ean13Ext5BarcodeWidget,
        Ean8BarcodeWidget,
        Ean5BarcodeWidget,
        UPCA,
        QrCodeWidget,
    ):
        codeName = widget.codeName
        codes[codeName] = widget

    return codes

reportlab.graphics.barcode.getCodes = getCodes
barcode.eanbc.Ean13Ext5BarcodeWidget = Ean13Ext5BarcodeWidget


# You are given two non-empty linked lists representing two non-negative integers.
# The digits are stored in reverse order and each of their nodes contain a single digit.
# Add the two numbers and return it as a linked list.
#
# You may assume the two numbers do not contain any leading zero, except the number 0 itself.
#
# Example:
#   Input: (2 -> 4 -> 3) + (5 -> 6 -> 4)
#   Output: 7 -> 0 -> 8
# Explanation: 342 + 465 = 807.

# Definition for singly-linked list.
class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution:
    def addTwoNumbers(self, l1: ListNode, l2: ListNode) -> ListNode:
        if not l1:
            l1 = ListNode(0)
        if not l2:
            l2 = ListNode(0)

        this_val = l1.val + l2.val

        if this_val >= 10:
            this_val = this_val % 10
            if l1.next:
                l1.next.val += 1
            elif l2.next:
                l2.next.val += 1
            else:
                l1.next = ListNode(1)

        this_node = ListNode(this_val)

        if l1.next or l2.next:
            this_node.next = self.addTwoNumbers(l1.next, l2.next)

        return this_node

if __name__ == "__main__":
    # test1
    x1_2 = ListNode(2)
    x1_4 = ListNode(4)
    x1_3 = ListNode(3)
    x1_4.next = x1_3
    x1_2.next = x1_4

    x2_5 = ListNode(5)
    x2_6 = ListNode(6)
    x2_4 = ListNode(4)
    x2_6.next = x2_4
    x2_5.next = x2_6

    print(Solution().addTwoNumbers(x1_2, x2_5).val)

    # test2
    y5_a = ListNode(5)
    y5_b = ListNode(5)

    print(Solution().addTwoNumbers(y5_a, y5_b).val)


# mouse_chase.py

import pygame, sys, random
from pygame.locals import *

import math

# set up pygame

pygame.init()
mainClock = pygame.time.Clock()
# Set up the window.
WINDOWWIDTH = 500
WINDOWHEIGHT = 500
windowSurface = pygame.display.set_mode((WINDOWWIDTH, WINDOWHEIGHT),
                                       0, 32)
pygame.display.set_caption('Mouse Chase')

# set up the colors for the background and the ball
WHITE = (255, 255, 255)
RED = (255, 0, 0)
BLUE = (0, 0, 255)

SPEED = 4 # controls how quickly ball moves
c_x = 250 # current x value for the ball initially 250
c_y = 250 # current y value for the ball initially 250

# run the game loop forever
while True:
    # check for a quit event
    for event in pygame.event.get():
        if event.type == QUIT:
            pygame.quit()
            sys.exit()

    # colors the background of the surface
    windowSurface.fill(WHITE)

    x = pygame.mouse.get_pos()[0]
    y = pygame.mouse.get_pos()[1]

    # updates the position of the ball by moving it slightly toward the mouse
    dy = y - c_y
    dx = x - c_x
    r = math.sqrt(dx**2 + dy**2)+0.0001
    cos = dx/r
    sin = dy/r
    x_speed = int(SPEED*cos)
    y_speed = int(SPEED*sin)
    c_x += x_speed
    c_y += y_speed

    # redraws the ball and updates the window
    pygame.draw.circle(windowSurface, RED, (c_x, c_y), 20)
    pygame.display.update()

    mainClock.tick(100)


import logging
import bpy
from pathlib import Path
from . import fm_paths

def is_valid_session_directory(path):
    freemocap_session_path = Path(bpy.path.abspath(path))
    if not freemocap_session_path.is_dir():
        logging.error(
            f"Given path doesn't point to a directory {freemocap_session_path}.")
        return False

    data_arrays = freemocap_session_path / fm_paths.DATA_DIR
    if not data_arrays.is_dir():
        logging.error(
            f"Given path doesn't contain a DataArrays directory {freemocap_session_path}.")
        return False

    mediaPipeSkel_3d_smoothed = data_arrays / fm_paths.SMOOTHED_MEDIAPIPE_DATA
    if not mediaPipeSkel_3d_smoothed.is_file():
        logging.error(
            f"Data Arrays don't contain a mediaPipeSkel_3d_smoothed.npy file. {data_arrays}.")
        return False

    # mediaPipeSkel_reprojErr = data_arrays / fm_paths.MEDIAPIPE_DATA_REPROJ_ERR
    # if not mediaPipeSkel_reprojErr.is_file():
    #     logging.error(
    #         f"Data Arrays don't contain a mediaPipeSkel_reprojErr.npy file. {data_arrays}.")

    logging.debug(f"Path to freemocap session: {freemocap_session_path}")
    return True


with open('input1.txt') as f:
    data = f.read().rstrip()

puzzle = data.split('\n')

calories = []
elf = []
for i, line in enumerate(puzzle):
    if line != '':
        elf.append(int(line))
        if i != len(puzzle) - 1:
            continue
    calories.append(elf)
    elf = []

totals = sorted([sum(x) for x in calories])
print(totals[-1])
tops = totals[-3:]
print(sum(tops))


from aiohttp import ClientSession
import discord
import requests
from discord.ext import commands
import random
import os
import asyncio
import json
import io
import contextlib
import datetime
import aiohttp

async def get(session: object, url: object) -> object:
    async with session.get(url) as response:
        return await response.text()

class Fun(commands.Cog):

    def __init__(self, bot):
        self.bot = bot

    @commands.command(name="cat")
    async def cat(self, ctx):
        async with aiohttp.ClientSession() as session:
            request = await session.get('https://some-random-api.ml/img/cat') # Make a request
            dogjson = await request.json() # Convert it to a JSON dictionary
        embed = discord.Embed(title="Cat!", color=discord.Color.purple()) # Create embed
        embed.set_image(url=dogjson['link']) # Set the embed image to the value of the 'link' key
        await ctx.send(embed=embed) # Send the embed

    @commands.command(name="rpanda")
    async def rpanda(self, ctx):
        async with aiohttp.ClientSession() as session:
            request = await session.get('https://some-random-api.ml/img/red_panda') # Make a request
            dogjson = await request.json() # Convert it to a JSON dictionary
        embed = discord.Embed(title="Red Panda!", color=discord.Color.purple()) # Create embed
        embed.set_image(url=dogjson['link']) # Set the embed image to the value of the 'link' key
        await ctx.send(embed=embed) # Send the embed

    @commands.command(name="panda")
    async def panda(self, ctx):
        async with aiohttp.ClientSession() as session:
            request = await session.get('https://some-random-api.ml/img/panda') # Make a request
            dogjson = await request.json() # Convert it to a JSON dictionary
        embed = discord.Embed(title="Panda", color=discord.Color.purple()) # Create embed
        embed.set_image(url=dogjson['link']) # Set the embed image to the value of the 'link' key
        await ctx.send(embed=embed) # Send the embed

    @commands.command(name="bird")
    async def Bird(self, ctx):
        async with aiohttp.ClientSession() as session:
            request = await session.get('https://some-random-api.ml/img/birb') # Make a request
            dogjson = await request.json() # Convert it to a JSON dictionary
        embed = discord.Embed(title="Bird", color=discord.Color.purple()) # Create embed
        embed.set_image(url=dogjson['link']) # Set the embed image to the value of the 'link' key
        await ctx.send(embed=embed) # Send the embed
    @commands.command(name="koala")
    async def koala(self, ctx):
        async with aiohttp.ClientSession() as session:
            request = await session.get('https://some-random-api.ml/img/koala') # Make a request
            dogjson = await request.json() # Convert it to a JSON dictionary
        embed = discord.Embed(title="Koala", color=discord.Color.purple()) # Create embed
        embed.set_image(url=dogjson['link']) # Set the embed image to the value of the 'link' key
        await ctx.send(embed=embed) # Send the embed

    @commands.command(name="duck")
    async def duck(self, ctx):

        async with aiohttp.ClientSession() as session:
            request = await session.get('https://random-d.uk/api/random') # Make a request
            dogjson = await request.json() # Convert it to a JSON dictionary
        embed = discord.Embed(title="Duck!", color=discord.Color.purple()) # Create embed
        embed.set_image(url=dogjson['url']) # Set the embed image to the value of the 'link' key
        await ctx.send(embed=embed) # Send the embed
    @commands.command(name="fart")
    async def fart(self, ctx):
      embed = discord.Embed(title="ewww", description=f"Ewwww {ctx.author.name} farted!")
      await ctx.send(embed=embed)

    @commands.command(name="penguin")
    @commands.cooldown(rate=1, per=2)
    async def penguin(self, ctx: commands.Context):
        variable = ["https://media.discordapp.net/attachments/862873875082379347/863840280968953856/adorable-baby-penguins.png?width=483&height=676",
        "https://media.discordapp.net/attachments/862873875082379347/863840230859341834/entertainment-2013-11-penguins-5-main.png?width=1014&height=676",
        "https://media.discordapp.net/attachments/862873875082379347/863840181924003860/6eaf86e60f5d9450d2372eaa65b5b3fc.png?width=712&height=676",
        "https://media.discordapp.net/attachments/862873875082379347/863840140237864990/gy6mkbk5ffk61.png?width=450&height=676",
        "https://cms.hostelbookers.com/hbblog/wp-content/uploads/sites/3/2011/11/Macaroni-penguin-e1321612212491.jpg",
        "http://www.awesomelycute.com/gallery/2015/05/cute-baby-penguin-8.jpg",
        "https://external-preview.redd.it/A9Kme6x7dAoHZZ2Y6mg7KIAFMyJaZ0_o2brmjfAdoTk.jpg?auto=webp&s=b19dc5b45613af90e6b5b319d5d3ad45fcb06cd1",
        "https://images.unsplash.com/photo-1602587365437-5d02d274b3cc?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxleHBsb3JlLWZlZWR8OHx8fGVufDB8fHx8&w=1000&q=80",
        "https://img.designswan.com/2013/04/penguin/4.jpg",
        "https://becausebirds.com/wp-content/uploads/2015/03/penguin-waddle.jpg",
        "https://us.123rf.com/450wm/antantarctic/antantarctic1508/antantarctic150800017/44230587-three-emperor-penguin-chicks-together.jpg?ver=6",
        "https://www.wildrepublic.com/wp-content/uploads/2018/11/Penguin-Walking-xl-600x404.jpg",
        "https://ichef.bbci.co.uk/news/976/cpsprodpb/146EC/production/_115229638_gettyimages-603118336-gentoo-antarcticpeninsula.jpg"]
        embed = discord.Embed(title="Here u go!", color=discord.Color.light_gray())
        embed.set_image(url=random.choice(variable))
        await ctx.send(embed=embed)

async def setup(bot):
    await bot.add_cog(Fun(bot))


import re

class NegationInstance:
    """Class for representing a single negation instance.

    Cues, scopes and events are internally represented as lists of (index, word_form).
    """

    def __init__(self, i, sentence):
        self.id = i
        self.sentence = sentence

        # Collect tokens involved in the negation instance (cue, scope, event)
        self.cue = []
        self.affix_cue = False  #
        self.scope = []
        self.event = []

        for token in sentence:
            for cue_neg_inst_id, cue_form in token.cue:
                if cue_neg_inst_id == self.id:
                    self.cue.append((token.id, cue_form))
                    if cue_form != token._form:  # affix negation!
                        self.affix_cue = True
            for scope_neg_inst_id, scope_form in token.scope:
                if scope_neg_inst_id == self.id and not ispunct(scope_form):
                    self.scope.append((token.id, scope_form))
            for event_neg_inst_id, event_form in token.event:
                if event_neg_inst_id == self.id:
                    self.event.append((token.id, event_form))

        # Compute some characteristics of the negation instance
        self.multiword_cue = len(self.cue) > 1
        self.multiword_event = len(self.event) > 1
        self.scope_length = len(self.scope)
        self.has_event = bool(self.event)

        assert self.cue
        # assert not (self.affix_cue and self.multiword_cue)

    def __str__(self):
        s = '*** NEGATION INSTANCE ***'
        s += '\n* sentence: ' + " ".join([t._form for t in self.sentence])
        s += '\n* cue:      ' + " ".join([c[1] for c in self.cue])
        if self.multiword_cue:
            s += ' (multiword cue)'
        if self.affix_cue:
            s += ' (affix cue)'
        s += '\n* scope:    ' + " ".join([s[1] for s in self.scope])
        if self.event:
            s += '\n* event:    ' + " ".join([e[1] for e in self.event])
            if self.multiword_event:
                s += ' (multiword event)'
        s += "\n* Some stats:"
        s += "\n* scope length: " + str(self.scope_length)
        s += "\n* sentence length: " + str(len(self.sentence))
        return s

def read_negation_instances_from_corpus(conll_data):
    """Method for reading in negation instances from a corpus file.

    Args:
        conll_data: PyConll representation of the given corpus.

    Returns: A list of lists, each of which contains the NegationInstances
      for the corresponding sentence. (The lists may also be empty if the corresponding
      sentence does not have a negation instance.)
    """
    all_neg_instances = []  # Negation instances, grouped into lists by sentence

    for sentence in conll_data:
        # Collect negation instance IDs
        neg_inst_ids = set()
        for token in sentence:
            if token.cue:
                for i, _ in token.cue:
                    neg_inst_ids.add(i)

        # Create actual negation instances
        curr_neg_instances = []
        for i in sorted(neg_inst_ids):
            curr_neg_instances.append(NegationInstance(i, sentence))

        all_neg_instances.append(curr_neg_instances)

    # Single negation instance in the sentence or several?
    for sentence_neg_instances in all_neg_instances:
        num_neg_instances_in_sent = len(sentence_neg_instances)
        for neg_instance in sentence_neg_instances:
            neg_instance.num_neg_instances_in_sent = num_neg_instances_in_sent

    return all_neg_instances

def ispunct(token):
    return not bool(re.search("\w", token))


from Models import *
from DAO import *

class CategoriaController:
    def Salvar(self, novaCategoria):
        with open('Categoria.txt', 'r') as arq:
            existe = False
            for i in arq.readlines():
                if i.lower().replace('\n', '') == novaCategoria.lower():
                    existe = True
            if existe == False:
                if len(str(novaCategoria).replace(' ', '')) < 4 or len(str(novaCategoria)) > 10:
                    print('Categoria inválida!')
                else:
                    CategoriaDAO.Salvar(novaCategoria)
                    print('Categoria ', novaCategoria, ' cadastrada com sucesso!')
            else:
                print('Categoria já existe!')

    def Remover(self, removerCategoria):
        with open('Categoria.txt', 'r') as arq:
            existe = False
            categorias = arq.readlines()
            for i in categorias:
                if i.lower().replace('\n', '') == removerCategoria.lower():
                    existe = True
            if existe:
                with open('Categoria.txt', 'w') as arq1:
                    for i in categorias:
                        if i.lower().replace('\n', '') != removerCategoria.lower():
                            CategoriaDAO.Salvar(i.replace('\n', ''))
                print('Categoria ', removerCategoria, ' removida com sucesso!')
            else:
                print('Categoria não encontrada!')

    def Alterar(self, categoriaantiga, categorianova):
        with open('Categoria.txt', 'r') as arq:
            categorias = arq.readlines()
            existeAntiga = False
            existeNova = False
            for i in categorias:
                if i.lower().replace('\n', '') == str(categoriaantiga).lower():
                    existeAntiga = True
                if i.lower().replace('\n', '') == str(categorianova).lower():
                    existeNova = True
            if not existeAntiga:
                print('Categoria ', categoriaantiga, ' não encontrada!')
            elif existeNova:
                print('Categoria ', categorianova, ' já existe!')
            else:
                with open('Categoria.txt', 'w') as arq:
                    for i in categorias:
                        if i.lower().replace('\n', '') != categoriaantiga.lower():
                            CategoriaDAO().Salvar(i.replace('\n', ''))
                    CategoriaDAO().Salvar(categorianova)
                    print('Categora ', categoriaantiga, ', alterada para ', categorianova,' com sucesso!')

    def Ler(self):
        with open('Categoria.txt', 'r') as arq:
            for i in arq.readlines():
                print(i.replace('\n', ''))

class ProdutosController:
    def Salvar(self, novoproduto: Produtos):
        with open('Produtos.txt', 'r') as arq:
            existe = False
            produtos = arq.readlines()
            for i in produtos:
                if i.lower().split('|')[0] == str(novoproduto.nome).lower():
                    existe = True
            if existe:
                return 'Produto já existente!'
            else:
                ProdutosDAO().Salvar(Produtos(novoproduto.nome, novoproduto.categoria, novoproduto.preco))
                return print('Produto', novoproduto.nome, ' salvo com sucesso!')

    def Remover(self, removerproduto):
        with open('Produtos.txt', 'r') as arq:
            produtos = arq.readlines()
            existe = False
            for i in produtos:
                if i.lower().split('|')[0] == str(removerproduto).lower():
                    existe = True
            if not existe:
                print(removerproduto, ' não foi encontrado!')
            else:
                with open('Produtos.txt', 'w') as arq:
                    for i in produtos:
                        if i.lower().split('|')[0] != removerproduto.lower():
                            ProdutosDAO().Salvar(Produtos(i.split('|')[0], i.split('|')[1], i.split('|')[2].replace('\n', '')))
                print(removerproduto, ' removido com sucesso!')

class EstoqueController():
    def Salvar(self, produto:Produtos, quantidade):
        e = []
        c = []
        for i in EstoqueDAO.Ler():
            e.append(str(i.produto.nome).lower())
        for i in CategoriaDAO.Ler():
            c.append(i.lower())
        if str(produto.nome).lower() in e:
            print('O produto', produto.nome, 'já consta no estoque!')
        elif str(produto.categoria).lower() not in c:
            print('A categoria', produto.categoria, 'não existe!')
        else:
            EstoqueDAO.Salvar(Estoque(Produtos(produto.nome, produto.categoria, produto.preco), quantidade))

    def Ler(self):
        print('-'*8, 'ESTOQUE', '-'*8)
        for i in EstoqueDAO.Ler():
            print('Produto: ', i.produto.nome, ' - Categoria: ', i.produto.categoria, ' - Quantidade: ', i.quantidade.replace('\n', ''))

    def Remover(self, estoqueRemover):
        existe = False
        estoque = EstoqueDAO.Ler()
        for i in estoque:
            if i.produto.nome.lower() == estoqueRemover.lower():
                existe = True
        if not existe:
            print(estoqueRemover, 'não foi encontrado!')
        else:
            with open('Estoque.txt', 'w') as arq:
                for i in estoque:
                    if i.produto.nome.lower() != estoqueRemover.lower():
                        EstoqueDAO.Salvar(Estoque(Produtos(i.produto.nome, i.produto.categoria, i.produto.preco),
                                                  i.quantidade.replace('\n', '')))
            print(estoqueRemover, 'removido com sucesso!')

class VendaController():
    def Salvar(self, venda: Venda):
        vendas = VendaDAO.Ler()
        existe = False
        for i in vendas:
            if i.itemVendido.nome.lower() == venda.itemVendido.nome.lower():
                existe = True
        if existe:
            print(venda.itemVendido.nome, 'já está cadastrado!')
        else:
            VendaDAO.Salvar(Venda(Produtos(venda.itemVendido.nome, venda.itemVendido.categoria, venda.itemVendido.preco),
                                  venda.quantidadeVendida, venda.vendedor, venda.comprador))
            print(venda.itemVendido.nome, 'cadastrado com sucesso!')

a = VendaController()
a.Salvar(Venda(Produtos('Feijão', 'Cereais', '6.99'), 2,
               'David Benner', 'Pedro Ismael'))