import pandas as pd               # Para la manipulación y análisis de datos
import numpy as np                # Para crear vectores y matrices n dimensionales
import matplotlib
import matplotlib.pyplot as plt   # Para la generación de gráficas a partir de los datos
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
import seaborn as sns             # Para la visualización de datos basado en matplotlib
from sklearn.preprocessing import StandardScaler, MinMaxScaler
import scipy.cluster.hierarchy as shc
from sklearn.cluster import AgglomerativeClustering
import secrets
import hashlib
import json
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances_argmin_min
from kneed import KneeLocator

class Clustering():
    @staticmethod
    def generar_hash_aleatorio():
        # Genera una cadena aleatoria
        cadena_aleatoria = secrets.token_hex(16)  # Genera una cadena hexadecimal de 16 bytes (32 caracteres)

        # Calcula el hash SHA256 de la cadena aleatoria
        sha256_hash = hashlib.sha256(cadena_aleatoria.encode()).hexdigest()
        return sha256_hash

    @staticmethod
    def generaMatrizCorrlaciones(file=None, dataset='datos_prueba/clustering/Hipoteca.csv'):
        if file:
            data = pd.read_csv(file)
        else:
            data = pd.read_csv(dataset)

        # eliminando columnas no numericas
        columnas_no_numericas = data.select_dtypes(exclude='number').columns
        data = data.drop(columnas_no_numericas, axis=1)
        columnas = data.columns.values.tolist()

        Correlaciones = data.corr(method='pearson')

        MatrizInf = np.triu(Correlaciones)
        heatmap = sns.heatmap(Correlaciones, cmap='RdBu_r', annot=True, mask=MatrizInf)

        # Ajusta el tamaño de la figura del mapa de calor
        fig = heatmap.get_figure()
        fig.set_size_inches(10, 8)

        # Guarda el mapa de calor en un archivo temporal
        temp_file = f'{Clustering.generar_hash_aleatorio()}.png'
        fig.savefig(f'image_tmp/{temp_file}')

        # Cierra la figura para liberar recursos
        plt.close(fig)

        return {
            "sucess": True,
            "data": {
                "image": f"http://localhost:8000/api/clustering/image/{temp_file}",
                'columnas': columnas
            }
        }

    @staticmethod
    def obtenerArbol(metrica, estandarizacion, columnas, file=None, dataset="datos_prueba/apriori/movies.csv"):
        if file:
            #print('recibiendo archivo')
            data = pd.read_csv(file)
        else:
            #print('default')
            data = pd.read_csv(dataset)

        data = np.array(data[columnas])

        print(columnas)

        if estandarizacion == 'StandardScaler':
            estandarizar = StandardScaler()
        else:
            estandarizar = MinMaxScaler()

        MEstandarizada = estandarizar.fit_transform(data)

        # Crear una figura

        plt.figure(figsize=(8, 7))

        plt.title("Dendograma")
        Arbol = shc.dendrogram(shc.linkage(MEstandarizada, method='complete', metric=metrica))
        temp_file = f'{Clustering.generar_hash_aleatorio()}.png'
        # Guardar la figura en un archivo de imagen
        plt.savefig(f'image_tmp/{temp_file}')
        plt.close()

        return {
            "sucess": True,
            "data": {
                "image": f"http://localhost:8000/api/clustering/image/{temp_file}"
            }
        }

    def obteniendoClusters(numero_clusters, metrica, estandarizacion, columnas, file=None, dataset="datos_prueba/apriori/movies.csv"):
        if file:
            #print('recibiendo archivo')
            data = pd.read_csv(file)
        else:
            #print('default')
            data = pd.read_csv(dataset)

        data_np = np.array(data[columnas])
        new_data = data[columnas]

        if estandarizacion == 'StandardScaler':
            estandarizar = StandardScaler()
        else:
            estandarizar = MinMaxScaler()

        MEstandarizada = estandarizar.fit_transform(data_np)
        MJerarquico = AgglomerativeClustering(n_clusters=int(numero_clusters), linkage='complete', metric=metrica)
        MJerarquico.fit_predict(MEstandarizada)

        new_data['clusterH'] = MJerarquico.labels_
        centroidesH = new_data.groupby('clusterH').mean()

        return {
            "sucess": True,
            "centroides": centroidesH.to_json(orient='records')
        }

    @staticmethod
    def obtenerCodo(estandarizacion, columnas, file=None, dataset="datos_prueba/apriori/movies.csv"):
        if file:
            #print('recibiendo archivo')
            data = pd.read_csv(file)
        else:
            #print('default')
            data = pd.read_csv(dataset)

        # agarramos solo las columnas que seleccionamos
        data = np.array(data[columnas])

        if estandarizacion == 'StandardScaler':
            estandarizar = StandardScaler()
        else:
            estandarizar = MinMaxScaler()

        MEstandarizada = estandarizar.fit_transform(data)

        #Definición de k clusters para K-means
        #Se utiliza random_state para inicializar el generador interno de números aleatorios
        SSE = []
        for i in range(2, 12):
            km = KMeans(n_clusters=i, random_state=0)
            km.fit(MEstandarizada)
            SSE.append(km.inertia_)

        #Se grafica SSE en función de k
        plt.figure(figsize=(10, 7))
        plt.plot(range(2, 12), SSE, marker='o')
        plt.xlabel('Cantidad de clusters *k*')
        plt.ylabel('SSE')
        plt.title('Elbow Method')

        temp_file = f'{Clustering.generar_hash_aleatorio()}.png'
        # Guardar la figura en un archivo de imagen
        plt.savefig(f'image_tmp/{temp_file}')
        plt.close()

        kl = KneeLocator(range(2, 12), SSE, curve="convex", direction="decreasing")
        plt.style.use('ggplot')
        temp_file2 = f'{Clustering.generar_hash_aleatorio()}.png'
        # Guardar la figura en un archivo de imagen
        kl.plot_knee()
        plt.savefig(f'image_tmp/{temp_file2}')
        plt.close()

        return {
            "sucess": True,
            "data": {
                "image": f"http://localhost:8000/api/clustering/image/{temp_file}",
                "knee_point": f"http://localhost:8000/api/clustering/image/{temp_file2}"
            }
        }

    def obteniendoClustersParticional(numero_clusters, estandarizacion, columnas, file=None, dataset="datos_prueba/apriori/movies.csv"):
        if file:
            #print('recibiendo archivo')
            data = pd.read_csv(file)
        else:
            #print('default')
            data = pd.read_csv(dataset)

        data_np = np.array(data[columnas])
        new_data = data[columnas]

        if estandarizacion == 'StandardScaler':
            estandarizar = StandardScaler()
        else:
            estandarizar = MinMaxScaler()

        MEstandarizada = estandarizar.fit_transform(data_np)
        MParticional = KMeans(n_clusters=int(numero_clusters), random_state=0).fit(MEstandarizada)
        MParticional.predict(MEstandarizada)

        new_data['clusterP'] = MParticional.labels_
        centroidesP = new_data.groupby('clusterP').mean()

        return {
            "sucess": True,
            "centroides": centroidesP.to_json(orient='records')
        }


 # Create Model

model = Sequential()
model.add(Dense(40, activation='relu', input_dim=12288))
model.add(Dropout(0.1))
model.add(Dense(25, activation='relu'))
# model.add(Dropout(0.1))
model.add(Dense(1, activation='sigmoid'))

# Compile model

model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

# Fit the model

model.fit(x_trn, y_trn, epochs=100, batch_size=10)

# Evaluate the model

scores = model.evaluate(x_trn, y_trn)
print("\ntraining %s: %.2f%%" % (model.metrics_names[1], scores[1]*100))

scores1 = model.evaluate(x_tst, y_tst)
print("\ntesting %s: %.2f%%" % (model.metrics_names[1], scores1[1]*100))


import scrapy
from ..CourseItems import CourseItem
#from scrapy.crawler import CrawlerProcess

class DmozSpider(scrapy.Spider):

    name = "dmoz"
    allowed_domains = ["imooc.com"]
    start_urls = [
        "http://www.imooc.com/course/list"
    ]

    def parse(self, response):
        #titles = response.xpath("//input[@type='submit']").extract()

        item = CourseItem()
        for box in response.xpath('//div[@class="course-card-container"]'):
            # 获取每个div中的课程路径
            item['url'] = 'http://www.imooc.com' +  box.xpath('.//@href').extract()[0]
            # 获取div中的课程标题
            item['title'] = box.xpath('.//h3/text()').extract()[0]
            # 获取div中的标题图片url
            item['image_url'] = box.xpath('.//@src').extract()[0]
            # 获取div中的标题图片标签
            item['image_lable'] = box.xpath('.//label/text()').extract()
            # 获取div中的学生人数
            item['student'] = box.xpath(
                './/div[@class="course-card-info"]/span[2]/text()').extract()[0].strip()
            # 获取div中的课程简介
            item['introduction'] = box.xpath('.//p/text()').extract()[0].strip()
            # 返回信息

            yield item
       #url = response.xpath("//a[contains(text(),'下一页')]/@href").extract()
       #if url :
       #    #将信息组合成下一页的url
       #    page = 'http://www.imooc.com' + url[0]
       #    #返回url
       #    yield scrapy.Request(page, callback=self.parse)

#process = CrawlerProcess({
#    'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
#})
#
#process.crawl(DmozSpider)
#process.start()


from collections import defaultdict

###http://surprise.readthedocs.io/en/stable/FAQ.html#how-to-get-the-top-n-recommendations-for-each-user
def get_top_n(predictions, n=10):
    '''Return the top-N recommendation for each user from a set of predictions.

    Args:
        predictions(list of Prediction objects): The list of predictions, as
            returned by the test method of an algorithm.
        n(int): The number of recommendation to output for each user. Default
            is 10.

    Returns:
    A dict where keys are user (raw) ids and values are lists of tuples:
        [(raw item id, rating estimation), ...] of size n.
    '''

    # First map the predictions to each user.
    top_n = defaultdict(list)
    for uid, iid, true_r, est, _ in predictions:
        top_n[uid].append((iid, est))

    # Then sort the predictions for each user and retrieve the k highest ones.
    for uid, user_ratings in top_n.items():
        user_ratings.sort(key=lambda x: x[1], reverse=True)
        top_n[uid] = user_ratings[:n]

    return top_n

def get_items_greater_threshold(predictions, threshold=0):
    '''Return the recommendation for each user from a set of predictions.

    Args:
        predictions(list of Prediction objects): The list of predictions, as
            returned by the test method of an algorithm.
        threshold: recommend the item if est > threshold

    Returns:
    A dict where keys are user (raw) ids and values are lists of tuples:
        [(raw item id, rating estimation), ...] of size n.
    '''

    # First map the predictions to each user.
    recommendationlist = defaultdict(list)

    for uid, iid, true_r, est, _ in predictions:
        if est >= threshold:
            recommendationlist[uid].append((iid, est))
        else:
            # print('est < threshold')
            pass
    return recommendationlist

# def precision_recall_at_k(predictions, k=10, threshold=3.5):
#     '''Return precision and recall at k metrics for each user.'''

#     # First map the predictions to each user.
#     user_est_true = defaultdict(list)
#     for uid, _, true_r, est, _ in predictions:
#         user_est_true[uid].append((est, true_r))

#     precisions = dict()
#     recalls = dict()
#     for uid, user_ratings in user_est_true.items():

#         # Sort user ratings by estimated value
#         user_ratings.sort(key=lambda x: x[0], reverse=True)

#         # Number of relevant items
#         n_rel = sum((true_r >= threshold) for (_, true_r) in user_ratings)

#         # Number of recommended items in top k
#         n_rec_k = sum((est >= threshold) for (est, _) in user_ratings[:k])

#         # Number of relevant and recommended items in top k
#         n_rel_and_rec_k = sum(((true_r >= threshold) and (est >= threshold))
#                               for (est, true_r) in user_ratings[:k])

#         # Precision@K: Proportion of recommended items that are relevant
#         precisions[uid] = n_rel_and_rec_k / n_rec_k if n_rec_k != 0 else 1

#         # Recall@K: Proportion of relevant items that are recommended
#         recalls[uid] = n_rel_and_rec_k / n_rel if n_rel != 0 else 1

#     return precisions, recalls

def prediction_for_userids(algo, trainset, testdic, userids):
    user_est_true = defaultdict(list)
    for u in userids:
        u_numid = trainset.to_inner_uid(u)
        user_items = set([j for (j, _) in trainset.ur[u_numid]])
        user_est_true[u] = []
        for i in trainset.all_items():
            if i not in user_items :
                if trainset.to_raw_iid(i) in testdic[u]:
                    user_est_true[u].append((algo.estimate(u_numid,i)-algo.trainset.offset,1))
                else:
                    user_est_true[u].append((algo.estimate(u_numid,i)-algo.trainset.offset,0))
    return user_est_true

def precision_recall_at_k(user_est_true, testdict, k=10, threshold=0):
    '''Return precision and recall at k metrics'''
    precisions = dict()
    recalls = dict()

    for uid, user_ratings in user_est_true.items():

        user_ratings_greater_threshold = [(est, true) for (est,true) in user_ratings if est > threshold]

        n_rel = len(testdict[uid])

        n_greater_threshold = len(user_ratings_greater_threshold)

        # Number of recommended items in top k
        if n_greater_threshold > k:
            n_rec_k = k
        else:
            n_rec_k = n_greater_threshold

        # Number of relevant and recommended items in top k
        if n_greater_threshold > k:
            user_ratings_greater_threshold.sort(key=lambda x: x[0], reverse=True)
            n_rel_and_rec_k = sum((true_r for (est, true_r) in user_ratings_greater_threshold[:k]))
        else:
            try:
                n_rel_and_rec_k = sum((true_r for (est, true_r) in user_ratings_greater_threshold))
            except IndexError:
                n_rel_and_rec_k = 0

        # Precision@K: Proportion of recommended items that are relevant
        precisions[uid] = n_rel_and_rec_k / n_rec_k if n_rec_k != 0 else 1

        # Recall@K: Proportion of relevant items that are recommended
        recalls[uid] = n_rel_and_rec_k / n_rel if n_rel != 0 else 1

    return precisions, recalls


from rest_framework import serializers
from models import Country, City, Airport, Artist, Show

class CountrySerializer(serializers.ModelSerializer):
    class Meta:
        model = Country
        fields = ('id', 'iata', 'name', 'cities')

class CitySerializer(serializers.ModelSerializer):
    class Meta:
        model = City
        fields = ('id', 'iata', 'name', 'country')

class AirportSerializer(serializers.ModelSerializer):
    class Meta:
        model = Airport
        fields = ('id', 'iata', 'name', 'city')

class ArtistSerializer(serializers.ModelSerializer):
    genres = serializers.SerializerMethodField()

    def get_genres(self, obj):
        return [genre.name for genre in obj.genres.all()]

    class Meta:
        model = Artist
        fields = ('id', 'spotify_id', 'name', 'popularity', 'genres', 'image')

class ShowSerializer(serializers.ModelSerializer):
    city = CitySerializer(many=False, read_only=True)
    artists = ArtistSerializer(many=True, read_only=True)
    name = serializers.SerializerMethodField()

    def get_name(self, obj):
        return unicode(obj)

    class Meta:
        model = Show
        fields = ('id', 'name', 'description', 'city', 'lat', 'lon', 'datetime', 'artists', 'address', 'price', 'image')


################################################################
# Angka Primer Sieve                                                                                #
# Editor : Matius Celcius Sinaga                                                                         #
# Author : # http://inventwithpython.com/hacking (BSD Licensed)                         #
################################################################

import math

def adalahPrimer(angka):
    #menghasilkan True apabila angka adalah angka primer
    #contohnya adalah bahwa angka dibawah 2 adalah bukan primer
    #menghasilkan True apabila angka yang dimaksud angka primer, vice versa
    if angka < 2:
        return False

    #jika sebuah angka dapat dibagi dua dengan setiap angka hingga
    #perpangkatan dua akar dari angka tersebut
    for i in range(2, int(math.sqrt(nomor)) + 1):
        if angka % i == 0:
            return False
    return True

def SievePrimer(ukuranSieve):
    #menghasilkan sebuah daftar angka primer yang dapat dihitung
    #menggunakan algoritma Sieve dari Eratosthenes
    sieve = [True] * ukuranSieve
    #angka nol dan satu (dibawah dua) bukanlah angka primer
    sieve[0] = False
    sieve[1] = False

    #proses menghasilkan sieve
    for i in range(2, int(math.sqrt(ukuranSieve)) + 1):
        pointer = i * 2
        while pointer < ukuranSieve:
            sieve[pointer] = False
            pointer += i
    #mengkompilasi daftar angka primer
    primes = []
    for i in range(ukuranSieve):
        if sieve[i] == True:
            primes.append(i)

    return primes


import os
import shutil
import zstandard

import time

def dump_zstd() -> float:
    # Emptying the folder first
    try:
        shutil.rmtree("zstd")
    except FileNotFoundError:
        pass

    compressor = zstandard.ZstdCompressor(level=3)

    os.makedirs("zstd")

    start = time.time()

    for file in os.listdir("raw"):
        with open(f"raw/{file}", "r") as match_file:
            data = bytearray(match_file.read(), "utf-8")

        with open(f"zstd/{file}.zst", "wb+") as file:
            file.write(compressor.compress(data=data))

    end = time.time()

    return end - start

def dump_zstd_dict() -> float:
    # Emptying the folder first
    try:
        shutil.rmtree("zstd_dict")
    except FileNotFoundError:
        pass

    os.makedirs("zstd_dict")

    # # We take some time to create the dictionary first
    # samples = []

    # for file in os.listdir("raw"):
    #     with open(f"raw/{file}", "r") as match_file:
    #         samples.append(bytearray(match_file.read(), "utf-8"))

    # compression_dict = zstandard.train_dictionary(1000 * 1000, samples=samples)

    dict_data = None

    start = time.time()

    for file in os.listdir("raw"):
        with open(f"raw/{file}", "r") as match_file:
            data = bytearray(match_file.read(), "utf-8")

        if not dict_data:
            dict_data = zstandard.ZstdCompressionDict(data)
            compressor = zstandard.ZstdCompressor(level=3, dict_data=dict_data)

        with open(f"zstd_dict/{file}.zst", "wb+") as file:
            file.write(
                compressor.compress(
                    data=data,
                )
            )

    end = time.time()

    return end - start


from fn_siemplify.lib.siemplify_common import SiemplifyCommon, PACKAGE_NAME
from resilient_circuits import AppFunctionComponent, app_function, FunctionResult
from resilient_lib import validate_fields

FN_NAME = "siemplify_sync_comment"

class FunctionComponent(AppFunctionComponent):
    """Component that implements function 'siemplify_sync_comment'"""

    def __init__(self, opts):
        super(FunctionComponent, self).__init__(opts, PACKAGE_NAME)

    @app_function(FN_NAME)
    def _app_function(self, fn_inputs):
        """
        Function: Create a Siemplify Case comment
        Inputs:
            -   fn_inputs.siemplify_case_id
            -   fn_inputs.siemplify_comment
            -   fn_inputs.siemplify_alert_id
        """

        yield self.status_message("Starting App Function: '{0}'".format(FN_NAME))

        fn_inputs_dict = fn_inputs._asdict()

        # validate app.config settings
        validate_fields([
                {"name": "api_key"},
                {"name": "base_url"}
            ],
            self.app_configs._asdict())

        validate_fields([
                {"name": "siemplify_case_id"},
                {"name": "siemplify_alert_id"},
                {"name": "siemplify_comment"}
            ],
            fn_inputs_dict)

        siemplify_env = SiemplifyCommon(self.rc, self.app_configs)
        results, error_msg = siemplify_env.sync_insight(fn_inputs_dict)

        yield self.status_message("Finished running App Function: '{0}'".format(FN_NAME))

        yield FunctionResult(results, success=isinstance(error_msg, type(None)), reason=error_msg)


# This program mirrors the meteor program in C++
# getting input from a GUI (ref page 542/543)
# This is for the STEM Festival programs

import tkinter
from random import randrange, randint
from PIL import ImageTk, Image
import time
import math
import meteorStemStatus
import meteorStemUtils

class MeteorGUI:
    def __init__(self):
        # create the main window
        self.main_window = tkinter.Tk()
        w, h = self.main_window.winfo_screenwidth(), self.main_window.winfo_screenheight()
        # self.main_window.overrideredirect(1)           # potentially malicious line
        self.main_window.title("Meteor Simulation")
        self.main_window.minsize(width=w, height=h)  # set the window size

        # create frames for widgets....................see page 549
        # set up a frame for each widget to position them correctly
        self.earth_y = 400
        self.earth_x = 1000

        self.top_frame = tkinter.Frame(self.main_window)
        self.heading_frame = tkinter.Frame(self.main_window)  # has the title
        self.canvas_h = 800
        self.canvas_w = 2000
        self.earth_canvas = tkinter.Canvas(self.main_window, width=self.canvas_w, height=self.canvas_h, bg="black")
        self.explosion_image = Image.open('explosion.png')
        self.explosion_photo_image = ImageTk.PhotoImage(self.explosion_image)
        self.earth_image = Image.open('earth.png')
        self.earth_photo_image = ImageTk.PhotoImage(self.earth_image)
        self.meteor_image = Image.open('meteor.png')
        self.meteor_photo_image = ImageTk.PhotoImage(self.meteor_image)
        self.meteor_x = 0
        self.meteor_y = 0
        self.meteor_dx = 1
        self.meteor_dy = 1
        self.meteor_deltaerr = self.earth_y/self.earth_x
        self.meteor_err = self.meteor_deltaerr - 0.5
        self.meteor_drr = 1
        self.meteor_h = 250
        self.meteor_w = 250
        self.rocket_x = 1050
        self.rocket_y = 400
        self.rocket_h = 10
        self.rocket_w = 20
        self.rocket_dy = 5
        self.rocket_dx = 5

        # self.blank_frame1 = tkinter.Frame(self.main_window)
        self.diameter_frame = tkinter.Frame(self.main_window)
        # self.blank_frame2 = tkinter.Frame(self.main_window)
        self.distance_frame = tkinter.Frame(self.main_window)
        # self.blank_frame3 = tkinter.Frame(self.main_window)

        self.bottom_frame = tkinter.Frame(self.main_window)  # bottom frame for buttons

        # create widgets for the top frame
        # self.blank_label1 = tkinter.Label(self.top_frame, text=' ')
        self.heading_label = tkinter.Label(self.heading_frame, \
                                           text='Meteor Defense System Simulation', font=("Helvetica", 18), fg="blue")

        # self.blank_label2 = tkinter.Label(self.blank_frame1, text=' ')
        # self.blank_label3 = tkinter.Label(self.blank_frame2, text=' ')

        # the Enry widgets
        self.diameter_label = tkinter.Label(self.diameter_frame, \
                                            text='Enter the meteor diameter in meters:  ', font=("Helvetica", 10))
        self.diameter_entry = tkinter.Entry(self.diameter_frame, width=20)
        self.diameter_entry.insert(0, 3)

        self.distance_label = tkinter.Label(self.distance_frame, \
                                            text='     Enter the meteor distance from Earth in miles:  ',
                                            font=("Helvetica", 10))

        self.distance_entry = tkinter.Entry(self.distance_frame, width=20)
        self.distance_entry.insert(0, 300)

        # pack (position and order) the top frame's widgets
        # self.blank_label1.pack()
        self.heading_label.pack()
        # self.blank_label2.pack()

        self.diameter_label.pack(side='left')
        self.diameter_entry.pack(side='left')
        # self.blank_label3.pack()

        self.distance_label.pack(side='left')
        self.distance_entry.pack(side='left')

        # create the button widgets and label for the bottom frame
        # self.blank_label4 = tkinter.Label(self.bottom_frame, text=' ')
        self.runSim_button = tkinter.Button(self.bottom_frame, \
                                            state='normal', text='Run Simulation', command=self.processData)
        # self.runSim_button = tkinter.Button(self.bottom_frame, \
        # text='Run Simulation', command=self.processData)
        # self.blank_label5 = tkinter.Label(self.bottom_frame, text=' ')
        self.quit_button = tkinter.Button(self.bottom_frame, text='Quit', \
                                          command=self.main_window.destroy)
        # self.blank_label6 = tkinter.Label(self.bottom_frame, text=' ')

        # pack the blank label and buttons in the bottom frame
        # self.blank_label4.pack()
        self.runSim_button.pack()
        # self.blank_label5.pack()
        self.quit_button.pack()
        # self.blank_label6.pack()

        # pack the frames
        self.top_frame.pack()
        self.heading_frame.pack()
        self.earth_canvas.pack()
        self.earth_canvas_image = self.earth_canvas.create_image(self.earth_x, self.earth_y, image=self.earth_photo_image)
        self.meteor_canvas_image = self.earth_canvas.create_image(int(self.meteor_x), int(self.meteor_y),
                                                                  image=self.meteor_photo_image)
        # self.blank_frame1.pack()
        self.diameter_frame.pack()
        # self.blank_frame2.pack()
        self.distance_frame.pack()
        # self.blank_frame3.pack()
        self.bottom_frame.pack()

        tkinter.mainloop()

    def animate_meteor(self, cancel=False):
        if cancel==False:
            self.draw_one_meteor_frame()
            self.distance = self.distance - (self.meteorSpeed / 60/10)
            self.distance_label.config(text='Distance is now {:d}'.format(int(self.distance)))
            if (int(self.distance) >= 1200 and int(self.distance) <= 1400):
                self.runSim_button.config(background='green', command=self.animate_rocket_success)
            else:
                self.runSim_button.config(background='red',command=self.animate_rocket_fail)
            self.animate_meteor_id = self.main_window.after(50, self.animate_meteor)
        else:
            self.explosion_image = Image.open('explosion.png')
            self.explosion_image = self.explosion_image.resize((self.meteor_w+40, self.meteor_h+40), Image.ANTIALIAS)
            self.explosion_photo_image = ImageTk.PhotoImage(self.explosion_image)
            self.explosion_canvas_image = self.earth_canvas.create_image(int(self.meteor_x), int(self.meteor_y),
                                                                         image=self.explosion_photo_image)
            self.main_window.after_cancel(self.animate_meteor_id)

    def animate_rocket_fail(self, cancel=False):
        self.runSim_button.config(state='disabled')
        self.rocket_h += 2
        self.rocket_w += 2
        self.rocket_x -= 5
        self.rocket_y -= 5
        self.rocket_image = Image.open('rocket.png')
        self.rocket_image = self.rocket_image.resize((self.rocket_h, self.rocket_w), Image.ANTIALIAS)
        self.rocket_photo_image = ImageTk.PhotoImage(self.rocket_image)
        self.rocket_canvas_image = self.earth_canvas.create_image(int(self.rocket_x), int(self.rocket_y),
                                                                  image=self.rocket_photo_image)
        self.animate_rocket_fail_id =self.main_window.after(100, self.animate_rocket_fail)

    def animate_rocket_success(self, cancel=False):
        if self.rocket_h == 10:
            self.runSim_button.config(state='disabled')
        if cancel==False:
            #while abs(self.meteor_x - self.rocket_x) > 10 and abs(self.meteor_y - self.rocket_y) > 10:
            self.rocket_h += 2
            self.rocket_w += 2
            if self.meteor_x > self.rocket_x:
                self.rocket_x += self.rocket_dx
            elif self.meteor_x < self.rocket_x:
                self.rocket_x -= self.rocket_dx
            if self.meteor_y > self.rocket_y:
                self.rocket_y += self.rocket_dy
            elif self.meteor_y < self.rocket_y:
                self.rocket_y -= self.rocket_dy
            if abs(self.meteor_x - self.rocket_x) < 20 and abs(self.meteor_y - self.rocket_y) < 20:
                self.main_window.after_cancel(self.animate_rocket_success_id)
                self.animate_meteor(True)
                return                      # appears to be needed to cancel after loop
            self.rocket_image = Image.open('rocket.png')
            self.rocket_image = self.rocket_image.resize((self.rocket_h, self.rocket_w), Image.ANTIALIAS)
            self.rocket_photo_image = ImageTk.PhotoImage(self.rocket_image)
            self.rocket_canvas_image = self.earth_canvas.create_image(int(self.rocket_x), int(self.rocket_y),
                                                                          image=self.rocket_photo_image)
            self.animate_rocket_success_id =self.main_window.after(50, self.animate_rocket_success)

    def draw_one_meteor_frame(self):
        self.meteor_image = Image.open('meteor.png')
        self.meteor_image = self.meteor_image.resize((self.meteor_h, self.meteor_w), Image.ANTIALIAS)
        self.meteor_photo_image = ImageTk.PhotoImage(self.meteor_image)
        self.meteor_err = self.meteor_err + self.meteor_deltaerr
        self.meteor_canvas_image = self.earth_canvas.create_image(int(self.meteor_x), int(self.meteor_y),
                                                                  image=self.meteor_photo_image)
        self.meteor_x += self.meteor_dx
        if abs(self.meteor_err) >= 0.5:
            self.meteor_y += self.meteor_dy
            self.meteor_w -= 1
            self.meteor_h -= 1
            self.meteor_err -= self.meteor_drr

    # Retrieve the data from the text box and call the function in meteorCalc
    def processData(self):
        # have to calculate distance relative to Earth, add
        self.distance = 1500
        #self.distance = int((self.distance_entry.get())) + self.earth_x
        self.diam = int((self.diameter_entry.get()))
        self.meteorSpeed = float(120 * self.diam)
        self.meteor_y = randint(0,self.canvas_h)
        self.meteor_x = math.floor(math.sqrt(abs(pow(self.canvas_h, 2) - pow(self.meteor_y, 2))))    # Pythagoras's Identity
        self.meteor_deltaerr = (self.earth_y-self.meteor_y)/(self.earth_x-self.meteor_x)
        self.runSim_button.config(state='normal', text='Launch', background='red', command=self.animate_rocket_fail)
        if self.meteor_x > self.earth_x:
            self.meteor_dx = -self.meteor_dx
        #if self.meteor_x == self.earth_x:
        #    self.meteor_dx = 0
        if self.meteor_y > self.earth_y:
            self.meteor_dy = -self.meteor_dy
            self.meteor_drr = -self.meteor_drr
        #if self.meteor_y == self.earth_y:
        #    self.meteor_dy = 0
        if (self.meteor_x > self.canvas_w):
            self.meteor_x = self.canvas_w
        elif (self.meteor_x < 0):
            self.meteor_x = 0
        if(self.meteor_y > self.canvas_h):
            self.meteor_y = self.canvas_h
        elif (self.meteor_y < 0):
            self.meteor_y = 0
        self.animate_meteor()

        self.diam = (float(self.diam))
        #distance = float(distance)
        # = randrange(800)
        # = int(math.sqrt(math.pow(int(distance*20),2)- math.pow(self.meteor_y, 2)))

        badDiam = False
        badDistance = False

        self.diameter_notice_label = tkinter.Label(self.diameter_frame, \
                                                   text='NOTICE Heding into ifs now and diam is: {:d}'.format(diam))
        self.distance_notice_label = tkinter.Label(self.distance_frame, \
                                                   text='NOTICE Heading into ifs now and distance is: {:d}'.format(
                                                       self.distance))
        self.diameter_notice_label.pack()
        self.distance_notice_label.pack()

        # tkinter.messagebox.showinfo('NOTICE', 'Heading into ifs now and diam is: ' + str(diam))
        # tkinter.messagebox.showinfo('NOTICE', 'Heading into ifs now and distance is: ' + str(distance))

        if meteorStemUtils.validInput(diam) == False:
            diam = '3'
            badDiam = True
            self.badDiam_label = tkinter.Label(self.diameter_frame, \
                                               text='Bad Input Provided badDiam now equals ' + str(badDiam))
            self.badDiam_label.pack()
            # tkinter.messagebox.showinfo('Bad Input Provided ', 'badDiam now equals ' + str(badDiam))

        # tkinter.messagebox.showinfo('BACK IN MAIN', 'Checking for valid distance now and badDistance is: ' + str(badDistance))
        self.badDistance_label = tkinter.Label(self.distance_frame, \
                                               text='BACK IN MAIN Checking for valid distance now and badDistance is: ' + str(
                                                   badDistance))
        self.badDistance_label.pack()
        self.badDistance_label.config(text='TEST')  # this is how I'll update a label within a loop
        for i in range(math.ceil(distance * 60 / meteorSpeed)):
            distance = distance - (meteorSpeed / 60/10)
            self.distance_label.config(text='Distance is now ' + str(distance))
            if (distance >= 1200 and distance <= 1400):
                self.runSim_button.config(background='green')
            elif (distance <= 0):
                self.main_window.destroy()
            else:
                self.runSim_button.config(background='red')
            self.main_window.update()
            # self.main_window.after_idle(self.badDistance_label.config(text='Distance is now: '.format(distance)))
            time.sleep(.1)
        if meteorStemUtils.validInput(distance) == False:
            distance = '300'
            badDistance = True
            self.badDistance_label.config(text='Bad Input Provided badDistance now equals ' + str(badDistance))

            # tkinter.messagebox.showinfo('Bad Input Provided ', 'badDistance now equals ' + str(badDistance))

            # if badDiam == True or badDistance == True:
            # tkinter.messagebox.showinfo('Bad Input Provided!', 'Bad input received.\n  Diameter:  "' + str(diam) +
            # '" \n  Distance: "' + str(distance) + '" \n\nRunning with 3 meters at 500 miles.')

        simRunning = False  # inserted for iconify and de-iconify, could use pack_forget, and then pack() again

        # disable the Run Simulation button until it completes
        # self.runSim_button['state']='disabled'

        # iconify the main window
        # self.main_window.iconify()

        meteorStemStatus.runSimulation(diam, distance, simRunning)

        # Enter the tkinter main loop

        tkinter.self.main_window.mainloop()

        # after the sim runs, de iconify
        if simRunning == False:
            self.main_window.deiconify()

    def runSimulation(diam, distance, simRunning):

        simRunning = True  # to let main loop know to deiconify the entry window

        # tkinter.messagebox.showinfo('Now in runSim function and boolean flag is ', str(simRunning))

        # type conversion page 64
        diamFloat = float(diam)
        distanceFloat = float(distance)

        # boolean vars page 111
        meteorInbound = True
        meteorSpeed = float(120 * diamFloat)
        meteorDistance = distanceFloat

        # showinfo page 538, output formatting page 69
        # tkinter.messagebox.showinfo('Now in runSim function', 'The speed is ' + format(meteorSpeed, '.2f'))

        # Generating another window for status output
        # status_window = tkinter.Tk()
        # status_window.title("System Status")
        # status_window.minsize(width=450,height=200)

        # status_window.geometry('600x400+50-100')   # width, ht, 50 pixels in from left, and 100 up from bottom

        # create frames for widgets....................see page 549
        # set up a frame for each widget to position them correctly

        # status_window.top_frame = tkinter.Frame(status_window)
        # status_window.bottom_frame = tkinter.Frame(status_window)

        # create widgets for the top frame, creating stringVars for text updates and output
        status_window.distanceValue = tkinter.StringVar()
        # convert distance to string and store it - page 546
        status_window.distanceValue.set(distanceFloat)

        status_window.blank_label1 = tkinter.Label(status_window.top_frame, text=' ')
        status_window.distance_label = tkinter.Label(status_window.top_frame, \
                                                     text='Meteor Distance = {:d}' + format(distanceFloat))
        status_window.distanceFloatValue = tkinter.Label(status_window.top_frame, \
                                                         textvariable=status_window.distanceValue)  # page 546

        # pack (position and order) the top frame's widgets
        status_window.blank_label1.pack(side='top')
        status_window.distance_label.pack(side='left')
        status_window.distanceFloatValue.pack(side='left')

        # create the button widget and label for the bottom frame
        status_window.blank_labelb = tkinter.Label(status_window.bottom_frame, text=' ')
        status_window.updateSim_button = tkinter.Button(status_window.bottom_frame, \
                                                        text='Update Simulation', command=simUpdate(distance, diam))

        # pack the blank label and buttons in the bottom frame
        status_window.blank_labelb.pack()
        status_window.updateSim_button.pack()

        # pack the frames
        status_window.top_frame.pack()
        status_window.bottom_frame.pack()

        status_window.lift()  # make it on top
        # tkinter.mainloop()

        # loops page 122, boolean vars page 111

        # call back function for the update button

    def simUpdate(distance, diam):
        distanceData = float(distance)
        diamData = float(diam)
        meteorSpeed = float(120 * diamData)
        # tkinter.messagebox.showinfo('In while loop now', 'The distance is ' + format(distanceData, '.2f'))

        # update the status
        while distanceData > 0:
            distanceData = distanceData - (meteorSpeed / 60)
            # tkinter.messagebox.showinfo('In while loop now', 'The distance is ' + format(distanceData, '.2f'))

        simRunning = False

# create an instance of the class
meteor_sim = MeteorGUI()


import json
import pandas as pd
from statsmodels.formula.api import ols
import numpy as np
import matplotlib
matplotlib.use('pdf')
import matplotlib.pyplot as plt

'''
ts = pd.Series(np.random.randn(1000), index=pd.date_range('1/1/2000', periods=1000))
ts = ts.cumsum()
ts.plot()
plt.savefig('test.pdf')
'''
finaldata = {
    'capacity_multiplier': [],
    'number_of_iterations': [],
};
with open('./export.json', 'r') as f:
    data = json.load(f)
for run in data:
    finaldata["capacity_multiplier"].append(data[run]["label"]);
    finaldata["number_of_iterations"].append(data[run]["avgItrs"]);
df = pd.DataFrame(data=finaldata)
df = df.sort_values(by='capacity_multiplier', ascending=True)
test = ols("capacity_multiplier ~ number_of_iterations", data=df).fit()
df.plot.bar(x='capacity_multiplier', y='number_of_iterations', rot=0)
plt.savefig('test.pdf')
print(test.params)
print(test.summary())


'''
NOTE:

This script provides a demonstration of how the GLM
models in the published paper were fit. This script
is intended ONLY as a demonstration, as there is not
enough training data supplied to produce high-quality
GLM model fits.

There is not much value in using the fits that result
from this script, as they will be massively overfit
to the training data and will not generalize

We use the same hyperparameters here as were used to
fit the models in the paper.
'''

import argparse
import pickle
from typing import Dict, List, Tuple, Union, Optional
import math
import tqdm

import numpy as np
import torch

from encoding_models.poisson_encoder import ScaledPoissonFittedParams, SharedStimMulticellScaleOnlyPoisson

from convex_solver_base.unconstrained_optim import FistaSolverParams, UnconstrainedSolverParams, \
    batch_parallel_unconstrained_solve

from data_util.matched_cells_struct import OrderedMatchedCellsStruct
from data_util.load_data import load_typed_dataset, load_cell_ordering, load_cropped_stas_by_type, \
    compute_stimulus_onset_spikes, get_fit_cell_binned_spikes_lnp

def parallel_no_window_fit_scaled_poissons(all_images: np.ndarray,
                                           cell_response_vector: np.ndarray,
                                           filters_for_cell: np.ndarray,
                                           solver_params: UnconstrainedSolverParams,
                                           device: torch.device,
                                           batch_size=16) -> List[ScaledPoissonFittedParams]:

    n_trials, frame_height, frame_width = all_images.shape
    n_cells = len(sta_list)

    # we have to do this in batches otherwise we run out of GPU space for the midgets
    # also might speed the whole thing up

    full_stimuli = torch.tensor(all_images.reshape(n_trials, -1), dtype=torch.float32, device=device)

    fitted_model_list = []  # type: List[ScaledPoissonFittedParams]

    pbar = tqdm.tqdm(total=int(math.ceil(n_cells / batch_size)))
    for low in range(0, n_cells, batch_size):
        high = min(low + batch_size, n_cells)
        curr_batch_size = high - low

        relevant_filters = filters_for_cell[low:high, ...].reshape((curr_batch_size, -1))
        relevant_spikes = cell_response_vector[:, low:high].T

        relevant_spikes_torch = torch.tensor(relevant_spikes, dtype=torch.float32, device=device)

        # make the model
        multicell_poisson_model = SharedStimMulticellScaleOnlyPoisson(
            relevant_filters,
        ).to(device)  # type: SharedStimMulticellScaleOnlyPoisson

        losses = batch_parallel_unconstrained_solve(
            multicell_poisson_model,
            solver_params,
            verbose=True,
            shared_stimulus=full_stimuli,
            multicell_spikes=relevant_spikes_torch
        ).detach().cpu().numpy()

        # then extract all of the fitted parameters
        trained_filters, trained_biases = multicell_poisson_model.get_filters_and_biases()
        for idx in range(curr_batch_size):
            fitted_model_list.append(ScaledPoissonFittedParams(
                trained_filters[idx, ...].reshape((frame_height, frame_width)),
                trained_biases[idx],
                losses[idx],
            ))

        pbar.update(1)

        del relevant_filters
        del multicell_poisson_model

    del full_stimuli

    pbar.close()

    return fitted_model_list

if __name__ == '__main__':
    parser = argparse.ArgumentParser('Fit scaled single-bin LNP models to flashed trial data')
    parser.add_argument('save_path', type=str, help='path to save model fits')
    parser.add_argument('-gpu', '--gpu', action='store_true', default=False, help='use GPU')
    args = parser.parse_args()

    use_gpu = args.gpu
    device = torch.device('cuda') if use_gpu else torch.device('cpu')

    ################################################################
    # get the data
    # IMPORTANT: THIS IS A TINY SUBSET OF THE ACTUAL DATASET THAT WAS
    # USED TO FIT THE GLM; THESE FITS ARE NOT GOING TO BE VERY GOOD BY
    # DEFINITION
    train_frames, train_spikes_dict = load_typed_dataset()
    train_spikes_lnp_dict = {cell_type: compute_stimulus_onset_spikes(glm_spikes)
                             for cell_type, glm_spikes in train_spikes_dict.items()}

    ################################################################
    # load the STA prior
    all_prior_filters = load_cropped_stas_by_type()

    ################################################################
    # Load the cell types and matching
    cells_ordered = load_cell_ordering()  # type: OrderedMatchedCellsStruct
    ct_order = cells_ordered.get_cell_types()

    #### Train the scaled Poisson for every cell #################################
    typed_model_dict = {}  # type: Dict[str, List[ScaledPoissonFittedParams]]
    for cell_type in ct_order:
        print("Fitting {0}".format(cell_type))

        reference_id_list = cells_ordered.get_reference_cell_order(cell_type)
        cell_response_vector = get_fit_cell_binned_spikes_lnp(train_spikes_lnp_dict,
                                                              cell_type,
                                                              reference_id_list,
                                                              cells_ordered)

        sta_list = all_prior_filters[cell_type]

        all_fits = parallel_no_window_fit_scaled_poissons(
            train_frames,
            cell_response_vector,
            sta_list,
            FistaSolverParams(initial_learning_rate=1.0,
                              max_iter=100,
                              backtracking_beta=0.5,
                              converge_epsilon=1e-6),
            device,
            batch_size=32  # in reality, we use 2 for the full dataset
        )

        typed_model_dict[cell_type] = all_fits

    with open(args.save_path, 'wb') as pfile:
        pickle.dump(typed_model_dict, pfile)


import multiprocessing
import os
import pathlib
import platform
import re
import shutil
import sys
from dataclasses import dataclass
from getpass import getpass
from pathlib import Path
from typing import Iterable, List, Optional

from craft_cli import emit
from craft_parts.sources.git_source import GitSource

from snapcraft import errors

@dataclass
class OSPlatform:
    """Platform definition for a given host."""

    system: str
    release: str
    machine: str

    def __str__(self) -> str:
        """Return the string representation of an OSPlatform."""
        return f"{self.system}/{self.release} ({self.machine})"

# architecture translations from the platform syntax to the deb/snap syntax
# These two architecture mappings are almost inverses of each other, except one map is
# not reversible (same value for different keys)
_ARCH_TRANSLATIONS_PLATFORM_TO_DEB = {
    "aarch64": "arm64",
    "armv7l": "armhf",
    "i686": "i386",
    "ppc": "powerpc",
    "ppc64le": "ppc64el",
    "x86_64": "amd64",
    "AMD64": "amd64",  # Windows support
    "s390x": "s390x",
    "riscv64": "riscv64",
}

# architecture translations from the deb/snap syntax to the platform syntax
_ARCH_TRANSLATIONS_DEB_TO_PLATFORM = {
    "arm64": "aarch64",
    "armhf": "armv7l",
    "i386": "i686",
    "powerpc": "ppc",
    "ppc64el": "ppc64le",
    "amd64": "x86_64",
    "s390x": "s390x",
    "riscv64": "riscv64",
}

_32BIT_USERSPACE_ARCHITECTURE = {
    "aarch64": "armv7l",
    "armv8l": "armv7l",
    "ppc64le": "ppc",
    "x86_64": "i686",
}

def get_os_platform(
    filepath=pathlib.Path(  # noqa: B008 Function call in arg defaults
        "/etc/os-release"
    ),
):
    """Determine a system/release combo for an OS using /etc/os-release if available."""
    system = platform.system()
    release = platform.release()
    machine = platform.machine()

    if system == "Linux":
        try:
            with filepath.open("rt", encoding="utf-8") as release_file:
                lines = release_file.readlines()
        except FileNotFoundError:
            emit.debug("Unable to locate 'os-release' file, using default values")
        else:
            os_release = {}
            for line in lines:
                line = line.strip()  # noqa PLW2901
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, value = line.rstrip().split("=", 1)
                if value[0] == value[-1] and value[0] in ('"', "'"):
                    value = value[1:-1]
                os_release[key] = value
            system = os_release.get("ID", system)
            release = os_release.get("VERSION_ID", release)

    return OSPlatform(system=system, release=release, machine=machine)

def get_host_architecture():
    """Get host architecture in deb format suitable for base definition."""
    os_platform_machine = get_os_platform().machine

    if platform.architecture()[0] == "32bit":
        userspace = _32BIT_USERSPACE_ARCHITECTURE.get(os_platform_machine)
        if userspace:
            os_platform_machine = userspace

    return _ARCH_TRANSLATIONS_PLATFORM_TO_DEB.get(
        os_platform_machine, os_platform_machine
    )

def convert_architecture_deb_to_platform(architecture: str) -> str:
    """Convert an architecture from deb/snap syntax to platform syntax.

    :param architecture: architecture string in debian/snap syntax
    :return: architecture in platform syntax
    :raises InvalidArchitecture: if architecture is not valid
    """
    platform_arch = _ARCH_TRANSLATIONS_DEB_TO_PLATFORM.get(architecture)
    if not platform_arch:
        raise errors.InvalidArchitecture(architecture)

    return platform_arch

def strtobool(value: str) -> bool:
    """Convert a string representation of truth to true (1) or false (0).

    :param value: a True value of 'y', 'yes', 't', 'true', 'on', and '1'
        or a False value of 'n', 'no', 'f', 'false', 'off', and '0'.
    :raises ValueError: if `value` is not a valid boolean value.
    """
    parsed_value = value.lower()

    if parsed_value in ("y", "yes", "t", "true", "on", "1"):
        return True
    if parsed_value in ("n", "no", "f", "false", "off", "0"):
        return False

    raise ValueError(f"Invalid boolean value of {value!r}")

def is_managed_mode() -> bool:
    """Check if snapcraft is running in a managed environment."""
    managed_flag = os.getenv("SNAPCRAFT_MANAGED_MODE", "n")
    return strtobool(managed_flag)

def get_managed_environment_home_path():
    """Path for home when running in managed environment."""
    return pathlib.Path("/root")

def get_managed_environment_project_path():
    """Path for project when running in managed environment."""
    return get_managed_environment_home_path() / "project"

def get_managed_environment_log_path():
    """Path for log when running in managed environment."""
    return pathlib.Path(
        "/tmp/snapcraft.log"  # noqa: S108 Probable insecure use of temp file
    )

def get_managed_environment_snap_channel() -> Optional[str]:
    """User-specified channel to use when installing Snapcraft snap from Snap Store.

    :returns: Channel string if specified, else None.
    """
    return os.getenv("SNAPCRAFT_INSTALL_SNAP_CHANNEL")

def get_effective_base(
    *,
    base: Optional[str],
    build_base: Optional[str],
    project_type: Optional[str],
    name: Optional[str],
) -> Optional[str]:
    """Return the base to use to create the snap.

    Returns build-base if set, but if not, name is returned if the
    snap is of type base. For all other snaps, the base is returned
    as the build-base.
    """
    if build_base is not None:
        return build_base

    return name if project_type == "base" else base

def get_parallel_build_count() -> int:
    """Obtain the number of concurrent jobs to execute.

    Try different strategies to obtain the number of parallel jobs
    to execute. If they fail, assume the safe default of 1. The
    number of concurrent build jobs can be limited by setting the
    environment variable ``SNAPCRAFT_MAX_PARALLEL_BUILD_COUNT``.

    :return: The number of parallel jobs for the current host.
    """
    try:
        build_count = len(os.sched_getaffinity(0))
        emit.debug(f"CPU count (from process affinity): {build_count}")
    except AttributeError:
        # Fall back to multiprocessing.cpu_count()...
        try:
            build_count = multiprocessing.cpu_count()
            emit.debug(f"CPU count (from multiprocessing): {build_count}")
        except NotImplementedError:
            emit.progress(
                "Unable to determine CPU count; disabling parallel builds",
                permanent=True,
            )
            build_count = 1

    max_count_env = os.environ.get("SNAPCRAFT_MAX_PARALLEL_BUILD_COUNT", "")
    try:
        max_count = int(max_count_env)
        if max_count > 0:
            build_count = min(build_count, max_count)
    except ValueError:
        emit.debug(f"Invalid SNAPCRAFT_MAX_PARALLEL_BUILD_COUNT {max_count_env!r}")

    return build_count

def confirm_with_user(prompt_text, default=False) -> bool:
    """Query user for yes/no answer.

    If stdin is not a tty, the default value is returned.

    If user returns an empty answer, the default value is returned.
    returns default value.

    :returns: True if answer starts with [yY], False if answer starts with [nN],
        otherwise the default.
    """
    if is_managed_mode():
        raise RuntimeError("confirmation not yet supported in managed-mode")

    if not sys.stdin.isatty():
        return default

    choices = " [Y/n]: " if default else " [y/N]: "

    with emit.pause():
        reply = str(input(prompt_text + choices)).lower().strip()

    if reply and reply[0] == "y":
        return True

    if reply and reply[0] == "n":
        return False

    return default

def prompt(prompt_text: str, *, hide: bool = False) -> str:
    """Prompt and return the entered string.

    :param prompt_text: string used for the prompt.
    :param hide: hide user input if True.
    """
    if is_managed_mode():
        raise RuntimeError("prompting not yet supported in managed-mode")

    if not sys.stdin.isatty():
        raise errors.SnapcraftError("prompting not possible with no tty")

    if hide:
        method = getpass
    else:
        method = input  # type: ignore

    with emit.pause():
        return str(method(prompt_text))

def humanize_list(
    items: Iterable[str],
    conjunction: str,
    item_format: str = "{!r}",
    sort: bool = True,
) -> str:
    """Format a list into a human-readable string.

    :param items: list to humanize.
    :param conjunction: the conjunction used to join the final element to
                        the rest of the list (e.g. 'and').
    :param item_format: format string to use per item.
    :param sort: if true, sort the list.
    """
    if not items:
        return ""

    quoted_items = [item_format.format(item) for item in items]

    if sort:
        quoted_items = sorted(quoted_items)

    if len(quoted_items) == 1:
        return quoted_items[0]

    humanized = ", ".join(quoted_items[:-1])

    if len(quoted_items) > 2:
        humanized += ","

    return f"{humanized} {conjunction} {quoted_items[-1]}"

def get_common_ld_library_paths(
    prime_dir: Path, arch_triplet: Optional[str]
) -> List[str]:
    """Return common existing PATH entries for a snap.

    :param prime_dir: Path to the prime directory.
    :param arch_triplet: Architecture triplet of target arch. If None, the list of paths
    will not contain architecture-specific paths.

    :returns: List of common library paths in the prime directory that exist.
    """
    paths = [
        prime_dir / "lib",
        prime_dir / "usr" / "lib",
    ]

    if arch_triplet:
        paths.extend(
            [
                prime_dir / "lib" / arch_triplet,
                prime_dir / "usr" / "lib" / arch_triplet,
            ]
        )

    return [str(p) for p in paths if p.exists()]

def get_ld_library_paths(prime_dir: Path, arch_triplet: Optional[str]) -> str:
    """Return a usable in-snap LD_LIBRARY_PATH variable.

    :param prime_dir: Path to the prime directory.
    :param arch_triplet: Architecture triplet of target arch. If None, LD_LIBRARY_PATH
    will not contain architecture-specific paths.

    :returns: The LD_LIBRARY_PATH environment variable to be used for the snap.
    """
    paths = ["${SNAP_LIBRARY_PATH}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"]
    # Add the default LD_LIBRARY_PATH
    paths += get_common_ld_library_paths(prime_dir, arch_triplet)

    ld_library_path = ":".join(paths)

    return re.sub(str(prime_dir), "$SNAP", ld_library_path)

def get_host_tool(command_name: str) -> str:
    """Return the full path of the given host tool.

    :param command_name: the name of the command to resolve a path for.
    :return: Path to command

    :raises SnapcraftError: if command_name was not found.
    """
    tool = shutil.which(command_name)
    if not tool:
        raise errors.SnapcraftError(
            f"A tool snapcraft depends on could not be found: {command_name!r}",
            resolution="Ensure the tool is installed and available, and try again.",
        )
    return tool

def get_snap_tool(command_name: str) -> str:
    """Return the path of a command found in the snap.

    If snapcraft is not running as a snap, shutil.which() is used
    to resolve the command using PATH.

    :param command_name: the name of the command to resolve a path for.
    :return: Path to command

    :raises SnapcraftError: if command_name was not found.
    """
    if os.environ.get("SNAP_NAME") != "snapcraft":
        return get_host_tool(command_name)

    snap_path = os.getenv("SNAP")
    if snap_path is None:
        raise RuntimeError(
            "The SNAP environment variable is not defined, but SNAP_NAME is?"
        )

    command_path = _find_command_path_in_root(snap_path, command_name)

    if command_path is None:
        raise errors.SnapcraftError(
            f"Cannot find snap tool {command_name!r}",
            resolution="Please report this error to the Snapcraft maintainers.",
        )

    return command_path

def _find_command_path_in_root(root: str, command_name: str) -> Optional[str]:
    for bin_directory in (
        "usr/local/sbin",
        "usr/local/bin",
        "usr/sbin",
        "usr/bin",
        "sbin",
        "bin",
    ):
        path = os.path.join(root, bin_directory, command_name)
        if os.path.exists(path):
            return path

    return None

def process_version(version: Optional[str]) -> str:
    """Handle special version strings."""
    if version is None:
        raise ValueError("version cannot be None")

    new_version = version
    if version == "git":
        emit.progress("Determining the version from the project repo (version: git).")
        new_version = GitSource.generate_version()

    if new_version != version:
        emit.progress(f"Version has been set to {new_version!r}", permanent=True)

    return new_version

def is_snapcraft_running_from_snap() -> bool:
    """Check if snapcraft is running from the snap."""
    return os.getenv("SNAP_NAME") == "snapcraft" and os.getenv("SNAP") is not None


from argparse import ArgumentParser
from .model_loader import *
from .fad_batch import cache_embedding_files

def main():
    """
    Launcher for caching embeddings of directories using multiple models.
    """
    models = {m.name: m for m in get_all_models()}

    agupa = ArgumentParser()

    # Accept multiple models and directories with distinct prefixes
    agupa.add_argument('-m', '--models', type=str, choices=list(models.keys()), nargs='+', required=True)
    agupa.add_argument('-d', '--dirs', type=str, nargs='+', required=True)

    # Add optional arguments
    agupa.add_argument('-w', '--workers', type=int, default=8)
    agupa.add_argument('-s', '--sox-path', type=str, default='/usr/bin/sox')

    args = agupa.parse_args()

    for model_name in args.models:
        model = models[model_name]
        for d in args.dirs:
            log.info(f"Caching embeddings for {d} using {model.name}")
            cache_embedding_files(d, model, workers=args.workers)

if __name__ == "__main__":
    main()


# **************************************************************************************************************
# Date: December 8, 2020
# Author: Changzhen Wang
# Description: This script implements the ScLouvain and ScLeiden methods
# **************************************************************************************************************

import networkx as nx
import copy
import numpy as np
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import reverse_cuthill_mckee

state1 = -1 # no flows to adjacent community

# Compute the modularity of a graph partition
def modularity(partition, graph):
    if type(graph) != nx.Graph:
        raise TypeError("Bad graph type, use only non directed graph")

    inc = dict([]) # sum of weights of all edges within each community c
    deg = dict([]) # sum of weights of all edges connected with a node in each community c
    links = graph.size(weight='weight') # return total of all edge weights.
    if links == 0:
        raise ValueError("A graph without link has an undefined modularity")

    for node in graph:
        if bool(partition.get(node)):
            com = partition[node]
            deg[com] = deg.get(com, 0.) + graph.degree(node, weight='weight')
            for neighbor, datas in graph[node].items():
                weight = datas.get("weight", 1)
                if bool(partition.get(neighbor)):
                    if partition[neighbor] == com:
                        if neighbor == node:
                            inc[com] = inc.get(com, 0.) + float(weight) #inc[com] is the sum of weights of all edges within the community com.
                        else:
                            inc[com] = inc.get(com, 0.) + float(weight) / 2.

    res = 0.
    for com in set(partition.values()):
        res += (inc.get(com, 0.) / links) - ((deg.get(com, 0.) / (2. * links)) ** 2)  # Q=∑c[Σin2m−(Σtot2m)2]; annotation should be ∑(in/links-r*(ds/2*links)**2) by Wang 12-15-2018
    return res

# split each community by spatial adjacency matrix and update the status
def __refine_graph(current_graph, com2node_dict, status, adjmx, graph):
    com2node_dict_temp = copy.deepcopy(com2node_dict)

    for node in current_graph.nodes():
        #com_node = status.node2com[node]
        node_original_list = com2node_dict_temp[node]
        sub_coms = dict()
        if len(node_original_list) > 1:
            sub_coms = split_com_by_adj(node_original_list, adjmx)
        if len(sub_coms) > 1: # exist multiple communities, then remove and extend the current com2node_dict_temp
            for i in range(1,len(sub_coms)):
                temp_pop = 0
                for sub_com_node in sub_coms[i]:
                    com2node_dict_temp[node].remove(sub_com_node)
                    temp_pop +=  graph.nodes[sub_com_node]['pop']
                ext_com = {max(com2node_dict_temp.keys()) + 1 : sub_coms[i]}
                com2node_dict_temp.update(ext_com)
                status.com2node.update({max(status.com2node.keys()) + 1 : len(status.com2node)})
                status.node2com.update({max(status.node2com.keys()) + 1 : node})
                status.com_attr1.update({max(status.com_attr1.keys()) + 1: temp_pop})
                status.com_attr1[node] -= temp_pop

    partition_init = __com2node_partition(com2node_dict_temp)
    current_graph = induced_graph(partition_init, graph, None, status)
    return current_graph, com2node_dict_temp

# split a list of nodes by spatial adjacent matrix
def split_com_by_adj(nodeslist, adjmx):
    sort_nodeslist = sorted(nodeslist)
    n = len(sort_nodeslist)
    nodes_dict = {k : sort_nodeslist[k] for k in range(0, len(sort_nodeslist))}
    adj_new = np.array([[0]*n for x in range(n)])
    rows_bl = np.isin(adjmx[0], sort_nodeslist)
    col_bl = np.isin(adjmx[1], sort_nodeslist)

    rows_index = np.nonzero(rows_bl)
    cols_index = np.nonzero(col_bl)
    comm_index = np.intersect1d(rows_index, cols_index)

    dd1 = np.take(adjmx[0], comm_index)
    dd2 = np.take(adjmx[1], comm_index)

    for lvalue, rvalue in zip(dd1, dd2):
        lindex = sort_nodeslist.index(lvalue)
        rindex = sort_nodeslist.index(rvalue)
        adj_new[lindex][rindex] = adj_new[rindex][lindex] = 1
    mlabel = adj2cluster(adj_new)
    sub_partition = { k: [] for k in mlabel.keys()}
    for key, value in mlabel.items():
        for lower_value in value:
            sub_partition[key].append(nodes_dict[lower_value])
    return sub_partition

# It is a very important and efficient function to construct cluster/island
def adj2cluster(A):
    # symmetrize adjacency matrix
    S = A + A.transpose()
    #print(S)
    # transform dense matrix into sparse matrix
    G_sparse = csr_matrix(S)
    #print(G_sparse)
    # Reverse Cuthill-McKee ordering
    r = np.flip(reverse_cuthill_mckee(G_sparse))
    #print(r)
    # Get the clusters
    mlabel = dict()
    clusterNum = 0
    mlabel[clusterNum] = [r[0]]
    for i in range(1, len(r)):
        if S[mlabel[clusterNum], r[i]].any():
            c = np.append(mlabel[clusterNum],r[i])
            mlabel[clusterNum] = c
        else:
            clusterNum = clusterNum + 1
            mlabel[clusterNum] = [r[i]]
    return mlabel

# calculate LI, average travel time and the number of destination nodes
# partition is the final partition
# part_init is the initial partition
def calculateindices(digraph, partition, part_init):
    internalvols = dict([])
    outgoingvols = dict([])
    sum_weight = dict()
    sum_weighttime = dict()
    dest_nodes = dict()

    links = digraph.size(weight = "weight")
    if links == 0:
        raise ValueError("A directed graph without link is invalid!")

    alledgeattr = list(list(digraph.edges(data=True))[0][-1].keys())
    for node in digraph.nodes():
        com = partition[node]
        com_init = part_init[node]
        if com_init != state1:
            if bool(dest_nodes.get(com)):
                dest_nodes[com] += 1
            else:
                dest_nodes[com] = 1
        for neighbor, datas in digraph[node].items():
            odweight = 0
            odesttime = 0
            if datas is not None:
                odweight = datas["weight"]
                if "estTime" in alledgeattr:
                    odesttime = datas["estTime"]
                sum_weight[com] = sum_weight.get(com, 0) + odweight
                sum_weighttime[com] = sum_weighttime.get(com, 0) + odweight * odesttime
            if partition[neighbor] == com:  # node and its neighbor are in the sam community
                internalvols[com] = internalvols.get(com, 0) + float(odweight)
                outgoingvols[com] = outgoingvols.get(com, 0)
            else:
                outgoingvols[com] = outgoingvols.get(com, 0) + float(odweight)
                internalvols[com] = internalvols.get(com, 0)

    values = sorted(set(partition.values()))
    res = {k: [] for k in list(values)}

    for com in values:
        if not bool(internalvols.get(com)):
            internalvols[com] = 0
        if not bool(outgoingvols.get(com)):
            outgoingvols[com] = 0
        if not bool(sum_weight.get(com)):
            sum_weight[com] = 0
        internalvol = internalvols[com]
        LI = -1
        if internalvol != links:
            if internalvol + outgoingvols[com] > 0:
                LI = internalvol/(internalvol + outgoingvols[com])
        res[com].append(LI)
        weightedtime = 0
        if sum_weight[com] != 0:
            weightedtime = sum_weighttime[com] / sum_weight[com]
        res[com].append(weightedtime)
        if bool(dest_nodes.get(com)):
            res[com].append(dest_nodes[com])
        else:
            res[com].append(0)
    return res

# make a new graph where discovered communities become its nodes,
# links between communities become its edges (weights are summed).
def induced_graph(partition, graph, com_old_vs_com_new, status):
    # com_old_vs_com_new is a dict that stores the relationship between old community id vs. new id after renumbering
    # status is the current status
    ret = nx.Graph()
    ret.add_nodes_from(partition.values(), pop =0) # partition.values() has the community number of each node after renumbering
    #ret.add_nodes_from(partition.values())
    # add and update the attribute1 (e.g., pop) of the new communities after renumbered
    if status is not None:
        if com_old_vs_com_new is not None:
            for com_old in com_old_vs_com_new.keys():
                com_new = com_old_vs_com_new[com_old]
                com_attr1 = status.com_attr1[com_old]
                ret.nodes[com_new]['pop'] = com_attr1
        else:
            for com, popvalue in status.com_attr1.items():
                if ret.has_node(com):
                    ret.nodes[com]['pop'] = popvalue
    else:
        sum_pop1 = 0
        for node in graph.nodes():
            com = partition[node]
            if ret.has_node(com):
                ret.nodes[com]['pop'] += graph.nodes[node]['pop']
                sum_pop1 += graph.nodes[node]['pop']

    for node1, node2, datas in graph.edges(data=True): #  If data=True, then return edge attribute dict in 3-tuple (u,v,ddict)
        weight = datas.get("weight", 1) # an edge weight should have a column name 'weight'!!!!! if 'weight' column is not found, then return 1 for this variable
        com1 = partition[node1] # community of node 1 of an edge
        com2 = partition[node2] # community of node 2 of an edge
        w_prec = ret.get_edge_data(com1, com2, {"weight": 0}).get("weight", 1) # Return the attribute dictionary associated with edge (u,v); {"weight": 0} is the value to return if the edge (com1,com2) is not found in the network ret.
        ret.add_edge(com1, com2, weight = w_prec + weight)

    return ret

# Renumber the values of the dictionary from 1 to n
# It is primarily used to renumber the community id in the partition result list
# Also, it updates the community id in the global dict com2node_globaldict after its renumbering
def __renumber(partition_dict, com2node_dict):
    count = 0 # new community id after renumber starts from 0
    re_part = partition_dict.copy()
    new_values = dict([]) # a dict that stores the relationship between old and new community id, see below
    com2node_tempdict = dict([]) # a temporal dict of com2node

    for key in partition_dict.keys():
        value = partition_dict[key]
        new_value = new_values.get(value, -1)
        if new_value == -1:
            new_values[value] = count # new_values is a dict that stores {526: 1, 486: 2...} where 526 is the
            # original community name (i.e., value) and 1 is the community name after renumbering (i.e., count)
            com2node_tempdict[count] = com2node_dict[value] # get ids that a community contains
            new_value = count   # pass value to the outside of the loop
            count += 1
        re_part[key] = new_value

    return re_part, com2node_tempdict, new_values

# has to change as the gain (even negative) is useless, merge the population to the o
def __second_level_pop(graph, status, attr1_limit, resolution, com2node_dict, isloop, adjmx):
    """
        merge all communities (i.e., new nodes in the current graph) whose attribute
        (e.g., total pop) are less than the predefined threshold to their neighbors
    """
    com2node_dict_temp = copy.deepcopy(com2node_dict)
    for node in graph.nodes():
        com_node = status.node2com[node]
        com_node_attr1 = status.com_attr1[com_node]
        if com_node_attr1 < attr1_limit:
            degc_totw = status.gdegrees.get(node, 0.) / (status.total_weight * 2.)
            edgeneigh_communities, toponeigh_communities = __toponeighcom(node, graph, status, com2node_dict, adjmx)
            __remove(node, com_node, edgeneigh_communities.get(com_node, 0.), status) # remove the current node from its original community
            best_com = com_node
            best_increase = 0
            if isloop is False:
                if len(toponeigh_communities) == 1:
                    best_com = list(toponeigh_communities.keys())[0]
                else:
                    if all(value == 0 for value in toponeigh_communities.values()):
                        com_pop_dict = {key: status.com_attr1[key] for key in toponeigh_communities.keys()}
                        com_pop_dict_sort = dict(sorted(com_pop_dict.items(), key=lambda value: value[1]))
                        neigh_coms = {key : value for (key, value) in com_pop_dict_sort.items() if value >= attr1_limit}
                        if len(neigh_coms) == 0:
                            best_com = list(com_pop_dict_sort.keys())[-1]
                        else:
                            best_com = [key for (key, value) in neigh_coms.items() if value == min(neigh_coms.values())][0]
                    else:
                        incr = 0
                        for neigh_com, dnc in toponeigh_communities.items():
                            incr = dnc - resolution * status.degrees.get(neigh_com, 0.) * degc_totw
                            if incr > best_increase:
                                best_increase = incr
                                best_com = neigh_com
            else:
                com_pop_dict = {key: status.com_attr1[key] for key in toponeigh_communities.keys()}
                com_pop_dict_sort = dict(sorted(com_pop_dict.items(), key=lambda value: value[1]))
                neigh_coms = {key : value for (key, value) in com_pop_dict_sort.items() if value >= attr1_limit}
                if len(neigh_coms) == 0:
                    best_com = list(com_pop_dict_sort.keys())[-1]
                else:
                    best_com = [key for (key, value) in neigh_coms.items() if value == min(neigh_coms.values())][0]
            __insert(node, best_com, edgeneigh_communities.get(best_com, 0.), status) # assign this node to the community with the max deltaQ
            if best_com != com_node:
                node_origin_list = com2node_dict[node][:]  # only pass value not reference (another way is to use deepcopy)
                for nd in node_origin_list:
                    com2node_dict_temp[com_node].remove(nd)  # remove
                    com2node_dict_temp[best_com].append(nd)  # update the com2node_dict
    com2node_dict = copy.deepcopy(com2node_dict_temp)
    return com2node_dict

def __ensure_one_more_destnodes(graph, status, resolution, com2node_dict, com_no_dnodes, isloop, adjmx):
    com2node_dict_temp = copy.deepcopy(com2node_dict)

    for node in graph.nodes():
        com_node = status.node2com[node]
        if com_node in com_no_dnodes:
            degc_totw = status.gdegrees.get(node, 0.) / (status.total_weight * 2.)
            edgeneigh_communities, toponeigh_communities = __toponeighcom(node, graph, status, com2node_dict, adjmx)
            __remove(node, com_node, edgeneigh_communities.get(com_node, 0.), status) # remove the current node from its original community
            best_com = com_node
            best_increase = 0
            if isloop is False:
                if len(toponeigh_communities) == 1:
                    best_com = list(toponeigh_communities.keys())[0]
                else:
                    if all(value == 0 for value in toponeigh_communities.values()): # no flows to topocommunity
                        com_has_dnodes = [key for key in toponeigh_communities.keys() if key not in com_no_dnodes] # find topocommunity with dnodes
                        if len(com_has_dnodes) > 0 :
                            com_pop_dict = {key: status.com_attr1[key] for key in com_has_dnodes} # find the population
                            com_pop_dict_sort = dict(sorted(com_pop_dict.items(), key=lambda value: value[1])) # ascending order
                            best_com = list(com_pop_dict_sort.keys())[0] # get com with the smallest population size
                    else:
                        incr = 0
                        for neigh_com, dnc in toponeigh_communities.items():
                            if neigh_com not in com_no_dnodes:
                                incr = dnc - resolution * status.degrees.get(neigh_com, 0.) * degc_totw
                                if incr > best_increase:
                                    best_increase = incr
                                    best_com = neigh_com
            else:
                # just in case the dead loop and no optimial topocommunity is found from above scripts
                com_has_dnodes = [key for key in toponeigh_communities.keys() if key not in com_no_dnodes] # find topocommunity with dnodes
                if len(com_has_dnodes) > 0 :
                    com_pop_dict = {key: status.com_attr1[key] for key in com_has_dnodes} # find the population
                    com_pop_dict_sort = dict(sorted(com_pop_dict.items(), key=lambda value: value[1])) # ascending order
                    best_com = list(com_pop_dict_sort.keys())[0] # get com with the smallest population size
            __insert(node, best_com, edgeneigh_communities.get(best_com, 0.), status) # assign this node to the community with the max deltaQ
            if best_com != com_node:
                node_original_list = com2node_dict[node][:]  # only pass value not reference (another way is to use deepcopy)
                for nd in node_original_list:
                    com2node_dict_temp[com_node].remove(nd)  # remove
                    com2node_dict_temp[best_com].append(nd)  # update the com2node_dict

    com2node_dict = copy.deepcopy(com2node_dict_temp)
    return com2node_dict

# ensure all community has at least one destination node
def __ensure_nonzero_LI(graph, status, resolution, com2node_dict, isloop, adjmx):
    com2node_dict_temp = copy.deepcopy(com2node_dict)
    for node in graph.nodes():
        com_node = status.node2com[node]
        incflows = status.internals[com_node] # get internal flows
        if incflows == 0:
            degc_totw = status.gdegrees.get(node, 0.) / (status.total_weight * 2.)
            edgeneigh_communities, toponeigh_communities = __toponeighcom(node, graph, status, com2node_dict, adjmx)
            __remove(node, com_node, edgeneigh_communities.get(com_node, 0.), status) # remove the current node from its original community
            best_com = com_node
            best_increase = 0
            if isloop is False:
                if len(toponeigh_communities) == 1:
                    best_com = list(toponeigh_communities.keys())[0]
                else:
                    if all(value == 0 for value in toponeigh_communities.values()):
                        com_pop_dict = {key: status.com_attr1[key] for key in toponeigh_communities.keys() if status.internals[key] != 0 }
                        com_pop_dict_sort = dict(sorted(com_pop_dict.items(), key=lambda value: value[1]))
                        best_com = list(com_pop_dict_sort.keys())[0]
                    else:
                        incr = 0
                        for neigh_com, dnc in toponeigh_communities.items():
                            incr = dnc - resolution * status.degrees.get(neigh_com, 0.) * degc_totw
                            if incr > best_increase:
                                best_increase = incr
                                best_com = neigh_com
            else:
                com_pop_dict = {key: status.com_attr1[key] for key in toponeigh_communities.keys() if status.internals[key] != 0}
                com_pop_dict_sort = dict(sorted(com_pop_dict.items(), key=lambda value: value[1]))
                if len(com_pop_dict_sort) > 0:
                    best_com = list(com_pop_dict_sort.keys())[0]

            __insert(node, best_com, edgeneigh_communities.get(best_com, 0.), status) # assign this node to the community with the max deltaQ
            if best_com != com_node:
                node_original_list = com2node_dict[node][:]  # only pass value not reference (another way is to use deepcopy)
                for nd in node_original_list:
                    com2node_dict_temp[com_node].remove(nd)  # remove
                    com2node_dict_temp[best_com].append(nd)  # update the com2node_dict

    com2node_dict = copy.deepcopy(com2node_dict_temp)
    return com2node_dict

# convert com2node to partition
def __com2node_partition(com2node):
    partition = {}
    for key, values in com2node.items():
        if len(values) > 0:
            for value in values:
                partition[value] = key
    return partition

# convert partition to com2node
def partition_com2node(partition):
    com2node_dict = {k: [] for k in list(set(partition.values()))}
    for node, com in partition.items():
        if bool(com2node_dict.get(com)):
            com2node_dict[com].append(node)
        else:
            com2node_dict[com] = [node]
    return com2node_dict

# part_init is the partition from network optimization method without any spatial constraints
# part_init2 is the partition based on the destination node
def refined_partition_network(graph, adjmx, attr1_limit, resolution, part_init, part_init2):
    if type(graph) != nx.Graph:
        raise TypeError("Bad graph type, use only non directed graph")

    status = Status()
    status.init(graph, part_init)
    com2node_globaldict = copy.deepcopy(status.com2node)
    current_graph = induced_graph(part_init, graph, None, status)
    # will decompose community by spatial adjacency matrix first
    current_graph, com2node_globaldict = __refine_graph(current_graph, com2node_globaldict, status, adjmx, graph)
    status.init(current_graph)

    # the following codes examine the attribute (e.g., total population) in each community, and merge the
    # community that is below the predefined threshold to its neighbor having the most modularity gain.
    minpop = min(status.com_attr1.values()) # the minimum pop of all communities of current status
    if attr1_limit is not None and attr1_limit > 0:
        if minpop < attr1_limit:
            isloop = False
            while True:
                num_coms = len(com2node_globaldict)
                com2node_globaldict = __second_level_pop(current_graph, status, attr1_limit, resolution, com2node_globaldict, isloop, adjmx)
                partition, com2node_globaldict, com_old_vs_com_new = __renumber(status.node2com, com2node_globaldict)
                current_graph = induced_graph(partition, current_graph, com_old_vs_com_new, status)
                status.init(current_graph)
                minpop = min(status.com_attr1.values())
                if minpop >= attr1_limit:
                    break
                else:
                    if num_coms == len(com2node_globaldict):
                        isloop = True # means infinite loop will begin, so merge strategy should be changed

    # find all destination nodes
    allvalues = [value for value in part_init2.values() if value != state1]
    # find com without destination nodes
    com_no_destnodes = [com for com, nodes in com2node_globaldict.items() if len(set(nodes).intersection(set(allvalues))) == 0]
    if len(com_no_destnodes) > 0:
        isloop = False
        while True:
            num_coms = len(com2node_globaldict)
            com2node_globaldict =__ensure_one_more_destnodes(current_graph, status, resolution, com2node_globaldict, com_no_destnodes, isloop, adjmx)
            partition, com2node_globaldict, com_old_vs_com_new = __renumber(status.node2com, com2node_globaldict)
            current_graph = induced_graph(partition, current_graph, com_old_vs_com_new, status)
            status.init(current_graph)
            com_no_destnodes = [com for com, nodes in com2node_globaldict.items() if len(set(nodes).intersection(set(allvalues))) == 0]
            if len(com_no_destnodes) == 0:
                break
            else:
                if num_coms == len(com2node_globaldict):
                    isloop = True

    # processing orphan nodes (a) no flows to all, (b) no internal flows
    miniinnflows = min(status.internals.values()) # ensure all LI >0
    if miniinnflows == 0:
        isloop = False
        while True:
            num_coms = len(com2node_globaldict)
            com2node_globaldict = __ensure_nonzero_LI(current_graph, status, resolution, com2node_globaldict, isloop, adjmx)
            partition, com2node_globaldict, com_old_vs_com_new = __renumber(status.node2com, com2node_globaldict)
            current_graph = induced_graph(partition, current_graph, com_old_vs_com_new, status)
            status.init(current_graph)
            miniinnflows = min(status.internals.values())
            if miniinnflows > 0:
                break
            else:
                if num_coms == len(com2node_globaldict):
                    isloop = True

    partition = __com2node_partition(com2node_globaldict)
    return partition

class Status(object):
    """
    To handle several data in one struct.

    Could be replaced by named tuple, but don't want to depend on python 2.6
    """
    node2com = {} # the community of each node
    com2node = {} # ids of nodes in each community, added by hu
    com_attr1 = {} # one attribute of each community, e.g., total population, added by hu
    node_attr1 = {} # one attribute of each node, e.g., total population, added by hu
    total_weight = 0 # sum of weights of all edges
    internals = {} # sum of weights of all edges within a community
    degrees = {} # degree list for all communities
    gdegrees = {} # degree list for all nodes in graph

    def __init__(self):
        self.node2com = dict([])
        self.com2node = dict([])
        self.com_attr1 = dict() # its value of each key is a number, not a list

        self.node_attr1 = dict()
        self.total_weight = 0
        self.degrees = dict([])
        self.gdegrees = dict([])
        self.internals = dict([]) # edge weights list for all communities
        self.loops = dict([]) # edge weights list for all nodes in graph

    def copy(self):
        """Perform a deep copy of status"""
        new_status = Status()
        new_status.node2com = copy.deepcopy(self.node2com)
        new_status.com2node = copy.deepcopy(self.com2node)
        new_status.com_attr1 = copy.deepcopy(self.com_attr1)
        new_status.node_attr1 = copy.deepcopy(self.node_attr1)
        new_status.internals = copy.deepcopy(self.internals)
        new_status.degrees = copy.deepcopy(self.degrees)
        new_status.gdegrees = copy.deepcopy(self.gdegrees)
        new_status.loops = copy.deepcopy(self.loops)
        new_status.total_weight = self.total_weight

        return new_status

    def init(self, graph, part=None):
        """Initialize the status of a graph with every node in one community"""
        count = 0 # new community id starts from 0
        self.node2com = dict([])
        self.com2node = dict([])
        self.com_attr1 = dict()
        self.node_attr1 = dict()
        self.total_weight = 0
        self.degrees = dict([])
        self.gdegrees = dict([])
        self.internals = dict([])
        self.total_weight = graph.size(weight='weight') # .size function returns the number of edges or sum of edge weights in the graph

        if part is None:
            self.com2node = {k: [] for k in range(0, graph.number_of_nodes())} # initialize a dictionary of empty list
            for node in graph.nodes(): # node is the node name
                self.node2com[node] = count # if no partition provided, each node belongs to one community
                self.node_attr1[node] = graph.nodes[node]['pop']
                if bool(self.com2node.get(count)): # if the same community exists, append nodes and plus attributes, changed by Wang 12-15-2018
                    self.com2node[count].append(node) # node is the node name. Other tips: g.node.keys()[i] return
                    # the ith key in the dict
                    self.com_attr1[count] += graph.nodes[node]['pop'] # may need to change the field name for other attributes
                else: # if no same community exists, append nodes and set attributes, changed by Wang 12-15-2018
                    self.com2node[count].append(node)
                    self.com_attr1[count] = graph.nodes[node]['pop']
                deg = float(graph.degree(node, weight='weight')) # calculate the degree of each node
                if deg < 0:
                    error = "Bad graph type ({})".format(type(graph))
                    raise ValueError(error)
                self.degrees[count] = deg
                self.gdegrees[node] = deg
                edge_data = graph.get_edge_data(node, node, {"weight": 0})
                self.loops[node] = float(edge_data.get("weight", 1))
                self.internals[count] = self.loops[node]
                count += 1
        else: # when a partition is provided as input
            #self.com2node = {k: [] for k in range(min(list(set(part.values()))), max(list(set(part.values()))) + 1 )}
            self.com2node = {k: [] for k in list(set(part.values()))}
            for node in graph.nodes():
                com = part[node]
                self.node2com[node] = com
                self.node_attr1[node] = graph.nodes[node]['pop']
                if bool(self.com2node.get(com)):
                    self.com2node[com].append(node)
                    self.com_attr1[com] += graph.nodes[node]['pop']
                else:
                    self.com2node[com].append(node)
                    self.com_attr1[com] = graph.nodes[node]['pop']
                deg = float(graph.degree(node, weight='weight'))
                self.degrees[com] = self.degrees.get(com, 0) + deg
                self.gdegrees[node] = deg

                inc = 0.
                for neighbor, datas in graph[node].items():
                    edge_weight = datas.get("weight", 1)
                    if edge_weight <= 0:
                        raise ValueError("Bad graph type, use positive weights")
                    if part[neighbor] == com: # if one of this node's neighbors is also in the same community
                        if neighbor == node: # if this neighbor is itself (i.e., a loop)
                            inc += float(edge_weight) # inc indicates the sum of edge weights inside a community
                        else:
                            inc += float(edge_weight) / 2

                self.internals[com] = self.internals.get(com, 0) + inc

def __isadj(nodelist, neighborlist, adjmx):
    rows_bl = np.isin(adjmx[0], nodelist)
    col_bl = np.isin(adjmx[1], neighborlist)

    rows_index = np.nonzero(rows_bl)
    cols_index = np.nonzero(col_bl)
    comm_index = np.intersect1d(rows_index, cols_index)
    if(len(comm_index) > 0):
        return True
    else:
        return False

'''
    Compute the communities of the neighborood of the node in the graph given
    with the decomposition node2com; note that the neighborhood of the node is defined
    more accurately by considering the spatial adjacency matrix 'adjmx' calculated
    from the polygon features. Note that the adjmx starts index from 0 rather than 1.
    The input com2node_dict (i.e., com2node_globaldict) is used to
    find the original node names. com_node is the community id of this node.

    In other words, a neighbor of one node i in the graph is a node j that has an
    edge ij between them as well as polygon i and polygon j are spatially touched.
'''
def __toponeighcom(node, graph, status, com2node_dict, adjmx):
    weights_EDGEneighbor = {} # it stores results for neighbors defined based on edges only
    weights_TOPOneighbor = {} # it stores results for neighbors defined based on topology only
    # sometimes, a node is only connected with nontopological neighbors
    nodelist = com2node_dict[node]
    neighcom_list = list()
    for neighbor, datas in graph[node].items(): # graph[node].items() returns the node name
        # and edge weight of each edge that has one end being this node
        if neighbor != node: # if one of this node's neighbors is not itself
            weight = datas.get("weight", 1)
            neighborcom = status.node2com[neighbor] # the community of its neighbor
            neighborlist = com2node_dict[neighborcom]
            weights_EDGEneighbor[neighborcom] = weights_EDGEneighbor.get(neighborcom, 0) + weight
            isadj = __isadj(nodelist, neighborlist, adjmx)
            if isadj:
                #isConnected_toponeigh = True
                weights_TOPOneighbor[neighborcom] = weights_TOPOneighbor.get(neighborcom, 0) + weight
                neighcom_list.append(neighborcom)
    # get spatial adjacent neighbors
    sortnodelist = sorted(nodelist)
    rows_bl = np.isin(adjmx[0], sortnodelist)
    rows_index = np.nonzero(rows_bl)
    node_toponbr_list = np.take(adjmx[1], rows_index)[0]
    diff_list = set(node_toponbr_list) - set(sortnodelist)
    toponbr_lists = set(diff_list)

    '''
    for node_i in nodelist:
        node_toponbr_list = np.nonzero(adjmx[node_i-1])[0]
        node_toponbr_list = [value + 1 for value in node_toponbr_list]
        diff_list = set(node_toponbr_list) - set(nodelist)
        toponbr_lists = list(set(toponbr_lists) | set(diff_list))
    '''
    toponeighcom_list = set([key for key, value in com2node_dict.items() if len(list(set(value).intersection(set(toponbr_lists))))]) - set(neighcom_list) - set([node])
    for neighborcom in toponeighcom_list:
        edge_data = graph.get_edge_data(node, neighborcom)
        weight = 0
        if edge_data is not None:
            weight = edge_data.get("weight")
        weights_TOPOneighbor[neighborcom] = weights_TOPOneighbor.get(neighborcom, 0) + weight
        #weights_EDGEneighbor[neighborcom] = 0
    return weights_EDGEneighbor, weights_TOPOneighbor # it is a dictionary which stores the community of its neighbor and the weight of edge between them

def __remove(node, com, weight, status):
    """ Remove node from community com and modify status"""
    status.degrees[com] = (status.degrees.get(com, 0.)
                           - status.gdegrees.get(node, 0.))
    status.internals[com] = float(status.internals.get(com, 0.) -
                                  weight - status.loops.get(node, 0.))
    status.node2com[node] = -1 # then this node has no community assigned
    status.com2node[com].remove(node) # remove this node ('node' is the node name here not the index)
    # from the original community
    status.com_attr1[com] -= status.node_attr1[node] # update the attr1 of this community

def __insert(node, com, weight, status):
    """ Insert node into community and modify status"""
    status.node2com[node] = com
    if bool(status.com2node.get(com)): # if there is at least a value of key=com in the dictionary
        status.com2node[com].append(node)
        status.com_attr1[com] += status.node_attr1[node]
    else:
        # status.com2node[com] = [node] # make it a list
        status.com2node[com].append(node)
        status.com_attr1[com] = status.node_attr1[node]
    status.degrees[com] = (status.degrees.get(com, 0.) +
                           status.gdegrees.get(node, 0.))
    status.internals[com] = float(status.internals.get(com, 0.) +
                                  weight + status.loops.get(node, 0.))

# Fast compute the modularity of the partition of the graph using status precomputed
def __modularity(status, resolution):
    links = float(status.total_weight)
    result = 0.
    for community in set(status.node2com.values()):
        in_degree = status.internals.get(community, 0.) # Σin, i.e., sum of weights of edges within a community
        degree = status.degrees.get(community, 0.) # Σtot, i.e., sum of weights of edges that have one end node inside this community
        if links > 0:
            result += in_degree / links - resolution * ((degree / (2. * links)) ** 2)
    return result


""" URL config for board views """

from django.urls import path
from .views import notice_boards, display_single_board, set_favourite_board, create_notice_board

urlpatterns = [
    path('', notice_boards, name="notice_boards"),
    path('<int:board_pk>/', display_single_board, name="display_single_board"),
    path('<int:board_pk>/set_as_favourite/', set_favourite_board, {'set_unset':True}, name="set_favourite_board"),
    path('<int:board_pk>/unset_as_favourite/', set_favourite_board, name="unset_favourite_board"),
    path('create_board/', create_notice_board, name="create_notice_board"),
]


"""PyFMD example

This example demonstrate how to build a structure using PyFMD interface for ASE and merge them together.
"""

from ase.visualize import view
import numpy as np
from ase.build import bulk
from ase.build.supercells import make_supercell

from pyfmd import *

# Create a super cell via ASE package
def make_supercell_cu():
    crys = bulk('Cu', 'fcc', a=3.6316, orthorhombic=True)
    P = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) * 10
    return make_supercell(crys, P)

# Create molecular frames 1 (group-id = 0)
mf1 = MolecularFrame('Cu crystal 1').import_from(make_supercell_cu()).set_group_id(0)
# view(mf1.export())

# Create molecular frames 2 (group-id = 1)
mf2 = MolecularFrame('Cu crystal 2').import_from(make_supercell_cu()).set_group_id(1)
# view(mf2.export())

# Clone mf2 and translate atoms along given direction
mf2_moved = mf2.translate_atoms((30,-15,-15), inplace=False)

# Define region
def sphere(x, y, z):
    """sphere region"""
    if np.sqrt(x**2+y**2+z**2) < 10.0:
        return True
    else:
        return False

# Translate and select atoms within predefined region
mf1_sphere = mf1.translate_atoms((-10,-10,-10), inplace=False).select(Select().region(sphere))
# view(mf1_sphere.export())

# Merge two molecular frames
mf = mf1_sphere + mf2_moved
mf.translate_atoms((50, 50, 50))

# Define box and/or state header
box = Box((150, 100, 100))
state = StateHeader(simulation_time=0.0, number_of_atoms=mf.number_of_atoms, box=box, pbc=(False, False, False))
mf.set_sections([box, state])

# print(mf)
mf.write("system.stt")  # This state file can base used FMD simulation.
mf.read("system.stt")
view(mf.export())


from flask import Flask, request, render_template
app = Flask(__name__)

@app.route('/')
def hello():
    q = request.args.get('q')
    print(q)
    return { "message": "Hello!" }, 201

@app.route('/number/<num>')
def showNumber(num):
    q = request.args.get('q')
    print(q)
    return { "message": num }, 201

@app.route('/showWeb/<name>')
def showWebSite(name=None):
    return render_template('test.html', name=name)

@app.route('/login', methods=['GET', 'POST'])
def login():
    if request.method == "POST":
        return { "message" : 'method post เด้อ'}, 201
    elif request.method == "GET":
        return { "message" : 'method GET เด้อ'}, 201

if __name__=='__main__':
    app.run(debug=True)