#!/usr/bin/python
# -*- coding: utf-8 -*-

import urllib
import json
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler

class MyHandler(BaseHTTPRequestHandler):

    def do_GET(self):
        self.send_response(200)
        self.send_header('Content-Type', 'text/plain')
        self.end_headers()
        container = ''
        freeme = ''
        useme = ''
        state = ''
        task = ''

        j = dict()

        # print type(j)

        if self.path == '/metrics':
            urls = [  # -> FOR NODE STATIC
                'http://11.11.22.56:8088/ws/v1/cluster/nodes',
                'http://11.11.22.56:8088/ws/v1/cluster/appstatistics',
                'http://11.11.22.56:50070/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState'
                    ,
                'http://11.11.22.56:50070/jmx?qry=java.lang:type=Memory'
                    ,
                'http://11.11.22.56:50070/jmx?qry=Hadoop:service=NameNode,name=JvmMetrics'
                    ,
                'http://11.11.22.56:8088/ws/v1/cluster/metrics',
                ]

                     #  -> FOR APP

            gc = ''

            for url in urls:
                f = urllib.urlopen(url)
                data = json.load(f)

                # print data

                d1 = dict()
                if 'beans' in data:
                    for (idx, val) in enumerate(data['beans']):
                        d1.update(val)
                    for i in d1:
                        if type(d1[i]) == int or type(d1[i]) == float:
                            mt = 'HD_GC_' + str(i) + '{val="' + str(i) \
                                + '"} ' + str(d1[i]) + '\n'
                            gc += mt

                j.update(data)

            # print type(j)

            for i in j['nodes']['node']:
                node = str.replace(str.replace(str(i['nodeHTTPAddress'
                                   ]), 'data-node', ''), ':8042', '')
                if str(i['state']) == 'RUNNING':
                    x = 1
                else:
                    x = 0
                mt_availMemoryMB = 'HD_availMemoryMB{node="' + node \
                    + '",code="5xx"} ' + str(i['availMemoryMB']) + '\n'
                mt_numContainers = 'HD_numContainers{node="' + node \
                    + '",code="5xx"} ' + str(i['numContainers']) + '\n'
                mt_usedMemoryMB = 'HD_usedMemoryMB{node="' + node \
                    + '",code="5xx"} ' + str(i['usedMemoryMB']) + '\n'
                mt_state = 'HD_state{node="' + node + '",code="5xx"} ' \
                    + str(x) + '\n'
                freeme += mt_availMemoryMB
                container += mt_numContainers
                useme += mt_usedMemoryMB
                state += mt_state
                node = '#HDFS METRICS - toanpt3' + '\n' + freeme \
                    + container + useme + state
            for i in j['appStatInfo']['statItem']:
                mt_taskinf = 'HD_task{state="' + str(i['state']) \
                    + '",code="5xx"} ' + str(i['count']) + '\n'
                task += mt_taskinf
            t = ''
            for i in j['clusterMetrics']:

                # print j['clusterMetrics'][i]

                mt2 = 'HD_MT_' + str(i) + '{val="' + str(i) + '"} ' \
                    + str(j['clusterMetrics'][i]) + '\n'

                t += mt2

            # print  task
                # HD_task{state="ACCEPTED",code="5xx"} 1
                # HD_task{state="RUNNING",code="5xx"} 28

            total = node + task + gc + t
            self.wfile.write(total)

HTTPServer(('', 8089), MyHandler).serve_forever()


#!/usr/bin/python3
"""
nqueens backtracking program to print the cordinates of n queens
on an n x n chessboard
"""

from sys import argv

if __name__ == "__main__":
    a = []
    if len(argv) != 2:
        print("Usage: nqueens N")
        exit(1)

    if not argv[1].isdigit():
        print("N must be a number")
        exit(1)

    n = int(argv[1])
    if n < 4:
        print("N must be at least 4")
        exit(1)

    for i in range(n):
        a.append([i, None])

    def already_exists(y):
        """Check that a queen does not already exist in that y value"""
        for x in range(n):
            if y == a[x][1]:
                return True
        return False

    def reject(x, y):
        """determines whether or not to reject the solution"""
        if (already_exists(y)):
            return False
        i = 0
        while(i < x):
            if abs(a[i][1] - y) == abs(i - x):
                return False
            i += 1
        return True

    def clear_a(x):
        """clears the answers from the point of failure on"""
        for i in range(x, n):
            a[i][1] = None

    def nqueens(x):
        """backtracking function"""
        if x == n:
            print(a)
        else:
            for y in range(n):
                if reject(x, y):
                    a[x][1] = y
                    nqueens(x + 1)
                    clear_a(x)

    nqueens(0)


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def averageOfLevels(self, root: Optional[TreeNode]) -> List[float]:
        queue = deque([root])
        output = []
        while queue:
            lenQ = len(queue)
            level = 0
            for i in range(lenQ):
                curr = queue.popleft()
                level += curr.val
                if curr.left:
                    queue.append(curr.left)
                if curr.right:
                    queue.append(curr.right)
            output.append(level/lenQ)
        return output


import tensorflow as tf

from models.losses import *

def normalize(img):
    # convert img vals from [0, 255] to [-1, 1]
    img = tf.cast(img, tf.float32)
    img = (img / 127.5) - 1.0
    return img

def denormalize(img):
    # convert img vals from [-1, 1] to [0, 1]
    return img / 2.0 + 0.5

def random_horizontal_flip(img):
    if tf.random.uniform(()) > 0.5:
        img = tf.image.flip_left_right(img)
    return img

def resize(img, size=128):
    img = tf.image.resize(img,
                          [size, size],
                          method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
    return img

def read_and_decode_img(img_path):
    # from image path to string
    img_string = tf.io.read_file(img_path)
    # from jpg-encoded image to a uint8
    img = tf.io.decode_jpeg(img_string)
    return img

def preprocess_img(img,
                   use_aug=True,
                   do_normalize=True):

    if use_aug:
        img = random_horizontal_flip(img)
    if do_normalize:
        img = normalize(img)
    return img

def preprocess_for_training(img, ini_cond):
    c_dim = 17
    # For image preprocessing
    img = preprocess_img(img)
    # Generate fin cond labels randomly
    fin_cond = tf.random.shuffle(ini_cond)
    fin_cond += tf.random.uniform([c_dim], -0.1, 0.1)

    return img, ini_cond, fin_cond

@tf.function
def train_disc(disc,
               gen,
               x_real,
               label_ini_cond,
               label_fin_cond,
               lambda_cond,
               lambda_gp,
               opt):

    with tf.GradientTape() as tape:
        # Compute loss with real images
        real_out_vals, real_out_cond = disc(x_real, training=True)
        d_loss_real = - get_mean_for_loss(real_out_vals)
        d_loss_cond = get_conditional_expression_loss(label_ini_cond, real_out_cond)
        # Compute loss with fake images
        x_fake, fake_attn_mask = gen(x_real, label_fin_cond, training=False)
        x_fake_masked = fake_attn_mask * x_real + (1 - fake_attn_mask) * x_fake
        fake_out_vals, _ = disc(x_fake_masked, training=True)
        d_loss_fake = get_mean_for_loss(fake_out_vals)
        # Compute loss for gradient penalty
        d_loss_gp = get_gradient_penalty(x_real, x_fake_masked, disc)
        # Compute the total loss for the discriminator
        d_loss = lambda_cond * d_loss_cond + d_loss_real + d_loss_fake + lambda_gp * d_loss_gp

    # Calculate the gradients and update params for the discriminator and the generator
    disc_gradients = tape.gradient(d_loss, disc.trainable_variables)
    opt.apply_gradients(zip(disc_gradients, disc.trainable_variables))

    return d_loss, d_loss_cond, d_loss_real, d_loss_fake, d_loss_gp

@tf.function
def train_gen(disc,
              gen,
              x_real,
              label_ini_cond,
              label_fin_cond,
              lambda_cond,
              lambda_rec,
              lambda_attn,
              lambda_tv,
              opt):

    with tf.GradientTape() as tape:
        # Compute loss for ini-to-fin
        x_fake, fake_attn_mask = gen(x_real, label_fin_cond, training=True)
        x_fake_masked = fake_attn_mask * x_real + (1 - fake_attn_mask) * x_fake
        fake_gen_out_vals, fake_gen_out_cond = disc(x_fake_masked, training=False)
        g_loss_fake = - get_mean_for_loss(fake_gen_out_vals)
        g_loss_cond = get_conditional_expression_loss(label_fin_cond, fake_gen_out_cond)
        # Compute loss for fin-to-ini
        x_gen_rec, rec_attn_mask = gen(x_fake_masked, label_ini_cond, training=True)
        x_rec_masked = rec_attn_mask * x_fake_masked + (1 - rec_attn_mask) * x_gen_rec
        # Compute the cycle and attention loss
        g_loss_rec = get_l1_loss(x_real, x_rec_masked)
        g_fake_attn_mask_loss, g_rec_attn_mask_loss, g_fake_tv_loss, g_rec_tv_loss = \
            get_attention_loss(fake_attn_mask, rec_attn_mask)
        # Compute the total loss for the generator
        g_loss = g_loss_fake + lambda_cond * g_loss_cond + lambda_rec * g_loss_rec + \
                 lambda_attn * g_fake_attn_mask_loss + lambda_attn * g_rec_attn_mask_loss + \
                 lambda_tv * g_fake_tv_loss + lambda_tv * g_rec_tv_loss

    gen_gradients = tape.gradient(g_loss, gen.trainable_variables)
    opt.apply_gradients(zip(gen_gradients, gen.trainable_variables))

    return g_loss, g_loss_fake, g_loss_cond, g_loss_rec, g_fake_attn_mask_loss, g_rec_attn_mask_loss, g_fake_tv_loss, g_rec_tv_loss

@tf.function
def update_lr(gen_opt, disc_opt, max_epoch, epoch, g_lr=0.0001, d_lr=0.0001):
    if g_lr != d_lr:
        decayed_lr = get_lr_decay_factor(epoch, max_epoch, g_lr)
        gen_opt.lr.assign(decayed_lr)
        disc_opt.lr.assign(decayed_lr)
        # to debug
        tf.print("decayed lr G: {}, D: {}".format(gen_opt.lr, disc_opt.lr))
    else:
        g_decayed_lr =  get_lr_decay_factor(epoch,
                                            max_epoch,
                                            g_lr)
        d_decayed_lr =  get_lr_decay_factor(epoch,
                                            max_epoch,
                                            d_lr)
        gen_opt.lr.assign(g_decayed_lr)
        disc_opt.lr.assign(d_decayed_lr)
        # to debug
        print("decayed lr G: {}, D: {}".format(gen_opt.lr, disc_opt.lr))

@tf.function
def update_lr_by_iter(gen_opt, disc_opt, iteration, diff_iter, g_lr=0.0001, d_lr=0.0001):
    if g_lr != d_lr:
        decayed_lr = get_lr_decay_factor_by_iter(iteration, diff_iter, g_lr)
        gen_opt.lr.assign(decayed_lr)
        disc_opt.lr.assign(decayed_lr)
        # to debug
        #tf.print("decayed lr G: {}, D: {}".format(gen_opt.lr, disc_opt.lr))
    else:
        g_decayed_lr =  get_lr_decay_factor_by_iter(iteration,
                                                    diff_iter,
                                                    g_lr)
        d_decayed_lr =  get_lr_decay_factor_by_iter(iteration,
                                                    diff_iter,
                                                    d_lr)
        gen_opt.lr.assign(g_decayed_lr)
        disc_opt.lr.assign(d_decayed_lr)

def get_lr_decay_factor(epoch, max_epoch, init_lr=0.0001):
    return tf.cast(2.0 * init_lr * (- tf.cast(epoch, tf.float32) / tf.cast(max_epoch, tf.float32) + 1.0), dtype=tf.float32)

def get_lr_decay_factor_by_iter(iteration, diff_iter, init_lr=0.0001):
    return tf.cast(init_lr * (- tf.cast(iteration, tf.float32) / tf.cast(diff_iter, tf.float32) + 2.0), dtype=tf.float32)

def print_log(epoch, start, end, d_losses, g_losses):
    tf.print("\nTime taken for epoch {} is {:.3f} sec\n".format(epoch,
                                                                round(end - start)))
    d_log = "d_loss: {:.3f} (d_loss_cond: {:.3f}, d_loss_real: {:.3f}, " \
            "d_loss_fake: {:.3f}, d_loss_gp: {:.3f})"
    g_log = "g_loss: {:.3f} (g_loss_fake: {:.3f}, g_loss_cond: {:.3f}, " \
            "g_loss_rec: {:.3f}, g_fake_attn_mask_loss: {:.3f}, " \
            "g_rec_attn_mask_loss: {:.3f}, g_fake_tv_loss: {:.3f}, " \
            "g_rec_tv_loss: {:.3f})"
    tf.print(d_log.format(d_losses[0],
                          d_losses[1],
                          d_losses[2],
                          d_losses[3],
                          d_losses[4]))
    tf.print(g_log.format(g_losses[0],
                          g_losses[1],
                          g_losses[2],
                          g_losses[3],
                          g_losses[4],
                          g_losses[5],
                          g_losses[6],
                          g_losses[7]))

def unpack_and_stack_img_vertically(img):
    img = tf.raw_ops.Unpack(value=img, num=img.shape[0])

    return tf.concat(img, axis=0)

def concat_tensors_horizontally(t1, t2):
    return tf.concat([t1, t2], axis=1)

def get_final_fake_img(color_mask, attn_mask, real_img):
    return (1 - attn_mask) * color_mask + attn_mask * real_img

def postprocess_to_plot(color_mask, attn_mask, real_img):
    fin_fake_img = get_final_fake_img(color_mask, attn_mask, real_img)
    col_real_img = unpack_and_stack_img_vertically(real_img)
    col_fake_img = unpack_and_stack_img_vertically(fin_fake_img)
    combined_img = concat_tensors_horizontally(col_real_img, col_fake_img)
    combined_img = denormalize(combined_img)
    combined_img = tf.cast(combined_img * 255, dtype=tf.uint8)

    return combined_img

def save_img(tensor, fpath):
    bstr = tf.io.encode_jpeg(tensor)
    with open(fpath, "wb") as f:
        f.write(bstr.numpy())

def save_test_results(model, img, fin_cond, save_path):
    color_mask, attn_mask = model(img, fin_cond)
    combined_img = postprocess_to_plot(color_mask, attn_mask, img)
    save_img(combined_img, save_path)


import os
from pprint import pprint
from work_class.work_with_vacancies import Work_With_Vacancies
from work_class.JSON_LD import JSONLD

class JobSearch:
    def __init__(self):
        self.wwv = Work_With_Vacancies()
        self.jld = JSONLD()
        self.hh = self.wwv.save_hh()
        self.sj = self.wwv.save_sj()
        self.all_job_listings = self.wwv.job_hh + self.wwv.job_sj

    @staticmethod
    def display_menu():
        """Функция выбора пользователем дальнейших действий"""
        print("=== Меню поиска работы ===")
        print("1. Получить список вакансий")
        print("2. Поиск вакансий")
        print("3. Получить топ N вакансий по зарплате(указать количество вакансий)")
        print("4. Получить отсортированный список вакансий(указать количество вакансий)")
        print("5. Поиск вакансий по ключевым словам")
        print("6. Добавление новой вакансии")
        print("7. Удаление вакансии")
        print("8. Сохранить вакансию в файл")
        print("0. Выход")

    def run_listings(self):
        """Функция для записи списков рабочего класса """
        self.wwv.save_hh()
        self.wwv.save_sj()

    def fetch_job_listings(self):
        """Функция для получения вакансий в полном формате"""
        pprint(self.all_job_listings)

    def search_job_listings(self, query):
        """Функция для поиска по названию вакансии"""
        if not query:
            print("Вы не ввели поисковый запрос. Пожалуйста, попробуйте ещё раз.")
            return
        matching_listings = [item for item in self.all_job_listings if
                             query.lower() in item["название вакансии"].lower()]
        if matching_listings:
            print(f"Найдено {len(matching_listings)} вакансий, соответствующих запросу '{query}':")
            for item in matching_listings:
                print(f"Должность: {item['название вакансии']}\n"
                      f"Ссылка для отклика: {item['ссылка на вакансию']}\n"
                      f"Размер заработной платы: {item['зарплата']} {item['валюта']}\n"
                      f"Адрес: {item['адрес']}\n")
        else:
            print("Вакансий, соответствующих поисковому запросу, не найдено.")

    def get_top_job_listings_by_salary(self, n):
        """ Получение отсортированного списка зарплат
        n - число вывода вакансий (Топ)"""
        if not isinstance(n, int):
            print("Введено недопустимое значение. Пожалуйста, введите число.")
            return
        sorted_listings = sorted(self.all_job_listings, key=lambda x: int(x["зарплата"])
            if x.get("зарплата") else 0, reverse=True)
        top_n_listings = sorted_listings[:n]

        if top_n_listings:
            print(f"Топ {n} вакансий по зарплате "
                  f"(указана средняя зарплата, для получения более точной информации перейти по ссылке):")
            for num, item in enumerate(top_n_listings):
                print(f"Номер {num + 1}\n"
                      f"Должность: {item['название вакансии']}\n"
                      f"Ссылка для отклика: {item['ссылка на вакансию']}\n"
                      f"Размер заработной платы: {item['зарплата']} {item['валюта']}\n"
                      f"Адрес: {item['адрес']}\n")
        else:
            print("Вакансий не найдено.")

    def get_sorted_job_listings(self, n):
        """Функция выводит отсортированный список вакансий"""

        sorted_listings = sorted(self.all_job_listings, key=lambda x: x["название вакансии"])
        top_n_listings = sorted_listings[:int(n)]

        if top_n_listings:
            print("Отсортированный список вакансий:")
            for item in top_n_listings:
                print(f"Должность: {item['название вакансии']}\n"
                      f"Ссылка для отклика: {item['ссылка на вакансию']}\n"
                      f"Размер заработной платы: {item['зарплата']} {item['валюта']}\n"
                      f"Адрес: {item['адрес']}\n")
        else:
            print("Вакансий не найдено.")

    def search_job_listings_by_keywords(self, keywords):
        """Функция для поиска по ключевым словам"""
        if not keywords:
            print("Вы не ввели ключевые слова. Пожалуйста, попробуйте ещё раз.")
            return
        matching_listings = []
        for item in self.all_job_listings:
            item_values = [value for value in item.values()]
            if all(keyword.strip().capitalize() in item_values for keyword in keywords):
                matching_listings.append(item)

        if matching_listings:
            print(f"Найдено {len(matching_listings)} вакансий, соответствующих ключевым словам:")
            for item in matching_listings:
                print(f"Должность: {item['название вакансии']}\n"
                      f"Ссылка для отклика: {item['ссылка на вакансию']}\n"
                      f"Размер заработной платы: {item['зарплата']} {item['валюта']}\n"
                      f"Адрес: {item['адрес']}\n")
        else:
            print("Вакансий, соответствующих ключевым словам, не найдено.")

    def add_job_listing(self, pick: int, job_listing: dict):
        """Функция добавляет вакансию в список"""
        self.jld.json_add(pick, job_listing)

    def remove_job_listing(self, pick: int, job_listing: dict):
        """Функция удаляет вакансию из списка"""
        self.jld.json_del(pick, job_listing)

    def run(self):
        """Функция для взаимодействия с пользователем"""
        self.run_listings()
        while True:
            self.display_menu()
            choice = int(input("Введите ваш выбор: "))

            if choice == 0:
                print("Выход...")
                break

            elif choice == 1:
                self.fetch_job_listings()

            elif choice == 2:
                query = input("Введите поисковый запрос"
                              "(указать название вакансии или ключевое слово в названии): ")
                self.search_job_listings(query)

            elif choice == 3:
                n = int(input("Введите количество вакансий: "))
                self.get_top_job_listings_by_salary(n)

            elif choice == 4:
                n = input("Введите число вакансий для вывода: ")
                self.get_sorted_job_listings(n)

            elif choice == 5:
                keywords = input("Введите ключевые слова (через ;): ").split(";")
                print("вывод вакансий будет по всем содержащимся ключевым словам в вакансии")
                self.search_job_listings_by_keywords(keywords)

            elif choice == 6:
                if not os.path.exists(__class__.directory):
                    print("С начало сохраните файл")
                else:
                    print("введите файл вакансии:\n"
                          "1 - HH\n"
                          "2 - SuperJob")
                    pick = int(input())
                    print("введите данные для новой вакансии")

                    job_listing = \
                        {"название вакансии": input("название вакансии:"),
                         "ссылка на вакансию": input("ссылка на вакансию:"),
                         "зарплата": int(input("зарплата:")), "валюта": input("валюта:"),
                         "требования": input("требования:"), "обязанности": input("обязанности:"), "адрес": input("адрес:"),
                         "метро": input("метро:"), "требуемый опыт": input("требуемый опыт:"),
                         "название компании": input("название компании:"), "город вакансии": input("город вакансии:")
                         }
                    self.add_job_listing(pick, job_listing)

            elif choice == 7:
                if not os.path.exists(__class__.directory):
                    print("С начало сохраните файл")
                else:
                    print("введите файл вакансии:\n"
                          "1 - HH\n"
                          "2 - SuperJob")
                    pick = int(input())
                    print("введите данные для удаления вакансии")

                    job_listing = \
                        {"название вакансии": input("название вакансии:"),
                         "ссылка на вакансию": input("ссылка на вакансию:"),
                         "зарплата": int(input("зарплата:")), "валюта": input("валюта:"),
                         "требования": input("требования:"), "обязанности": input("обязанности:"),
                         "адрес": input("адрес:"), "метро": input("метро:"),
                         "требуемый опыт": input("требуемый опыт:"), "название компании": input("название компании:"),
                         "город вакансии": input("город вакансии:")
                         }

                    self.remove_job_listing(pick, job_listing)

            elif choice == 8:
                print("Сохранить файл с вакансиями:\n"
                      "1 - HH\n"
                      "2 - SuperJob")
                pick = int(input("введите номер:"))
                self.jld.json_save(pick)

            else:
                print("Недопустимый выбор. Пожалуйста, попробуйте ещё раз.")


class Solution:
    # @param A : string
    # @return an integer
    def solve(self, A):
        target = A.lower()
        vowel = ['a','e','i','o','u']
        amazing_num, l = 0, len(A)
        for idx, i in enumerate(target):
            if i in vowel:
                amazing_num += (l - idx)
        return amazing_num % 10003


## COPY THIS FILE TO stable-diffusion/ldm/data/music.py

import json
from pathlib import Path
import random
import math

import numpy as np
import torchaudio
import torch
from torch.utils.data import Dataset
from PIL import Image
from torchvision import transforms
from einops import rearrange

class MusicData(Dataset):
    def __init__(self,
        root_dir,
        caption_file,
        ) -> None:
        """Create a dataset from a folder of music.
        """
        self.log_file = open("/devel/missing.log", "a+")
        self.root_dir = Path(root_dir)
        with open(caption_file, "rt") as f:
            lines = f.readlines()
            lines = [json.loads(x) for x in lines]
            captions = {x["file_name"]: x["text"].strip("\n") for x in lines}
            self.captions = captions
        self.file_extensions = ["_drums.mp3", "_bass.mp3", "_other.mp3"]

        self.sample_rate = 44100
        self.n_fft = 17640
        self.hop_length = 441
        self.win_length = 4410
        self.n_mels = 171
        self.min_freq = 20
        self.max_freq = 20000
        self.mel_scale_norm = None
        self.mel_scale_type = "htk"

        self.spec_scaler = 1000
        self.audio_seconds = 5.11
        self.sample_length = int(self.audio_seconds * self.sample_rate)

        self.spectrogram_func = torchaudio.transforms.Spectrogram(
            n_fft=self.n_fft,
            hop_length=self.hop_length,
            win_length=self.win_length,
            pad=0,
            window_fn=torch.hann_window,
            power=None,
            normalized=False,
            wkwargs=None,
            center=True,
            pad_mode="reflect",
            onesided=True,
        )

        self.mel_scale_func = torchaudio.transforms.MelScale(
            n_mels=self.n_mels,
            sample_rate=self.sample_rate,
            f_min=self.min_freq,
            f_max=self.max_freq,
            n_stft=self.n_fft // 2 + 1,
            norm=self.mel_scale_norm,
            mel_scale=self.mel_scale_type,
        )

        self.to_tensor_transform = transforms.ToTensor()

    def __len__(self):
        return len(self.captions.keys())

    def __getitem__(self, index):
        data = {}
        chosen = list(self.captions.keys())[index]
        caption = self.captions.get(chosen, None)
        filename_base = self.root_dir/chosen

        song_length = int(float(caption.split(' ')[-2]))
        if song_length <= self.audio_seconds:
            print(filename_base, file=self.log_file, flush=True)
            return self.__getitem__((index + 1) % self.__len__())
        sample_point = random.randint(0, (song_length - math.ceil(self.audio_seconds)))
        sample_start = sample_point * self.sample_rate

        specs = []
        try:
            for file_extension in self.file_extensions:
                waveform, _ = torchaudio.load(f"{filename_base}{file_extension}", normalize=True, frame_offset=sample_start, num_frames=self.sample_length)
                specs.append(self.gen_spectrogram(waveform))
        except:
            print(filename_base, file=self.log_file, flush=True)
            return self.__getitem__((index + 1) % self.__len__())

        im = self.concat_spectrograms(specs)
        im = self.process_im(im)

        data["image"] = im
        data["txt"] = caption + f" between position {sample_point} and {sample_point + self.audio_seconds}"

        return data

    def gen_spectrogram(self, waveform):
        spectrogram_complex = self.spectrogram_func(waveform)
        amplitudes = torch.abs(spectrogram_complex)
        spectrogram = self.mel_scale_func(amplitudes).numpy()

        data = spectrogram / self.spec_scaler
        data = np.power(data, .25)
        data = data * 255
        return data.astype(np.uint8)

    def concat_spectrograms(self, image_arrays):
        images = [Image.fromarray(image_array[0], mode="L") for image_array in image_arrays]
        index = 0
        for image in images:
            index += 1
        widths, heights = zip(*(image.size for image in images))

        total_height = sum(heights)
        max_width = max(widths)

        result_image = Image.new('L', (max_width, total_height))

        y_offset = 0
        for image in images:
            result_image.paste(image, (0, y_offset))
            y_offset += image.size[1]

        return result_image.convert("RGB")

    def process_im(self, im):
        im = im.resize((512, 512))
        im_tensor = self.to_tensor_transform(im)
        return rearrange(im_tensor * 2. - 1., 'c h w -> h w c')

if __name__ == '__main__':
    md = MusicData("/devel/data/sources/", "/devel/data/caption_files/debug.jsonl")
    for s in md:
        print(s["txt"])
        break


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import pandas as pd
import re
import time as timelib

pd.options.mode.chained_assignment = None

def retrieve_article_id(urlstring):

	# Looking for 'id_article' string
	id_position = urlstring.find('id_article=')
	if id_position>0:
		return (urlstring.split('id_article=')[1]);
	match = re.findall('a\d+\.html',urlstring)
	if len(match)==1:
		return (match[0].split('a')[1].split('.')[0])
	match = re.findall('actu\d+\.html',urlstring)
	if len(match)==1:
		return (match[0].split('actu')[1].split('.')[0])
	return '';

def extract(log_dataframe):
    urls = pd.DataFrame(columns = ["url","article_code"], dtype=str)
    requested_urls = log_dataframe['requested_url']
    referrer_urls = log_dataframe['referrer_url']
    urls['url'] = pd.concat([requested_urls,referrer_urls], axis = 0)
    urls.drop_duplicates(subset=['url'],inplace=True)
    urls.reset_index(inplace=True,drop=True)
    urls['article_code']=urls['url'].apply(retrieve_article_id)
    urls_with_code=urls[urls.article_code!=''].shape[0]
    print("     %d unique URLs found in the log file (%.1f%% of which are articles)."%(urls.shape[0],100*urls_with_code/urls.shape[0]))
    return urls;

def duplicity(urls, log_dataframe):
    urls['short_format']=urls.url.apply(lambda x: x.find('?id_article=')>0)
    grouped_articles=urls.groupby('article_code')
    duplicated_articles=grouped_articles.count()
    duplicated_articles=duplicated_articles[duplicated_articles.url>1]
    duplicated_articles_lookup_table=pd.Series()
    for code in duplicated_articles.index:
        #
        if code=='':
            continue

        group=grouped_articles.get_group(code)
        # skip if there all short or all long format
        if group.short_format.all() or not group.short_format.any():
            continue
        # add to lookup table

        short_url=group.url[group.short_format].iloc[0]
        long_url=group.url[~group.short_format].iloc[0]
        duplicated_articles_lookup_table=duplicated_articles_lookup_table.append(pd.Series([long_url],index=[short_url]))
        # delete the short format copy from new urls
        urls=urls[urls.url!=short_url]

    # changing detected URL with duplicity in the log table
    log_dataframe[log_dataframe.requested_url.isin(duplicated_articles_lookup_table.index)].requested_url=log_dataframe[log_dataframe.requested_url.isin(duplicated_articles_lookup_table.index)].requested_url.map(duplicated_articles_lookup_table, na_action='ignore')
    log_dataframe.loc[log_dataframe.requested_url.isin(duplicated_articles_lookup_table.index),'requested_url']=log_dataframe[log_dataframe.requested_url.isin(duplicated_articles_lookup_table.index)].requested_url.map(duplicated_articles_lookup_table, na_action='ignore')
    urls.drop(['short_format'],inplace=True,axis=1)
    return urls;

def format_pages(urls):
    start = timelib.time()
    urls['pageID']=str()
    urls['external']=str()
    for i, row in enumerate(urls.itertuples()):
        # Anonymise urls
        urls['pageID'].at[row.Index] = 'PAGE'+str(i)
        # Check if requests are from Melty
        if urls['id'].at[row.Index][0] == 'M' :
            urls['external'].at[row.Index] = 'False'
        else : urls['external'].at[row.Index] ='True'
    # TO AVOID NAN VALUES WHEN INFO IS MISSING LATER IN ANALYSIS :
    # FOLDER->0; MELTY_PAGE_TYPE->other; MELTY_THEMA_NAME->Other
    for row in urls.itertuples():
        if  row.melty_folder_id =='':
            urls['melty_folder_id'].at[row.Index] = 0
        if row.melty_page_type=='':
            urls['melty_page_type'].at[row.Index] = 'other'
        if row.melty_thema_name=='':
            urls['melty_thema_name'].at[row.Index] = 'Other'
        if  row.melty_folder_id =='null': # if this is not a folder
            urls['melty_folder_id'].at[row.Index] = 0
        if row.melty_thema_name=='null': # if this is not a folder
            urls['melty_thema_name'].at[row.Index] = 'Other'
    print("     URLs formatted in %0.2f seconds."%(timelib.time()-start))
    return urls;

def format_log(log_dataframe, urls):
    start = timelib.time()
    # Anonymise users
    log_dataframe['userID'] = str()
    user = list()
    for i in range(len(log_dataframe.user.unique())):
        user.append('USR'+str(i))
    log_dataframe['userID'] = log_dataframe.user.map(pd.Series(data=user,index=log_dataframe.user.unique()))

    # Identity url with pageID to anonymise
    log_dataframe['requested_pageID'] = 'Unknown'
    log_dataframe['referrer_pageID'] = 'Unknown'
    log_dataframe['requested_pageID'] = log_dataframe.requested_url.map(pd.Series(index = urls.url.values, data = urls.pageID.values))
    log_dataframe['referrer_pageID'] = log_dataframe.referrer_url.map(pd.Series(index = urls.url.values, data = urls.pageID.values))
    # Select useful entries
    output_log = pd.DataFrame(columns = ['userID','timestamp','requested_pageID','referrer_pageID','device'], index=None)
    output_log['userID'] = log_dataframe['userID']
    output_log['timestamp'] = log_dataframe['timestamp']
    output_log['requested_pageID'] = log_dataframe['requested_pageID']
    output_log['referrer_pageID'] = log_dataframe['referrer_pageID']
    output_log['device'] = log_dataframe['device']
    print("     Output log formatted in %0.2f seconds."%(timelib.time()-start))
    return output_log;


import json
from pymatgen.analysis.reaction_calculator import Reaction
from pymatgen.core.composition import Composition

__autor__ = 'Jherfson Castro Gomes'
__email__ = 'jherfson.castro@gmail.com'
__version__ = '2020.21.11'
__date__ = 'apr 24, 2020'

class Stoichiometric():
    """[summary]
    """
    def __init__(self, equation, mass_production):
        """[summary]

        Args:
            equation ([type]): [description]
            mass_production ([type]): [description]
        """
        self.equation = equation
        self.mass_production = mass_production

        # fazendo a separação da reagente do produto
        self.separate_rp = self.equation.split('->')

        # separando os reagente para passar para Classe Composition
        self.reagent = self.separate_rp[0].split('+')

        # separando os produtos para passar para Classe Composition
        self.producer = self.separate_rp[1].split('+')

        reactants = [Composition(i) for i in self.reagent]
        producers = [Composition(i) for i in self.producer]

        self.reaction = Reaction(reactants, producers)
        print(self.reaction)

        self.mol_reactant = []
        self.mol_production = []
        self.mass_reactant = []
        self.massa_production = []

        tam = int(len(producers))

        j = 0

        for i in range(len(self.reaction.coeffs)):
            if i < tam:
                mol = (self.reaction.coeffs[i]/self.reaction.coeffs[-tam]) * (mass_production/Composition(self.producer[-tam]).weight)
                self.mol_reactant.append(mol)
                mass = mol * Composition(self.reagent[i]).weight
                self.mass_reactant.append(mass)
            else:
                mol_ = (self.reaction.coeffs[i]/self.reaction.coeffs[-tam]) * (mass_production/Composition(self.producer[-tam]).weight)
                self.mol_production.append(mol_)
                mass_ = mol_ * Composition(self.producer[j]).weight
                j = j + 1
                self.massa_production.append(mass_)

    def to_dict(self):
        """[summary]

        Returns:
            [type]: [description]
        """
        # composto
        comp = self.reagent.copy()
        comp.extend(self.producer)

        # mol
        mol = self.mol_reactant.copy()
        mol.extend(self.mol_production)

        # mass
        massa = self.mass_reactant.copy()
        massa.extend(self.massa_production)

        chemical_compounds = []
        for i in range(len(self.reaction.coeffs)):
            x = {comp[i]: {'coefciente': round(self.reaction.coeffs[i], 4), 'peso molecular': round(Composition(comp[i]).weight, 4), 'mol': round(mol[i], 4), 'massa': round(massa[i], 4)}}
            chemical_compounds.append(x)

        return chemical_compounds

    def to_json(self):
        """[summary]

        Returns:
            [type]: [description]
        """
        return json.dumps(self.to_dict(), indent=4, separators=(", ", " : "))

    def print_mass(self):
        """[created a new function that prints the compound and the dough]

        Returns:
            [dict]: [compound and mass in grams. "Li2CO3 " : (-0.4466)g]
        """
        mass = self.to_dict()
        result = {}

        for i in range(len(mass)):
            for composto, massa in mass[i].items():
                result[composto] = massa['massa']

        return json.dumps(result, indent=4, separators=(", ", " : "))


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Support for running jobs that are invoked via the `run-task` script.
"""

import os

import attr
from voluptuous import Any, Optional, Required

from taskgraph.transforms.job import run_job_using
from taskgraph.transforms.job.common import support_vcs_checkout
from taskgraph.transforms.task import taskref_or_string
from taskgraph.util import path, taskcluster
from taskgraph.util.schema import Schema

EXEC_COMMANDS = {
    "bash": ["bash", "-cx"],
    "powershell": ["powershell.exe", "-ExecutionPolicy", "Bypass"],
}

run_task_schema = Schema(
    {
        Required("using"): "run-task",
        # if true, add a cache at ~worker/.cache, which is where things like pip
        # tend to hide their caches.  This cache is never added for level-1 jobs.
        # TODO Once bug 1526028 is fixed, this and 'use-caches' should be merged.
        Required("cache-dotcache"): bool,
        # Whether or not to use caches.
        Optional("use-caches"): bool,
        # if true (the default), perform a checkout on the worker
        Required("checkout"): Any(bool, {str: dict}),
        Optional(
            "cwd",
            description="Path to run command in. If a checkout is present, the path "
            "to the checkout will be interpolated with the key `checkout`",
        ): str,
        # The sparse checkout profile to use. Value is the filename relative to the
        # directory where sparse profiles are defined (build/sparse-profiles/).
        Required("sparse-profile"): Any(str, None),
        # The command arguments to pass to the `run-task` script, after the
        # checkout arguments.  If a list, it will be passed directly; otherwise
        # it will be included in a single argument to the command specified by
        # `exec-with`.
        Required("command"): Any([taskref_or_string], taskref_or_string),
        # Context to substitute into the command using format string
        # substitution (e.g {value}). This is useful if certain aspects of the
        # command need to be generated in transforms.
        Optional("command-context"): dict,
        # What to execute the command with in the event command is a string.
        Optional("exec-with"): Any(*list(EXEC_COMMANDS)),
        # Base work directory used to set up the task.
        Required("workdir"): str,
        # Whether to run as root. (defaults to False)
        Optional("run-as-root"): bool,
    }
)

def common_setup(config, job, taskdesc, command):
    run = job["run"]
    if run["checkout"]:
        repo_configs = config.repo_configs
        if len(repo_configs) > 1 and run["checkout"] is True:
            raise Exception("Must explicitly specify checkouts with multiple repos.")
        elif run["checkout"] is not True:
            repo_configs = {
                repo: attr.evolve(repo_configs[repo], **config)
                for (repo, config) in run["checkout"].items()
            }

        support_vcs_checkout(
            config,
            job,
            taskdesc,
            repo_configs=repo_configs,
            sparse=bool(run["sparse-profile"]),
        )

        vcs_path = taskdesc["worker"]["env"]["VCS_PATH"]
        for repo_config in repo_configs.values():
            checkout_path = path.join(vcs_path, repo_config.path)
            command.append(f"--{repo_config.prefix}-checkout={checkout_path}")

        if run["sparse-profile"]:
            command.append(
                "--{}-sparse-profile=build/sparse-profiles/{}".format(
                    repo_config.prefix,
                    run["sparse-profile"],
                )
            )

        if "cwd" in run:
            run["cwd"] = path.normpath(run["cwd"].format(checkout=vcs_path))
    elif "cwd" in run and "{checkout}" in run["cwd"]:
        raise Exception(
            "Found `{{checkout}}` interpolation in `cwd` for task {name} "
            "but the task doesn't have a checkout: {cwd}".format(
                cwd=run["cwd"], name=job.get("name", job.get("label"))
            )
        )

    if "cwd" in run:
        command.extend(("--task-cwd", run["cwd"]))

    taskdesc["worker"].setdefault("env", {})["MOZ_SCM_LEVEL"] = config.params["level"]

worker_defaults = {
    "cache-dotcache": False,
    "checkout": True,
    "sparse-profile": None,
    "run-as-root": False,
}

def script_url(config, script):
    if "MOZ_AUTOMATION" in os.environ and "TASK_ID" not in os.environ:
        raise Exception("TASK_ID must be defined to use run-task on generic-worker")
    task_id = os.environ.get("TASK_ID", "<TASK_ID>")
    # use_proxy = False to avoid having all generic-workers turn on proxy
    # Assumes the cluster allows anonymous downloads of public artifacts
    tc_url = taskcluster.get_root_url(False)
    # TODO: Use util/taskcluster.py:get_artifact_url once hack for Bug 1405889 is removed
    return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}"

@run_job_using(
    "docker-worker", "run-task", schema=run_task_schema, defaults=worker_defaults
)
def docker_worker_run_task(config, job, taskdesc):
    run = job["run"]
    worker = taskdesc["worker"] = job["worker"]
    command = ["/usr/local/bin/run-task"]
    common_setup(config, job, taskdesc, command)

    if run.get("cache-dotcache"):
        worker["caches"].append(
            {
                "type": "persistent",
                "name": "{project}-dotcache".format(**config.params),
                "mount-point": "{workdir}/.cache".format(**run),
                "skip-untrusted": True,
            }
        )

    run_command = run["command"]

    command_context = run.get("command-context")
    if command_context:
        run_command = run_command.format(**command_context)

    # dict is for the case of `{'task-reference': str}`.
    if isinstance(run_command, str) or isinstance(run_command, dict):
        exec_cmd = EXEC_COMMANDS[run.pop("exec-with", "bash")]
        run_command = exec_cmd + [run_command]
    if run["run-as-root"]:
        command.extend(("--user", "root", "--group", "root"))
    command.append("--")
    command.extend(run_command)
    worker["command"] = command

@run_job_using(
    "generic-worker", "run-task", schema=run_task_schema, defaults=worker_defaults
)
def generic_worker_run_task(config, job, taskdesc):
    run = job["run"]
    worker = taskdesc["worker"] = job["worker"]
    is_win = worker["os"] == "windows"
    is_mac = worker["os"] == "macosx"
    is_bitbar = worker["os"] == "linux-bitbar"

    if is_win:
        command = ["C:/mozilla-build/python3/python3.exe", "run-task"]
    elif is_mac:
        command = ["/tools/python36/bin/python3", "run-task"]
    else:
        command = ["./run-task"]

    common_setup(config, job, taskdesc, command)

    worker.setdefault("mounts", [])
    if run.get("cache-dotcache"):
        worker["mounts"].append(
            {
                "cache-name": "{project}-dotcache".format(**config.params),
                "directory": "{workdir}/.cache".format(**run),
            }
        )
    worker["mounts"].append(
        {
            "content": {
                "url": script_url(config, "run-task"),
            },
            "file": "./run-task",
        }
    )
    if worker.get("env", {}).get("MOZ_FETCHES"):
        worker["mounts"].append(
            {
                "content": {
                    "url": script_url(config, "fetch-content"),
                },
                "file": "./fetch-content",
            }
        )

    run_command = run["command"]

    if isinstance(run_command, str):
        if is_win:
            run_command = f'"{run_command}"'
        exec_cmd = EXEC_COMMANDS[run.pop("exec-with", "bash")]
        run_command = exec_cmd + [run_command]

    command_context = run.get("command-context")
    if command_context:
        for i in range(len(run_command)):
            run_command[i] = run_command[i].format(**command_context)

    if run["run-as-root"]:
        command.extend(("--user", "root", "--group", "root"))
    command.append("--")
    if is_bitbar:
        # Use the bitbar wrapper script which sets up the device and adb
        # environment variables
        command.append("/builds/taskcluster/script.py")
    command.extend(run_command)

    if is_win:
        worker["command"] = [" ".join(command)]
    else:
        worker["command"] = [
            ["chmod", "+x", "run-task"],
            command,
        ]


import numpy as np
import sys
from exputil import *
from read_pingmesh_csv import read_pingmesh_csv

def ping_plot(ping_plot_dir, logs_ns3_dir, from_id, to_id):

    # Read in the rtt_results
    rtt_results = read_pingmesh_csv(logs_ns3_dir)

    # Write results to temporary input file
    if (from_id, to_id) in rtt_results:
        rtts = rtt_results[(from_id, to_id)]
        with open(ping_plot_dir + "/temp_input.txt", "w+") as out_file:
            for val in rtts:
                if val[0]:
                    out_file.write(str(val[1]) + ",-100000000\n")
                else:
                    out_file.write(str(val[1]) + "," + str(val[2]) + "\n")
    else:
        raise ValueError("(%d, %d) is not in the results" % (from_id, to_id))

    # And perform the plotting
    local_shell = LocalShell()
    out_filename = logs_ns3_dir.replace("/", "\\/") + "\\/plot_ping_rtt_" + str(from_id) + "_to_" + str(to_id) + ".pdf"
    data_filename = (ping_plot_dir + "/temp_input.txt").replace("/", "\\/")
    local_shell.copy_file(ping_plot_dir + "/plot_ping_rtt.plt", ping_plot_dir + "/temp.plt")
    local_shell.perfect_exec("sed -i 's/\\[OUTPUT\\-FILE\\]/" + out_filename + "/g' " + ping_plot_dir + "/temp.plt")
    local_shell.perfect_exec("sed -i 's/\\[DATA\\-FILE\\]/" + data_filename + "/g' " + ping_plot_dir + "/temp.plt")
    local_shell.perfect_exec("gnuplot " + ping_plot_dir + "/temp.plt")
    print("Produced plot: " + out_filename.replace("\\", ""))
    local_shell.remove(ping_plot_dir + "/temp.plt")
    local_shell.remove(ping_plot_dir + "/temp_input.txt")

if __name__ == "__main__":
    args = sys.argv[1:]
    if len(args) != 4:
        print("Must supply exactly four arguments")
        print("Usage: python ping_plot.py [ping_plot directory] [logs_ns3 directory] [from_id] [to_id]")
        exit(1)
    else:
        ping_plot(
            args[0],
            args[1],
            int(args[2]),
            int(args[3])
        )


from datetime import datetime, timedelta
from os.path import exists
start = datetime(2014, 9, 10)
end = datetime(2015, 6, 1)
day = timedelta(1)
diff = (end - start).days
dt_osis_dict = {}
for _ in range(diff):
	start += day
	stpath = start.strftime('%Y_%m_%d') + '.csv'
	if exists(stpath):
		with open('logs/' + stpath) as f:
			osises = [int(i.strip()) for i in f.readlines()]
		dt_osis_dict[start] = osises
		ordered_oses.append(osises)
lengths = [len(i) for i in ordered_oses]
x = range(len(ordered_oses))
plt.plot(x, lengths)
plt.title('Attendance vs. Time')
plt.show()


import iris.tests as tests  # isort:skip

from pathlib import Path
from unittest import mock

import iris
from iris.cube import Cube

class TestSave(tests.IrisTest):
    def test_pathlib_save(self):
        file_mock = mock.Mock()
        # Have to configure after creation because "name" is special
        file_mock.configure_mock(name="string")

        find_saver_mock = self.patch(
            "iris.io.find_saver", return_value=(lambda *args, **kwargs: None)
        )

        def replace_expand(file_specs, files_expected=True):
            return file_specs

        # does not expand filepaths due to patch
        self.patch("iris.io.expand_filespecs", replace_expand)

        test_variants = [
            ("string", "string"),
            (Path("string/string"), "string/string"),
            (file_mock, "string"),
        ]

        for target, fs_val in test_variants:
            try:
                iris.save(Cube([]), target)
            except ValueError:
                print("ValueError")
                pass
            find_saver_mock.assert_called_with(fs_val)

if __name__ == "__main__":
    tests.main()


from os import getcwd, scandir
from PyPDF2 import PdfFileReader
from datetime import date

def extraer_texto_pdf(archivo):
    file=f"pdf/{archivo}"
    pdf=PdfFileReader(file)
    pageObj = pdf.getPage(0)
    text = pageObj.extractText()
    return text

def ls(ruta = getcwd()):
    return [arch.name for arch in scandir(ruta) if arch.is_file()]

def extraer_nombre_fecha_nac(text):
    posicion_incial_nombre=text.find("Nombre inscrito")+25
    posicion_final_nombre=text.find("R.U.N")
    nombre=text[posicion_incial_nombre:posicion_final_nombre]
    posicion_incial_fecha=text.find("Fecha nacimiento")+25
    posicion_final_fecha = text.find("Sexo")
    fecha_nacimiento = text[posicion_incial_fecha:posicion_final_fecha]
    fecha_format=formato_fecha_nacimiento(fecha_nacimiento)
    return f"{nombre}:{fecha_nacimiento}:{fecha_format}"

def days_between(d1, d2):
    return abs(d2 - d1).days

def formato_fecha_nacimiento(fecha_nac):
     meses={'Enero':1,'Febrero':2,'Marzo':3,'Abril':4,'Mayo':5,'Junio':6,'Julio':7,'Agosto':8,'Septiembre':9,'Octubre':10,'Noviembre':11,'Diciembre':12}
     dia=fecha_nac[0:2]
     mes=fecha_nac[3:-4]
     año=fecha_nac[-4:]
     d1 = date(int(año), int(meses[mes.strip()]), int(dia))
     d2 = date.today()
     return (int(days_between(d2, d1) / 365))


# [巡回セールスマン問題](http://codecrafthouse.jp/p/2016/08/traveling-salesman-problem/)
import numpy as np
import matplotlib.pyplot as plt
import math
import sys

plt.style.use('ggplot')

N = 80
MAP_SIZE = 100

# 総移動距離の計算をする関数
# 訪問順の評価指標
def calculate_total_distance(order, distance_matrix):
    """Calculate total distance traveled for given visit order"""
    idx_from = np.array(order)
    idx_to = np.array(order[1:] + [order[0]])
    distance_arr = distance_matrix[idx_from, idx_to]

    return np.sum(distance_arr)

# 訪問経路の可視化
def visualize_visit_order(order, city_xy):
    """Visualize traveling path for given visit order"""
    route = np.array(order + [order[0]])  # add point of departure
    x_arr = city_xy[:, 0][route]
    y_arr = city_xy[:, 1][route]

    plt.figure(figsize=(4, 4))
    plt.plot(x_arr, y_arr, 'o-')
    plt.show()

# 交換による総移動距離の差分を計算する関数
def calculate_2opt_exchange_cost(visit_order, i, j, distance_matrix):
    """Calculate the difference of cost by applying given 2-opt exchange"""
    n_cities = len(visit_order)
    a, b = visit_order[i], visit_order[(i + 1) % n_cities]
    c, d = visit_order[j], visit_order[(j + 1) % n_cities]

    cost_before = distance_matrix[a, b] + distance_matrix[c, d]
    cost_after = distance_matrix[a, c] + distance_matrix[b, d]
    return cost_after - cost_before

# ２つのパスの交換後の訪問順序を計算する
def apply_2opt_exchange(visit_order, i, j):
    """Apply 2-opt exhanging on visit order"""

    tmp = visit_order[i + 1: j + 1]
    tmp.reverse()
    visit_order[i + 1: j + 1] = tmp

    return visit_order

# 2-optによる近傍探索
def improve_with_2opt(visit_order, distance_matrix):
    """Check all 2-opt neighbors and improve the visit order"""
    n_cities = len(visit_order)
    cost_diff_best = 0.0
    i_best, j_best = None, None

    for i in range(0, n_cities - 2):
        for j in range(i + 2, n_cities):
            if i == 0 and j == n_cities - 1:
                continue

            cost_diff = calculate_2opt_exchange_cost(
                visit_order, i, j, distance_matrix)

            if cost_diff < cost_diff_best:
                cost_diff_best = cost_diff
                i_best, j_best = i, j

    if cost_diff_best < 0.0:
        visit_order_new = apply_2opt_exchange(visit_order, i_best, j_best)
        return visit_order_new
    else:
        return None

def local_search(visit_order, distance_matrix, improve_func):
    """Main procedure of local search"""
    cost_total = calculate_total_distance(visit_order, distance_matrix)

    while True:
        improved = improve_func(visit_order, distance_matrix)
        if not improved:
            break

        visit_order = improved

    return visit_order

def main():

    # 都市を表示
    np.random.seed(10)
    city_xy = np.random.rand(N, 2) * MAP_SIZE
    plt.figure(figsize=(4, 4))
    plt.plot(city_xy[:, 0], city_xy[:, 1], 'o')
    plt.show()

    x = city_xy[:, 0]
    y = city_xy[:, 1]
    # 都市間距離を計算
    distance_matrix = np.sqrt((x[:, np.newaxis] - x[np.newaxis, :]) ** 2 +
                              (y[:, np.newaxis] - y[np.newaxis, :]) ** 2)

    # 見やすくするために10都市分だけ出力。
    # distance_matrix[i, j]が都市iと都市jの距離。
    print(np.round(distance_matrix[:10, :10]))

    # 試しに距離を計算してみる
    test_order = list(np.random.permutation(N))
    print('訪問順序 = {}'.format(test_order))

    total = calculate_total_distance(test_order, distance_matrix)
    print('総移動距離 = {}'.format(total))

    # 可視化を試してみる
    # visualize_visit_order(test_order, city_xy)

    # 多スタート戦略
    N_START = 100
    order_best = None
    score_best = sys.float_info.max

    for _ in range(N_START):
        order_random = list(np.random.permutation(N))
        order_improved = local_search(
            order_random, distance_matrix, improve_with_2opt)
        score = calculate_total_distance(order_improved, distance_matrix)

        if score < score_best:
            score_best = score
            order_best = order_improved

    # 可視化を試してみる
    visualize_visit_order(order_best, city_xy)
    total_distance = calculate_total_distance(order_best, distance_matrix)
    print('多スタート戦略で得られた経路の総移動距離 = {}'.format(total_distance))

if __name__ == '__main__':
    main()


#!/usr/bin/python3

from statistics import stdev

import sys

CT_ALPHABET_SIZE = 83
CT_ALPHABET = bytes(range(CT_ALPHABET_SIZE))

READABLE_OFFSET = 32

def data_to_info(data, prefix=""):
    if prefix != "":
        prefix = prefix + "_"
    data = sorted(data)
    result = {
        prefix+"mean": sum(data) / len(data),
        prefix+"median": data[len(data) // 2],
        prefix+"minimum": min(data),
        prefix+"maximum": max(data),
        prefix+"stdev": stdev(data),
    }
    return result

def print_cipher_stack(cipher_stack):
    for cipher, params in cipher_stack:
        print(cipher.__name__ + ":")
        print("\t", params)

def get_stdin_texts():
    pts = sys.stdin.buffer.read().splitlines()
    if "--normalize-input" in sys.argv:
        return normalize_pts(pts)
    else:
        return [bytes([y - READABLE_OFFSET for y in x]) for x in pts]

def print_cts(cts):
    for ct in cts:
        print(bytes([l + READABLE_OFFSET for l in ct]).decode())

def get_alphabet(pts):
    return bytes(set(b"".join(pts)))

def normalize_pt(pt, alphabet):
    return bytes([alphabet.index(l) for l in pt])

def normalize_pts(pts):
    alphabet = get_alphabet(pts)
    return [normalize_pt(pt, alphabet) for pt in pts]

def get_isomorphs(ct, max_size):
    isomorphs = set()
    for i, l in enumerate(ct):
        offset = ct.find(l, i + 1, i + max_size + 1)
        while offset != -1:
            isomorphs.add((i, offset - i - 1))
            offset = ct.find(l, offset + 1, i + max_size + 1)
    return isomorphs

def autokeyer(pts, autokeyer_alphabet_fn, iv):
    autokey_alphabets = {}
    for l in CT_ALPHABET:
        autokey_alphabets[l] = autokeyer_alphabet_fn(l)
    cts = []
    for pt in pts:
        ct = []
        for i, l in enumerate(pt):
            ct.append(autokey_alphabets[iv if i == 0 else ct[-1]][l])
        cts.append(bytes(ct))
    return cts

def poly_substitute(pts, ct_alphabet_fn):
    max_len = max([len(pt) for pt in pts])
    ct_alphabets = [ct_alphabet_fn(i) for i in range(max_len)]
    results = []
    for pt in pts:
        results.append(bytes([ct_alphabets[i][l] for i, l in enumerate(pt)]))
    return results

def substitute(pts, ct_alphabet):
    results = []
    for pt in pts:
        results.append(bytes([ct_alphabet[l] for l in pt]))
    return results

# takes a ciphertext
# returns a dictionnary, key is letter, value is count
def get_letters_count(ct):
    counts = dict.fromkeys(range(CT_ALPHABET_SIZE), 0)
    for l in ct:
        counts[l] += 1
    return counts


import streamlit as st
import hydralit_components as hc
from multiapp import MultiApp
from pages import upload, visualisation, prediction, home
import time

st.set_page_config(page_title='FABI App', page_icon=':evergreen_tree:', layout='wide',initial_sidebar_state='collapsed')

# specify the primary menu definition
menu_data = [
    {'id': 'home','icon': "home", 'label':"Home"},
    {'id': 'prediction','icon':"bug",'label':"Prediction"},
    {'id': 'visualisation','icon': "graph", 'label':"Visualisation"},
    {'id': 'upload','icon': "graph", 'label':"Upload"}]

over_theme = {'txc_inactive': '#FFFFFF'}
menu_id = hc.nav_bar(
    menu_definition=menu_data,
    override_theme=over_theme,
    hide_streamlit_markers=False,
    sticky_nav=True,
    sticky_mode='pinned')

app = MultiApp()

#pages
if menu_id == 'home':
    app.add_app("Home", home.app)

if menu_id == 'prediction':
    app.add_app("Prediction", prediction.app)

if menu_id == 'visualisation':
    app.add_app("Visualisaton", visualisation.app)

if menu_id == 'upload':
    app.add_app("Upload", upload.app)
# The main app
app.run()


"""
CCC '00 S2 - Babbling Brooks
Find this problem at:
https://dmoj.ca/problem/ccc00s2
"""

# Initial number and flow of streams on the top of the mountain
init_streams = int(input())
streams = [int(input()) for i in range(init_streams)]
while True:
    line = int(input())
    # We have reached the bottom!
    if line == 77:
        break
    # Combining the value of two streams into one
    elif line == 88:
        num_stream = int(input())
        streams[num_stream-1] += streams.pop(num_stream)
    # Separating the value of stream index n-1
    # into stream index n-1 and stream index n
    elif line == 99:
        num_stream, percent = (int(input()), int(input()))
        cur_val = streams.pop(num_stream-1)
        streams.insert(num_stream-1, cur_val*percent/100)
        streams.insert(num_stream, cur_val*(100-percent)/100)
# Round and print out the value of all streams at the bottom
print(' '.join(str(int(i)) for i in streams))


import sys
import os

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from mdpkg.rwfile import read_dat, Dat

R = 8
ratio = 48
A = -50

try:
    surf_con = float(sys.argv[1])
except IndexError:
    surf_con = 1.0

snap_time_t = 163
snap_time_s = 94

separation = 8

path_to_save = os.getcwd()
path_to_data = '/home/luishcc/hdd/free_thread_results/'
# path_to_data = '/home/luishcc/hdd/free_thread_new/'
path_to_data = '/home/luishcc/hdd/'
# path_to_data = '/media/luis/luis-backup/hdd1-panos3/hdd/free_thread_results/'
# path_to_data = '/home/luishcc/test/'

path_to_data = '/home/luishcc/hdd/surfactant/new/'

def get_snap(dir):
    with open(dir+'/breaktime.txt', 'r') as fd:
        snap = int(fd.readline())
    return snap

# sim_case = f'R{R}_ratio{ratio}_A{abs(A)}'
sim_case = f'R{R}-{surf_con}'

dir_in = path_to_data + sim_case

def path_to_file_t(case, snap):
    # return dir_in + '-' + str(case) + f'/cluster/{snap+snap_time_t}.csv'
    return dir_in + '/' + str(case) + f'/cluster/{snap+snap_time_t}.csv'

def path_to_file_s(case, snap):
    # return dir_in + '-' + str(case) + f'/cluster/{snap+snap_time_s}.csv'
    return dir_in + '/' + str(case) + f'/cluster/{snap+snap_time_s}.csv'

case = 1
ff = dir_in + '-' + str(case)
ff = dir_in + '/' + str(case)

sum = 0
sumsq = 0
n = 0

print(ff)

while os.path.isdir(ff):
    ss = get_snap(ff)
    file_t = path_to_file_t(case, ss)
    file_s = path_to_file_s(case, ss)

    df = pd.read_csv(file_t)
    df2 = pd.read_csv(file_s)

    df.drop(df[df['size'] <= 8].index, inplace=True)
    df.drop(df[df['anisotropy'] > 0.25].index, inplace=True)

    df2.drop(df2[df2['size'] <= 6].index, inplace=True)
    df2.drop(df2[df2['anisotropy'] > 0.25].index, inplace=True)

    satellite = df2[df2['radius'] < separation]
    main = df[df['radius'] > separation]

    num_total = df.shape[0]
    # num_total = main.shape[0]
    num_satellite = satellite.shape[0]

    sum += num_satellite/num_total
    sumsq += (num_satellite/num_total)**2
    n += 1

    case += 1
    # ff = dir_in + '-' + str(case)
    ff = dir_in + '/' + str(case)

avg = sum/n
std = sumsq/n - avg**2
print("Avg: ", avg, "Var: ", std, "N: ", n)


import pandas as pd
import numpy as np
from tqdm import tqdm
import pickle
import glob
from scipy import sparse
import torch
import gc
import subprocess
from torch_geometric import utils, data
pd.set_option('mode.chained_assignment',None)

# Get zones
zones = [int(z[3:]) for z in pd.read_csv('SimpleNNData.csv', index_col=0).filter(regex = 'lz').columns]

# Make datasets for PTG
def make_PTG(graph, zones):
    attr, adj = graph

    # Filter out
    if (attr.time_to_reservation.values[-1] >= 48) or ~attr.next_customer[-1]:
        return None

    if attr.leave_zone[-1] not in zones:
        return None

    # Slice
    _, labels = sparse.csgraph.connected_components(csgraph=adj, directed=False, return_labels=True)
    newl = labels[-1]
    indices = labels == newl

    attr = attr[indices]
    adj = adj[indices,:].tocsc()[:,indices].tocsr()

    # Time variables
    attr['weekend'] = attr.time.dt.weekday//5

    def circle_transform(col, max_val=86400):
        tot_sec = ((col - col.dt.normalize()) / pd.Timedelta('1 second')).astype(int)
        cos_val = np.cos(2*np.pi*tot_sec/max_val)
        sin_val = np.sin(2*np.pi*tot_sec/max_val)
        return cos_val, sin_val

    attr['Time_Cos'], attr['Time_Sin'] = [x.values for x in circle_transform(attr.time)]

    # drop
    attr.drop(columns=['park_location_lat', 'park_location_long', 'leave_location_lat', 'leave_location_long', 'park_fuel', 'park_zone', 'moved', 'movedTF', 'time', 'prev_customer', 'next_customer'], inplace = True)

    # One hot encoding
    attr['leave_zone'] = pd.Categorical(attr['leave_zone'], categories=zones)
    attr = pd.get_dummies(attr, columns = ['leave_zone'], prefix='lz')

    attr['engine']= pd.Categorical(attr['engine'], categories=['118I', 'I3', 'COOPER', 'X1'])
    attr = pd.get_dummies(attr, columns = ['engine'], prefix='eng')

    # Normalize fuel and dist
    attr['leave_fuel'] = attr['leave_fuel']/100
    #df['dist_to_station'] = df['dist_to_station']/5320

    # Get edges
    edge_index, edge_weight = utils.convert.from_scipy_sparse_matrix(adj)

    # Make pytorch data type
    d = data.Data(x = torch.tensor(attr.drop(columns = ['time_to_reservation']).to_numpy(dtype = 'float')).float(), edge_index=edge_index, edge_attr=edge_weight.float(), y = torch.tensor(attr.time_to_reservation.values).float())

    return d

# Get files
files = glob.glob("Graphs/*")

dataset = []
with open(files[0], 'rb') as f:
    graph_collection = pickle.load(f)

for g in graph_collection.values():
    res = make_PTG(g,zones)
    if res:
        dataset.append(res)

train_val_size = int(0.8 * len(dataset))
val_test_size = len(dataset)-train_val_size
train_val_data, test_data = torch.utils.data.random_split(dataset, [train_val_size, val_test_size])
train_size = train_val_size-val_test_size
train_data, val_data = torch.utils.data.random_split(train_val_data, [train_size, val_test_size])
del train_val_data

for file in tqdm(files[1:]):
    #dataset = []
    with open(file, 'rb') as f:
        graph_collection = pickle.load(f)

    for g in graph_collection.values():
        res = make_PTG(g,zones)
        if res:
            dataset.append(res)

    train_val_size = int(0.8 * len(dataset))
    val_test_size = len(dataset)-train_val_size
    train_val_data_tmp, test_data_tmp = torch.utils.data.random_split(dataset, [train_val_size, val_test_size])
    train_size = train_val_size-val_test_size
    train_data_tmp, val_data_tmp = torch.utils.data.random_split(train_val_data_tmp, [train_size, val_test_size])

    train_data = torch.utils.data.ConcatDataset([train_data,train_data_tmp])
    val_data = torch.utils.data.ConcatDataset([val_data,val_data_tmp])
    test_data = torch.utils.data.ConcatDataset([test_data,test_data_tmp])

print(subprocess.run(['free', '-m'], stdout=subprocess.PIPE).stdout.decode('utf-8'))
print('Deleting other variables')
del zones, files, dataset, res, train_val_size, val_test_size, train_size, train_val_data_tmp, train_data_tmp, val_data_tmp, test_data_tmp
gc.collect()
dataset = 0
train_val_data_tmp = 0
train_data_tmp = 0
val_data_tmp = 0
test_data_tmp = 0
print(subprocess.run(['free', '-m'], stdout=subprocess.PIPE).stdout.decode('utf-8'))

with open(f'GNNDatasets/Val_data.pickle', 'wb') as handle:
    pickle.dump(val_data, handle, pickle.HIGHEST_PROTOCOL)
print('Val dumped\n')
del val_data
gc.collect()
val_data = 0
print(subprocess.run(['free', '-m'], stdout=subprocess.PIPE).stdout.decode('utf-8'))

with open(f'GNNDatasets/Test_data.pickle', 'wb') as handle:
    pickle.dump(test_data, handle, pickle.HIGHEST_PROTOCOL)
print('Test dumped\n')
del test_data
gc.collect()
test_data = 0
print(subprocess.run(['free', '-m'], stdout=subprocess.PIPE).stdout.decode('utf-8'))

with open(f'GNNDatasets/Train_data.pickle', 'wb') as handle:
    pickle.dump(train_data, handle, pickle.HIGHEST_PROTOCOL)
print('Train dumped')


import argparse
import configparser
import logging
import sys
import os
from pathlib import Path

import torch
from torch.utils.data import DataLoader

import config as cfg
from datagen.algorithms import info, gen_multi_algo_data
from datagen.graphgen import gen_erdos_renyi, gen_barabasi_albert, gen_twod_grid
import dataloaders as dl
from loss import LossAssembler, create_loss_class
import model as arch
import initialisation as init
from train import train, train_metadata
import evaluate as ev
import pdb
def main():
    ## read config and cmd line arguments & set up logging
    args, logger = pre_main()
    args = vars(args)

    # determine cuda device
    device = torch.device('cuda' if args['cuda'] and torch.cuda.is_available() else 'cpu')

    # for reproducibility set the seed
    torch.manual_seed(args['seed'])

    '''if args['runtests']:
        test.run(logger)
        logger.info("Done with tests")
        sys.exit(0)'''

    transfer = False
    if (args['transfertask'] != '') and (args['transfertask'] != 'None'):
        transfer = True
        transfertasks = args['transfertask'].split(' ')
        transfer_algo_names = list(map(lambda x: x.split('_')[-1], transfertasks))
        transfer_metadata = info(logger, transfer_algo_names)

    # getting metadata about the task
    task_list = args['task'].split(' ')
    algo_names = list(map(lambda x: x.split('_')[-1], task_list))
    metadata = info(logger, algo_names)
    logger.info("Task metadata: " + str(metadata))
    pdb.set_trace()
    if args['datagen']:
        rand_gen = torch.distributions.Uniform(0.0, 1.0)
        ngraphs = args['ngraphs'].split(',')
        nnodes = args['nnodes'].split(',')
        #filename = args['datagraphsetname'].split(',')
        for graphtype, params in zip(args['graphtypes'].split(','), args['graphparams'].split(',')):
            for i in range(len(ngraphs)):
                logger.info("Generating {} graphs".format(graphtype))
                if graphtype == 'ErdosRenyi':
                    directed = True if params == "True" else False
                    gen_erdos_renyi(rand_gen, int(ngraphs[i]), int(nnodes[i]), args['datasavefp']+filename[i]+"_", directed)
                    src_nodes = torch.argmax(rand_gen.sample((int(ngraphs[i]), int(nnodes[i]))), dim=1)
                    gen_multi_algo_data(
                        args['datasavefp']+filename[i]+"_"+"erdosrenyi" + ngraphs[i] + "_"+ nnodes[i] + ".pt",
                        src_nodes,
                        task_list,
                        args['hidekeys']
                    )
                elif graphtype == "BarabasiAlbert":
                    m = int(params)
                    gen_barabasi_albert(rand_gen, int(ngraphs[i]), int(nnodes[i]), m, args['datasavefp']+filename[i]+"_")
                    src_nodes = torch.argmax(rand_gen.sample((int(ngraphs[i]), int(nnodes[i]))), dim=1)
                    gen_multi_algo_data(
                        args['datasavefp']+filename[i]+"_"+"barabasialbert" + ngraphs[i] + "_"+ nnodes[i] + ".pt",
                        src_nodes,
                        task_list,
                        args['hidekeys']
                    )
                elif graphtype == "TwoDGrid":
                    gen_twod_grid(rand_gen, int(ngraphs[i]), int(nnodes[i]), args['datasavefp']+filename[i]+"_")
                    src_nodes = torch.argmax(rand_gen.sample((int(ngraphs[i]), int(nnodes[i]))), dim=1)
                    gen_multi_algo_data(
                        args['datasavefp']+filename[i]+"_"+"twodgrid" + ngraphs[i] + "_"+ nnodes[i] + ".pt",
                        src_nodes,
                        task_list,
                        args['hidekeys']
                    )
                else:
                    logger.exception("Unknown graphtype {}")
        logger.info("Data generation done")

    # collect relevant stuff for training
    dset = dl.MultiAlgo
    collate_batch = dl.collate_multi_algo
    metrics = [m for t in task_list for m in ev.get_metrics(t)]
    # loss_train, loss_val, loss_test, metrics, dset, collate_batch = loss_and_data_fns(logger, task_list)

    algos = []
    for task in task_list:
        algos.append(
            create_loss_class(task, args)
        )
    loss_module = LossAssembler(device, logger, algos, args)

    # normalise the training loss or not
    cfg.normaliseloss = args['normaliseloss']

    # loading train data
    data_stream = DataLoader(dset(logger,args['datafp'].split(' '),algo_names,"Train"),
                             shuffle = True,
                             batch_size = args['batchsize'],
                             num_workers = args['nworkers'],
                             collate_fn = collate_batch,
                             drop_last = False
                             )
    # loading val data
    val_stream = DataLoader(dset(logger,args['valdatafp'].split(' '),algo_names,"Validation"),
                            shuffle = False,
                            batch_size = args['batchsize'],
                            num_workers = args['nworkers'],
                            collate_fn = collate_batch,
                            drop_last = False
                            )
    # loading test data
    test_stream = []
    for task, fp in zip(args['evaltask'].split(','),args['evaldatafp'].split(' ')):
        test_stream.append(DataLoader(dset(logger,[fp],algo_names,task),
                                      shuffle = False,
                                      batch_size = args['evalbatchsize'],
                                      num_workers = args['nworkers'],
                                      collate_fn = collate_batch,
                                      drop_last = False
                                      )
                           )
    logger.info("Dataloaders initialised")

    # potentially hoist model creation code to function when adding more models
    model = arch.create_model(args['arch'], device, args, metadata)
    train_params = {k:args[k] for k in train_metadata()}

    if args['optimal']:
        logger.info("Using optimal model")
        model = arch.optimal_model(device, logger, metadata, task_list)

    logger.info("Model initialised")
    if args['loadmodel'] != '':
        logger.info("Loading model from " + args['loadmodel'])
        checkpoint = torch.load(args['loadmodel'], map_location=torch.device('cuda') if args['cuda'] else torch.device('cpu'))
        model.load_state_dict(checkpoint['model_state'])
    else:
        recorder = None
        # creating the training parameter dict
        model_state, val_loss = train(logger,
                                      device,
                                      data_stream,
                                      val_stream,
                                      model,
                                      train_params,
                                      loss_module,
                                      recorder
                                      )
        if args['savemodel'] != '':
            torch.save({'model_state': model_state}, args['savemodel'])

    # evaluation, i.e. test time loss + fancy stuff
    ev.evaluate(logger,
                device,
                test_stream,
                model,
                loss_module,
                metrics
                )

    if transfer:
        ## it is currently assumed that at transfer time we always use the no_algo loss
        ## to change that in the future register them as separate tasks in the future
        import no_algo_loss as loss
        # collect relevant stuff for training
        dset = dl.MultiAlgo
        collate_batch = dl.collate_multi_algo
        metrics = [m for t in transfertasks for m in ev.get_metrics(t)]
        # loss_train, loss_val, loss_test, metrics, dset, collate_batch = loss_and_data_fns(logger, task_list)

        algos = []
        for task in transfertasks:
            algos.append(
                create_loss_class(task, args)
            )
        loss_module = LossAssembler(device, logger, algos, args)

        # loading train data
        data_stream = DataLoader(dset(logger,args['datafp'].split(' '),transfer_algo_names,"Train"),
                                 shuffle = True,
                                 batch_size = args['batchsize'],
                                 num_workers = args['nworkers'],
                                 collate_fn = collate_batch,
                                 drop_last = False
                                 )
        # loading val data
        val_stream = DataLoader(dset(logger,args['valdatafp'].split(' '),transfer_algo_names,"Validation"),
                                shuffle = False,
                                batch_size = args['batchsize'],
                                num_workers = args['nworkers'],
                                collate_fn = collate_batch,
                                drop_last = False
                                )
        # loading test data
        test_stream = []
        for task, fp in zip(args['evaltask'].split(','),args['evaldatafp'].split(' ')):
            test_stream.append(DataLoader(dset(logger,[fp],transfer_algo_names,task),
                                          shuffle = False,
                                          batch_size = args['evalbatchsize'],
                                          num_workers = args['nworkers'],
                                          collate_fn = collate_batch,
                                          drop_last = False
                                          )
                               )
        arch_type = args['transferarch'] if args['transferarch'] != '' else args['arch']
        transfer_model = arch.create_model(arch_type, device, args, transfer_metadata)
        model_state = model.state_dict()
        init.processor_init(transfer_model, model_state)
        train_params['lr'] = args['tunelr']
        _ , val_loss = train(logger,
                            device,
                            data_stream,
                            val_stream,
                            transfer_model,
                            train_params,
                            loss_module
                            )
        target_model = transfer_model
        ev.evaluate(logger,
                    device,
                    test_stream,
                    target_model,
                    loss_module,
                    metrics
                    )

    return None

### boilerplate code for logging and commandline arguments
def pre_main():
    def str2bool(v):
        if isinstance(v, bool):
            return v
        if v.lower() in ("yes", "true", "t", "y", "1"):
            return True
        elif v.lower() in ("no", "false", "f", "n", "0"):
            return False
        else:
            raise argparse.ArgumentTypeError("Boolean value expected.")

    ## parse the config file
    conf_parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
        add_help=False
    )
    conf_parser.add_argument('-c', '--config',
                             default='./options.conf',
                             help='Path to config file',
                             metavar="FILE"
    )
    args, remaining_argv = conf_parser.parse_known_args()

    # default values as specified by the config file
    defaults = {}

    '''if args.config:
        config = configparser.ConfigParser()
        config.read([args.config])
        for key, val in config.items('Defaults'):
            if val == 'True' or val == 'False':
                defaults[key] = config.getboolean('Defaults', key)
            else:
                defaults[key] = val'''

    # parse the remaining arguments
    # this Parser also generates the help message
    parser = argparse.ArgumentParser(
        # inherit options from config_parser
        parents=[conf_parser]
    )
    # add all the command line arguments here
    parser.add_argument('-d', '--dataset',
                        type=str,
                        default='',
                        help='Dataset to use'
    )
    parser.add_argument('-f', '--datafp',
                        type=str,
                        default='./data',
                        help='Path to dataset'
    )
    parser.add_argument('--valdatafp',
                        type=str,
                        default='./data/val',
                        help='Path to validation dataset'
    )
    parser.add_argument('--evaldatafp',
                        type=str,
                        default='./data/eval',
                        help='Path to test dataset'
    )
    parser.add_argument('--evaltask',
                        type=str,
                        default='Test',
                        help='Name of test data'
    )
    parser.add_argument('--transfertask',
                        type=str,
                        default='',
                        help='List of tasks to test transfer of algorithmic alignment on. Empy List means no transfer testing.'
    )
    parser.add_argument('--transferarch',
                        type=str,
                        default='',
                        help='Transfer architecture to use'
    )
    parser.add_argument('--datasavefp',
                        type=str,
                        default='./',
                        help='Path to save dataset'
    )
    parser.add_argument('--savemodel',
                        type=str,
                        default='',
                        help='Path to save model checkpoint, ignored if empty string.'
    )
    parser.add_argument('--loadmodel',
                        type=str,
                        default='',
                        help='Path to load the model checkpoint from, ignored if empty string'
    )
    parser.add_argument('--graphtypes',
                        type=str,
                        default='ErdosRenyi',
                        help='The graphtype to generate if datagen is set to True. Options are: ErdosRenyi, BarabasiAlbert'
    )
    parser.add_argument('--graphparams',
                        type=str,
                        default='False',
                        help='The graphparams for the graphtypes once per graphsetname. ' + \
                             'For ErdosRenyi: directed (bool), BarabasiAlbert: m (int)'
    )
    parser.add_argument('--datagen',
                        type=str2bool,
                        default=False,
                        help='Generate data'
    )
    parser.add_argument('--hidekeys',
                        type=str2bool,
                        default=True,
                        help='Generate data, while hiding the keys'
    )
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        help='If verbose flag is set then all the logging output goes to std_out as well'
    )
    parser.add_argument('--debug',
                        action='store_true',
                        default=False,
                        help='If debug flag is set then debug messages be printed'
    )
    parser.add_argument('-l', '--logfile',
                        type=str,
                        default='./last_run.log',
                        help='File path to logfile'
    )
    parser.add_argument('--cuda',
                        type=str2bool,
                        default=torch.cuda.is_available(),
                        help='Decides whether to use GPU or CPU'
    )
    parser.add_argument('--seed',
                        type=int,
                        default=0,
                        help='Torch seed'
    )
    parser.add_argument('--optimal',
                        type=str2bool,
                        default=False,
                        help='Use optimal parameters for pre-determined encoding dim, not all tasks have them'
    )
    parser.add_argument('-t', '--task',
                        type=str,
                        default='',
                        help='The task to run on.'
    )
    parser.add_argument('-a', '--arch',
                        type=str,
                        default='NeuralExec',
                        help='The architecture to use for the model. Current options are: NeuralExec, NCNeuralExec.'
    )
    parser.add_argument('--layers',
                        type=int,
                        default=1,
                        help='Number of layers for the GNN arch'
    )
    # train_params
    parser.add_argument('--optimizer',
                        type=str,
                        default='adam',
                        help='Optimizer choice'
    )
    parser.add_argument('--epochs',
                        type=int,
                        default=0,
                        help='Number of max epochs for training'
    )
    parser.add_argument('--ksamples',
                        type=int,
                        default=1,
                        help='Number of trajectories to sample'
    )
    parser.add_argument('--lr',
                        type=float,
                        default=0.0001,
                        help='Learning rate'
    )
    parser.add_argument('--weightdecay',
                        type=float,
                        default=0.00001,
                        help='L2 regularisation also called weightdecay'
    )
    parser.add_argument('--warmup',
                        type=int,
                        default=0,
                        help='How many warm-up epochs'
    )
    parser.add_argument('--earlystop',
                        type=str2bool,
                        default=True,
                        help='Whether or not to do Early Stopping'
    )
    parser.add_argument('--earlytol',
                        type=float,
                        default=5e-6,
                        help='The tolerance for the early stopping criterion'
    )
    parser.add_argument('--patience',
                        type=int,
                        default=1,
                        help='Patience during early stopping'
    )
    parser.add_argument('--schedpatience',
                        type=int,
                        default=0,
                        help='Scheduler patience before reducing lr'
    )
    parser.add_argument('--tempinit',
                        type=float,
                        default=1.0,
                        help='Initial temperature for softmax'
    )
    parser.add_argument('--temprate',
                        type=float,
                        default=1.0,
                        help='Temperature rate (multiplier)'
    )
    parser.add_argument('--tempmin',
                        type=float,
                        default=1.0,
                        help='Temperature minium value'
    )
    parser.add_argument('--normaliseloss',
                        type=str2bool,
                        default=False,
                        help='Whether or not to normalise the loss to log(#nodes)'
    )
    # transfer params
    parser.add_argument('--iterations',
                        type=int,
                        default=1,
                        help='Number of transfer exploit and explore iterations'
    )
    parser.add_argument('--transferepochs',
                        type=int,
                        default=1,
                        help='Number of transfer exploit and explore iterations'
    )
    parser.add_argument('--tunelr',
                        type=float,
                        default=0.0001,
                        help='Learning rate'
    )
    # model parameters
    parser.add_argument('--latentdim',
                        type=int,
                        default=32,
                        help='Size of the latent dimension in the processor network'
    )
    parser.add_argument('--encdim',
                        type=int,
                        default=32,
                        help='Size of the encoding dimension in the encoder network'
    )
    parser.add_argument('--noisedim',
                        type=int,
                        default=0,
                        help='Size of to inject for each node, this serves to help the GNN identify nodes uniquely'
    )
    # dataloader a params
    parser.add_argument('--batchsize',
                        type=int,
                        default=32,
                        help='Size of batches'
    )
    parser.add_argument('--evalbatchsize',
                        type=int,
                        default=32,
                        help='Size of eval batches'
                        )
    parser.add_argument('--nworkers',
                        type=int,
                        default=0,
                        help='Number of workers in the dataloader'
    )
    # datagen params
    parser.add_argument('--ngraphs',
                        type=str,
                        default='',
                        help='List of number of graphs'
    )
    parser.add_argument('--nnodes',
                        type=str,
                        default='',
                        help='List of number of nodes per graph'
    )
    parser.add_argument('--datagraphsetnam',
                        type=str,
                        default='',
                        help='List of number of nodes per graph'
    )

    # note: set_default won't work with any variables with dashes/underscores in the name
    parser.set_defaults(**defaults)
    args = parser.parse_args(remaining_argv)

    ## set up logging
    logFormatter = logging.Formatter(
        '%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s'
    )
    rootLogger = logging.getLogger()
    # set debugging level
    if args.debug:
        level = logging.DEBUG
    else:
        level = logging.INFO
    rootLogger.setLevel(level)

    # file logging to logfile
    fileHandler = logging.FileHandler(args.logfile)
    fileHandler.setFormatter(logFormatter)
    rootLogger.addHandler(fileHandler)

    # logging to std out
    consoleHandler = logging.StreamHandler(sys.stdout)
    consoleHandler.setFormatter(logFormatter)
    # if verbose is set we print level else we print warnings only
    if args.verbose:
        consoleHandler.setLevel(level)
    else:
        consoleHandler.setLevel(logging.WARNING)
    rootLogger.addHandler(consoleHandler)

    # print options to log
    rootLogger.info(args)

    return args, rootLogger

if __name__ == '__main__':
    main()