from numpy.lib import type_check
# from tensorflow.python.keras.engine.base_layer import default
from builtins import help, print
import argparse
import os
from posixpath import join
import random
from torch.distributed.distributed_c10d import destroy_process_group
import yaml
import numpy as np
from shutil import copyfile
# from subprocess import PIPE
import time, datetime
from time import strftime
import logging
import warnings
import sys
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.nn.functional as F
import torch.backends.cudnn as cudnn
import torch.distributed as dist
import torch.optim as optim
import torch.utils.data
import torch.utils.data.distributed
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torchvision.models as models
import torch.multiprocessing as mp
from multiprocessing import Process, Pipe, Lock

# from models.vanilla_resnet_slow import *
# resnet34S = resnet34
# resnet18S = resnet18
# from models.vanilla_resnet import *
# from models.small_model import Small_Net, Small_Net_2
# from models.resnet20_imagenet import resnet20
# from models.vgg_grasp import vgg11, vgg13, vgg16, vgg19
from set_dataset import *
from models.vanilla_resnet_slow import resnet18S, resnet34S
from models.vanilla_resnet import resnet18, resnet34, resnet50, resnet101, resnet152
from models.vgg_grasp import *
from models.small_model import *
from models.resnet20_imagenet import resnet20
from models.builder import build_model

import torch.autograd.profiler as profiler

def conn_message(mesg_type, mesg_data=None):
    return [str(datetime.datetime.now()), mesg_type, mesg_data]

class application():
    def __init__(self, appid, arch, depth, batch, workers, work_space, port, \
                arrival_time=0, cuda_device="0, 1", start_iter=0, start_epoch=1, \
                end_iter=500, end_epoch=1):
        self.appid = appid
        self.arch = arch
        self.depth = depth
        self.batch = batch
        self.workers = workers
        self.work_space = work_space
        self.port = port
        self.arrival_time = arrival_time
        self.cuda_device = cuda_device
        self.process = None
        self.start_iter = start_iter
        self.start_epoch = start_epoch
        self.end_iter = end_iter
        self.end_epoch = end_epoch
        self.checkpoint = False
        self.finish_time = -1
        self.subprocess_conns = {}
        self.paused_counter = 0
        self.finished_counter = 0
        self.init_model()
        # self.best_acc1 = 0

    def print_info(self):
        print("appid: {}, model: {}{}, batch: {}, workers: {}, output: {}, port: {}, cuda_device: {}" \
            .format(self.appid, self.arch, self.depth, self.batch, self.workers, self.work_space, self.port, self.cuda_device))

    def init_model(self):
        device = 'cuda' if torch.cuda.is_available() else 'cpu'

        model_names = sorted(name for name in models.__dict__
                            if name.islower() and not name.startswith("__")
                            and callable(models.__dict__[name]))

        parser = argparse.ArgumentParser(description='PyTorch ImageNet Training')
        parser.add_argument('--data', default='/work/yanzhi_group/datasets/imagenet/',
                            help='path to dataset')
        parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18',
                            # choices=model_names,
                            help='model architecture: ' +
                                ' | '.join(model_names) +
                                ' (default: resnet18)')
        parser.add_argument('--depth', default=None, type=int,
                            help='depth of the neural network, 16,19 for vgg; 18, 50 for resnet')
        parser.add_argument('-j', '--workers', default=8, type=int, metavar='N',
                            help='number of data loading workers (default: 8)')
        parser.add_argument('--iteration', default=500, type=int, )
        parser.add_argument('--epochs', default=1, type=int, metavar='N',
                            help='number of total epochs to run')
        parser.add_argument('-b', '--batch-size', default=256, type=int, metavar='N',
                            help='mini-batch size (default: 256), this is the total '
                                'batch size of all GPUs on the current node when '
                                'using Data Parallel or Distributed Data Parallel')
        parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, metavar='LR',
                            help='initial learning rate', dest='lr')
        parser.add_argument('--lr-decay', type=int, default=30, metavar='LR_decay',
                            help='how many every epoch before lr drop (default: 30)')
        parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
                            help='momentum')
        parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, metavar='W',
                            help='weight decay (default: 1e-4)', dest='weight_decay')
        parser.add_argument('-p', '--print-freq', default=100, type=int, metavar='N',
                            help='print frequency (default: 10)')
        parser.add_argument('--world-size', default=1, type=int,
                            help='number of nodes for distributed training')
        parser.add_argument('--rank', default=0, type=int,
                            help='node rank for distributed training')
        parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str,
                            help='url used to set up distributed training')
        parser.add_argument('--dist-backend', default='nccl', type=str,
                            help='distributed backend')
        parser.add_argument('--seed', default=None, type=int,
                            help='seed for initializing training. ')
        parser.add_argument('--gpu', default=None, type=int,
                            help='GPU id to use.')
        parser.add_argument('--multiprocessing-distributed', action='store_true',
                            help='Use multi-processing distributed training to launch '
                                'N processes per node, which has N GPUs. This is the '
                                'fastest way to use PyTorch for either single node or '
                                'multi node data parallel training')
        parser.add_argument('--optimizer', type=str, default='sgd',
                            help='define optimizer')
        parser.add_argument('--lr-scheduler', type=str, default='default',
                            help='define lr scheduler')

        parser.add_argument('--trace', type=str, default='trace',
                            help='plot trace')
        parser.add_argument('--profile', action='store_true',
                            help='Use profiler')
        parser.add_argument('--profile-epochs', default=0, type=int,
                            help='Use profiler')
        parser.add_argument('--profile-name', type=str, default='test',
                            help='name of profile test')
        parser.add_argument('--record-shapes', action='store_true',
                            help='define lr scheduler')
        parser.add_argument('--profile-memory', action='store_true',
                            help='Use profiler')
        parser.add_argument('--use-cuda', action='store_true',
                            help='define lr scheduler')
        parser.add_argument('--profile-batches', default=-1, type=int,
                            help='Use profiler')
        parser.add_argument('--appid', default=1, type=int, help="Application Unique ID")
        parser.add_argument('--dataset', type=str, default='imagenet',
                            help='define dataset')
        parser.add_argument('--checkpoint', dest='checkpoint', action='store_false',
                            help="if resume frome a checkpoint")
        parser.add_argument('--start_iteration', default=0, type=int)
        parser.add_argument("--mode", "-m")

        args = parser.parse_args()
        args.appid = self.appid
        args.arch = self.arch
        args.depth = self.depth
        args.batch_size = self.batch
        args.workers = self.workers
        args.work_space = self.work_space
        args.dist_url = 'tcp://127.0.0.1:{}'.format(self.port)
        args.multiprocessing_distributed = True
        args.iteration = self.end_iter
        args.epochs = self.end_epoch
        args.checkpoint = self.checkpoint
        args.start_iteration = self.start_iter
        args.cuda_device = self.cuda_device
        args.print_freq = 10
        return args

def send_pause_to_subprocess(app):
    try:
        for sub_idx in app.subprocess_conns:
            if sub_idx != 0:
                master_conn, _ = app.subprocess_conns[sub_idx]
                master_conn.send(conn_message("Pause"))
        sub_idx = 0
        master_conn, _ = app.subprocess_conns[sub_idx]
        master_conn.send(conn_message("Pause"))
    except Exception as e:
            print("send pause to {}".format(sub_idx), e)

def send_exit_to_subprocess(app):
    try:
        for sub_idx in app.subprocess_conns:
            master_conn, _ = app.subprocess_conns[sub_idx]
            master_conn.send(conn_message("Exit"))
    except Exception as e:
            print("send exit to {}".format(sub_idx), e)

def send_exit_to_rank0(app):
    try:
        master_conn, _ = app.subprocess_conns[0]
        master_conn.send(conn_message("Exit"))
    except Exception as e:
        print("send exit to rank0", e)

def app_messeger(app, child_conn):
    while 1:
        if child_conn.poll():
            ws_event = child_conn.recv()
            if ws_event[1] == "Pause":
                send_pause_to_subprocess(app)
            else:
                print("unknow message")
        for sub_idx in app.subprocess_conns:
            master_conn, _ = app.subprocess_conns[sub_idx]
            if master_conn.poll():
                sub_event = master_conn.recv()
                if sub_event[1] == "Paused":
                    app.paused_counter += 1
                    print("{}: worker: {} receive a pause signal, now counter is {}".format(datetime.datetime.now(), app.appid, app.paused_counter))
                    if app.paused_counter == len(app.cuda_device):
                        # child_conn.send(conn_message("Paused", {"iter": sub_event[2]["iter"] , "epoch":sub_event[2]["epoch"]}))
                        send_exit_to_subprocess(app)
                        print("{}: worker: {} send an exit signal".format(datetime.datetime.now(), app.appid))
                        child_conn.send(conn_message("Paused", {"iter": sub_event[2]["iter"] , "epoch":sub_event[2]["epoch"]}))
                        return
                # if sub_event[1] == "Exited":

                elif sub_event[1] == "Finished":
                    app.finished_counter += 1
                    if app.finished_counter == len(app.cuda_device):
                        for s_idx in app.subprocess_conns:
                            master_conn, worker_conn = app.subprocess_conns[s_idx]
                            master_conn.close()
                            worker_conn.close()
                        try:
                            dist.destroy_process_group()
                        except Exception as e:
                            print("destroy_process_group failed", e)

                        child_conn.send(conn_message("Finished"))
                        return
                elif sub_event[1] == "HeartBeat":
                    child_conn.send(sub_event)

def save_train_model(epoch, iter, model, optimizer, loss, scheduler, args, model_path=''):
    try:
        if model_path=='':
            model_path = os.path.join(args.work_space, "app-{}-{}.tar".format(
                        args.appid, "gpu{}".format(args.gpu) if args.gpu!=None else "cpu{}".format(args.rank)))
        # if model_path=='':
        #     model_path = os.path.join(args.work_space, "app-{}.tar".format(args.appid))
        torch.save({
                'iter': iter,
                'epoch': epoch,
                'model_state_dict': model.state_dict(),
                'optimizer_state_dict': optimizer.state_dict(),
                'loss': loss,
                # 'scheduler': scheduler
                }, model_path)
        # copyfile(model_path, os.path.join(args.work_space, "app-{}-{}.tar".format(args.appid, datetime.datetime.now())))
    except Exception as e:
        print("app: {} save model".format(args.appid), e)

def resume_train_model(model, optimizer, args, model_path=''):
    if model_path=='':
        latest_ts = 0
        for f in os.listdir(args.work_space):
            if "app-{}".format(args.appid) in f:
                if latest_ts<os.path.getctime(os.path.join(args.work_space, f)):
                    model_path = os.path.join(args.work_space, f)

        # model_path = os.path.join(args.work_space, "app-{}-{}.tar".format(
        #             args.appid, "gpu1"))
    # if model_path=='':
    #     model_path = os.path.join(args.work_space, "app-{}.tar".format(args.appid))
    try:
        print("{} resumes from {}".format(args.appid, model_path))
        checkpoint = torch.load(model_path)
        model.load_state_dict(checkpoint['model_state_dict'], strict=False)
        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
        epoch = checkpoint['epoch']
        iter = checkpoint['iter']
        loss = checkpoint['loss']
        # scheduler = checkpoint['scheduler']
        return model, optimizer, epoch, iter, loss#, scheduler
    except Exception as e:
        print(e)
        raise

def main_worker(gpu, ngpus_per_node, app, args):
    criterion = None
    scheduler = None
    optimizer = None
    epoch = 1

    args.gpu = gpu

    if args.gpu is not None:
        print("Use GPU: {} for training".format(args.gpu))

    try:
        if args.distributed:
            if args.dist_url == "env://" and args.rank == -1:
                args.rank = int(os.environ["RANK"])
            if args.multiprocessing_distributed:
                # For multiprocessing distributed training, rank needs to be the
                # global rank among all the processes
                args.rank = args.rank * ngpus_per_node + gpu
            # os.environ['NCCL_ASYNC_ERROR_HANDLING'] = 1
            dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url,
                                    world_size=args.world_size, rank=args.rank)
    except Exeception as e:
        print("distributed cluster: ", e)

    try:
        print("=> creating model '{}'".format(args.arch))
        model = build_model(args.arch, args.depth)
        if args.distributed:
            # For multiprocessing distributed, DistributedDataParallel constructor
            # should always set the single device scope, otherwise,
            # DistributedDataParallel will use all available devices.
            if args.gpu is not None:
                torch.cuda.set_device(args.gpu)
                model.cuda(args.gpu)
                # When using a single GPU per process and per
                # DistributedDataParallel, we need to divide the batch size
                # ourselves based on the total number of GPUs we have
                args.batch_size = int(args.batch_size / ngpus_per_node)
                args.workers = int(args.workers / ngpus_per_node)
                model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu])
            else:
                model.cuda()
                # DistributedDataParallel will divide and allocate batch_size to all
                # available GPUs if device_ids are not set
                model = torch.nn.parallel.DistributedDataParallel(model)
                print("1")
        elif args.gpu is not None:
            torch.cuda.set_device(args.gpu)
            model = model.cuda(args.gpu)
        else:
            # DataParallel will divide and allocate batch_size to all available GPUs
            if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
                model = torch.nn.DataParallel(model)
                model.cuda()
                print("2")
            else:
                model = torch.nn.DataParallel(model).cuda()
                print("3")

    except Exception as e:
        print("transport model to GPU", e)

    cudnn.benchmark = True

    try:
        #warmup
        optimizer_init_lr = args.lr

        if (args.optimizer == 'sgd'):
            optimizer = torch.optim.SGD(model.parameters(), lr=optimizer_init_lr, momentum=args.momentum, weight_decay=args.weight_decay)
        elif (args.optimizer == 'adam'):
            optimizer = torch.optim.Adam(model.parameters(), optimizer_init_lr)
    except Exeception as e:
        print("initialize model: ", e)

    # if this is a paused app to resmue
    try:
        if app.checkpoint:
            model, optimizer, epoch, args.start_iteration, criterion = resume_train_model(model, optimizer, args)
            criterion.cuda(args.gpu)
            print("resumes at epoch {} iter{}".format(epoch, args.start_iteration))
    except Exception as e:
        print("read checkpoint: ", e)

    if scheduler == None:
        try:
            #  scheduler
            if args.lr_scheduler == 'cosine':
                scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs, eta_min=4e-08)
            elif args.lr_scheduler == 'default':
                # my learning rate scheduler for cifar, following https://github.com/kuangliu/pytorch-cifar
                epoch_milestones = [60, 120]
                """Set the learning rate of each parameter group to the initial lr decayed
                    by gamma once the number of epoch reaches one of the milestones
                """
                scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=epoch_milestones, gamma=0.1)
            else:
                raise Exception("unknown lr scheduler")
        except Exception as e:
            print("scheduler: ", e)

    if criterion==None:
        # define loss function (criterion)
        criterion = nn.CrossEntropyLoss().cuda(args.gpu)

        # Data loading code
        # traindir = os.path.join(args.data, 'train')
        # valdir = os.path.join(args.data, 'val')
        # normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
        #                                  std=[0.229, 0.224, 0.225])

        # train_dataset = datasets.ImageFolder(
        #     traindir,
        #     transforms.Compose([
        #         transforms.RandomResizedCrop(224),
        #         transforms.RandomHorizontalFlip(),
        #         transforms.ToTensor(),
        #         normalize,
        #     ]))

        # if args.distributed:
        #     train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
        # else:
        #     train_sampler = None

        # train_loader = torch.utils.data.DataLoader(
        #     train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None),
        #     num_workers=args.workers, pin_memory=True, sampler=train_sampler)

        # val_loader = torch.utils.data.DataLoader(
        #     datasets.ImageFolder(valdir, transforms.Compose([
        #         transforms.Resize(256),
        #         transforms.CenterCrop(224),
        #         transforms.ToTensor(),
        #         normalize,
        #     ])),
        #     batch_size=args.batch_size, shuffle=False,
        #     num_workers=args.workers, pin_memory=True)

        ########################## Get Dataset ###############

    try:
        if "cifar10" in args.arch:
            args.dataset="cifar10"
        else:
            args.dataset="imagenet"
        train_loader, test_loader, train_sampler = get_dataset(args)
    except Exception as e:
        print("dataloading: ", e)

    try:
        # training
        _, sub_conn = app.subprocess_conns[args.rank]
        if args.profile == False:
            for e in range(epoch, args.epochs + 1):
                epoch_start = time.time()
                if args.distributed:
                    train_sampler.set_epoch(epoch)

                # reset start_iteration for new epoch
                if epoch!=e:
                    args.start_iteration = 0
                # train for one epoch
                last_iter = train(train_loader, model, criterion, optimizer, scheduler, e, sub_conn, args)

                if last_iter != args.iteration:
                    epoch_end = time.time()
                    print("app{}-{}: epoch_time for {} iter: {}".format(args.appid, args.rank, last_iter, epoch_end - epoch_start))
                    while 1:
                        if sub_conn.poll(timeout=0.01):
                            message_type = sub_conn.recv()[1]
                            if message_type == "Exit":
                                print("{}: worker: {}-{} exit at epoch {} iter {}".format(datetime.datetime.now(), args.appid, args.rank, e, last_iter))
                                return
                    # return

                # adjust learning rate
                scheduler.step()

                epoch_end = time.time()
                print("app{}-{}: epoch_time: {}".format(args.appid, args.rank, epoch_end - epoch_start))

        sub_conn.send(conn_message("Finished"))

    except Execption as e:
        print("training procedure: ", e)

    # else:
    #     if args.profile_batches == -1:
    #         args.profile_batches = len(train_loader)

    #     with profiler.profile(record_shapes=args.record_shapes, profile_memory=args.profile_memory, use_cuda=args.use_cuda) as prof:
    #         with profiler.record_function(expriment_name):
    #             for epoch in range(1, args.profile_epochs + 1):
    #                 epoch_start = time.time()
    #                 if args.distributed:
    #                     train_sampler.set_epoch(epoch)

    #                 # train for one epoch
    #                 train(train_loader, model, criterion, optimizer, epoch, conn, args)

    #                 # adjust learning rate
    #                 scheduler.step()

    #                 epoch_end = time.time()
    #                 print("epoch_time: {}".format(epoch_end - epoch_start))

    #     # save profile results
    #     profile_name = args.profile_name
    #     trace_name = "{}/{}.json".format(args.work_space, args.rank)
    #     prof.export_chrome_trace(trace_name)

def train(train_loader, model, criterion, optimizer, scheduler, epoch ,conn, args):
    try:
        batch_time = AverageMeter()
        data_time = AverageMeter()
        forward_time = AverageMeter()
        backward_time = AverageMeter()
        update_time = AverageMeter()
        losses = AverageMeter()
        datatrans_time = AverageMeter()
        lossupdate_time = AverageMeter()
        # switch to train mode
        model.train()

        end = time.time()
        iter_inx =  args.start_iteration
        for i, (input, target) in enumerate(train_loader):
            if iter_inx != -1 and iter_inx == args.iteration:
                break
            # measure data loading time
            dataloading_ts = time.time()
            if args.gpu is not None:
                input = input.cuda(args.gpu, non_blocking=True)
            target = target.cuda(args.gpu, non_blocking=True)
            # measure data transfer time
            datatransfer_ts = time.time()
            output = model(input)
            ce_loss = criterion(output, target)
            forward_ts = time.time()
            # get loss from GPU, deprecated for performance concern
            #losses.update(ce_loss.item(), input.size(0))
            loss_ts = time.time()
            # compute gradient and do SGD step
            # replace optimizer.zero_grad(), taking less memory oprations
            for param in model.parameters():
                param.grad = None
            ce_loss.backward()
            backward_ts = time.time()
            # update
            optimizer.step()
            update_ts = time.time()
            # measure elapsed time with breakdown
            start = end
            end = time.time()
            batch_time.update(end - start)
            data_time.update(dataloading_ts - start)
            datatrans_time.update(datatransfer_ts - dataloading_ts)
            forward_time.update(forward_ts - datatransfer_ts)
            lossupdate_time.update(loss_ts - forward_ts)
            backward_time.update(backward_ts - loss_ts)
            update_time.update(update_ts- backward_ts)

            if iter_inx % args.print_freq == 0:
                print("iteration {} takes {}".format(i, end-start))
                conn.send(conn_message("HeartBeat",
                            {"epoch":epoch, "iter": iter_inx ,
                            "iter_time": np.array(batch_time.tracker[-10:len(batch_time.tracker)]).mean(),
                            "dataloading_time: ": np.array(data_time.tracker[-10:len(batch_time.tracker)]).mean()}))

            iter_inx += 1

            # check if to pause in each iteration
            if conn.poll(timeout=0.01):
                message_type = conn.recv()[1]
                if message_type == "Pause":
                    if args.rank == 1 or len(args.cuda_device)==1:
                        save_train_model(epoch, iter_inx, model, optimizer, criterion, scheduler, args)
                    else:
                        time.sleep(2)
                    conn.send(conn_message("Paused", {"iter":iter_inx, "epoch": epoch}))
                    print("{}: worker: {}-{} send a pause signal".format(datetime.datetime.now(), args.appid, args.rank))
                    break

        # df = pd.DataFrame(columns=["iteration","dataloading","datatransfer", "forward","lossupdate", "backward","update","loss"])
        # df["iteration"] = np.array(batch_time.tracker)
        # df["dataloading"] = np.array(data_time.tracker)
        # df["datatransfer"] = np.array(datatrans_time.tracker)
        # df["forward"] = np.array(forward_time.tracker)
        # df["lossupdate"] = np.array(lossupdate_time.tracker)
        # df["backward"] = np.array(backward_time.tracker)
        # df["update"] = np.array(update_time.tracker)
        # #df["loss"] = np.array(losses.tracker[:len(df)])
        # trace_path = os.path.join(args.work_space, "device_{}.csv".format(args.rank))
        # if not os.path.exists(trace_path):
        #     df.to_csv(trace_path, index=False)
        # else:
        #     previous_df = pd.read_csv(trace_path, header=0)
        #     pd.concat([df, previous_df]).to_csv(trace_path, index=False)

        return iter_inx

    except Exception as e:
        raise e

def validate( val_loader, model, criterion, args):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    top5 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    with torch.no_grad():
        end = time.time()
        for i, (input, target) in enumerate(val_loader):
            if args.gpu is not None:
                input = input.cuda(args.gpu, non_blocking=True)
            target = target.cuda(args.gpu, non_blocking=True)

            # compute output
            output = model(input)
            loss = criterion(output, target)

            # measure accuracy and record loss
            acc1, acc5 = accuracy(output, target, topk=(1, 5))
            losses.update(loss.item(), input.size(0))
            top1.update(acc1[0], input.size(0))
            top5.update(acc5[0], input.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            if i % args.print_freq == 0:
                print('Test: [{0}/{1}]\t'
                    'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                    'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                    'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t'
                    'Acc@5 {top5.val:.3f} ({top5.avg:.3f})'.format(
                    i, len(val_loader), batch_time=batch_time, loss=losses,
                    top1=top1, top5=top5))

        print(' * Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f}'
            .format(top1=top1, top5=top5))

    return top5.avg

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0
        self.tracker = []

    def update(self, val, n=1):
        self.val = val
        self.tracker.append(val)
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def accuracy(output, target, topk=(1,)):
    """Computes the accuracy over the k top predictions for the specified values of k"""
    with torch.no_grad():
        maxk = max(topk)
        batch_size = target.size(0)

        _, pred = output.topk(maxk, 1, True, True)
        pred = pred.t()
        correct = pred.eq(target.reshape(1, -1).expand_as(pred))

        res = []
        for k in topk:
            correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)
            res.append(correct_k.mul_(100.0 / batch_size))
        return res

def app_main(conn, app: application):
    os.environ["CUDA_VISIBLE_DEVICES"] = " ".join( str(d)+',' for d in app.cuda_device)[:-1]
    args = app.init_model()

    if args.seed is not None:
        random.seed(args.seed)
        torch.manual_seed(args.seed)
        cudnn.deterministic = True
        warnings.warn('You have chosen to seed training. '
                    'This will turn on the CUDNN deterministic setting, '
                    'which can slow down your training considerably! '
                    'You may see unexpected behavior when restarting '
                    'from checkpoints.')

    if args.gpu is not None:
        warnings.warn('You have chosen a specific GPU. This will completely '
                    'disable data parallelism.')

    if args.dist_url == "env://" and args.world_size == -1:
        args.world_size = int(os.environ["WORLD_SIZE"])

    args.distributed = args.world_size > 1 or args.multiprocessing_distributed
    # ngpus_per_node = torch.cuda.device_count()
    ngpus_per_node = len(app.cuda_device)
    if args.multiprocessing_distributed:
        # Since we have ngpus_per_node processes per node, the total world_size
        # needs to be adjusted accordingly
        args.world_size = ngpus_per_node * args.world_size
        os.environ['MKL_THREADING_LAYER'] = 'GNU'
        for i in range(ngpus_per_node):
            master_conn, sub_conn = Pipe()
            app.subprocess_conns[i] = (master_conn, sub_conn)
        p = Process(target=app_messeger, args=(app, conn, ), name="app_messeger_{}".format(app.appid), daemon=False)
        p.start()
        # Use torch.multiprocessing.spawn to launch distributed processes: the
        # main_worker process function
        mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, app, args), join=True)
    else:
        # Simply call main_worker function
        main_worker(args.gpu, ngpus_per_node, app, args)

# def exec_single_app(app: application, conns):
#     try:
#         _, c_conn = conns
#         # p = Process(target=app_main, args=(child_conn, app,), name="app-{}".format(app.appid), daemon=False)
#         p = Process(target=app_main, args=(c_conn, app, ), name="app-{}".format(app.appid), daemon=False)
#         app.process = p
#         app.process.start()
#     except Exception as e:
#         print("exec app: {}, ".format(app.appid), e)

if __name__ == '__main__':
    app_main()
    # from dl_scheduler import submitter
    # submit_path = "/tmp/home/danlinjia/torchloader/torchloader/dl_scheduler-test.conf.csv"
    # ws_conn, sb_conn = Pipe()
    # sb = submitter(submit_path, sb_conn, time_window=5)
    # sb.read_app_submissions()
    # app = sb.app_infos[0]
    # p_conn, c_conn = Pipe()
    # app_main(c_conn, app)


"""
Este programa encuentra la mayor cantidad de unos en la suma de un numero.
"""
numero = 10

def dec_to_bina(n):
    binario = ""
    while n != 0:
        binario = str(n % 2) + binario
        n = n // 2
    return binario

def cuenta_unos(st):
    return st.count("1")

def maximo(num):
    maxcandy = 0
    maxcandy_n1 = 0
    maxcandy_n2 = 0
    for i in range(1, num+1):
        n1 = i
        n2 = num - i
        bn1 = dec_to_bina(n1)
        bn2 = dec_to_bina(n2)
        cu_bn1 = cuenta_unos(bn1)
        cu_bn2 = cuenta_unos(bn2)
        if cu_bn1 + cu_bn2 > maxcandy:
            maxcandy = cu_bn1 + cu_bn2
            maxcandy_n1 = n1
            maxcandy_n2 = n2
    return maxcandy_n1, maxcandy_n2

print(maximo(numero))


'''
cgat_script_template.py - template for CGAT scripts
====================================================

:Author:
:Release: $Id$
:Date: |today|
:Tags: Python

Purpose
-------

.. Overall purpose and function of the script>

Usage
-----

.. Example use case

Example::

   python cgat_script_template.py

Type::

   python cgat_script_template.py --help

for command line help.

Command line options
--------------------

'''

import sys
import CGAT.Experiment as E
import pyximport

import collections
from CGAT import GTF, IndexedFasta, IOTools
from iCLIP import TranscriptCoordInterconverter
from CGAT.Genomics import complement

import pysam

pyximport.install()
from _simple_align import simple_align

def convert_alignment(read, converter):

    try:
        pos =  converter.genome2transcript((read.reference_start,
                                            read.reference_end))
        if converter.strand == "-":
            pos = (pos[1] + 1 , pos[0] + 1)
        return pos

    except ValueError as e:
        if "not in transcript" in e.message:
            return None
        else:
            raise

def format_alignment(a, l, unaligned_is_5p):
    ''' a is an alignement of structure (score, [start]), (read.start, read.end)
        l is the length of the unaligned segment that has been mapped'''

    alignment, read_pos = a
    a_score, a_start = alignment

    results = ((a_start[0], a_start[0] + l), read_pos)
    if not unaligned_is_5p:
        results = results[::-1]

    return results

def main(argv=None):
    """script main.
    parses command line options in sys.argv, unless *argv* is given.
    """

    if argv is None:
        argv = sys.argv

    # setup command line parser
    parser = E.OptionParser(version="%prog version: $Id$",
                            usage=globals()["__doc__"])
    parser.add_option("-g", "--gtffile", dest="gtffile", type="string",
                      help="GTF file with transcript models to search in")
    parser.add_option("-f", "--fastafile", dest="fastafile", type="string",
                      help="Indexed Fasta file with genome sequence for"
                      "transcripts")
    parser.add_option("-j", "--junctions-file", dest="junctions", type="string",
                      help="Tabix index bed file with junctions to add on"
                      "top of provided GTF file")
    parser.add_option("-m", "--mismatches", dest="mismatches", type="float",
                      default=2,
                      help="Number of mismatches to allow when aligning read"
                      "sections")
    parser.add_option("--minimum-unaligned", dest="min_unaligned", type="int",
                      default=10,
                      help="Minimum number of unaligned bases before"
                      "considering read for realignment")
    parser.add_option("-d", "--minimum-distance", dest="min_dist", type="int",
                      default=5,
                      help="Minimum distance between end of primary alignment"
                      "and novel alignment. This filters out short indels")
    parser.add_option("-t", "--test", dest="test", type="string",
                      help="supply help")
    parser.add_option("-p", "--out-prefix", dest="out_prefix", type="string",
                      default=None,
                      help="Prefix for output files, if not specified bam come out of stdout"
                      "and all others will use input bam name")

    # add common options (-h/--help, ...) and parse command line
    (options, args) = E.Start(parser, argv=argv)

    bamfile = pysam.AlignmentFile(args[0])
    fasta = IndexedFasta.IndexedFasta(options.fastafile)
#    fasta.setConverter(IndexedFasta.getConverter("zero-both-open"))
    junctions = pysam.TabixFile(options.junctions, parser=pysam.asBed())

    stats = collections.Counter()
    inner_dist_dist = collections.Counter()
    xl_dist_dist = collections.Counter()
    gene_counts = collections.Counter()

    if options.out_prefix:
        out_bam_name = options.out_prefix + ".bam"
    elif not options.stdout == sys.stdout:
        options.stdout.close()
        out_bam_name = options.stdout.name
    else:
        out_bam_name = "-"
    out_bam = pysam.AlignmentFile(out_bam_name, "wb", template=bamfile)

    if not options.out_prefix:
        if args[0].endswith(".bam"):
            options.out_prefix = args[0][:-4]
        else:
            options.out_prefix = args[0]

    for gene in GTF.gene_iterator(GTF.iterator(IOTools.openFile(
            options.gtffile))):

        transcripts = list(gene)
        transcripts = [[exon for exon in transcript if exon.feature == "exon"]
                       for transcript in transcripts]
        start = min(exon.start for transcript in gene for exon in transcript)
        end = max(exon.end for transcript in gene for exon in transcript)
        contig = transcripts[0][0].contig
        strand = transcripts[0][0].strand

        gene_exon = GTF.Entry().fromGTF(transcripts[0][0])
        gene_exon.start = start
        gene_exon.end = end
        gene_exon.transcript_id = "gene"

        transcripts.append([gene_exon])

        reads = list(bamfile.fetch(contig, start, end))
        if len(reads) == 0:
            continue

        converters = [TranscriptCoordInterconverter(transcript)
                      for transcript in transcripts]

        transcript_sequences = [GTF.toSequence(transcript, fasta).upper()
                                for transcript in transcripts]
        transcript_junctions = junctions.fetch(contig, start, end, strand)
        junction_starts = [j.start for j in transcript_junctions]
        junction_ends = [j.end for j in transcript_junctions]

        for read in reads:

            if sum(stats.itervalues()) % 1000 == 0:
                E.debug("Analysed %i reads" % sum(stats.itervalues()))

            # Basic read filters ###
            if not read.is_reverse == (strand == "-"):
                # read is on different strand to gene
                stats["wrong strand"] += 1
                continue

            if not (read.reference_start > start and read.reference_end < end):
                stats["not in gene"] += 1
                continue

            if read.is_secondary or read.is_supplementary:
                stats["not primary"] += 1
                continue

            # Find any unaligned sequence in the read ###
            unaligned_start, unaligned_seq, unaligned_is_5p = \
                get_softclipped(read, options.min_unaligned)

            if unaligned_start is None:
                stats["no softclipped"] += 1
                continue

            if not float(options.mismatches).is_integer():
                mm_thresh = options.mismatches * len(unaligned_seq)
            else:
                mm_thresh = options.mismatches

            if (read.reference_end in junction_starts and
                 unaligned_start != 0) or \
               (read.reference_start in junction_ends and
                 unaligned_start == 0):
                stats["junction"] += 1
                continue

            # Get the transcripts and transcript coordinates ###
            aligned_seq_pos, filtered_sequences, filtered_converters = \
                get_read_transcripts(read, transcript_sequences,
                                     converters, start, end, strand)

            # Do the alignments and format results ###
            alignments = [simple_align(s, unaligned_seq.upper())
                          for s in filtered_sequences]

            # add coordinates
            alignments = zip(alignments, aligned_seq_pos)

            # Filter the alignments ###
            min_mm = min(a[0][0] for a in alignments)

            if min_mm > mm_thresh:
                # no good alignments
                stats["No good alignment"] += 1
                continue

            filtered_converters = [c for c, a in zip(filtered_converters,
                                                     alignments)
                                   if a[0][0] == min_mm]
            alignments = [a for a in alignments if a[0][0] == min_mm]

            if any([len(a[0][1]) > 1 for a in alignments]):
                stats["Multiple best hits"] += 1
                continue

            # reformat the alignments in start finish pairs in read order.
            f_alignments = [format_alignment(a, len(unaligned_seq),
                                             unaligned_is_5p)
                            for a in alignments]

            # Find index of alignment with minimum inner distance
            best_inner_dist, best_alignment_index = \
                select_alignement(f_alignments, options.min_dist)

            if best_inner_dist < options.min_dist:
                stats["probable indel"] += 1
                continue

            stats["good"] += 1

            # use alignment with the smallest inner distance as THE alignment

            aligned_converter = filtered_converters[best_alignment_index]
            alignment = alignments[best_alignment_index]
            f_alignment = f_alignments[best_alignment_index]

            # This read pair contains two cross links. One is at the
            # 5' end of the 5' fragment (f_alignment[0]) the other is
            # somewhere in the 3' fragment, (f_alignment[1]). best
            # guess - the centre.

            xl_dist = int(f_alignment[1][1] + f_alignment[1][0])/2 - \
                f_alignment[0][0]

            new_read = build_read(alignment,
                                  len(unaligned_seq),
                                  aligned_converter,
                                  unaligned_start,
                                  best_inner_dist,
                                  xl_dist,
                                  read)

            out_bam.write(read)
            out_bam.write(new_read)
            gene_counts[transcript[0].gene_id] += 1
            inner_dist_dist[best_inner_dist] += 1
            xl_dist_dist[xl_dist] += 1

    inner_dist_fn = options.out_prefix + ".inner_distance_distribution.tsv.gz"
    xl_dist_fn = options.out_prefix + ".crosslink_distance_distribution.tsv.gz"
    gene_counts_fn = options.out_prefix + ".gene_counts.tsv.gz"

    IOTools.writeLines(inner_dist_fn, sorted(inner_dist_dist.iteritems()),
                       ["Distance", "Count"])
    IOTools.writeLines(xl_dist_fn, sorted(xl_dist_dist.iteritems()),
                       ["Distance", "Count"])
    IOTools.writeLines(gene_counts_fn, sorted(gene_counts.iteritems()),
                       ["gene_id", "Count"])

    stat_lines = [map(str, line) for line in stats.iteritems()]
    stat_lines = sorted(stat_lines)
    stat_lines = ["\t".join(line) for line in stat_lines]
    E.info("\n".join(stat_lines))

    # write footer and output benchmark information.
    E.Stop()

def select_alignement(alignments, min_dist):
    '''Returns the index of the alignment with the minimum
    inner distance. Raises NoGoodAlignment if this is less than
    min_dist'''

    inner_dist = [abs(a[1][0]-a[0][1]) for a in alignments]
    min_inner_dist = min(inner_dist)
    min_dist_i = inner_dist.index(min_inner_dist)

    return min_inner_dist, min_dist_i

def get_softclipped(read, min_unaligned):
    '''Finds any parts of the read where the there are more than
    min_unaligned bases are softclipped and returns the start of the
    unaligned sequence on the read, the sequenced of the unaligned
    part of the read and whether this is on the 5' end of the read
    **in physical read coordinates**.

    unaligned sequence on the 5' end of the read is prioritised.

    Returns None, None, None if sufficient unaligned sequence is not
    found'''

    unaligned_is_5p = None
    unaligned_start = None
    unaligned_seq = None

    if read.is_reverse:

        if read.query_length - read.query_alignment_end > min_unaligned:
            unaligned_is_5p = True
            unaligned_start = read.query_alignment_end
            unaligned_seq = complement(read.seq[read.query_alignment_end:])
        elif read.query_alignment_start > min_unaligned:
            unaligned_is_5p = False
            unaligned_start = 0
            unaligned_seq = complement(read.seq[:read.query_alignment_start])
    else:
        if read.query_alignment_start > min_unaligned:
            unaligned_is_5p = True
            unaligned_start = 0
            unaligned_seq = read.seq[:read.query_alignment_start]
        elif read.query_length - read.query_alignment_end > min_unaligned:
            unaligned_is_5p = False
            unaligned_start = read.query_alignment_end
            unaligned_seq = read.seq[read.query_alignment_end:]

    return unaligned_start, unaligned_seq, unaligned_is_5p

def get_read_transcripts(read, transcript_sequences, converters,
                         start, end, strand):
    '''Find which transcripts from the gene the alignment is contained in,
    return a list of the positions in these transcripts, the sequences of
    these transcripts, and the converters for those transcripts'''

    aligned_seq_pos = [convert_alignment(read, converter)
                       for converter in converters]

    assert aligned_seq_pos[-1] is not None

    filtered_sequences = [t for t, c
                          in zip(transcript_sequences, aligned_seq_pos)
                          if c is not None]
    filtered_converters = [c for c, p
                           in zip(converters, aligned_seq_pos)
                           if p is not None]
    aligned_seq_pos = [x for x in aligned_seq_pos if x is not None]

    return aligned_seq_pos, filtered_sequences, filtered_converters

def build_read(alignment, length, converter, unaligned_start,
               inner_dist, xl_dist, old_read):
    '''Build a new read to represent the new supplimentary alignment.'''

    new_read = pysam.AlignedSegment()
    new_read.query_name = old_read.query_name
    new_read.seq = old_read.seq
    new_read.query_qualities = old_read.query_qualities
    new_read.flag = old_read.flag
    new_read.is_supplementary = True
    new_read.reference_id = old_read.reference_id

    if old_read.is_reverse:
        new_read.reference_start = \
                    converter.transcript2genome(alignment[0][1]) - length + 1
    else:
        new_read.reference_start = converter.transcript2genome(alignment[0][1])

    cigar = list()
    if unaligned_start > 0:
        cigar.append((4, unaligned_start))

    cigar.append((0, length))

    if (length + unaligned_start) < old_read.query_length:
        cigar.append((4, old_read.query_length - (length + unaligned_start)))

    cigar = tuple(cigar)
    new_read.cigartuples = cigar

    new_read.template_length = old_read.template_length

    new_read.tags = (("XL", int(xl_dist)),
                     ("XI", int(inner_dist)),
                     ("NM", int(alignment[0][0])))

    return new_read

if __name__ == "__main__":
    sys.exit(main(sys.argv))


#MenuTitle: New Tab with Missing Kerning Pairs
# -*- coding: utf-8 -*-
from __future__ import division, print_function, unicode_literals
__doc__ = """
Compares two glyphs files and opens a new tab with missing kerning pairs for the current master
"""

Glyphs.clearLog()

# generate glyph names to display in the tabs
def glyphStr(f,g):
    if "@" not in g:
        # if it's not a kerning group, it's a glyph id.
        # iterate through the glyphs in the font to find the name.
        return "/" + f.glyphForId_(g).name
    elif "@MMK" in g:
        # if it's a group, just get the last characters in the name
        return "/" + str(g)[7:]

def getID(g1,g2):
	print(g1, g2)
	# print(type(font1.glyphs[g1]), type(font1.glyphs[g2]))

	# if font1.glyphs[g1] is not None:
	if font1.glyphs[g1].rightKerningKey:
		g1 = font1.glyphs[g1].rightKerningKey
# if font1.glyphs[g2] is not None:
	if font1.glyphs[g2].leftKerningKey:
		g2 = font1.glyphs[g2].leftKerningKey
	# g1 = font1.glyphs[g1].rightKerningKey
	# g2 = font1.glyphs[g2].leftKerningKey
	# if "@" not in g1:
	# 	g1 = font1.glyphs[g1].id
	# if "@" not in g2:
	# 	g2 = font1.glyphs[g2].id
	return g1, g2

font1 = Glyphs.fonts[0]
font2 = Glyphs.fonts[1]

masterID = font1.selectedFontMaster.id
print(masterID)

kerningFront = font1.kerning[ masterID ]
kerningBack = font2.kerning[ masterID ]

text = []

pairsFront = []
pairsBack = []

print(kerningFront)

for leftG in kerningFront:
	for rightG in kerningFront[leftG]:
		currentPair = [ glyphStr(font1, leftG), glyphStr(font1, rightG) ]
		currentPair = ''.join(currentPair)
		pairsFront.append(currentPair)

for leftG in kerningBack:
	for rightG in kerningBack[leftG]:
		currentPair = [ glyphStr(font2, leftG), glyphStr(font2, rightG) ]
		currentPair = ''.join(currentPair)
		pairsBack.append(currentPair)

for pair in pairsBack:
	if pair not in pairsFront:
		glyphs = pair[1:].split("/")
		leftG, rightG = getID( glyphs[0], glyphs[1] )
		print(leftG, rightG)
		kernValue = font1.kerningForPair( masterID, leftG, rightG )
		if kernValue > 100000:
			text.append("/n/n" + pair + "/n/n\n")

text = ''.join(text)
font1.newTab(text)

Metrics and Kerning/New Tab with Missing Kerning Pairs.py


import random
from ..components.industry_info import INDUSTRIES
#from ..components.company import Company
from .company import Company

class Industry(object):
    def __init__(self, name, economy):
        self.name = ""
        I = INDUSTRIES[name]
        sizes = I["Sizes"]
        self.size_ranges = []
        for size in sizes:
            self.size_ranges.extend([size for _ in range(sizes[size])])
        avg_keys = ("Revenue per Employee",
                           "Payroll per Employee",
                           "Expense Percentage")
        self.averages = {k: I[k] for k in avg_keys}
        self.make_companies(I["Names"], economy)

    def make_companies(self, names, economy):
        self.companies = []
        for name, symbol in names:
            size = random.randint(*random.choice(self.size_ranges))
            c = Company(name, symbol, size, self, economy)
            self.companies.append(c)


graph = {
    'A': {'B': 2, 'D': 4},
    'B': {'C': 3, 'D': 3},
    'C': {'E': 2},
    'D': {'C': 3, 'E': 4},
    'E': {}
}

'''
Definiujemy listę kosztów i rodziców.
Algorytm wygląda następująco:
1. Wypełniamy listę wierzchołków inf, żeby waga każdego nieznanego wierzchołka była mniejsza.

2. Stoimy na wierzchołku aktualnym wierzchołku (na początku początkowy) i sprawdzamy gdzie możemy iść.
Uzupełniamy tabele najtańszym kosztem.

3. Dodajemy wierzchołek do przeprocesowanych.

4. Teraz musimy znaleźć kolejny wierzchołek o najniższym koszcie oraz taki w którym nie byliśmy.
Przekazujemy koszt (szukamy najmniejszych) oraz przeprocesowane nody (nocy których nie byliśmy).

5. Bierzemy listę kosztów (ten sam case co z tablica inf) porównujemy koszty aby znaleźć najtańszy node
oraz sprawdzamy czy juz tam byliśmy.

6. Zwracamy jego nazwę i zaczynamy od początku z nowy nodem.
Wracamy do punktu 2, robimy tak długo aż (tak jak w tym przypadku dojdziemy do końca grafu.

'''
costs = {}
parents = {}

def get_cheapest_node(costs_dict, processed_list):
    '''5. Z listy kosztów (cost_dic) musimy wybrać najtańszy node którego nie ma w przeprocesowanych.
    Aby to zrobić musimy porównać z nieskończonością i schodzić do najniższego kosztu.
    Nazwa najtańszego wierzchołka defoultowo jest None. W razie, gdyby był to ostatni wierzchołek
    zwróci None i pętla while się zakończy.'''
    cheapest_cost = float('inf')
    cheapest_key = None

    for name_, cost in costs_dict.items():
        if name_ not in processed_list and cost < cheapest_cost:
            cheapest_cost = cost
            cheapest_key = name_
    return cheapest_key

for name in graph.keys():
    '''1.Uzupełniamy tablice kosztów na infinity.'''
    costs[name] = float('inf')

'''1.Ustawiamy tablice odwiedzonych wierzchołków oraz wierzchołek startowy.'''
processed = []
current_node = 'A'
costs[current_node] = 0
parents[current_node] = None

'''2.Wypełniamy koszt sąsiadów jeśli jest najmniejszy, czyli jest to taki wierzchołek do którego możemy się dostać nie był on
przetworzony oraz jest on najtańszy. Warunkiem kontrolnym jest current_node, czyli jeśli jest znany, jeśli nie jest tzn.
że skończyliśmy'''
while current_node:

    for neighbour in graph[current_node]:
        '''2.Sprawdzamy gdzie możemy isć i ewentualnie aktualizujemy cenę - tylko jeśli znaleźliśmy tańszą drogę.
        Dlatego inf na początku.'''
        if costs[neighbour] > costs[current_node] + graph[current_node][neighbour]:
            costs[neighbour] = costs[current_node] + graph[current_node][neighbour]
            parents[neighbour] = current_node
    '''4. Gdy odwiedzimy wszystkie dzieci wierzchołka dodajemy go do odwiedzonych.
    Następnie ustalamy następny najtańszy węzeł (z sumy poprzednich węzełów). '''
    processed.append(current_node)
    current_node = get_cheapest_node(costs, processed)

for d, c in costs.items():
    print(f'The cost for {d} is {c}')


import pytest
from token_thrift.token_thrift import TokenThrift
from token_thrift.api_client.mock_api_client import MockAPIClient
from unittest.mock import MagicMock
import math

class TestTokenThriftRequestProcessing:

    @classmethod
    def setup_class(cls):
        cls.api_key = "sample_api_key"
        cls.budget_in_dollars = 500
        cls.mock_response = "Mock response"
        cls.mock_api_client = MockAPIClient(cls.mock_response)

    def test_process_requests_sequentially(self):
        thrift = TokenThrift(
            self.api_key, self.budget_in_dollars, api_client=self.mock_api_client)
        mock_callback = MagicMock()

        # enqueue some requests
        prompt1 = "Translate the following text to French: 'Hello, world!'"
        prompt2 = "Summarize the book 'To Kill a Mockingbird'"

        total_cost_1 = thrift.convert_tokens_to_dollars(
            thrift.token_count(prompt1), thrift.token_count(self.mock_response))

        total_cost_2 = thrift.convert_tokens_to_dollars(
            thrift.token_count(prompt2), thrift.token_count(self.mock_response))

        thrift.enqueue_request(prompt1)
        thrift.enqueue_request(prompt2)

        thrift.process_requests_sequentially(mock_callback)

        assert mock_callback.call_count == 2  # assert callback was called twice
        assert math.isclose(
            thrift.get_total_dollar_spent(),
            total_cost_1 + total_cost_2,
            rel_tol=1e-9
        )

    # def test_process_requests_concurrently(self):
    #     thrift = TokenThrift(
    #         self.api_key, self.budget_in_dollars, api_client=self.mock_api_client)

    #     # enqueue multiple requests
    #     for _ in range(3):
    #         prompt = "Translate the following text to French: 'Hello, world!'"
    #         estimated_tokens = thrift.estimate_token_count(prompt)
    #         thrift.enqueue_request(prompt, estimated_tokens)

    #     thrift.process_requests_concurrently()

    #     # total tokens spent should be 3 times the estimated tokens per request
    #     assert thrift.get_total_token_spent() == 3 * estimated_tokens


import numpy as np

from scipy import optimize

def _initial_bracket_spinodal_right(C,ID,mu_in,efac=1.0,
                                    dmu=0.5,vmax=1e20,ntry=20,step=+1,reweight_kwargs={},
                                    DeltabetaE_kwargs={}):
    """
    find initial bracketing lnpi_phases of phaseID==ID bracketing point where DeltabetaE_phaseIDS()[ID]==efac

    Parameters
    ----------
    C : lnPi_collection
        initial estimates to work from

    ID : int
        phaseID to work with

    mu_in : list
        list with value of static chem pot, and None for variable. e.g.,
        mu_in=[None,0.0] implies mu[0] is variable, and mu[1]=0.0

    efac : float (Default 1.0)
        cutoff value for spinodal

    dmu : float (Default 0.5)
        factor to kick back if C doesn't already have left and right bounds

    vmax : float (default 1e20)
        value indicating no transition, but phaseID==ID present

    ntry : int (Default 20)
        number of times to try kicking forward/backwards to find bracket

    step : int (Default +1)
        if +1, step forward
        if -1, step backward

    reweight_kwargs : dict
        extra arguments to reweight

    DeltabetaE_kwargs : dict
        extra arguemtns to lnPi.DeltabetaE_phaseIDs

    Returns
    -------
    left,right: lnpi_phases
        left and right bracketing lnPi_phases objects

    """

    #use lnpi_phase reference
    ref = C[0]

    reweight_kwargs = dict(dict(ZeroMax=True),**reweight_kwargs)

    #mu which varies
    mu_idx = mu_in.index(None)

    #delta E
    dE = C.DeltabetaE_phaseIDs(**DeltabetaE_kwargs)[:,ID]
    Omegas = C.Omegas_phaseIDs()[:,ID]

    #find locations where have 'ID'
    msk = C.has_phaseIDs[:,ID]
    if msk.sum()==0:
        raise ValueError('no phaseID %i'%ID)

    w = np.where(msk)[0]

    #left
    left = None
    for i in w[-1::-1]:
        if dE[i]>efac and np.isfinite(Omegas[i]):
            left = C[i]
            break

    if left is None:
        #need to find a new value mu bounding thing
        new_mu = mu_in[:]
        new_mu[mu_idx] = C[w[0]].mu[mu_idx]
        new_mu = np.asarray(new_mu)

        for i in range(ntry):
            new_mu[mu_idx] -= step*dmu

            t = ref.reweight(new_mu,**reweight_kwargs)

            if t.DeltabetaE_phaseIDs(**DeltabetaE_kwargs)[ID]>efac and \
               np.isfinite(t.Omegas_phaseIDs()[ID]):
                left = t
                break
        if left is None:
            raise RuntimeError('could not find left')

    #right
    #first, is there a value in C where has_phase and dE<efac?
    right = None
    for i in w[-1::-1]:
        if dE[i]<efac:
            right = C[i]
            break

    if right is None:
        if w[-1]+1<len(C):
            right = C[w[-1]+1]
        else:
            new_mu = mu_in[:]
            new_mu[mu_idx] = C[w[-1]].mu[mu_idx]
            new_mu = np.asarray(new_mu)

            for i in range(ntry):
                new_mu[mu_idx] += step*dmu

                t = ref.reweight(new_mu,**reweight_kwargs)

                if not t.has_phaseIDs[ID]:
                    right = t
                    break
            if right is None:
                raise RuntimeError('could not find right')

    return left,right

def _refine_bracket_spinodal_right(L,R,ID,efac=1.0,nmax=30,vmax=1e20,vmin=0.0,
                                   reweight_kwargs={},DeltabetaE_kwargs={},close_kwargs={}):
    """
    find refined bracket with efac<DeltabetaE_left<vmax and vmin<DeltabetaE_right<efac

    Parameters
    ----------
    L,R : lnpi_phases objects
        left and right initial brackets

    ID : int
        phaseID to work with

    efac : float (Default 1.0)
        cutoff value for spinodal

    nmax : int (Default 30)
        max number of interations

    vmin,vmax : see above

    reweight_kwargs : dict
        extra arguments to reweight

    DeltabetaE_kwargs : dict
        extra arguments to lnPi_phases.DeltabetaE_phaseIDs

    close_kwargs : dict
        arguments to np.allclose

    Returns
    -------
    left,right : lnpi_phases objects
        left and right phases bracketing spinodal

    r : scipy.optimize.zeros.RootResults object
    """

    ref = L
    reweight_kwargs = dict(dict(ZeroMax=True),**reweight_kwargs)

    doneLeft=False
    doneRight=False

    left = L
    right = R

    close_kwargs = dict(dict(atol=1e-5),**close_kwargs)

    for i in range(nmax):
        v = left.DeltabetaE_phaseIDs()[ID]
        if v<vmax and v>efac:
            doneLeft=True

        v = right.DeltabetaE_phaseIDs(**DeltabetaE_kwargs)[ID]
        if v>vmin and v<efac:
            doneRight=True

        #########
        #checks
        if doneLeft and doneRight:
            #find bracket
            r = optimize.zeros.RootResults(root=None,iterations=i,function_calls=i,flag=1)
            return left,right,r

        ########
        #converged?
        if np.allclose(left.mu,right.mu,**close_kwargs):
            #we've reached a breaking point
            if doneLeft:
                #can't find a lower bound to efac, just return where we're at
                r = optimize.zeros.RootResults(root=left.mu,iterations=i+1,function_calls=i,flag=0)
                setattr(r,'left',left)
                setattr(r,'right',right)
                setattr(r,'info','all close and doneLeft')
                return left,right,r

            #elif not doneLeft and not doneRight:
            else:
                #all close, and no good on either end -> no spinodal
                r = optimize.zeros.RootResults(root=None,iterations=i+1,function_calls=i,flag=1)
                setattr(r,'left',left)
                setattr(r,'right',right)
                setattr(r,'doneLeft',doneLeft)
                setattr(r,'doneRight',doneRight)
                setattr(r,'info','all close and not done Left')
                return None,None,r

        mu_mid = 0.5*(left.mu+right.mu)
        #print(mu_mid)
        mid = ref.reweight(mu_mid,**reweight_kwargs)

        v = mid.DeltabetaE_phaseIDs(**DeltabetaE_kwargs)[ID]

        if v>=efac and np.isfinite(mid.Omegas_phaseIDs()[ID]):
            left = mid
        else:
            right = mid

    print('i',i)
    print(ID)
    print('left mu',left.mu)
    print('right mu',right.mu)
    print('dmu',left.mu-right.mu)

    print('left DE',left.DeltabetaE_phaseIDs())
    print('right DE',right.DeltabetaE_phaseIDs())

    print('doneleft',doneLeft)
    print('doneright',doneRight)
    print('close',np.allclose(left.mu,right.mu,**close_kwargs))

    raise RuntimeError('did not finish')

def _solve_spinodal(ref,ID,mu_in,a,b,efac=1.0,
                reweight_kwargs={},
                argmax_kwargs={},
                phases_kwargs={},
                ftag_phases = None,
                DeltabetaE_kwargs={},
                **kwargs):

    idx = mu_in.index(None)
    reweight_kwargs = dict(dict(ZeroMax=True),**reweight_kwargs)

    def f(x):
        mu = mu_in[:]
        mu[idx] = x
        c = ref.reweight(mu,**reweight_kwargs)

        f.lnpi = c

        return c.DeltabetaE_phaseIDs(**DeltabetaE_kwargs)[ID] - efac

    xx,r = optimize.brentq(f,a,b,full_output=True,**kwargs)

    r.residual = f(xx)
    mu = f.lnpi.mu

    return mu,r,f.lnpi

def _get_step(C,ID,**kwargs):
    """
    find step value on

    if DeltabetaE[-1,ID] - DeltabetaE[0,ID]<0 -> step=+1 (go right)

    else step = -1
    """

    delta = C[-1].DeltabetaE_phaseIDs(**kwargs)[ID] - \
            C[0].DeltabetaE_phaseIDs(**kwargs)[ID]

    if delta == 0:
        raise ValueError('could not determine step, delta==0')

    elif delta < 0.0:
        #go right
        return +1

    else:
        #go left
        return -1

def get_spinodal(C,ID,efac=1.0,
                 dmu=0.5,vmin=0.0,vmax=1e20,ntry=20,step=None,
                 nmax=20,
                 reweight_kwargs={},DeltabetaE_kwargs={},
                 close_kwargs={},
                 solve_kwargs={},full_output=False):
    """
    locate spinodal point for a given phaseID

    Parameters
    ----------
    C : lnPi_collection
        initial estimates to work from.  Function assumes C is in mu sorted order

    ID : int
        phaseID to work with

    mu_in : list
        list with value of static chem pot, and None for variable. e.g.,
        mu_in=[None,0.0] implies mu[0] is variable, and mu[1]=0.0

    efac : float (Default 1.0)
        cutoff value for spinodal

    dmu : float (Default 0.5)
        factor to kick back if C doesn't already have left and right bounds

    vmin : float (default 0.0)
        value denoting vmin, i.e., value of DeltabetaE if phaseID does not exist

    vmax : float (default 1e20)
        value indicating no transition, but phaseID==ID present

    ntry : int (Default 20)
        number of times to try kicking forward/backwards to find bracket

    step : int or None (Default None)
        if +1, step forward
        if -1, step backward
        if None, try to determine step

    nmax : int (Default 20)
        max number of steps to refine bracket

    reweight_kwargs : dict
        extra arguments to reweight

    DeltabetaE_kwargs : dict
        extra arguemtns to lnPi.DeltabetaE_phaseIDs

    close_kwargs : dict
        arguments to np.allclose

    solve_kwargs : dict
        extra arguments to scipy.optimize.brentq

    full_output : bool (Default False)
        if true, return output info object

    Returns
    -------
    out : lnPi_phases object at spinodal point

    r : output info object (optional, returned if full_output is True)

    """
    assert(len(C)>1)

    if step is None:
        step = _get_step(C,ID,**DeltabetaE_kwargs)

    if step == +1:
        CC = C
    elif step == -1:
        CC = C[-1::-1]
    else:
        raise ValueError('bad step')

    msk = C[0].mu != C[1].mu
    assert msk.sum() == 1

    mu_idx = np.where(msk)[0][0]
    mu_in = list(C[0].mu[:])
    mu_in[mu_idx] = None

    #get initial bracket
    L,R = _initial_bracket_spinodal_right(CC,ID,mu_in,efac=efac,
                                          dmu=dmu,vmax=vmax,ntry=ntry,step=step,
                                          reweight_kwargs=reweight_kwargs,
                                          DeltabetaE_kwargs=DeltabetaE_kwargs)

    left,right,rr = _refine_bracket_spinodal_right(L,R,ID,efac=efac,nmax=nmax,
                                                  vmin=vmin,vmax=vmax,
                                                  reweight_kwargs=reweight_kwargs,
                                                  DeltabetaE_kwargs=DeltabetaE_kwargs)

    if left is None and right is None:
        #no spinodal found and left and right are close
        spin = None
        r = rr

    elif rr.converged:
        #converged to a solution

        spin = left

        r = rr
        setattr(r,'bracket_iteration',rr.iterations)
        setattr(r,'from_solve',False)

    else:
        #solve
        if step == -1:
            left,right = right,left

        a,b = left.mu[mu_idx],right.mu[mu_idx]

        mu,r,spin = _solve_spinodal(C[0],ID,mu_in,a,b,
                                    efac=efac,
                                    reweight_kwargs=reweight_kwargs,
                                    argmax_kwargs=C[0]._argmax_kwargs,
                                    phases_kwargs=C[0]._phases_kwargs,
                                    ftag_phases=C[0]._ftag_phases,
                                    DeltabetaE_kwargs=DeltabetaE_kwargs,
                                    **solve_kwargs)

        setattr(r,'bracket_iterations',rr.iterations)
        setattr(r,'from_solve',True)

    if full_output:
        return spin,r
    else:
        return spin


from user_inputs.room_form import FormComponent

def add_form(root, content_frame, forms):
    # Create a new form component
    new_form = FormComponent(content_frame, index=len(forms))
    new_form.pack()

    # Append the new form to the list
    forms.append(new_form)

    # Update the geometry to accommodate the new forms
    root.update_idletasks()
    root.geometry(f"400x{len(forms) * 280}")

def pack_time_label(time_label, forms, hotel_details, cta_btns, time):
    for form in forms:
        form.destroy()

    hotel_details.destroy()

    cta_btns.destroy()

    time_label.pack()
    time_label.set_time(time)


import json
from functools import reduce
from pathlib import Path

import httpx
from dotenv import dotenv_values
from tenacity import retry
from tenacity import stop_after_attempt
from tenacity import wait_random_exponential

from ice.cache import diskcache
from ice.settings import settings
from ice.utils import deep_merge

script_dir = Path(__file__).parent
root_dir = script_dir.parent.parent.parent
config = dotenv_values(root_dir / ".env")

@diskcache()
# try 5 times, because sometimes preview apps take a while to start responding again after they have an issue
@retry(stop=stop_after_attempt(5), wait=wait_random_exponential(2), reraise=True)
def send_elicit_request(*, request_body, endpoint: str):
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {settings.ELICIT_AUTH_TOKEN}",
    }
    chunks = []

    with httpx.stream(
        "POST",
        endpoint,
        json=request_body,
        headers=headers,
        timeout=40,
    ) as r:
        r.raise_for_status()
        for chunk in r.iter_text():
            chunks.append(chunk)

    joined = "".join(chunks)
    split_on_newlines = joined.split("\n")
    as_json = [json.loads(line) for line in split_on_newlines if line]
    merged = reduce(deep_merge, as_json)
    if "detail" in merged:
        raise ValueError(merged["detail"])
    return merged


import os
import random
import shutil
import hashlib
import itertools
import socket

import urllib.request
from urllib import parse
from urllib.error import URLError
from bauta.utils.EnvironmentUtils import EnvironmentUtils
from bauta.utils.SystemUtils import SystemUtils
from bauta.Constants import constants
from random import shuffle

class DatasetGenerator():

    def __init__(self, data_path):
        socket.setdefaulttimeout(10)
        self.data_path = data_path
        self.system = SystemUtils()
        if not os.path.exists(data_path):
            os.makedirs(data_path)
        self.environment = EnvironmentUtils(self.data_path)
        self.logger = self.system.getLogger(self)

    def createConfigurationFile(self, class_names, background_classes):
        try:
            class_names_without_background = list(filter(lambda class_name: class_name != constants.background_label, class_names))
            class_names_without_background.sort()
            configuration_file_path = os.path.join(self.data_path, "config.yaml")
            if not os.path.isfile(configuration_file_path):
                with open(configuration_file_path, 'w') as config_yaml_file:
                    classes_as_string = [f'  - {class_name}\n' for class_name in class_names_without_background]
                    classes_yaml  = 'classes:\n'
                    config_yaml_file.write(classes_yaml)
                    [config_yaml_file.write(class_as_string) for class_as_string in classes_as_string]
                    background_classes_yaml = 'background_classes:\n'
                    config_yaml_file.write(background_classes_yaml)
                    [config_yaml_file.write(f'  - {background_class}\n') for background_class in background_classes]
                    self.logger.info('config.yaml created')
        except BaseException as e:
            self.logger.error(f'Cannot open configuration file "{configuration_file_path}"', e)

    def generateDatasetFromListOfImages(self, images_path, split_test_proportion, download_batch_size, download_images, background_classes):
        self.makeDefaultDirs()
        if images_path is not None:
            if not os.path.isdir(images_path):
                self.logger.error(f'"{images_path}" is not an existing directory')
                return None
            class_names = [os.path.splitext(file_in_path)[0] for file_in_path in os.listdir(images_path) if os.path.isfile(os.path.join(images_path, file_in_path)) and os.path.splitext(file_in_path)[0] != 'config' ]
            for background_class in background_classes:
                if not background_class in class_names:
                    self.logger.error(f'Specified noise class {background_class} is not in the directory {images_path}')
                    return None
            foreground_class_names = set(class_names) - set(background_classes)

            if constants.background_label not in foreground_class_names:
                 self.logger.error(f'The class "background" is compulsory but it was not found. Thus dataset cannot be created.', e)
                 return None
            if len(foreground_class_names) <= 1:
                self.logger.error(f'There should be at least one class besides the "background" one (e.g. "cat" and "background").', e)
                return None
            self.createConfigurationFile(foreground_class_names, background_classes)
            image_paths = [os.path.join(images_path, file_in_path) for file_in_path in os.listdir(images_path) \
                if self.system.hasExtension(os.path.join(images_path, file_in_path), ['txt'])]
            class_names = []
            for image_path in image_paths:
                file_in_path = os.path.basename(image_path)
                class_name = os.path.splitext(file_in_path)[0]
                train_path = self.environment.objectsFolder(class_name, is_train=True)
                test_path  = self.environment.objectsFolder(class_name, is_train=False)
                self.system.makeDirIfNotExists(train_path)
                self.system.makeDirIfNotExists(test_path)

                existing_train_images = self.readImages( os.path.join(train_path, f'{class_name}.txt'), True )
                existing_test_images = self.readImages( os.path.join(test_path, f'{class_name}.txt'), True )
                images = self.readImages(image_path)
                training_set, test_set = self.testTrainSplit(images, existing_train_images, existing_test_images, split_test_proportion)
                shuffle(training_set)
                shuffle(test_set)
                self.writeImages(f'{train_path}/{class_name}.txt', training_set)
                self.writeImages(f'{test_path}/{class_name}.txt', test_set)
                class_names.append(class_name)
        if download_images:
            self.downloadImages(download_batch_size)
        return class_names

    def downloadImages(self, download_batch_size):
        datasets_with_attributes = []
        class_names = []
        for is_train in [True, False]:
            for (class_name, class_path) in self.environment.classesInDatasetFolder(is_train):
                class_file = os.path.join(class_path, f'{class_name}.txt')
                dataset = self.readImages(class_file, True)
                datasets_with_attributes.append({'dataset': self.createGroupedDataset(dataset, download_batch_size), 'class': class_name, 'is_train': is_train})
                class_names.append(class_name)

        datasets = [dataset_with_attributes['dataset'] for dataset_with_attributes in datasets_with_attributes]
        for zipped_dataset in itertools.zip_longest(*datasets):
            for (index, batch_dataset) in enumerate(zipped_dataset):
                if batch_dataset is not None:
                    batch_dataset = list(filter(lambda batch_element: type(batch_element) == type(list()), batch_dataset))
                    self.retrieveImages(batch_dataset, datasets_with_attributes[index]['class'], datasets_with_attributes[index]['is_train'])
        return class_names

    def makeDefaultDirs(self):
        essential_paths = [
            f'{self.data_path}/dataset/augmentation/test',
            f'{self.data_path}/dataset/augmentation/train',
            f'{self.data_path}/dataset/test',
            f'{self.data_path}/dataset/train',
            f'{self.data_path}/dataset/validation',
            f'{self.data_path}/models'
        ]
        [self.system.makeDirIfNotExists(essential_path) for essential_path in essential_paths]

    def createGroupedDataset(self, dataset, download_batch_size):
        return [ dataset[batch : batch + download_batch_size ] for batch in range(0, len(dataset), download_batch_size)]

    def readImages(self, txt_file_path, read_id=False):
        try:
            if not os.path.isfile(txt_file_path):
                return []
            else:
                with open(txt_file_path, 'r') as txt_file:
                    lines = txt_file.read().splitlines()
                    if read_id == False:
                        return lines
                    else:
                        return [line.split(',') for line in lines]
        except BaseException as e:
            self.logger.error(f'Cannot open and parse txt file "{txt_file_path}"', e)
            return []

    def writeImages(self, txt_file_path, image_urls):
        try:
            with open(txt_file_path, 'w') as txt_file_buffer:
                [txt_file_buffer.write(f'{image_id},{image_url}\n') for (image_id, image_url) in image_urls]
        except BaseException as e:
            self.logger.error(f'Cannot open and parse txt file "{txt_file_path}"', e)
            return False
        return True

    def testTrainSplit(self, images, existing_train_images, existing_test_images, split_test_proportion):
        ids_to_images = [(hashlib.md5(bytes(image, encoding='utf-8')).hexdigest(), image) for image in images]
        existing_image_ids = [os.path.splitext(image[0])[0] for image in existing_train_images + existing_test_images]
        new_ids_to_images = list(filter(lambda id_to_image: id_to_image[0] not in existing_image_ids, ids_to_images))
        wanted_test_size = min( len(new_ids_to_images), \
            int( ( len(new_ids_to_images) + len(existing_test_images) + len(existing_train_images) ) * split_test_proportion ) )
        current_test_size = len(existing_test_images)
        test_to_add = wanted_test_size - current_test_size
        if test_to_add > 0:
            ids_to_train_images, ids_to_test_images = new_ids_to_images[test_to_add:], new_ids_to_images[:test_to_add]
        else:
            ids_to_train_images, ids_to_test_images = new_ids_to_images, []
        return ids_to_train_images + existing_train_images, ids_to_test_images + existing_test_images

    def retrieveImages(self, images, class_name, is_train):
        for (image_id, image_url) in images:
            parsed_url = urllib.parse.urlparse(image_url)
            image_url_path = parsed_url.path
            name_and_extension = os.path.splitext(image_url_path)
            if len(name_and_extension) == 2:
                image_extension = name_and_extension[1]
                if image_extension == '':
                    image_extension = '.jpg'
            else:
                image_extension = '.png'
            local_file_path = os.path.join(self.environment.objectsFolder(class_name, is_train), f'{image_id}{image_extension}')
            if os.path.isfile(local_file_path):
                continue
            self.logger.info(f'Retrieving image "{image_url}" for class "{class_name}" and saving it in "{local_file_path}"')
            def retrieveAndStoreImage():
                if parsed_url.scheme:
                    try:
                        urllib.request.urlretrieve(image_url, local_file_path)
                    except URLError as e:
                        self.logger.error(f'Error retrieving image with {image_url}')
                else:
                    shutil.copyfile(image_url, local_file_path)
                return local_file_path
            def validateStoredImage(file_path_destination):
                return os.path.isfile(file_path_destination)
            try:
                self.system.tryToRun(retrieveAndStoreImage, validateStoredImage, 2, silent=True)
            except BaseException as e:
                self.logger.error(f'Cannot download and/or move image id "{image_id}" with URI "{image_url}"', e)


import logging
import signal
import sys
import time
from typing import NamedTuple

import prctl

from . import __version__, settings
from .settings import SETTINGS

logger = logging.getLogger(__name__)

class Server:
    """
    Application server. Intended to be used as singleton. Runs main application
    thread, handles signals and makes sure everything starts up and shuts down
    cleanly.

    Arguments:
        environment: name of runtime environment (ie. 'test', 'development',
            'production')
        cmdline_args: parsed and validated command line arguments.
    """
    def __init__(self, environment: str, cmdline_args: NamedTuple = None):
        settings.init_module(environment, cmdline_args)
        prctl.set_proctitle(SETTINGS.instance_name)

    def before_startup(self):
        """Executed before main thread loop is started."""
        pass

    def startup(self):
        """Starts main thread loop."""
        logger.debug(
            "Starting up '%s' (v%s) application server...",
            SETTINGS.instance_name,
            __version__,
        )

        for signame in ('SIGINT', 'SIGTERM', 'SIGHUP', 'SIGQUIT'):
            signal.signal(
                getattr(signal, signame), lambda signum, frame: self.shutdown(signame)
            )

        self.before_startup()

        logger.info(
            "Started '%s' (v%s) application server...",
            SETTINGS.instance_name,
            __version__,
        )

        while True:
            time.sleep(0.1)

    def before_shutdown(self):
        """Executed before main thread loop is terminated."""
        settings.cleanup_module()

    def shutdown(self, signame):
        """Terminates main thread loop."""
        logger.debug(
            "Initiating shut down of '%s' application server...", SETTINGS.instance_name
        )

        self.before_shutdown()

        logger.info(
            "Application server '%s' (v%s) was shut down. So Long, and Thanks for "
            "All the Fish!",
            SETTINGS.instance_name,
            __version__,
        )

        sys.exit(0)

def create_app(cmdline_args):
    """
    `.Server` factory.

    Arguments:
        cmdline_args(collections.NamedTuple): parsed command line
            arguments.
    """
    return Server(cmdline_args.environment, cmdline_args)


#
# @lc app=leetcode id=842 lang=python3
#
# [842] Split Array into Fibonacci Sequence
#

# @lc code=start
class Solution:
    def splitIntoFibonacci(self, num: str) -> List[int]:
        grps = []
        U32_MAX = 2**31 - 1

        def dfs(seq):
            if not seq:
                return len(grps) >= 3

            if seq[0] == "0":
                if len(grps) < 2 or grps[-1] + grps[-2] == 0:
                    grps.append(0)
                    if dfs(seq[1:]):
                        return True
                    grps.pop()
                return False

            for i in range(1, len(seq) + 1):
                cur = int(seq[:i])
                if cur > U32_MAX:
                    break

                if len(grps) < 2 or cur == grps[-1] + grps[-2]:
                    grps.append(cur)
                    if dfs(seq[i:]):
                        return True
                    grps.pop()
            return False

        dfs(num)
        return grps

# @lc code=end


import requests

URL = "http://www.cl.ecei.tohoku.ac.jp/nlp100/data/neko.txt"

# 本文を抜き出す
response = requests.get(URL)
response.encoding = response.apparent_encoding  # SHIFT_JIS

# 確認
print(response.text)


from django.core.exceptions import ValidationError
from parameterized import parameterized

from recipes.models import Recipe

from .test_recipe_base import RecipeBaseTest

class RecipeTestModelRecipe(RecipeBaseTest):
    def setUp(self) -> None:
        self.recipe = self.make_recipe()

        return super().setUp()

    # Teste individual
    # Para campos semelhantes, há o teste parametrizado logo abaixo
    # def test_recipe_saving_with_title_above_the_allowed(self):
    #     self.recipe.title = 'a'*70

    #     with self.assertRaises(ValidationError):
    #         self.recipe.full_clean()

    #     self.recipe.save()

    # maneira simples e nativa de usar teste parametrizado
    # Desvantagem: não cria contexto por campos
    # def test_recipe_max_length_fields(self):
    #     fields_to_test = [
    #         ('title', 65),
    #         ('description', 255),
    #         ('preparation_time_unit', 65),
    #         ('servings_unity', 65),
    #     ]

    #     for field, max_length in fields_to_test:
    #         setattr(self.recipe, field, 'A' * (max_length + 1))

    #         with self.assertRaises(ValidationError):
    #             self.recipe.full_clean()

    #         self.recipe.save()

    # Maneira com package 'parameterized'
    # https://pypi.org/project/parameterized/
    @parameterized.expand([
        ('title', 65),
        ('description', 255),
        ('preparation_time_unit', 65),
        ('servings_unit', 65),
    ])
    def test_recipe_max_length_fields(self, field, max_length):
        setattr(self.recipe, field, 'A' * (max_length + 1))
        with self.assertRaises(ValidationError):
            self.recipe.full_clean()

    def make_recipe_without_default_fields(
        self,
        author_data=None,
        category_data=None,
        title='Recipe Title',
        description='Recipe Description',
        slug='recipe-slug',
        preparation_time='10',
        preparation_time_unit='Minutos',
        servings='30',
        servings_unit='Pratos',
        preparation_steps='Prepation Steps',
    ):
        if author_data is None:
            author_data = {}

        if category_data is None:
            category_data = {}

        return Recipe.objects.create(
            author=self.make_author(**author_data),
            category=self.make_category(**category_data),
            title=title,
            description=description,
            slug=slug,
            preparation_time=preparation_time,
            preparation_time_unit=preparation_time_unit,
            servings=servings,
            servings_unit=servings_unit,
            preparation_steps=preparation_steps
        )

    def test_recipe_field_preparation_steps_is_html_is_saved_as_default(self):
        recipe = self.make_recipe_without_default_fields(
            category_data={
                'name': 'Category 2'
            },
            author_data={
                'username': 'author-02'
            }
        )

        self.assertFalse(
            recipe.preparation_steps_is_html,
            f'Esperado que "preparation_steps_is_html" seja False\
                mas foi salvo como {recipe.preparation_steps_is_html}'
        )

    def test_recipe_field_is_published_saved_as_default_false(self):
        recipe = self.make_recipe_without_default_fields(
            category_data={
                'name': 'Category 2'
            },
            author_data={
                'username': 'author-02'
            }
        )

        self.assertFalse(
            recipe.is_published,
            f'Esperado que "is_published" seja False\
            mas foi salvo como {recipe.is_published}'
        )

    def test_recipe_if_return_str_cast_is_title_recipe(self):
        needed_title = 'Recipe title to compare'
        self.recipe.title = needed_title
        self.recipe.save()
        self.assertEqual(str(self.recipe), needed_title)


from abc import abstractmethod
from os import close
import re

from sky import getSkyHeadline, getSkyOthers
from bbc import getBbcHeadline, getBbcOthers
from register import getRegister

stories = []

def getAll(list):
        list.append(getSkyHeadline())
        for story in getSkyOthers():
            list.append(story)

        list.append(getBbcHeadline())
        for story in getBbcOthers():
            list.append(story)

        for story in getRegister():
            list.append(story)

def getStories(onlyNew=False):
    if onlyNew == False:    # ie: get all stories
        getAll(stories)

    else:   # ie : get only the stories that have come up since last time it was ran
        with open('previousStories.txt', 'r') as prevStoriestxt:
            prevStoriesRaw = prevStoriestxt.read()
            prevStories = prevStoriesRaw.split('\n')
            allStories = []
            getAll(allStories)
            for story in allStories:
                isValid = True
                for pStory in  prevStories:
                    if pStory == story.title:
                        isValid = False
                        break
                if isValid:
                    stories.append(story)

def printTitles():
    for number, story in enumerate(stories):
        if story.isHeadline:
            print(f"{number + 1}.   HEADLINE:   {story.title}\n")
        elif not story.isHeadline:
            print(f"{number + 1}.    {story.title}\n")

if __name__ == "__main__":
    def showStories(onlyNew=False):
        getStories(onlyNew)
        if len(stories) == 0 and onlyNew == True:
            print("\nThere are no new stories! Come back soon! :)         (or I'm broken)\n\n")

        else:
            printTitles()
            def getFollowInput(maxStory):
                storiesToFollowRaw = input("Please enter the numbers of the stories you would like to follow up on! Enter multiples with spaces inbetween, like so: '2 5 12 3'\n")
                # print("\n")
                follows = storiesToFollowRaw.split(" ")
                isValid = True
                for inputted in follows:
                    # work here
                    if re.search('[a-zA-Z]', inputted) or int(inputted) > maxStory:
                        isValid = False
                        break

                if not isValid:
                    print(f"Please ONLY enter NUMBERS (as numerals eg 1 8 19) between 1 and {maxStory}")
                    return getFollowInput(maxStory)        # this reruns the fn, taking user back to input phase

                else:
                    return follows

                # if re.search('[a-zA-Z]', storyToFollowRaw):
                #     print(f"Please ONLY enter a number between 1 and {maxStory}")
                #     return int(getFollowInput(maxStory))
                # else:
                #     return storyToFollowRaw     # this reruns the fn, taking user back to input phase

            toFollow : list = getFollowInput(maxStory=len(stories))
            for index, story in enumerate(toFollow):
                print(f"{stories[int(story) - 1].title}     {stories[int(story) - 1].link} ")

        # this writes all stories currently on the sites to the previousStories.txt

        with open('previousStories.txt', 'w') as prevStoriestxt:
            currentStories = []
            getAll(currentStories)
            for story in currentStories:
                prevStoriestxt.write(f"{story.title}\n")

    def getMode():
        mode = input("Select mode: \n1. Only show stories that have been published since last run (and are on the front page currently)\n2. Show all stories currently on the front page\n")
        print("\n")      # all this does is make it look nicer in terminal
        if re.search('[a-zA-Z]', mode):
            print(f"Please ONLY enter '1' or '2'")
            return int(getMode())
        else:
            return int(mode)

    mode = getMode()
    # mode = 1        # REMOVE ME
    if mode == 1:        # new stories
        showStories(onlyNew=True)

    elif mode == 2:        # all stories
        showStories()
    else:
        print("Something's gone seriously wrong.")
        close

# bug : bbc module adding a story as headline and normal story