import sys
from lab1.integral.CustomException import CustomException
from lab1.integral.InputReader import InputReader
from lab1.integral.InputValidator import InputValidator
from lab1.integral.TrIntegrator import TrIntegrator

__author__ = 'Sebastian Kubalski'

#class is manager of entire functionality
class ApplicationMgr:
    #constructor
    def __init__(self):
        self._result = None

    #method run entire logic
    def run(self):
        try:
            fileName = InputValidator.inputParamValidator(sys.argv)
            reader = InputReader(fileName)
            InputValidator.isFileValidator(reader.fileName)
            reader.loadData()
            InputValidator.inputDataValidator(reader.getInputData())
            InputValidator.inputTypeValidator(reader.getInputData())
            integralEngine = TrIntegrator(reader.getInputData())
            self._result = integralEngine.integrate()
            self._printResults()
        except KeyboardInterrupt:
            print('Program has been stopped')
        except CustomException as e:
            print(e.msg)
        except Exception:
            print("Something went wrong")

    #printing results of calculating integral
    def _printResults(self):
        print('Result: ' + str(self._result))


from django.shortcuts import get_object_or_404, render, redirect

from .forms import TodoListForm
from .models import TodoList

def home(request):
    if request.user.is_authenticated():
        return redirect('todolists:show_lists')
    return render(request, 'todolists/home.html')

def new_list(request):
    if request.method == 'POST':
        form = TodoListForm(request.POST)
        if form.is_valid():
            new_list = TodoList()
            new_list.title = form.cleaned_data['title']
            new_list.description = form.cleaned_data['description']
            new_list.is_active = True
            new_list.user = request.user
            new_list.save()
            return redirect('todolists:show_lists')
    else:
        form = TodoListForm()
    return render(request, 'todolists/new_list.html', {'form': form})

def show_lists(request, category=None):
    todolists = TodoList.objects.filter(is_active=True, user=request.user)
    return render(request, 'todolists/show_lists.html', {'todolists': todolists})

def list_detail(request, pk):
    list = get_object_or_404(TodoList, pk=pk)
    return render(request, 'todolists/list_detail.html', {'list': list})


import sys
import os
import csv
import h5py
import numpy as np
sys.path.append('../')
from data_parser import DataParser

class UnitTest:
    _data_path = 'unit_test/data'
    _baseline_path = 'unit_test/baseline'
    _result_path = 'unit_test/results'

    def __init__(self, link):
        self._link = link

    def test_DataParser_parse_sample(self):
        data_parser = DataParser(' ')
        for line in self._link:
            try:
                """ generate results """
                dicom_file_path = os.path.join(self._data_path, line)
                contour_file_path = os.path.join(self._data_path, self._link[line])
                shape, img, mask, contour = data_parser.parse_sample(dicom_file_path, contour_file_path)
                np.savez(os.path.join(self._result_path, os.path.splitext(line)[0]) +'.npz', shape=shape, img=img, mask=mask, contour=contour)

                """ Load baseline and do the comparison """
                baseline = np.load(os.path.join(self._baseline_path, os.path.splitext(line)[0]) +'.npz')
                equal1 = np.array_equal(baseline['shape'], shape)
                equal2 = np.array_equal(baseline['img'], img)
                equal3 = np.array_equal(baseline['mask'], mask)
                equal4 = np.array_equal(baseline['contour'], contour)
                if(not(equal1 and equal2 and equal3 and equal4)):
                    return False
            except:
                return False

        return True

    def test_DataParser_save_sample(self):
        data_parser = DataParser(' ')
        data_parser.set_result_path(self._result_path)
        for line in self._link:
            try:
                """ generate results """
                dicom_file_path = os.path.join(self._data_path, line)
                contour_file_path = os.path.join(self._data_path, self._link[line])
                shape, img, mask, contour = data_parser.parse_sample(dicom_file_path, contour_file_path)
                data_parser.save_sample(os.path.splitext(line)[0], shape, img, mask, contour)

                """ Load baseline and do the comparison """
                with h5py.File(os.path.join(self._baseline_path, 'data.h5'), 'r') as hf:
                    sample = hf[os.path.splitext(line)[0]]
                    equal1 = np.array_equal(np.array(sample['shape']), shape)
                    equal2 = np.array_equal(np.array(sample['img']), img)
                    equal3 = np.array_equal(np.array(sample['mask']), mask)
                if(not(equal1 and equal2 and equal3)):
                    return False
            except:
                return False

        return True

def parseLink(link_file):
    with open(link_file) as f:
        reader = csv.reader(f, delimiter=',')
        link_pairs = list(reader)
        link = {}
        for line in link_pairs:
            if len(line) == 2:
                link[line[0]] = line[1]
        return link

if __name__ == '__main__':
    link = parseLink('./unit_test/link.csv')

    unit_test = UnitTest(link);

    if unit_test.test_DataParser_parse_sample():
        print('Succeed    DataParser::parse_sample()')
    else:
        print('Failed     DataParser::parse_sample()')

    if unit_test.test_DataParser_save_sample():
        print('Succeed    DataParser::save_sample()')
    else:
        print('Failed     DataParser::save_sample()')


# In Processing Whitespace-Delimited Data, on page 193, we used the “For
# Line in File” technique to process data line by line, breaking it into pieces
# using the string method split. Rewrite the function process_file to skip the
# header as normal but then use the Read technique to read all the data
# at once.

def skip_header(reader):
    """ (file open for reading) -> str

    Skip the header in reader and return the first real piece of data.

    """
    # Read the description line.
    line = reader.readline()

    # Find the first non-comment line.
    line = reader.readline()
    while line.startswith('#'):
        line = reader.readline()

    # Now line contains the first real piece of data.
    return line

def process_file(reader):
    """ (file open for reading) -> NoneType

    Read and print the data from reader, which must start with a single
    description line, then a sequence of lines beginning with '#', then a
    sequence of data.

    """
    # Find and print the first piece of data.
    line = skip_header(reader).strip()
    print(line)

    # Read the rest of the data.
    # for line in reader:
    #     line = line.strip()
    #     print(line)
    data = reader.read().replace(' ', '')
    print(data)

if __name__ == '__main__':
    with open('hopedale.txt', 'r') as input_file:
        process_file(input_file)


pessoas = ['Clark Kent', 'Bruce Wayne', 'Bruce Banner',
           'Steve Rogers', 'Barry Allen', 'Tony Stark']
herois = ['Superman', 'Batman', 'Hulk', 'Captain America',
          'Flash', 'Iron Man']

herois_dc = {'Superman', 'Batman', 'Wonder Woman', 'Flash',
             'Green Lantern', 'Nightwing'}

identidades_secretas = {}
for (pessoa, heroi) in zip(pessoas, herois):
    if heroi in herois_dc:
        identidades_secretas[heroi] = pessoa

print(identidades_secretas)

print({hero: person for person, hero in zip(pessoas, herois) if hero in herois_dc})

herois_marvel = {'Black Widow', 'Ant-Man', 'Hulk', 'Captain Marvel', 'Captain America', 'Iron Man', 'Vision'}


DIGIT_MAP = {
	'zero': '0',
	'one': '1',
	'two': '2',
	'three': '3',
	'four': '4',
	'five': '5',
	'six': '6',
	'seven': '7',
	'eight': '8',
	'nine': '9',
}

def convert(s):
    """Convert a string to an integer."""
    x = -1
    try:
        number = ''
        for token in s:
            number += DIGIT_MAP[token]
        x = int(number)
        print(f"Conversion succeeded! x = {x}")
    except (KeyError, TypeError):
        pass # this is a special statement that does nothing
    return x
# alternatively, we can do away with the "x = -1" and "pass" above, and simply put "return -1" after the except:
# except (KeyError, TypeError):
#     return -1


from torchvision import transforms, datasets
from modules import lenet
import torch
from tqdm import tqdm
from torch.utils.tensorboard import SummaryWriter

writer = SummaryWriter()
data_path = '/mnt'
batch_size = 256
learning_rate = 1e-3

data_transform = transforms.Compose([
    transforms.Resize((32,32)),
    transforms.ToTensor(),
])

train_data = datasets.MNIST(data_path, train=True, download=True, transform=data_transform)
val_data = datasets.MNIST(data_path, train=False, download=True, transform=data_transform)

train_loader = torch.utils.data.DataLoader(
    train_data,
    batch_size = batch_size,
    shuffle = True,
    num_workers = 4,
)

val_loader = torch.utils.data.DataLoader(
    val_data,
    batch_size = batch_size,
    num_workers = 4,
)

model = lenet.LeNet().cuda()
criterion = torch.nn.CrossEntropyLoss(reduction='sum')
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=2, eta_min=1e-5)

def accuracy(output, target, topk=(1,)):
    """Computes the accuracy over the k top predictions for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)
    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.reshape(1, -1).expand_as(pred))

    return [correct[:k].reshape(-1).float().sum(0) * 100. / batch_size for k in topk]

def train(epoch, log_rate, loader, model, criterion, optimizer, scheduler):
    model.train()
    step = len(loader) // log_rate
    for it, (input, target) in enumerate(tqdm(loader, leave=False)):
        it = len(loader) * epoch + it
        input = input.cuda()
        target = target.cuda()

        output = model(input)

        loss = criterion(output, target)

        optimizer.zero_grad()
        loss.backward()

        optimizer.step()
        scheduler.step()

        lr = optimizer.param_groups[0]["lr"]
        wd = optimizer.param_groups[0]["weight_decay"]
        if it % step == 0:
            writer.add_scalar("train/loss", loss.item(), it)
            writer.add_scalar("train/lr", lr, it)
            writer.add_scalar("train/wd", wd, it)

    print(f"Epoch: [{epoch}] loss: {loss.item()}, lr: {lr}, wd: {wd}")

def validate(epoch, log_rate, loader, model, criterion):
    model.eval()
    step = len(loader) // log_rate

    acc1_global = 0
    acc5_global = 0
    for it, (input, target) in enumerate(tqdm(loader, leave=False)):
        it = len(loader) * epoch + it
        with torch.no_grad():
            input = input.cuda()
            target = target.cuda()

            output = model(input)

        loss = criterion(output, target)
        acc1, acc5 = accuracy(output, target, topk=(1,5))

        if it % step == 0:
            writer.add_scalar("val/loss", loss.item(), it)
            writer.add_scalar("val/acc1", acc1.item(), it)
            writer.add_scalar("val/acc5", acc5.item(), it)

        acc1_global += acc1.item() / len(loader)
        acc5_global += acc5.item() / len(loader)
    print(f"Epoch: [{epoch}] loss: {loss.item()}, acc1: {acc1.item()}, acc5: {acc5.item()}")

    return acc1_global, acc5_global

def main(epoch, train_loader, val_loader, model, criterion, optimizer, scheduler):
    print(f"Total epoch: {epoch}")
    _acc1 = 0.
    _acc5 = 0.
    validate(-1, 1, val_loader, model, criterion)
    for it in range(epoch):
        train(it, 10, train_loader, model,criterion, optimizer, scheduler)
        acc1, acc5 = validate(it, 10, val_loader, model, criterion)
        if it%1 == 0:
            torch.save(model.state_dict(), f'lenet_mnist_{it}.pth')
            print(f"Save ckpt file: lenet_mnist_{it}.pth")

        if acc1 >= _acc1:
            _acc1 = acc1
            _acc5 = acc5

main(100, train_loader, val_loader, model, criterion, optimizer, scheduler)


import os
import glob
import random
import torch
import yaml
import json
import argparse
import numpy as np
import pandas as pd
from tqdm import tqdm
from pprint import pprint
import torch.nn as nn

import asteroid
from asteroid.metrics import get_metrics
from asteroid.losses import *

from asteroid.models import save_publishable
from asteroid.utils import tensors_to_device

from asteroid.models import DPRNNTasNet
from my_data import make_dataloader,gatherCelltypes
from sklearn.metrics import balanced_accuracy_score, roc_auc_score, roc_curve, f1_score
from sklearn.metrics import auc,precision_recall_curve, r2_score
from sklearn.metrics import mean_squared_error, r2_score
from scipy.stats import pearsonr
import seaborn as sns
import pandas as pd
from test_functions import *
from utils import get_logger, parse #device,
from src_ssl.models import *
from src_ssl.models.sepformer_tasnet import SepFormerTasNet, SepFormer2TasNet
import sys
sys.path.append("../")
from my_data import *
from comparison_model import *

parser = argparse.ArgumentParser()
parser.add_argument('--parent_dir', default='final.pth.tar',
                    help='Location to save best validation model')
parser.add_argument("--model", default="ConvTasNet", choices=["ConvTasNet", "DPRNNTasNet", "DPTNet", "SepFormerTasNet", "SepFormer2TasNet", "FC_MOR", "NL_MOR"])
parser.add_argument("--gpu", default="2")
parser.add_argument("--ckpt_path", default="best_model.pth", help="Experiment checkpoint path")
parser.add_argument("--resume", action="store_true", help="Resume-training")
parser.add_argument("--use_gpu", type=int, default=0, help="Whether to use the GPU for model execution")
parser.add_argument("--out_dir", type=str, default="results/best_model", help="Directory in exp_dir where the eval results will be stored")
parser.add_argument('--testset', default="test",
                        help='partition to use')
parser.add_argument('--limit', default=None)
parser.add_argument('--custom_testset', default=None,
                        help='Use a custom testset')
parser.add_argument('--other_testset_name', default=None,
                        help='Use a custom testset')

parser.add_argument('--without_MIC', action='store_true',
                        help='binarize output')
parser.add_argument('--without_OPC', action='store_true',
                        help='binarize output')

parser.add_argument('--without_AST', action='store_true',
                        help='binarize output')
parser.add_argument('--binarize', action='store_true',
                        help='binarize output')
parser.add_argument('--masking', action='store_true',
                        help='binarize output')
parser.add_argument('--pure', action='store_true',
                        help='Test on pure')
parser.add_argument('--save', action='store_true',
                        help='Test on pure')
parser.add_argument("--dataset", default="Brain")

def main(args):
    parent_dir = args.parent_dir
    name = "cvtest"
    if args.dataset == "Brain":
        list_ids = ["13_0038","13_0419",
                    "11_0393",
                    "09_1589", "14_0586",
                    "14_1018",
                    "09_35",
                    "04_38",
                    "06_0615",
                 "03_39",
                    "11_0311",
                    "13_1226"
                    ]
    elif args.dataset == "PBMC":
        list_ids = ["10k_pbmc", "pbmc_10k_nextgem",
                    "pbmc_10k_v1", "SUHealthy_PBMC_B1T2",
                    "pbmc_5k_nextgem", "SUHealthy_PBMC1_B1T1",
                    "frozen_sorted_pbmc_5k", "frozen_unsorted_pbmc_5k",
                    "pbmc_5k_v1", "scATAC_PBMC_D10T1","scATAC_PBMC_D12T1",
                    "scATAC_PBMC_D12T2", "scATAC_PBMC_D12T3",
                    "scATAC_PBMC_D11T1", "pbmc_1k_nextgem",
                    "pbmc_1k_v1", "pbmc_500_nextgem", "pbmc_500_v1"]
    else:
            print("dataset not found %s"%args.dataset)
    res_cv_raw = None
    res_cv_mask = None
    opt_par = parse(args.parent_dir + "train.yml", is_tain=True)
    df_metrics_sub_list = []
    df_metrics_it_list = []
    gt_m = None
    opt_p = parse(args.parent_dir + "train.yml", is_tain=True)
    save_par_dir = args.parent_dir
    if not os.path.exists(save_par_dir):
        os.mkdir(save_par_dir)
    if args.other_testset_name is not None:
            save_par_dir = os.path.join(save_par_dir,args.other_testset_name)
    if not os.path.exists(save_par_dir):
        os.mkdir(save_par_dir)
    for s_id in list_ids:
        args.model_path = os.path.join(parent_dir,
                                    "exp_%s/"%(s_id))
        print(args.model_path)
        model_path = os.path.join(args.model_path, args.ckpt_path)
        savedir = os.path.join(args.model_path, args.testset )
        if not os.path.exists(savedir):
            os.mkdir(savedir)
        if not os.path.exists(savedir):
            os.mkdir(savedir)
        if args.ckpt_path != "best_model.pth":
            savedir = os.path.join(savedir,
                    args.ckpt_path.split("/")[-1].split(".")[0])
        opt = parse(args.model_path + "train.yml", is_tain=True)
        celltypes = opt["datasets"]["celltype_to_use"]
        opt["datasets"]["hdf_dir"]  = os.path.join(opt_p["datasets"]["hdf_dir"],  "_%s"%s_id)
        if args.other_testset_name is not None:
            opt["datasets"]["name"] = args.other_testset_name
            opt["datasets"]["hdf_dir"]  = os.path.join(opt_p["datasets"]["hdf_dir"],  "kp_%s"%s_id)
            savedir = os.path.join(savedir,args.other_testset_name)
            if args.without_AST:
                savedir += "wo_AST"
            if args.without_MIC:
                savedir += "wo_MIC"
            if not os.path.exists(savedir):
                os.mkdir(savedir)
        opt["datasets"]["dataset_dir"]  = opt_p["datasets"]["dataset_dir"]
        num_spk = len(celltypes)

        # all resulting files would be saved in eval_save_dir
        eval_save_dir = os.path.join(args.model_path, args.out_dir)
        if args.custom_testset is not None:
            eval_save_dir = os.path.join(eval_save_dir, args.custom_testset)
        os.makedirs(eval_save_dir, exist_ok=True)

        #limit = 1000
        if args.limit is not None:
            limit = eval(args.limit)
        else:
            limit = None
        if not os.path.exists(os.path.join(eval_save_dir, "final_metrics.json")):
            if args.ckpt_path == "best_model.pth":
                # serialized checkpoint
                model = getattr(asteroid.models, args.model).from_pretrained(model_path)
                print(model_path)
            else:
                # non-serialized checkpoint, _ckpt_epoch_{i}.ckpt, keys would start with
                # "model.", which need to be removed
                model = getattr(asteroid.models, args.model)(**opt["filterbank"], **opt["masknet"])
                all_states = torch.load(args.ckpt_path, map_location="cpu")
                print(args.ckpt_path)
                state_dict = {k.split('.', 1)[1]: all_states["state_dict"][k] for k in all_states["state_dict"]}
                model.load_state_dict(state_dict)
                # model.load_state_dict(all_states["state_dict"], strict=False)

            # Handle device placement
            if args.use_gpu:
                model.cuda()
            model_device = next(model.parameters()).device
            print(model)
            if args.testset == "train":
                use_train = True
            else:
                use_train = False
            val_loader = make_dataloader("test",
                                 is_train=False,
                                    data_kwargs=opt['datasets'],
                                    num_workers=opt['datasets'] ['num_workers'],
                                       batch_size=opt['training']["batch_size"],
                                       limit=limit,
                                       without_MIC=args.without_MIC,
                                       without_AST=args.without_AST,
                                       use_train=use_train,
                                       custom_testset=args.custom_testset,
                                       pure=args.pure)#.data_loader
    #
            series_list = []
            binary = False
            perfs = []
            torch.no_grad().__enter__()

            if limit is None:
                limit=len(val_loader)
            #for idx in tqdm(range(mixtures.shape[0])):
            mixture_ = []
            separate_ = []
            separate_pred = []
            label_ = []
            for i, batch in enumerate(val_loader):
            #if i*len(batch)<limit:
                with torch.no_grad():
                    mix = batch[0]
                    sources = batch[1]
                    label = batch[2]
                # Forward the network on the mixture.
                    mix = tensors_to_device(mix, device=model_device)
                    sources = tensors_to_device(sources, device=model_device)
                    est_sources = model(mix)
                    sig = [0]*len(celltypes)
                    mix_np = mix.cpu().data.numpy()
                    sources_np = sources.cpu().data.numpy()
                    est_sources_np = est_sources.cpu().data.numpy()
                    if args.masking:
                        est_sources_np = est_sources_np*bin_matrix #mask
                    mixture_.append(mix_np)
                    separate_.append(sources_np)
                    separate_pred.append(est_sources_np)
                    label_.append(label)
            mixtures = np.concatenate(mixture_)
            separates = np.concatenate(separate_)
            separates_pred = np.concatenate(separate_pred)
            if not os.path.exists(args.model_path + "MASK.npy"):
                mask = defineMask(separates,
                                    separates_pred,
                                    celltypes,
                                    savedir,
                                    name + "MASK_TRAINSET")
                np.save(args.model_path + "MASK.npy",mask)
            else:
                mask = np.load(args.model_path + "MASK.npy")
                name += "mask"
           # if not val_loader.dataset.gene_filtering is None:
            del model
            separates_pred = separates_pred*mask
            separates = separates*mask
            torch.cuda.empty_cache()
            name = args.testset + "_" + str(args.pure)
            if args.custom_testset is not None:
                name += args.custom_testset
            if args.masking:
                name += "_masking_with_input_"
            if args.save:
                print("Saving ...")
                labels = np.concatenate(label_)
                ppp = parent_dir.split("/")[-2]
                print(ppp)
                tmp = os.path.join("/home/eloiseb/experiments/deconv_rna/",
                                    ppp)
                tmp = os.path.join(tmp,
                                    "exp_kfold_%s/"%(s_id))
                if not os.path.exists(tmp):
                    os.mkdir(tmp)
                np.savez_compressed(os.path.join(tmp, "test_predictions_"
                                                + name+ ".npz"),
                                    mat=separates_pred)
                np.savez_compressed(os.path.join(tmp, "label_"
                                                + name+ ".npz"),
                                    mat=labels)
                np.savez_compressed(os.path.join(tmp, "test_true_"
                                                + name+ ".npz"),
                                    mat=separates)
                np.savez_compressed(os.path.join(tmp, "test_mixtures_"
                                                + name+ ".npz"),
                                    mat=mixtures)
            print(opt['datasets']["hdf_dir"])
            os.remove(os.path.join(opt['datasets']["hdf_dir"],
                                "test.hdf5"))

        separates[np.isnan(separates)] = 0
        separates_pred[np.isnan(separates_pred)] = 0
        if args.without_MIC:
            separates_pred = separates_pred[:, [0,1,2,4,5], :]
            separates = separates[:, [0,1,2,4,5], :]
            print(celltypes)
            celltypes.remove("MIC")
        if args.without_OPC:
            separates_pred = separates_pred[:, [0,1,2,3,5], :]
            separates = separates[:, [0,1,2,3,5], :]
            celltypes.remove("OPCs")
        df_metrics_per_subject= compute_metrics_per_subject(separates_pred,
                            separates,
                            celltypes,
                            label_)
        df_metrics_per_subject["fold"] = "fold_%s"%str(s_id)
        df_metrics_per_it= compute_metrics(separates_pred,
                            separates,
                            celltypes)
        df_metrics_per_it["fold"] = "fold_%s"%str(s_id)
        df_metrics_per_subject.to_csv(os.path.join(savedir, "metrics_per_subjects.csv"))
        df_metrics_per_it.to_csv(os.path.join(savedir, "metrics_per_it.csv"))
        df_metrics_sub_list.append(df_metrics_per_subject)
        df_metrics_it_list.append(df_metrics_per_it)
    df_metrics = pd.concat(df_metrics_it_list)
    df_metrics.to_csv(os.path.join(save_par_dir,
                        "metrics_all_per_it.csv"),
                    index=None)
    df_metrics = pd.concat(df_metrics_sub_list)
    df_metrics.to_csv(os.path.join(save_par_dir,
                        "metrics_all_per_sub.csv"),
                    index=None)

if __name__ == "__main__":

    args = parser.parse_args()
    print(args)
    main(args)


import os
import numpy as np
import torch
import torchvision

import augmentations
from skimage import io

def gpu_check():
    return torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def count_files(dir):
    counter = 0
    for content in os.walk(dir):
        files = content[2]
        counter += len(files)
    return counter

def random_img_split(img):
    h, w = img.shape[0], img.shape[1]
    random_point = np.random.uniform(low=0, high=1)
    h_split, w_split = round(h * random_point), round(w * random_point)

    if h_split - 47 <= 0:
        h_split = h_split + abs(h_split - 48)

    if w_split - 27 <= 0:
        w_split = w_split + abs(w_split - 28)

    new_img = img[h_split - 47:h_split, w_split - 27:w_split]
    return new_img

def combine_images(bg, symb):
    bg_crop = random_img_split(bg)
    augmented_symb = augmentations.symbol_transform(symb).reshape((47, 27))
    augmented_bg_small = augmentations.background_transform(bg_crop).reshape((47, 27))

    bg_small_avg_color = torch.mean(bg_crop)

    if torch.round(bg_small_avg_color) == 0:
        tone_mult = 1

    else:
        tone_mult = 0

    # Actually this one somehow combines these two images
    main_img = (augmented_symb * augmented_bg_small - 0.5) + augmented_symb * (-tone_mult)
    augmented_main_image = augmentations.final_transform(main_img)

    return augmented_main_image

def load_image(path):
    img = torchvision.transforms.ToTensor()(io.imread(path))
    return img


import torch
import torch.nn as nn
from math import sqrt

'''
Pytorch 使用nn.Module定义模型的方法
1.必须继承nn.Module这个类，要让PyTorch知道这个类是一个Module
2.在init(self)中设置好需要的"组件"(如conv,pooling,Linear,BatchNorm等)
3.最后，在forward(self,x)中定义好的“组件”进行组装，就像搭积木，把网络结构搭建出来，这样一个模型就定义好了
'''

# 中间层
class Conv_ReLU_Block(nn.Module):
    def __init__(self):
        super(Conv_ReLU_Block, self).__init__()
        self.conv = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1, bias=False)
        self.relu = nn.ReLU(inplace=True)

    def forward(self, x):
        return self.relu(self.conv(x))

class VDSR(nn.Module):
    def __init__(self):
        super(VDSR, self).__init__()  # 找到Net的父类，并将其初始化。这里继承的是nn.Module类

        # 创建中间层 中间层有18个block,每个block为 Conv + Relu
        self.residual_layer = self.make_layer(Conv_ReLU_Block, 18)

        # 输入的第一个卷积层。 不使用bias,并且先扩大一圈保证输出还是原图大小。  输入为灰度图
        self.input_conv = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=3, stride=1, padding=1, bias=False)

        # 输出卷积层
        self.output_conv = nn.Conv2d(in_channels=64, out_channels=1, kernel_size=3, stride=1, padding=1, bias=False)
        self.relu = nn.ReLU(inplace=True)  # inplace为True，将会改变输入的数据 ，否则不会改变原输入，只会产生新的输出

        for m in self.modules():  # self.modules(), 他会返回该网络中的所有modules.
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels  # k^2 * c
                m.weight.data.normal_(0, sqrt(2. / n))  # He的方法，权值初始化

    def make_layer(self, block, num_of_layer):
        layers = []
        for _ in range(num_of_layer):
            layers.append(block())
        return nn.Sequential(*layers)  # 模块将按照构造函数中传递的顺序添加到模块中

    def forward(self, x):
        ILR = x  # 这里的residual应该是原图
        hidden = self.relu(self.input_conv(x))  # 第一层
        hidden = self.residual_layer(hidden)  # 中间层
        out = self.output_conv(hidden)  # 输出层，无激活函数
        out = torch.add(out, ILR)  # 预测的残差结果和原图相加
        return out


from collections import namedtuple

from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.db.models import Q
from django.dispatch import receiver
from morango.models.core import Store

from kolibri.core.auth.models import AbstractFacilityDataModel

ContentAssignment = namedtuple(
    "ContentAssignment", ["contentnode_id", "source_model", "source_id", "metadata"]
)

DeletedAssignment = namedtuple("ContentAssignment", ["source_model", "source_id"])

# a global registry of usages of the `ContentAssignmentManager` so we can find models where it
# was implemented and gather their assignments
CONTENT_ASSIGNMENT_MANAGER_REGISTRY = {}

class ContentAssignmentManager(object):
    """
    The ContentAssignmentManager is attached to a model, similar to a Django manager, but it's
    purpose is to allow introspection of changes to the model that occur during a sync, and to
    generate 'assignments' that are translated into `ContentRequest`s
    """

    __slots__ = (
        "model",
        "name",
        "one_to_many",
        "filters",
        "lookup_field",
        "lookup_func",
    )

    def __init__(
        self, one_to_many=False, filters=None, lookup_field=None, lookup_func=None
    ):
        """
        :param one_to_many: indicating that the associated model maps to multiple content nodes
        :type one_to_many: bool
        :param filters: dict of QuerySet filters to apply for determining validity of assignments
        :type filters: dict
        :param lookup_field: the model field which maps the content node
        :type lookup_field: str
        :param lookup_func: a function which receives the value of `lookup_field` to return nodes
            and metadata
        :type lookup_func: callable<tuple[contentnode_id: str, metadata: None|dict]>
        """
        self.model = None
        self.name = None
        self.one_to_many = one_to_many
        self.filters = filters
        self.lookup_field = lookup_field
        self.lookup_func = lookup_func

    def contribute_to_class(self, model, attribute_name):
        """
        This is a special method associated with Django. When an instance of this class is
        set on a Django model, it will call this method, which gives the ability to make the
        association with the model

        :type model: AbstractFacilityDataModel
        :type attribute_name: str
        """
        if not issubclass(model, AbstractFacilityDataModel):
            raise ImproperlyConfigured(
                "{} is only valid on AbstractFacilityDataModels".format(
                    self.__class__.__name__
                )
            )

        if self.one_to_many and self.lookup_func is None:
            raise ImproperlyConfigured(
                "One-to-many assignment models need a lookup function"
            )

        self.model = model
        self.name = attribute_name
        setattr(model, attribute_name, self)
        CONTENT_ASSIGNMENT_MANAGER_REGISTRY.update({id(model): self})

    @classmethod
    def on_any_downloadable_assignment(cls, callable_func):
        """
        Connects the provided callable to the post_save signal of all the associated models.
        Executes the callable with any downloadable assignments from the model instances
        if they match the filters (if defined).

        :param callable_func: The callable function to be executed with the new assignments.
        :type callable_func: callable
        """
        for manager in CONTENT_ASSIGNMENT_MANAGER_REGISTRY.values():
            manager.on_downloadable_assignment(callable_func)

    @classmethod
    def on_any_removable_assignment(cls, callable_func):
        """
        Connects the provided callable to the post_save signal of all associated models.
        Executes the callable with any removable assignments from the model instances
        if they match the filters (if defined).

        :param callable_func: The callable function to be executed with the new assignments.
        :type callable_func: callable
        """
        for manager in CONTENT_ASSIGNMENT_MANAGER_REGISTRY.values():
            manager.on_removable_assignment(callable_func)

    @classmethod
    def find_all_downloadable_assignments(
        cls, dataset_id=None, transfer_session_id=None
    ):
        """
        :param dataset_id: optional argument to filter assignments by dataset
        :param transfer_session_id:
        :rtype: list of ContentAssignment
        """
        if (dataset_id is None) == (transfer_session_id is None):
            raise ValueError(
                "One parameter needs specified: dataset_id and transfer_session_id"
            )

        for manager in CONTENT_ASSIGNMENT_MANAGER_REGISTRY.values():
            for assignment in manager.find_downloadable_assignments(
                dataset_id, transfer_session_id
            ):
                yield assignment

    @classmethod
    def find_all_removable_assignments(cls, dataset_id=None, transfer_session_id=None):
        """
        :param dataset_id: optional argument to filter assignments by dataset_id
        :param transfer_session_id:optional argument to filter assignments by transfer_session_id
        :rtype: list of ContentAssignment or DeletedAssignment
        """
        if (dataset_id is None) == (transfer_session_id is None):
            raise ValueError(
                "One parameter needs specified: dataset_id and transfer_session_id"
            )

        for manager in CONTENT_ASSIGNMENT_MANAGER_REGISTRY.values():
            for assignment in manager.find_removable_assignments(
                dataset_id, transfer_session_id
            ):
                yield assignment

    def _get_modified_store(self, transfer_session_id):
        """
        Queryset for finding the Store records affected by a sync
        """
        return Store.objects.filter(
            model_name=self.model.morango_model_name,
            last_transfer_session_id=transfer_session_id,
        )

    def _get_assignments(self, model_qs):
        """
        Shared method that iterates of a model queryset, finding `lookup_field` and calling
        `lookup_func` if necessary for each result

        :param model_qs: The queryset to find `lookup_field` and sources
        :type model_qs: django.db.models.QuerySet
        :rtype: list of ContentAssignment
        """
        for source_id, lookup_field_value in model_qs.values_list(
            "id", self.lookup_field
        ):
            # avoid duplicate contentnode_ids for same source
            contentnode_ids = set()
            assignments = []

            # when one_to_many, we could get more than one contentnode assignment per source
            if self.one_to_many:
                assignments.extend(self.lookup_func(lookup_field_value))
            elif self.lookup_func is not None:
                assignments.append(self.lookup_func(lookup_field_value))
            else:
                assignments.append((lookup_field_value, None))

            for contentnode_id, metadata in assignments:
                if contentnode_id not in contentnode_ids:
                    yield ContentAssignment(
                        contentnode_id,
                        self.model.morango_model_name,
                        source_id,
                        metadata,
                    )
                    contentnode_ids.update([contentnode_id])

    def on_downloadable_assignment(self, callable_func):
        """
        Connects the provided callable to the post_save signal of the attached model.
        Executes the callable with the downloadable assignments from the model instance
        if it matches the filters (if defined).

        :param callable_func: The callable function to be executed with the new assignments.
        :type callable_func: callable
        """
        # since this is a local function, we need use `weak=False` to prevent garbage collection
        @receiver(models.signals.post_save, sender=self.model, weak=False)
        def on_save(sender, instance, **kwargs):
            queryset = self.model.objects.filter(pk=instance.pk)
            if self.filters:
                queryset = queryset.filter(**self.filters)
            if queryset.exists():
                assignments = self._get_assignments(queryset)
                callable_func(instance.dataset_id, assignments)

    def on_removable_assignment(self, callable_func):
        """
        Connects the provided callable to the post_save signal of the associated models.
        Executes the callable with the removable assignments from the model instance
        if it matches the filters (if defined).

        :param callable_func: The callable function to be executed with the new assignments.
        :type callable_func: callable
        """
        # since these are local functions, we need use `weak=False` to prevent garbage collection
        @receiver(models.signals.post_save, sender=self.model, weak=False)
        def on_save(sender, instance, **kwargs):
            queryset = self.model.objects.filter(pk=instance.pk)
            if self.filters:
                queryset = queryset.exclude(**self.filters)
            if queryset.exists():
                assignments = self._get_assignments(queryset)
                callable_func(instance.dataset_id, assignments)

        @receiver(models.signals.post_delete, sender=self.model, weak=False)
        def on_delete(sender, instance, **kwargs):
            callable_func(
                instance.dataset_id,
                [DeletedAssignment(self.model.morango_model_name, instance.pk)],
            )

    def find_downloadable_assignments(self, dataset_id=None, transfer_session_id=None):
        """
        :param dataset_id: optional dataset_id to filter records by
        :param transfer_session_id: optional transfer_session_id to filter records by transfer_session_id
        :return: yields ContentAssignment tuples
        :rtype: list of ContentAssignment
        """
        if (dataset_id is None) == (transfer_session_id is None):
            raise ValueError(
                "One parameter needs specified: dataset_id and transfer_session_id"
            )

        model_qs = self.model.objects.all()
        if transfer_session_id:
            modified_store = self._get_modified_store(transfer_session_id).exclude(
                Q(deleted=True) | Q(hard_deleted=True) | ~Q(deserialization_error="")
            )
            model_qs = model_qs.filter(
                pk__in=modified_store.values_list("id", flat=True)
            )
        elif dataset_id:
            model_qs = model_qs.filter(dataset_id=dataset_id)

        if self.filters:
            model_qs = model_qs.filter(**self.filters)

        for assignment in self._get_assignments(model_qs):
            yield assignment

    def find_removable_assignments(self, dataset_id=None, transfer_session_id=None):
        """
        :param dataset_id: the ID of the dataset to filter records by dataset_id
        :param transfer_session_id: the ID of the transfer session to filter records by transfer_session_id
        :return: yields ContentAssigment or DeletedAssignment tuples
        :rtype: list of ContentAssignment or DeletedAssignment
        """
        if (dataset_id is None) == (transfer_session_id is None):
            raise ValueError(
                "One parameter needs specified: dataset_id and transfer_session_id"
            )

        model_qs = self.model.objects.all()
        if transfer_session_id:
            modified_store = self._get_modified_store(transfer_session_id)
            model_qs = model_qs.filter(
                pk__in=modified_store.values_list("id", flat=True)
            )

            # models that were deleted
            deleted_qs = modified_store.filter(Q(deleted=True) | Q(hard_deleted=True))
            for source_id in deleted_qs.values_list("id", flat=True):
                yield DeletedAssignment(self.model.morango_model_name, source_id)
        elif dataset_id:
            model_qs = model_qs.filter(dataset_id=dataset_id)

        if self.filters:
            # modified models that do not match filters
            model_qs = model_qs.exclude(**self.filters)

        for assignment in self._get_assignments(model_qs):
            yield assignment


import time
import requests

url = "http://www.momoshop.com.tw/search/"

fh = {"user-agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36"}

r = requests.get(url + "searchShop.jsp?keyword=HTC", headers=fh)

if r.status_code == requests.codes.ok :
    r.encoding = "big5"
    print (r.text)
else :
    print ("HTTP請求錯誤 :"+ url)


from os import listdir
from os.path import isfile,join
import cv2

path = '/home/limon/workspace/datasets/trafficLabels/'
Read_path = path + 'labels/'
img_path = path + 'images/'

for f in listdir(Read_path):
    if isfile(join(Read_path, f)):
        name,ext = f.split('.')
        img = cv2.imread(img_path + name + '.jpg')
        h,w,_ = img.shape
        print(name)
        f = open(Read_path+f, "r")
        lines = f.readlines()
        for line in lines:
            data = line.split(",")
            obj = data[0]
            x1 = int(data[1])
            y1 = int(data[2])
            x2 = int(data[3])
            y2 = int(data[4])
            cv2.rectangle(img,(x1,y1),(x2,y2),(0,255,0),3)

        f.close()
        cv2.imshow('image',img)
        k = cv2.waitKey(0)
        if k == 27:         # wait for ESC key to exit
            exit()
        elif k == ord('s'):
            cv2.destroyAllWindows()


import scrapy
#import requests
import re

class EG(scrapy.Spider):
    name = "eg1"
    start_urls = ['https://www.shoppersstop.com/haute-curry-womens-tie-up-neck-printed-churidar-suit/p-203566837']

    def parse(self, response):
        import pdb;pdb.set_trace()
        cookies = response.headers.get('Set-Cookie','')
        j_session = ''.join(re.findall('JSESSIONID=(.*?) \Path',cookies))
        j_session = j_session.replace(';','')
        csrf_token = ''.join(response.xpath('//div[@class="user-container user-icons log-member"]/input[@id="ajaxCSRF"]/@value').extract())
        cookies={}
        cookie = literal_eval(json.loads(json.dumps(str(headers)))).get('Set-Cookie', [])
        for i in cookie:
            data_key  = i.split(';')[0]
            if data_key:
                try: key, val = data_key.split('=', 1)
                except : continue
                cookies.update({key.strip():val.strip()})
        headers = {
            'origin': 'https://www.shoppersstop.com',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'en-US,en;q=0.9,te;q=0.8',
            'x-requested-with': 'XMLHttpRequest',
            'save-data': 'on',
            'pragma': 'no-cache',
            'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
            'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
            'accept': '*/*',
            'cache-control': 'no-cache',
            'authority': 'www.shoppersstop.com',
            'referer': 'https://www.shoppersstop.com/haute-curry-womens-tie-up-neck-printed-churidar-suit/p-203566837',
        }

        data = {
          'qty': '1',
          'baseProductCode': '203566837',
          'productCodePost': '203566842',
          'CSRFToken': csrf_token,
        }
        response = requests.post('https://www.shoppersstop.com/cart/add', headers=headers, data=data)


import sys
import os
import re
from threading import Thread
try:
    from IPython.kernel.zmq.kernelapp import IPKernelApp
    ZMQ = True
except:
    # had trouble finding kernel.zmq.  Try:
    #   $ pip install -U ipython pyzmq
    ZMQ = False
try:
    import gdb
except:
    print("Either your gdb is not > gdb 7")
    print("Or you are trying to run this without gdb")
    print("Exiting . . .")
    sys.exit(1)

if not "ATP" in os.environ:
    print(os.environ)
    print("Must define ATP breakpoint locations. Exiting . . .")
    sys.exit(1)
else:
    # define dua and atp break point locations
    # must be in format of gdb break points (ie, file : line, *address, symbol_name)
    atp_loc = os.environ['ATP']

BUG_EFFECT_COUNT = 0
SIG_EVENT_COUNT = None
record_regex = re.compile(".*?Log contains ([0-9]+) instructions\..*",
                          re.MULTILINE)
event_regex = re.compile(".*?Current event: ([0-9]+).*", re.MULTILINE)

def get_instr_count():
    data = gdb.execute("info record", to_string=True)
    m = re.search(record_regex, data)
    if m == None:
        print("coulnd't find instruction count in [info record] command")
        print(data)
    return int(m.groups()[0])

def get_event_count():
    data = gdb.execute("when", to_string=True)
    m = re.search(event_regex, data)
    if m == None:
        print("coulnd't find event count in when command")
        print(data)
        assert 0
    return int(m.groups()[0])

# bp_num is int
def launch_debug_using_ipython():
    # run this from a gdb session in order to create a
    # ipython session one could connect to for gdb_python symbol
    # symbol support
    import IPython
    if not ZMQ:
        IPython.embed()
    else:
        IPython.embed_kernel()
        """
        After this you will see:
            NOTE: When using the `ipython kernel` entry point, Ctrl-C will not work.

            To exit, you will have to explicitly quit this process, by either
            sending
            "quit" from a client, or using Ctrl-\ in UNIX-like environments.

            To read more about this, see
            https://github.com/ipython/ipython/issues/2049

            To connect another client to this kernel, use:
                    --existing kernel-138767.json
        To connect to this ipython kernel, from another terminal on same machine
        type:
            $ ipython console --existing kernel-138767.json
        Note that the kernel number will change with each run of the debugging
        application
        To exit, type quit in the ipython terminal, which will give you control
        of the gdb session
        """

def get_bp_hits(bp_num):
    data = gdb.execute("info b {}".format(bp_num), to_string=True)
    hit_str = "breakpoint already hit "
    if not hit_str in data:
        return 0
    else:
        return int(data.split(hit_str)[1].split()[0])

EXIT_LOC = "exit"

class ATP_Breakpoint(gdb.Breakpoint):
    def stop(self):
        global SIG_EVENT_COUNT
        # if (gdb.execute("info record", to_string=True) ==
            # "No record target is currently active.\n"):
            # gdb.write("Starting recording process")
        # else:
            # gdb.write("Hit ATP again.  Restarting . . .")
            # gdb.execute("record stop")
        # gdb.execute("record full")
        gdb.execute("when")
        print("!! Hit ATP !!")
        # print "Instruction Count =", SIG_EVENT_COUNT - get_event_count()
        print("Events =", SIG_EVENT_COUNT, get_event_count())
        gdb.execute("q")
        sys.exit(0)
        return True

class Exit_Breakpoint(gdb.Breakpoint):
    def stop(self):
        ret_data = gdb.execute("info arg", to_string=True)
        ret_code = int(ret_data.split(" = ")[1])
        print("At program exit normal with status: {}".format(ret_code))
        print("Instruction Count = {}".format(get_instr_count()))
        gdb.execute("when")
        # print "Instruction Count = {}".format(get_instr_count())
        ATP_Breakpoint(atp_loc)
        # gdb.execute("set scheduler-locking on")
        gdb.post_event(lambda s: gdb.execute("reverse-continue"))
        gdb.post_event(lambda s: gdb.execute("reverse-continue"))
        # gdb.execute("set scheduler-locking off")
        # gdb.execute("reverse-continue")
        # gdb.execute("q")
        # sys.exit(1)

# class GdbCommand():
    # # def __init__(self, cmd):
        # # self.cmd = cmd
    # def (self):
        # cmd = self.args[0]
        # # gdb.execute(self.cmd)
        # # gdb.write("GDB MESSAGE")
        # print "HERE HERE"
        # print "About to execute: [{}]".format(self.cmd)
        # gdb.execute(self.cmd)

def event_handler (event):
    def handle_sig_event (event):
        if isinstance(event, gdb.SignalEvent):
            if event.stop_signal in ["SIGSEGV", "SIGABRT"]:
                print("Found a SIG {}".format(event.stop_signal))
                print(gdb.execute("p $_siginfo._sifields._sigfault.si_addr",
                            to_string=True))
                # print gdb.execute("info proc mappings", to_string=True)
                gdb.execute("when")
                # print "Instruction Count = {}".format(get_instr_count())
                ATP_Breakpoint(atp_loc)
                def print_fn(s):
                    print(s)
                    return True
                # gdb.post_event(lambda s: print_fn("hello") and gdb.execute("reverse-continue"))
                # gdb.post_event(lambda s: print_fn("hello2") and gdb.execute("reverse-continue"))
                # gdb.post_event(lambda s: print_fn("hello2") and gdb.execute("reverse-continue"))
                # gdb.execute("set scheduler-locking on")
                # gdb.post_event(lambda s: gdb.execute("reverse-continue"))
                # gdb.post_event(lambda s: gdb.execute("reverse-continue"))
                # gdb.execute("set scheduler-locking off")
                try:
                    global SIG_EVENT_COUNT
                    if SIG_EVENT_COUNT == None:
                        SIG_EVENT_COUNT = get_event_count()
                        print("SIG_EVENT_COUNT: {}".format(SIG_EVENT_COUNT))
                    gdb.execute("reverse-continue")
                    gdb.execute("reverse-continue")
                except gdb.error:
                    pass
                # gdb.execute("reverse-continue")
                # gdb.execute("q")
                # sys.exit(1)
            else:
                # print "Instruction Count = {}".format(get_instr_count())
                print("Reached unhandled signal event: {}".format(event.stop_signal))
                print("Exiting . . .")
                gdb.execute("q")
                sys.exit(1)

    # generic StopEvent handler.  We will assume that we only get here from gdb
    # def handle_stop_event (event):
        # if isinstance(event, gdb.StopEvent):
            # global BUG_EFFECT_COUNT
            # BUG_EFFECT_COUNT += 1
            # print "HANDLING STOP EVENT: STEP COUNT {}".format(BUG_EFFECT_COUNT)
            # if BUG_EFFECT_COUNT > 100000:
                # print "Instruction Count = {}".format(BUG_EFFECT_COUNT)
                # gdb.execute("q")
            # else:
                # # gdb.post_event(run_gdb_si)
                # print "posting an event a si thread from stop event"
                # run_gdb_si()
                # # thread = Thread(target=run_gdb_si)
                # # thread.start()

    if isinstance(event, gdb.SignalEvent):
        handle_sig_event(event)
    # assume we get here from beginning of rr thread stop point
    elif isinstance(event, gdb.StopEvent):
        gdb.execute("c")
        # pass
        # handle_stop_event(event)

gdb.execute("set breakpoint pending on", to_string=True)
gdb.execute("set pagination off", to_string=True)
# gdb.execute("set logging on", to_string=True)
# gdb.execute("set logging file /nas/ulrich/mygdb.log", to_string=True)
# gdb.execute("set verbose off", to_string=True)
gdb.execute("set confirm off", to_string=True)
gdb.execute("set width 0", to_string=True)
gdb.execute("set height 0", to_string=True)
# maybe this will redirect program output to dev/null
gdb.execute("tty /dev/null", to_string=True)

# gdb.execute("break " + atp_loc, to_string=True)
# ATP_Breakpoint(atp_loc)
# set breakpoints on normal exit of program and SEGFAULTS
Exit_Breakpoint(EXIT_LOC)
# gdb.execute("break " + EXIT_LOC, to_string=True)
# gdb.execute("handle SIGSEGV stop")
gdb.execute("handle all stop", to_string=True)

# establish callback on breakpoints
gdb.events.stop.connect(event_handler)

# uncomment this line of code to trigger an ipython kernel session that you can
# console into.  See launch_debug_using_ipython() for more info
# gdb.execute("c")
# import IPython; IPython.embed_kernel()
# gdb.execute("si")


'''
    plot some Twitter Places on a Folium map
'''
import csv
import math
import pandas as pd
import os.path

# Map related:
import folium

import branca.colormap as cm

from folium.plugins import FloatImage
url = 'img/accu-snow-weather-sm.jpg'

# data set-up:
locs = 'places_grouped.csv' # lats longs and names
dfin = pd.read_csv(locs)
df = dfin.set_index(['_id'])
all = df.index.tolist()
# print(df.head(3))
# print(df.columns)
# print(df.describe())
with_states = df['place.0.name'].str.contains('\s[A-Z][A-Z]$', na=False, regex=True)
num_states = with_states.sum()

# states = df[df[1].str.extract('\s[A-Z][A-Z]$')].sum()

# MAPS and their params
filename = os.path.basename(__file__)
output_osm = filename.split('.')[0] + '.html'
zoom_start = 7
centre = [df['place.0.bounding_box.coordinates.0.0.1'].mean(), df['place.0.bounding_box.coordinates.0.1.0'].mean()] # Pennsylvania-ish
df = df.sort_values(['place.0.place_type'], ascending=[True])

# Setup the map:
fmap = folium.Map(location=centre, max_zoom=30, zoom_start=zoom_start, control_scale=True, tiles="stamenterrain")

# box styles
styles = {'opacity' : 0.2, 'color' : '#323232', 'fill_color':'red' }
threshold = 5000 # max no of count we will allow
adjust_scale_color = 100
linear = cm.LinearColormap(['green', 'yellow', 'red'], vmin=1, vmax=round(threshold/adjust_scale_color)) # 'yellow',
linear.caption = 'Tweet intensity (scaled)'

# for point marker size
radius = 10

# different place_types for Places.
colors = {'admin' : 'black', 'city' : 'purple' , 'poi' : '#318633' , 'neighborhood' : 'blue', 'country' : 'white'}
idx = 0
remove = ['country']

feature_groups = colors.copy()
# make this have a group for each place_type so they can all be toggled.
for key, value in feature_groups.items():
    if key not in remove:
        feature_groups[key] = folium.FeatureGroup(name='enable place_type: ' + key)

marker_cluster = folium.MarkerCluster("enable place_type: point-of-interest").add_to(fmap)

# fixme - will calculate the color based on the count (tweet intensity)
for row in df.itertuples():
    idx += 1

    # popup text
    index, count, name, place_type = row[:4]

    # skip some values that just screw up the map
    if (not isinstance(place_type, str)) or (not isinstance(name, str)) or\
     (place_type in remove) or (name[-6:] == 'Canada') or (count > threshold):
        continue # i.e. omit all these ones

    # bounding box corners for every Place
    bottom_left = list()
    top_right = list()

    # boxes coords:
    bottom_left.append(row[5])
    bottom_left.append(row[4])
    top_right.append(row[9])
    top_right.append(row[8])

    # now build markers for map and their popups:
    string = '%s: %d tweets. Type:%s' % (name, count, place_type)
    if bottom_left[0] != top_right[0]:
        marker = folium.features.RectangleMarker(
            bounds=[bottom_left, top_right],
            color=colors[place_type],
            fill_color=linear(count),
            fill_opacity=styles['opacity'],
            popup=string)
    else:
        # icon=folium.Icon(color='red', icon='info-sign')
        # sometimes the Places are point coordinates!
        marker = folium.RegularPolygonMarker(
            location=bottom_left,
            radius=radius,
            color=colors[place_type],
            weight=0,
            fill_color=linear(count),
            number_of_sides=6,
            popup=string)

    if(place_type == 'poi'):
        marker.add_to(marker_cluster)
    else:
        feature_groups[place_type].add_child(marker)

fmap.add_child(linear)
for key, feature_group in feature_groups.items():
    if (key not in remove) and (key is not 'poi'):
        fmap.add_child(feature_group)
# folium.TileLayer('openstreetmap').add_to(fmap)
folium.TileLayer('Cartodb Positron').add_to(fmap)
folium.LayerControl(collapsed=False).add_to(fmap)

# todo: add in choro layer with states totals as per:
# https://nbviewer.jupyter.org/github/python-visualization/folium/blob/master/examples/Quickstart.ipynb
# need to make a df from States from full_name last letters to do this.
# output both maps
FloatImage(url, bottom=3, right=1).add_to(fmap)
fmap.save(output_osm)


# Ejercicio 1. Elementos de una sucesión menores o iguales a 100

## Enunciado

# Crea las funciones que devuelvan las listas con los elementos de las siguientes sucesiones que sean menores o iguales a 100. Utiliza para ello sólo range.

# - Primera sucesión: 1, 4, 7, 10, 13,...
# - Segunda sucesión: -12, -7, -2, 3, 8,...

def successions(start, step):
    """
    Crea una lista de números, desde inicio hasta 100, que sumen
    de step en step
    """
    finalList = []
    for numer in range(start, 100, step):
        finalList.append(numer)

    return finalList

print(successions(-8,10))


import torch
import math
from torch import nn
from torch.nn import functional as F

class MAtt(nn.Module):
    def __init__(self, N=5, K=1):
        super(MAtt, self).__init__()
        self.N = N
        self.K = K

    def forward(self, fspt, fqry):
        # fspt: [B, N*K, C, H, W]
        # fqry: [B, N*num_query, C, H, W]

        b, nk, c, h, w = fspt.size()
        nq = fqry.size(1)
        n = self.N

        prototype = fspt.view(b, self.N, self.K, c, h, w).mean(dim=2) # [B, N, C, H, W]
        prototype = prototype.view(b, n, c, -1).unsqueeze(1) # [B, 1, N, C, H*W]
        fqry = fqry.view(b, nq, c, -1).unsqueeze(2) # [B, N*num_query, 1, C, H*W]

        prototype_norm = F.normalize(prototype, p=2, dim=3).transpose(3, 4) # [B, 1, N, H*W, C]
        fqry_norm = F.normalize(fqry, p=2, dim=3) # [B, N*num_query, 1, C, H*W]

        correlation_mat = torch.matmul(prototype_norm, fqry_norm) # [B, N*num_query, N, H*W, H*W]

        ap = correlation_mat.mean(dim=4, keepdim=True).transpose(3, 4) # [B, N*num_query, N, 1, H*W]
        ap = F.softmax(ap/0.25, dim=4) + 1
        aq = correlation_mat.mean(dim=3, keepdim=True) # [B, N*num_query, N, 1, H*W]
        aq = F.softmax(aq/0.25, dim=4) + 1

        prototype_adp = (prototype*ap).view(b, nq, n, c, h, w)
        fqry_adp = (fqry*aq).view(b, nq, n, c, h, w)

        return prototype_adp, fqry_adp


import requests
from pathlib import Path
from bs4 import BeautifulSoup
import bs4
from docx import Document
from docx.shared import RGBColor
from docx.shared import Pt
from docx.enum.text import WD_PARAGRAPH_ALIGNMENT
from ebooklib import epub
import re

# def saveToTxt(contents:list):
#     fileName='唐诗三百首.txt'
#     with open(fileName,'a',encoding='utf-8') as f:
#         print('写入中...'+contents[0])
#         for rowContent in contents:
#             f.write(rowContent+'\n')

def getHtmlText(url:str)->str:
    try:
        simBrowser={
            'User-Agent':'Chrome/114.0.5735.106',
            'Accept-Encoding': 'gzip, deflate',
            'Accept': '*/*',
            'Connection': 'keep-alive'
        }
        requests.url
        r=requests.get(url,headers=simBrowser,timeout=30)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        return r.text
    except:
        return str()

def getPoetryContent(soup:BeautifulSoup,contentList:list,string:str):
    for br_tag in soup.find_all('br'):
        br_tag.replace_with('\n')
    for tag in soup.find_all(id='contson'+string):
        content=tag.text.strip()
        content=re.sub(r'(\(|（).*?(\)|）)','',content)
        contentList.append(content)
    return contentList

def clearDocx(fileName:str):
    path=Path(fileName)
    if path.exists():
        doc=Document(fileName)
        for element in doc.element.body:
            doc.element.body.remove(element)
        doc.save(fileName)

def addEmptyLine(doc:Document):
    text=doc.add_paragraph()
    format=text.paragraph_format
    #format.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
    format.line_spacing = 1.0  # 设置为1.0表示单倍行距
    format.space_before = 0
    format.space_after = format.space_before

def paragraphCommonFormat(run,format):
    run.font.name = 'SimSun'
    run.font.color.rgb = RGBColor(0, 0, 0)
    format.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER
    format.line_spacing = 1.0  # 设置为1.0表示单倍行距
    format.space_before = 0
    format.space_after = format.space_before

def setDocxTitle(fileName:str,string:str,rank:int):
    path=Path(fileName)
    doc:Document=Document()
    if path.exists():
        doc=Document(fileName)
    title=doc.add_paragraph(string,'Heading'+str(rank))
    titleRun=title.runs[0]
    titleParagraph_format = title.paragraph_format
    paragraphCommonFormat(titleRun,titleParagraph_format)
    #addEmptyLine(doc)
    doc.save(fileName)

def writeToDocx(fileName:str,contents:list):
    path=Path(fileName)
    doc:Document=Document()
    if path.exists():
        doc=Document(fileName)
    pos=0
    for content in contents:
        paragraph=None
        if pos==0:
            paragraph=doc.add_paragraph(content,'Heading3')
        # elif pos==1:
        #     paragraph=doc.add_paragraph(content,'Heading4')
        else:
            paragraph=doc.add_paragraph(content)
        textRun=paragraph.runs[0]
        Paragraph_format = paragraph.paragraph_format
        paragraphCommonFormat(textRun,Paragraph_format)
        pos+=1
    #addEmptyLine(doc)
    doc.save(fileName)

def saveToDocx(htmlText:str):
    bookName='唐诗三百首'
    saveFileName='/home/linux/Documents/books/{}.docx'.format(bookName)
    clearDocx(saveFileName)
    #setDocxTitle(saveFileName,bookName,1)
    subTitlePos=0
    regex = re.compile(r'(\(|（).*?(\)|）)')
    for div in BeautifulSoup(htmlText,'html.parser').find_all(attrs='typecont'):
        subTitlePos+=1
        setDocxTitle(saveFileName,str(subTitlePos)+'、'+div.div.text,1)
        for tag in div(name='span'):
            poetryName=tag.text.strip()
            author=regex.search(poetryName)
            poetryName=regex.sub('',poetryName)
            contentList=[]
            contentList.append(poetryName)
            if(author):
                contentList.append(author.group(0))
            newUrl='https://so.gushiwen.cn'+tag.a.get('href')
            getPoetryContent(BeautifulSoup(getHtmlText(newUrl),'html.parser'),contentList,newUrl[-17:-5])
            writeToDocx(saveFileName,contentList)
            print('爬取中...'+tag.text.strip())

def saveToEpub(htmlText:str):
    bookName='唐诗三百首'
    saveFileName='/home/linux/Documents/books/{}.epub'.format(bookName)
    book = epub.EpubBook()
    book.set_title(bookName)
    book.set_language('zh')
    subTitlePos=0
    regex = re.compile(r'(\(|（).*?(\)|）)')
    for div in BeautifulSoup(htmlText,'html.parser').find_all(attrs='typecont'):
        subTitlePos+=1
        mainTitle=str(subTitlePos)+'、'+div.div.text
        mainHtmlName='{}.xhtml'.format(mainTitle)
        mainHtmlContent="
{}
".format(mainTitle)
        mainChapter=epub.EpubHtml(title=mainTitle,file_name=mainHtmlName,content=mainHtmlContent)
        book.add_item(mainChapter)
        chapterList:list[epub.EpubHtml]=[]
        for titleTag in div(name='span'):
            subTitle=titleTag.text.strip()
            poetryName=subTitle
            author=regex.search(poetryName)
            poetryName=regex.sub('',subTitle)
            subHtmlName='{}.xhtml'.format(subTitle)
            suHtmlContent="
{}
".format(poetryName)
            suHtmlContent+='
'
            if(author):
                suHtmlContent+=author.group(0)+'
'
            newUrl='https://so.gushiwen.cn'+titleTag.a.get('href')
            for contentTag in BeautifulSoup(getHtmlText(newUrl),'html.parser').find_all(id='contson'+newUrl[-17:-5]):
                for br_tag in contentTag.find_all('br'):
                    br_tag.replace_with('@')
                content=re.sub(r'@', "
",contentTag.text.strip())
                content=regex.sub('',content)
                suHtmlContent+=content
            suHtmlContent+='
'
            subChapter=epub.EpubHtml(title=subTitle,file_name=subHtmlName,content=suHtmlContent)
            book.add_item(subChapter)
            chapterList.append(subChapter)
            print('爬取中...'+subTitle)
        book.toc.append((epub.Section(mainTitle,mainHtmlName),tuple(chapterList)))
        book.add_item(epub.EpubNcx())
        book.add_item(epub.EpubNav())
    epub.write_epub(saveFileName, book,{})

def CrawlTangPoetry(htmlText:str):
    #saveToDocx(htmlText)
    saveToEpub(htmlText)

def main():
    print('开始爬取！')
    url='https://so.gushiwen.cn/gushi/tangshi.aspx'
    htmlText=getHtmlText(url)
    if len(htmlText)>0:
        print('爬取成功！')
        CrawlTangPoetry(htmlText)
    else:
        print('爬取失败！')

main()


#This seems to be working 5/16/2020
import random
import time
import copy
deckinit = [
2,2,2,2,
3,3,3,3,
4,4,4,4,
5,5,5,5,
6,6,6,6,
7,7,7,7,
8,8,8,8,
9,9,9,9,
10,10,10,10,
10,10,10,10,
10,10,10,10,
10,10,10,10,
11,11,11,11,
]
purse = 100
decks = copy.copy(deckinit)
def single_deal(deck) :
    #INITIAL DEAL
    global purse
    total = 0
    dealer_total = 0
    print("Purse:", purse)
    while True :
        print("How much is your bet?")
        bet = input()
        bet = int(bet)
        if bet > purse or bet <= 0 :
            print('You can only bet up to your purse total\nYour purse is', purse)
        else:
            break
    purse -= bet
    card1 = deck.pop(random.randint(0, len(deck) - 1))
    dealer_hidden = deck.pop(random.randint(0, len(deck) - 1))
    card2 = deck.pop(random.randint(0, len(deck) - 1))
    dealer_shown = deck.pop(random.randint(0, len(deck) - 1))
    dealer_hand = [dealer_hidden, dealer_shown]
    player_hand = [card1, card2]
    print('Dealer: [  '+ str(dealer_shown) +']')
    print('Your Hand:', player_hand)

    #IF BLACKJACK
    if 10 in player_hand and 11 in player_hand and 11 not in dealer_hand :
        time.sleep(1)
        print('Dealer Hand: ', dealer_hand)
        time.sleep(1)
        print('Black Jack!')
        time.sleep(1)
        purse += bet * 2
        return
    # insurance
    if dealer_shown == 11:
        print('Would you like to buy insurance?')
        ins = input()
        if 'y' in ins and dealer_hidden == 10:
            print('Insurance Bet:', bet/2)
            time.sleep(1)
            print('Dealer Hand:', dealer_hand)
            purse += bet
            return
        elif 'y' in ins :
            print('Insurance Bet:', bet/2)
            purse -= bet/2
            print('Purse:', purse)

    #GAMEPLAY
    while sum(player_hand) <= 21 :
        time.sleep(1)
        hit = input('Hit or Stay? ')
        if 'h' in hit:
            time.sleep(1)
            dealt = deck.pop(random.randint(0, len(deck) - 1))
            print("Dealt:", dealt)
            player_hand.append(dealt)
            print('Your Hand:', player_hand)
            if 11 in player_hand and sum(player_hand) > 21:
                Ace = player_hand.index(11)
                player_hand.remove(11)
                player_hand.insert(Ace, 1)
                print('Your Hand:', player_hand)
            continue
        else:
            break

    #IF PLAYER BUSTS
    if sum(player_hand) > 21 :
        print("Bust! You Lose!")
        return
    print("Dealer:", dealer_hand)
    time.sleep(1)

    #DEALERS TURN
    while sum(dealer_hand) < 17 :
        dealer_card = deck.pop(random.randint(0, len(deck) - 1))
        dealer_hand.append(dealer_card)
        print('Dealer Hits:', dealer_card)
        time.sleep(1)
        print('Dealer Hand:',dealer_hand, "=", sum(dealer_hand))
        time.sleep(1)
    #DEALER BUSTS
    if sum(dealer_hand) > 21:
        print("Dealer Busts! You Win!")
        purse += bet * 2
        return
    #WHO WINS?
    if sum(dealer_hand) == sum(player_hand) :
        print("Push")
        return
    elif sum(player_hand) > sum(dealer_hand) :
        print("You win!")
        purse += bet *2
        return
    elif sum(dealer_hand) > sum(player_hand) :
        print('Dealer Wins!')
        return
    print('Your Hand:', player_hand, "=", sum(player_hand))
single_deal(decks)
print("Purse:",purse)
print('Play again?')
again = input('y/n')
while again == 'y' :
    single_deal(decks)
    decks = copy.copy(deckinit)
    print("Purse:",purse)
    print('Play again?')
    again = input('y/n: ')
quit()


import re

inp = "target area: x=20..30, y=-10..-5"
inp = "target area: x=111..161, y=-154..-101"

match = re.search("x=(-?\d+)\.\.(-?\d+), y=(-?\d+)..(-?\d+)", inp)
assert match
x1, x2, y1, y2 = map(int, match.groups())
x_target_min, x_target_max = sorted([x1, x2])
y_target_min, y_target_max = sorted([y1, y2])

def simulate(x_vel, y_vel):
    x = 0
    y = 0
    max_y = 0
    while x < x_target_max and y > y_target_min:
        x += x_vel
        y += y_vel
        max_y = max(max_y, y)
        x_vel = max(x_vel - 1, 0)
        y_vel -= 1

        if x_target_min <= x <= x_target_max and y_target_min <= y <= y_target_max:
            return max_y

    return None

max_height = max(simulate(x, y) or 0 for x in range(200) for y in range(200))
print("Part 1:", max_height)

total_hits = sum(simulate(x, y) is not None for x in range(200) for y in range(-200, 200))
print("Part 2:", total_hits)


"""BazaFilmowa3 URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path

from filmy import views

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', views.index),
    path('osoby/', views.show_osoby),
    path('osoby/<int:pk>/', views.osoba_detail),
    path('add_osoba/', views.add_osoba),
    path('filmy/', views.show_filmy),
    path('add_filmy/', views.add_filmy),
    path('film/<int:pk>/', views.film_detail),
    path('film/<int:film_pk>/add_actor/', views.AddActorToMovie.as_view()),
    path('add_genre/', views.add_genre ),
    path('genre/', views.genre),
    path('delete_osoby/<int:pk>/', views.delete_osoby ),
    path('set_session/', views.SessionFunView.as_view()),
    path('show_session/', views.ShowSession.as_view()),
    path('cookiemonster/', views.CookieeMonsterFunView.as_view()),
    path("delete_cookie/",views.DeleteCookiee.as_view())
]


from pysat.spectral.spectral_data import spectral_data
from pysat.regression import regression
from pysat.regression import cv
from pysat.plotting.plots import make_plot, pca_ica_plot
import pandas as pd
from PYSAT_UI_MODULES.Error_ import error_print
from PYSAT_UI_MODULES.del_layout_ import *
from PyQt4.QtCore import QThread
from PyQt4 import QtCore
import numpy as np

class pysat_func(QThread):
    taskFinished = QtCore.pyqtSignal()

    def __init__(self):
        QThread.__init__(self)
        self.leftOff = 0
        self.data = {}  # initialize with an empty dict to hold data frames
        self.datakeys = []
        self.models = {}
        self.modelkeys = []
        self.model_xvars = {}
        self.figs = {}
        self.fun_list = []
        self.arg_list = []
        self.kw_list = []
        self.greyed_modules = []

    """
    Getter and setter functions below
    """

    def set_fun_list(self, fun, replacelast=False):
        if replacelast:
            self.fun_list[-1] = fun
        else:
            self.fun_list.append(fun)

    def set_arg_list(self, args, replacelast=False):
        if replacelast:
            self.arg_list[-1] = args
        else:
            self.arg_list.append(args)

    def set_kw_list(self, kws, replacelast=False):
        if replacelast:
            self.kw_list[-1] = kws
        else:
            self.kw_list.append(kws)

    def set_greyed_modules(self, modules, replacelast=False):
        if replacelast:
            self.greyed_modules[-1] = modules
        else:
            self.greyed_modules.append(modules)

    def getDataKeys(self):
        return self.datakeys

    def getModelKeys(self):
        return self.modelkeys

    def getData(self):
        return self.data

    def getModels(self):
        return self.models

    """
    Work functions below
    """

    def set_file_outpath(self, outpath):
        try:
            self.outpath = outpath
            print("Output path folder has been set")
        except Exception as e:
            error_print(e)

    def get_data(self, filename, keyname):
        try:
            print('Loading data file: ' + str(filename))
            self.data[keyname] = spectral_data(pd.read_csv(filename, header=[0, 1]))
            self.datakeys.append(keyname)
            pass
        except Exception as e:
            error_print('Problem reading data: {}'.format(e))

    def removenull(self,datakey,colname):
        try:
            print(self.data[datakey].df.shape)
            self.data[datakey]=spectral_data(self.data[datakey].df.ix[-self.data[datakey].df[colname].isnull()])
            print(self.data[datakey].df.shape)

        except Exception as e:
            error_print(e)

    def do_mask(self, datakey, maskfile):
        try:
            self.data[datakey].mask(maskfile)
            print("Mask applied")
        except Exception as e:
            error_print(e)

    def do_interp(self, datakey_to_interp, datakey_ref):
        print(self.data[datakey_ref].df.columns.levels[0])
        try:
            self.data[datakey_to_interp].interp(self.data[datakey_ref].df['wvl'].columns)
        except Exception as e:
            error_print(e)

    def do_pca(self, datakey, nc, col, load_fit=None):
        print(self.data[datakey].df.columns.levels[0])
        try:
            self.data[datakey].pca(col, nc=nc, load_fit=load_fit)
        except Exception as e:
            error_print(e)

    def do_ica(self, datakey, nc, col, load_fit=None):
        try:
            self.data[datakey].ica(col, nc=nc, load_fit=load_fit)
        except Exception as e:
            error_print(e)

    def do_ica_jade(self, datakey, nc, col, load_fit=None, corrcols=None):
        try:
            self.data[datakey].ica_jade(col, nc=nc, load_fit=load_fit, corrcols=corrcols)
        except Exception as e:
            error_print(e)

    def do_norm(self, datakey, ranges):
        print("{}".format(ranges))
        try:
            print(self.data[datakey].df.columns.levels[0])
            self.data[datakey].norm(ranges)
            print(self.data[datakey].df.columns.levels[0])
            print("Normalization has been applied to the ranges: " + str(ranges))
        except Exception as e:
            error_print(e)

    def do_strat_folds(self, datakey, nfolds, testfold, colname):
        self.data[datakey].stratified_folds(nfolds=nfolds, sortby=colname)

        self.data[datakey + '-Train'] = self.data[datakey].rows_match(('meta', 'Folds'), [testfold], invert=True)
        self.data[datakey + '-Test'] = self.data[datakey].rows_match(('meta', 'Folds'), [testfold])
        self.datakeys = self.data.keys()

        print(self.data.keys())
        print(self.data[datakey + '-Test'].df.index.shape)
        print(self.data[datakey + '-Train'].df.index.shape)

    def do_regression_train(self, datakey, xvars, yvars, yrange, method, params, ransacparams, modelkey=None):
        try:
            if modelkey is None:
                modelkey = method+'-'+str(yvars)+' ('+str(yrange[0])+'-'+str(yrange([1])+') ')
            self.models[modelkey] = regression.regression([method], [yrange], [params], i=0, ransacparams=[ransacparams])
            self.modelkeys.append(modelkey)

            x = self.data[datakey].df[xvars]
            y = self.data[datakey].df[yvars]
            x=np.array(x)
            y=np.array(y)
            ymask = np.squeeze((y > yrange[0]) & (y < yrange[1]))
            y=y[ymask]
            x=x[ymask,:]
            self.models[modelkey].fit(x, y)
            self.model_xvars[modelkey] = xvars

        except Exception as e:
            error_print(e)

    def do_cv_train(self, datakey, xvars, yvars, method, params):

        try:
            cv_obj=cv.cv(params)
            self.data[datakey].df,self.cv_results=cv_obj.do_cv(self.data[datakey].df,xcols=xvars,ycol=yvars)
            self.data['CV Results']=self.cv_results

        except Exception as e:
            error_print(e)

    def do_regression_predict(self, datakey, modelkey, predictname):
        try:
            prediction = self.models[modelkey].predict(self.data[datakey].df[self.model_xvars[modelkey]])
            self.data[datakey].df[predictname] = prediction
            pass
        except Exception as e:
            error_print(e)

    def do_submodel_predict(self):
        pass

    def do_plot(self, datakey,
                xvar, yvar,
                figfile=None, xrange=None,
                yrange=None, xtitle='Reference (wt.%)',
                ytitle='Prediction (wt.%)', title=None,
                lbl=None, one_to_one=False,
                dpi=1000, color=None,
                annot_mask=None,
                cmap=None, colortitle='', figname=None, masklabel='',
                marker='o', linestyle='None'
                ):

        try:
            x = self.data[datakey].df[xvar]
            y = self.data[datakey].df[yvar]
        except:
            x=self.data[datakey][xvar]
            y=self.data[datakey][yvar]
        try:
            loadfig = self.figs[figname]
        except:
            loadfig = None
            # outpath=self.outpath
        try:
            # Alpha is missing, fix this!
            outpath = self.outpath
            self.figs[figname] = make_plot(x, y, outpath, figfile, xrange=xrange, yrange=yrange, xtitle=xtitle,
                                             ytitle=ytitle, title=title,
                                             lbl=lbl, one_to_one=one_to_one, dpi=dpi, color=color,
                                             annot_mask=annot_mask, cmap=cmap,
                                             colortitle=colortitle, loadfig=loadfig)
        except Exception as e:
            error_print(e)
            # dealing with the a possibly missing outpath
            outpath = './'
            self.figs[figname] = make_plot(x, y, outpath, figfile, xrange=xrange, yrange=yrange, xtitle=xtitle,
                                           ytitle=ytitle, title=title,
                                           lbl=lbl, one_to_one=one_to_one, dpi=dpi, color=color,
                                           annot_mask=annot_mask, cmap=cmap,
                                           colortitle=colortitle, loadfig=loadfig)

    def do_pca_ica_plot(self, datakey,
                        x_component,
                        y_component,
                        figfile,
                        colorvar=None,
                        cmap='viridis',
                        method='PCA'
                        ):
        pca_ica_plot(self.data[datakey], x_component, y_component, colorvar=colorvar, cmap=cmap, method=method,
                     figpath=self.outpath, figfile=figfile)

    def __del__(self):
        self.wait()

    def del_layout(self):
        del_qwidget_(self.greyed_modules[-1])
        if len(self.greyed_modules) > 0:
            del self.greyed_modules[-1]

    def run(self):
        # TODO this function will take all the enumerated functions and parameters and run them
        try:
            for i in range(self.leftOff, len(self.fun_list)):
                print(self.fun_list[i])
                self.fun_list[i](*self.arg_list[i], **self.kw_list[i])
                self.greyed_modules[i].setDisabled(True)
                self.leftOff = i + 1
                self.taskFinished.emit()
        except:
            self.taskFinished.emit()


import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore
import os

def get_Current_Path(file_target):
    current_directory = os.getcwd()
    port_path = current_directory.replace("\\", "/")
    real_path = port_path+file_target
    #real_path = file_path.replace("\\", "/")
    return real_path

# Set up
# cred = credentials.Certificate("C:/Users/Supakij/Documents/GitHub/testbase_back-end/backend/serviceAccountKey.json")
cred = credentials.Certificate(get_Current_Path("/backend/database/serviceAccountKey.json"))
firebase_admin.initialize_app(cred)

db = firestore.client()

# Add a document to the collection
# Replace "my-collection" with the desired collection name
data = ({'basicsubject':'test','coursecode':533433,'coursename':'null','credit':'null','teachername':'null','test':'test'})
db.collection('Subject test').add(data)

# Close the Firebase Admin SDK when done
firebase_admin.delete_app(firebase_admin.get_app())


import os
import asyncio
from gear import Database, transaction

async def main():
    scope = os.environ['HAIL_SCOPE']

    if scope in ('deploy', 'dev'):
        standing_worker_max_idle_time_secs = 7200  # 2 hours
    else:
        standing_worker_max_idle_time_secs = 300  # 5 minutes

    db = Database()
    await db.async_init()

    columns = [
        ('inst_colls', 'max_new_instances_per_autoscaler_loop', 'INT', 10),  # n * 16 cores / 15s = excess_scheduling_rate/s = 10/s => n ~= 10
        ('inst_colls', 'autoscaler_loop_period_secs', 'INT', 15),
        ('inst_colls', 'worker_max_idle_time_secs', 'INT', 30),
        ('pools', 'standing_worker_max_idle_time_secs', 'INT', standing_worker_max_idle_time_secs),
        ('pools', 'job_queue_scheduling_window_secs', 'INT', 150),  # 2.5 minutes is approximately worker start up time
    ]

    @transaction(db)
    async def insert_and_update(tx):
        for table, col, typ, value in columns:
            await tx.just_execute(f'ALTER TABLE {table} ADD COLUMN {col} {typ}, ALGORITHM=INSTANT;')
            await tx.execute_update(f'UPDATE {table} SET {col} = {value};')
            await tx.just_execute(f'ALTER TABLE {table} MODIFY COLUMN {col} {typ} NOT NULL;')

    await insert_and_update()  # pylint: disable=no-value-for-parameter

    await db.async_close()

loop = asyncio.get_event_loop()
loop.run_until_complete(main())


"""
Advent of Code 2020: Day 1
https://adventofcode.com/2020/day/1
"""

from typing import List, Tuple
from collections import Counter
from itertools import product

def find_pair_sums_to_N(nums: List[int], N: int) -> Tuple[int, int]:
    if (N % 2 == 0) and nums.count(N // 2) > 1:
        return (N // 2, N // 2)
    nums_set = set(nums)
    for n in nums_set:
        if N - n in nums_set and (n != N - n):
            return (n, N - n)
    raise ValueError("No pair found.")

def overcounted(arr: List, limits: Counter) -> bool:
    """Determines whether any item in arr appears more times than allowed by count_limits."""
    return any(arr.count(x) > limits[x] for x in set(arr))

def find_triplet_sums_to_N(nums: List[int], N: int) -> Tuple[int, int, int]:
    nums_counter = Counter(nums)
    if (N % 3 == 0) and not overcounted([N // 3] * 3, nums_counter):
        return (N // 3, N // 3, N // 3)
    nums_set = set(nums)
    for (m, n) in product(nums_set, nums_set):
        if N - m - n in nums_set and not overcounted([m, n, N - m - n], nums_counter):
            return (m, n, N - m - n)
    raise ValueError("No triplet found.")

#################

with open("input_1.txt") as f:
    x_input = [int(x) for x in f]

if __name__ == "__main__":
    x, y = find_pair_sums_to_N(x_input, 2020)
    print(f"Part 1: {x * y}")
    x, y, z = find_triplet_sums_to_N(x_input, 2020)
    print(f"Part 2: {x * y* z}")


#!/usr/bin/env python3
""" Plotting Task 6-bars"""
import numpy as np
import matplotlib.pyplot as plt

np.random.seed(5)
fruit = np.random.randint(0, 20, (4, 3))

plt.bar(["Farrah", "Fred", "Felicia"],
        fruit[:][0], color='red',
        label='apples', width=0.5)
plt.bar(["Farrah", "Fred", "Felicia"],
        fruit[:][1], bottom=fruit[0],
        color='yellow', label='bananas', width=0.5)
plt.bar(["Farrah", "Fred", "Felicia"],
        fruit[:][2],
        bottom=fruit[0] + fruit[1],
        color='#ff8000', label='oranges', width=0.5)
plt.bar(["Farrah", "Fred", "Felicia"],
        fruit[:][3],
        bottom=fruit[0] + fruit[1] + fruit[2],
        color='#ffe5b4', label='peaches', width=0.5)
plt.title("Number of Fruit per Person")
plt.yticks(range(0, 81, 10))
plt.ylabel("Quantity of Fruit")
plt.legend()
plt.show()


#Imports packages datetime and re
import datetime, re

#Constant that holds value of percentage of levy if customer pays with credit card
CREDIT_CARD_LEVY = 0.05

#Constant that holds value of reduction if customer pays with cash
CASH_REDUCTION = 0.05

#Constant that holds value of cost of booking one day
ONE_DAY_BILL = 260

#Constant that holds value of cost per day if the customer books from 2 to 4 days
TWO_FOUR_DAYS_BILL = 240

#Constant that holds value of cost per day if the customer books from 5 to 8 days
FIVE_EIGHT_DAYS_BILL = 210

#Constant that holds value of cost per day if the customer books for 9 or more days
NINE_LONGER_DAYS_BILL = 200

#Constant that holds value of cost per session musician booked per day
SESSION_MUSICIAN_COST = 100

#List that holds value of band member names
bandMemberNames=[]

#List that holds value of band member instruments
bandMemberInstruments=[]

#Variable that holds payment options menu presented to the customer
MENU = "Will you pay by\n1: Credit Card (5% Levy)\n2: Cash (5% Discount)\n"\
       "3: Cheque"

print('Welcome to start-up recording studio, we will start your aplication process!!!')

'''
Loop that takes customer contact name input and validates if lenght is higher than 1 character
'''
contactNameBoolean=False
while not contactNameBoolean:
	contactName = input('\nPlease insert your group\'s contact name:')
	contactNameBoolean=len(contactName)>1
	if not contactNameBoolean:
		print('\nplease insert a name with at least 2 characters')

'''
Loop that takes customer group manage email input and
validates if email as @ character and if has . character after @
with text after the . character also
'''
managerEmailBoolean=False
while not managerEmailBoolean:
	managerEmail = input('\nPlease insert the email of the group\'s manager:')
	if not re.match(r"[^@]+@[^@]+\.[^@]+", managerEmail):
		print('Email not valid, please insert a valid email.')
	else:
		managerEmailBoolean=True

'''
Loop that takes customer group manager phone number input and
validates if it's an integer and if length is higher than 9 this validation
takes in account irish phone numbers
'''
managerPhoneNumberBoolean=False
while not managerPhoneNumberBoolean:
	try:
		managerPhoneNumber = int(input('\nInsert the mobile phone number of the group\'s manager:'))
		managerPhoneNumberBoolean=len(str(managerPhoneNumber))>9
		if not managerPhoneNumberBoolean:
			print('\Please enter a valid phone number')
	except:
		print('\nYou didn\'t insert a valid numeric character, please enter a valid phone number')

'''
Loop that takes customer start date input and
validates if it's in the format dd/mm/yyyy and if
date is higher or equal to the current day
'''
requestDateBoolean=False
while not requestDateBoolean:
	try:
		requestDate = input('\nWhat start date are you looking for (dd/mm/yyyy)?\n')
		datetime.datetime.strptime(requestDate, '%d/%m/%Y')
		if requestDate>=datetime.datetime.now().strftime ("%d/%m/%Y"):
			requestDateBoolean=True
		else:
			print('The date you inserted is lower than the current day please insert a correct date.')

	except:
		print("Incorrect date format, please insert date in the following format dd/mm/yyyy, ex: 05/11/2017")

print("\nOur studio can only accomodate up to 8 musicians\n")

'''
Loop that takes customer total band members input and
validates if it's an integer and if total band members don't exceed the total members
and if the number of band member is lower than 0
'''
numberBandMembersBoolean=False
while not numberBandMembersBoolean:
	try:
		numberBandMembers = int(input('\nHow many members of the band?\n'))
		numberBandMembersBoolean=numberBandMembers>0 and numberBandMembers<=8
		if numberBandMembers>8:
			print('\nSorry, but our studio can only accomodate up to 8 musicians')
		elif numberBandMembers<1:
			print('\nSorry, but you need to book at least for 1 musician')
	except:
		print('\nYou didn\'t insert a valid number, please enter total band members again')

'''
Loop that takes customer total booking days input and
validates if it's an integer and if booking days are higher than 0
'''
numberBookingDaysBoolean = False
while not numberBookingDaysBoolean:
	try:
		numberBookingDays = int(input('\nHow many days you want to book the studio?\n'))
		numberBookingDaysBoolean=numberBookingDays>0
		if not numberBookingDaysBoolean:
			print('\nYou have to book for at least one day')
	except:
		print('\nPlease enter a valid number of days')

#Condition that calculares cost of booking days
if numberBookingDays==1:
	bookingDaysCost = ONE_DAY_BILL
elif numberBookingDays>1 and numberBookingDays<=4:
	bookingDaysCost = TWO_FOUR_DAYS_BILL*numberBookingDays
elif numberBookingDays>4 and numberBookingDays<=8:
	bookingDaysCost = FIVE_EIGHT_DAYS_BILL*numberBookingDays
else:
	bookingDaysCost = NINE_LONGER_DAYS_BILL*numberBookingDays

'''
Loop that takes customer band member names and band member instrumet input and
validates if it's length of input inserted is higher than one character and
appends input to band member names and instruments lists
'''
bandAccumulator=1
while bandAccumulator <=numberBandMembers:
	bandMemberBoolean=False
	while not bandMemberBoolean:
		bandMember = input('\nWhat is band member #'+str(bandAccumulator)+'\'s name?\n')
		bandMemberBoolean=len(bandMember)>1
		if not bandMemberBoolean:
			print('\nplease insert a name with at least 2 character')
		else:
			bandMemberNames.append(bandMember)

	bandInstrumentBoolean = False
	while not bandInstrumentBoolean:
		bandInstrument = input('\nWhat is '+bandMemberNames[bandAccumulator-1] +'\'s instrument?\n')
		bandInstrumentBoolean=len(bandInstrument)>1
		if not bandInstrumentBoolean:
			print('\nplease insert an instrument with at least 2 character')
		else:
			bandMemberInstruments.append(bandInstrument)
	bandAccumulator+=1

spareMusicians=8-numberBandMembers#variable that holds value if there is still room for session musicans

'''
Condition that tests if there is still room for session musicans and
loop that if condition True takes user input on number of session musicians
booked validates if its a positive integer and calculates the session musicians
cost
'''
if spareMusicians>0:
	sessionMusiciansBoolean = False
	while not sessionMusiciansBoolean:
		try:
			sessionMusicians = int(input("\nThere is room for "+ str(spareMusicians) +" session musicans - how many do you want?\n"))
			sessionMusiciansBoolean = sessionMusicians >= 0 and sessionMusicians < spareMusicians
			if sessionMusicians < 0:
				print("\nHey there is no negative musicians, please insert a correct value!!!")
			elif sessionMusicians>spareMusicians:
				print("\nHey you have excedeed the amount of musicians allowed per session! You can only add up to "+str(spareMusicians)+"session musicians" )
		except:
			print('\nPlease enter a valid integer')

	sessionMusiciansCost = sessionMusicians*100*numberBookingDays
else:
	sessionMusiciansCost = 0
	sessionMusicians = 0

#Ouputs payment options
print(MENU)

'''
Loop that takes customer payment option input and
validates if it's a valid option
'''
menuBoleean = False
while not menuBoleean:
	try:
		menuChoice = int(input("\nPlease insert 1 to select Credit Card, 2 to select"\
		+"Cash and 3 to select Cheque as a payment method?\n"))
		menuBoleean = menuChoice>0 and menuChoice<=3
		if not menuBoleean:
			print("\nHey you select a non existent payment method")
	except:
		print("\nPlease insert only the correspondent number to select the payment option")

bookingApplication = "Booking application"
bandMembers = "Band Members"

#Creates txt file with customer contact name in append mode
bookingRequest = open(contactName+'.txt','a')

#Ouputs bands contact name, email, phone number and date requested to book
print("\n"+bookingApplication+"\n"+len(bookingApplication)*"-"+"\nRequested by: "+contactName\
+" (Contact: "+managerEmail+" & "+str(managerPhoneNumber)+")"+"\n\nDate requested --> "+requestDate)

#Writes to Txt file bands contact name, email, phone number and date requested to book
bookingRequest.write("\n"+bookingApplication+"\n"+len(bookingApplication)*"-"\
+"\nRequested by: "+contactName+" (Contact: "+managerEmail+" & "+str(managerPhoneNumber)+")"+"\n\nDate requested --> "+requestDate+"\n")

#Ouputs bands contact name, email, phone number and date requested to book
print("\n"+bandMembers+"\n"+len(bandMembers)*"-")

#Writes to Txt file bands contact name, email, phone number and date requested to book
bookingRequest.write("\n"+bandMembers+"\n"+len(bandMembers)*"-"+"\n")

'''
Loop that outputs and writes to txt file the band member names
and instruments
'''
bandMemberNamesAccumulator=1
for name in bandMemberNames:
	print("\n"+str(bandMemberNamesAccumulator)+": "+bandMemberNames[bandMemberNamesAccumulator-1]\
	+" - "+bandMemberInstruments[bandMemberNamesAccumulator-1])
	bookingRequest.write("\n"+str(bandMemberNamesAccumulator)+": "+bandMemberNames[bandMemberNamesAccumulator-1]\
	+" - "+bandMemberInstruments[bandMemberNamesAccumulator-1]+"\n")
	bandMemberNamesAccumulator+=1

#Ouputs and writes to txt file the number of session musicians included
print("\nIncludes "+str(sessionMusicians)+" session musicians per day.")
bookingRequest.write("\nIncludes "+str(sessionMusicians)+" session musicians per day.\n")

'''
Condition that checks payment option selected by user and calculates total cost accordingly
Outupts and writes to txt file the total cost and payment option
'''
if menuChoice==1:
	totalCost = (bookingDaysCost+sessionMusiciansCost)*(1+CREDIT_CARD_LEVY)
	print("\nPayment will be "+str(totalCost)+" Euros to be paid by Credit Card\n")
	bookingRequest.write("\nPayment will be "+str(totalCost)+" Euros to be paid by Credit Card\n")
elif menuChoice==2:
	totalCost = (bookingDaysCost+sessionMusiciansCost)*(1-CASH_REDUCTION)
	print("\nPayment will be "+str(totalCost)+" Euros to be paid by Cash")
	bookingRequest.write("\nPayment will be "+str(totalCost)+" Euros to be paid by Cash\n")
else:
	totalCost = bookingDaysCost+sessionMusiciansCost
	print("\nPayment will be "+str(totalCost)+" Euros to be paid by Cheque")
	bookingRequest.write("\nPayment will be "+str(totalCost)+" Euros to be paid by Cheque\n")

#Ouputs and writes to txt file the date of submission of booking request
print("\nDate of submission---->"+datetime.datetime.now().strftime ("%d/%m/%Y"))
bookingRequest.write("\nDate of submission---->"+datetime.datetime.now().strftime ("%d/%m/%Y"))

#Closes the txt file
bookingRequest.close()