import sys
from os import path
import timeit
from collections import OrderedDict

import torch.nn as nn
import torch.nn.functional as F
from torch import optim
from torch.utils import data as torch_data
from torchvision import transforms, utils
import segmentation_models_pytorch as smp
from tabulate import tabulate
from sklearn.metrics import confusion_matrix as confmatrix
import wandb
import matplotlib.pyplot as plt

from unet import UNet, Dpn92_Unet_Double, SeNet154_Unet_Double, SeResNext50_Unet_Double
from unet.dataloader import Xview2Detectron2DamageLevelDataset
from unet.augmentations import *

from experiment_manager.args import default_argument_parser
from experiment_manager.metrics import MultiClassF1
from experiment_manager.config import new_config
from experiment_manager.loss import *
from eval_unet_xview2 import inference_loop
from eval_util.xview2_scoring import RowPairCalculator, XviewMetrics

# import hp

def train_net(net,
              cfg):

    log_path = cfg.OUTPUT_DIR
    summarize_config(cfg)

    optimizer = optim.Adam(net.parameters(),
                           lr=cfg.TRAINER.LR,
                           weight_decay=0.0005)
    weighted_criterion = False
    if cfg.MODEL.LOSS_TYPE == 'CrossEntropyLoss':
        criterion = cross_entropy_loss
    elif cfg.MODEL.LOSS_TYPE == 'SoftDiceMulticlassLoss':
        criterion = soft_dice_loss_multi_class
    elif cfg.MODEL.LOSS_TYPE == 'SoftDiceMulticlassLossDebug':
        criterion = soft_dice_loss_multi_class_debug
    elif cfg.MODEL.LOSS_TYPE == 'GeneralizedDiceLoss':
        criterion = generalized_soft_dice_loss_multi_class
    elif cfg.MODEL.LOSS_TYPE == 'JaccardLikeLoss':
        criterion = jaccard_like_loss_multi_class
    elif cfg.MODEL.LOSS_TYPE == 'ComboLoss':
        criterion = combo_loss
        weighted_criterion = cfg.TRAINER.CE_CLASS_BALANCE.ENABLED
        weights = 1 / torch.tensor(cfg.TRAINER.CE_CLASS_BALANCE.WEIGHTS)
        weights = weights.cuda()

    if cfg.MODEL.PRETRAINED.ENABLED:
        net = load_pretrained(net, cfg)

    if torch.cuda.device_count() > 1:
        print(torch.cuda.device_count(), " GPUs!")
        net = nn.DataParallel(net)
    net.to(device)
    bg_class = cfg.MODEL.BACKGROUND.TYPE
    trfm = build_transforms(cfg, for_training=True, use_gts_mask=cfg.DATASETS.LOCALIZATION_MASK.TRAIN_USE_GTS_MASK)
    dataset = Xview2Detectron2DamageLevelDataset(cfg.DATASETS.TRAIN[0],
                                                 pre_or_post='post',
                                                 include_image_weight=True,
                                                 background_class=bg_class,
                                                 transform=trfm)

    dataloader_kwargs = {
        'batch_size': cfg.TRAINER.BATCH_SIZE,
        'num_workers': cfg.DATALOADER.NUM_WORKER,
        'shuffle':cfg.DATALOADER.SHUFFLE,
        'drop_last': True,
    }

    # sampler
    if cfg.AUGMENTATION.IMAGE_OVERSAMPLING_TYPE == 'simple':
        image_p = image_sampling_weight(dataset.dataset_metadata)
        sampler = torch_data.WeightedRandomSampler(weights=image_p, num_samples=len(image_p))
        dataloader_kwargs['sampler'] = sampler
        dataloader_kwargs['shuffle'] = False

    dataloader = torch_data.DataLoader(dataset, **dataloader_kwargs)

    max_epochs = cfg.TRAINER.EPOCHS
    global_step = 0
    for epoch in range(max_epochs):
        start = timeit.default_timer()
        print('Starting epoch {}/{}.'.format(epoch + 1, max_epochs))
        epoch_loss = 0

        net.train()
        loss_set, f1_set = [], []
        loss_component_set = []
        positive_pixels_set = [] # Used to evaluated image over sampling techniques
        for i, batch in enumerate(dataloader):
            x = batch['x'].to(device)
            y_gts = batch['y'].to(device)
            image_weight = batch['image_weight']

            optimizer.zero_grad()

            y_pred = net(x)
            ce_loss = 0
            dice_loss = 0
            if weighted_criterion:
                loss, (ce_loss, dice_loss) = criterion(y_pred, y_gts, weights)
            else:
                loss = criterion(y_pred, y_gts)
            epoch_loss += loss.item()

            loss.backward()
            optimizer.step()

            loss_set.append(loss.item())
            # loss_component_set.append(loss_component.cpu().detach().numpy())
            positive_pixels_set.extend(image_weight.cpu().numpy())

            if global_step % 10000 == 0 and global_step > 0:
                check_point_name = f'cp_{global_step}.pkl'
                save_path = os.path.join(log_path, check_point_name)
                torch.save(net.state_dict(), save_path)

            if global_step % 100 == 0 and global_step > 0:
                # time per 100 steps
                stop = timeit.default_timer()
                time_per_n_batches= stop - start

                max_mem, max_cache = gpu_stats()

                print(f'step {global_step},  avg loss: {np.mean(loss_set):.4f}, cuda mem: {max_mem} MB, cuda cache: {max_cache} MB, time: {time_per_n_batches:.2f}s',
                      flush=True)

                log_data = {
                    'loss': np.mean(loss_set),
                    'ce_component_loss': ce_loss,
                    'dice_component_loss': dice_loss,
                    'gpu_memory': max_mem,
                    'time': time_per_n_batches,
                    'total_positive_pixels': np.mean(positive_pixels_set),
                    'step': global_step,
                }

                wandb.log(log_data)

                loss_set = []
                positive_pixels_set = []
                start = stop

            # torch.cuda.empty_cache()
            global_step += 1

        # Evaluation for multiclass F1 score
        dmg_model_eval(net, cfg, device, max_samples=100, step=global_step, epoch=epoch)
        dmg_model_eval(net, cfg, device, max_samples=100, run_type='TRAIN', step=global_step, epoch=epoch)

def dmg_model_eval(net, cfg, device,
                   run_type='TEST',
                   max_samples = 1000,
                   step=0, epoch=0,
                   use_confusion_matrix=False,
                   include_component_f1=False,
                   include_disaster_type_breakdown = False):
    '''
    Runner that is concerned with training changes
    :param run_type: 'train' or 'eval'
    :return:
    '''
    measurer = MultiClassF1(ignore_last_class=cfg.MODEL.BACKGROUND.TYPE=='new-class')
    diaster_type_measurers = {}

    confusion_matrix_with_bg = []
    confusion_matrices_by_disaster_type = {}
    component_f1 = []
    allrows = []
    def evaluate(x, y_true, y_pred, img_filenames):

        # TODO Only enable me if testing pretrained winner model where bg is class 0 !!
        y_pred = y_pred[:,[1,2,3,4,0]]

        if cfg.MODEL.BACKGROUND.MASK_OUTPUT:
            # No background class, manually mask out background
            localization_mask = x[:,[3]] # 3 is a hard coded mask index
            y_pred = localization_mask * y_pred
        tp, tn, fp, fn = measurer.add_sample(y_true, y_pred)

        # === Component F1 score
        if include_component_f1:
            loss, loss_component = soft_dice_loss_multi_class_debug(y_pred, y_true)
            component_f1.append(loss_component.cpu().detach().numpy())

        # === Official F1 evaluation
        # y_pred_np = y_pred.argmax(dim=1).cpu().detach().numpy() # One-hot to not-one-hot
        # y_true_np = y_true.argmax(dim=1).cpu().detach().numpy()
        # # loc_row is not being used, but to keep original code intact, we keep it here
        #
        # row = RowPairCalculator.get_row_pair(np.zeros([2]), y_pred_np, np.zeros([2]), y_true_np)
        # allrows.append(row)

        # # # TODO DEBUG TP FP FN
        # for i in range(4):
        #     cls = i * 3
        #     print(f'TP {i}:', row[1][cls+0], int(tp[i].item()))
        #     print(f'FN {i}:', row[1][cls + 1], int(fn[i].item()))
        #     print(f'FP {i}:', row[1][cls + 2], int(fp[i].item()))
        #     print('')

        # === Confusion Matrix stuff
        if use_confusion_matrix:
            y_true_flat = y_true.argmax(dim=1).cpu().detach().numpy()
            y_pred_flat = torch.softmax(y_pred, dim=1).argmax(dim=1).cpu().detach().numpy()
            labels = [0, 1, 2, 3, 4] # 5 classes
            _mat = confmatrix(y_true_flat.flatten(), y_pred_flat.flatten(), labels = labels)
            confusion_matrix_with_bg.append(_mat)

        #=== Breakdown by image class
        # Disaster type
        if include_disaster_type_breakdown:
            for i, img_filename in enumerate(img_filenames):
                # Compute F1
                disaster_type = img_filename.split('_')[0]
                if disaster_type not in diaster_type_measurers:
                    diaster_type_measurers[disaster_type] = MultiClassF1(ignore_last_class=cfg.MODEL.BACKGROUND.TYPE == 'new-class')
                    confusion_matrices_by_disaster_type[disaster_type] = 0
                diaster_type_measurers[disaster_type].add_sample(y_true[[i]], y_pred[[i]])

                # Compute Confusion Matrix
                if use_confusion_matrix:
                    _mat = confmatrix(y_true_flat[i].flatten(), y_pred_flat[i].flatten(), labels=labels)
                    confusion_matrices_by_disaster_type[disaster_type] += _mat

    use_gts_mask = run_type == 'TRAIN' and cfg.DATASETS.LOCALIZATION_MASK.TRAIN_USE_GTS_MASK
    dset_source = cfg.DATASETS.TEST[0] if run_type == 'TEST' else cfg.DATASETS.TRAIN[0]

    # TODO some transforms shouldn't exist in evaluation
    trfm = build_transforms(cfg, use_gts_mask = use_gts_mask)
    bg_class = 'new-class' if cfg.MODEL.BACKGROUND.TYPE == 'new-class' else None
    dataset = Xview2Detectron2DamageLevelDataset(dset_source,
                                                 pre_or_post='post',
                                                 transform=trfm,
                                                 background_class=bg_class,)
    inference_loop(net, cfg, device, evaluate,
                   batch_size=cfg.TRAINER.INFERENCE_BATCH_SIZE,
                   run_type='TRAIN',
                   max_samples = max_samples,
                   dataset = dataset,
                   callback_include_x=True)

    # Summary gathering ===

    print('Computing F1 score ... ', end=' ', flush=True)
    # Max of the mean F1 score

    total_f1, f1_per_class = measurer.compute_f1(include_bg=False)
    all_fpr, all_fnr = measurer.compute_basic_metrics()
    print(total_f1, flush=True)

    set_name = 'test_set' if run_type == 'TEST' else 'training_set'
    log_data = {f'{set_name} total F1': total_f1,
               'step': step,
               'epoch': epoch,
               }

    if include_disaster_type_breakdown:
        for disaster_type, m in diaster_type_measurers.items():
            f1 = m.compute_f1(include_bg=False)[0]
            print(f'disaster_{disaster_type}_f1', f1)
            wandb.log({
                f'disaster-{disaster_type}-f1': f1
            })
    # official Xivew2 scoring
    # offical_score = XviewMetrics(allrows)
    # print(f'official_f1', offical_score.df1)
    # wandb.log({
    #     f'official-f1': offical_score.df1
    # })

    if include_disaster_type_breakdown and use_confusion_matrix:
        for disaster_type, cm in confusion_matrices_by_disaster_type.items():
            name = f'disaster {disaster_type}'
            plot_confmtx(name, cm)
            log_data[name] = plt

    if include_component_f1:
        loss_component_mean = np.mean(component_f1, axis=0)
        for comp_loss, cls in zip(loss_component_mean, ['no-dmg', 'minor-dmg', 'major-dmg', 'destroyed', 'bg']):
            log_data[f'{run_type}_{cls}_soft_dice'] = comp_loss

    damage_levels = ['no-damage', 'minor-damage', 'major-damage', 'destroyed']
    for f1, dmg in zip(f1_per_class, damage_levels):
        log_data[f'{set_name} {dmg} f1'] = f1

    damage_levels += ['negative class']
    for fpr, fnr, dmg in zip(all_fpr, all_fnr, damage_levels):
        log_data[f'{set_name} {dmg} false negative rate'] = fnr
        log_data[f'{set_name} {dmg} false positive rate'] = fpr

    # Plot confusion matrix
    if use_confusion_matrix:
        cm = np.sum(confusion_matrix_with_bg, axis=0)
        plot_confmtx(name=run_type, confusion_matrix=cm)
        log_data['confusion_matrix'] = plt

    wandb.log(log_data)

def plot_confmtx(name, confusion_matrix):
    '''
    Plots and saves the confusion matrix into the run directory
    :param name:
    :param confusion_matrix:
    :return:
    '''
    # print('confusion_matrix ', confusion_matrix)
    normalized_cm = confusion_matrix / confusion_matrix.sum(axis=-1, keepdims=True)
    # print('confusion_matrix normalized', normalized_cm)
    # normalized_cm = confusion_matrix_with_bg / confusion_matrix_with_bg.sum(axis=0, keepdims=True)
    labels = ['no-damage', 'minor-damage', 'major-damage', 'destroyed', 'background', ]

    fig, ax = plt.subplots()

    ax.matshow(normalized_cm, cmap='Blues')

    for (i, j), z in np.ndenumerate(normalized_cm):
        true_value = confusion_matrix[i, j]
        ax.text(j, i, '{:0.3f}\n{}'.format(z, true_value), ha='center', va='center',
                bbox=dict(boxstyle='round', facecolor='white', edgecolor='0.3'))
    ax.autoscale(False)
    # ax.set_yticks(np.arange(5))
    #
    ax.set_yticklabels([''] + labels)
    ax.set_xticklabels([''] + labels)
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")
    fig.tight_layout()
    plt.title(f'{name} confusion matrix')

    directory = f'{cfg.OUTPUT_DIR}/confusion_matrices/'
    if not os.path.exists(directory):
        os.makedirs(directory)

    plt.savefig( path.join(directory, f'{name}.png') )

def load_pretrained(net:nn.Module, cfg):
    p_cfg = cfg.MODEL.PRETRAINED
    dirs = os.path.abspath(cfg.OUTPUT_DIR).split('/')
    dirs = dirs[:-2]
    dirs += ['unet', p_cfg.NAME, p_cfg.CP_FILE] # remove ./dmg/run_name/ -> ./
    cp_path = '/'.join(dirs)

    loaded_dict = torch.load(cp_path)

    state = OrderedDict()
    for key, tensor in loaded_dict.items():
        # Skip output layer
        if key.startswith('outc'): continue
        # Skip input layer
        if not p_cfg.INCLUDE_INPUT_LAYER and key.startswith('inc'): continue
        # Skip decoder
        if p_cfg.ENCODER_ONLY and key.startswith('up_seq'): continue

        state[key] = tensor

    # Because merge state dict
    full_state = net.state_dict()
    full_state.update(state)
    net.load_state_dict(full_state)

    print('Pretrained model loaded! ', cp_path, flush=True)
    return net

def build_transforms(cfg, for_training=False, use_gts_mask = False):
    trfm = []
    # trfm.append(BGR2RGB())

    if cfg.DATASETS.LOCALIZATION_MASK.ENABLED: trfm.append(IncludeLocalizationMask(use_gts_mask))
    if cfg.DATASETS.INCLUDE_PRE_DISASTER: trfm.append(StackPreDisasterImage())
    if cfg.AUGMENTATION.RESIZE: trfm.append(Resize(scale=cfg.AUGMENTATION.RESIZE_RATIO))
    if cfg.AUGMENTATION.CROP_TYPE == 'uniform' and for_training:
        trfm.append(UniformCrop(crop_size=cfg.AUGMENTATION.CROP_SIZE))
    elif cfg.AUGMENTATION.CROP_TYPE == 'importance' and for_training:
        trfm.append(ImportanceRandomCrop(crop_size=cfg.AUGMENTATION.CROP_SIZE))
    if cfg.AUGMENTATION.RANDOM_FLIP_ROTATE and for_training:
        trfm.append(RandomFlipRotate())
    trfm.append(Npy2Torch())
    if cfg.AUGMENTATION.ENABLE_VARI: trfm.append(VARI())
    if cfg.AUGMENTATION.ZERO_MEAN: trfm.append(ZeroMeanUnitImage())
    trfm = transforms.Compose(trfm)
    return trfm

def combo_loss(p, y, class_weights=None):
    y_ = y.argmax(dim=1).long()
    ce = F.cross_entropy(p, y_, weight=class_weights)
    dice = soft_dice_loss_multi_class(p, y)
    loss =  ce + dice
    # only return component loss if using weighted ce
    if class_weights is not None: return loss, (ce, dice)
    return loss

def image_sampling_weight(dataset_metadata):
    print('performing oversampling...', end='', flush=True)
    EMPTY_IMAGE_BASELINE = 1000
    image_p = np.array([image_desc['post']['image_weight'] for image_desc in dataset_metadata]) + EMPTY_IMAGE_BASELINE
    print('done', flush=True)
    # normalize to [0., 1.]
    image_p = image_p
    return image_p

def gpu_stats():
    max_memory_allocated = torch.cuda.max_memory_allocated() / 1e6 # bytes to MB
    max_memory_cached = torch.cuda.max_memory_cached() /1e6
    return int(max_memory_allocated), int(max_memory_cached)

def summarize_config(cfg):
    run_config = {}
    run_config['CONFIG_NAME'] = cfg.NAME
    run_config['device'] = device
    run_config['log_path'] = cfg.OUTPUT_DIR
    run_config['training_set'] = cfg.DATASETS.TRAIN
    run_config['test set'] = cfg.DATASETS.TEST
    run_config['epochs'] = cfg.TRAINER.EPOCHS
    run_config['learning rate'] = cfg.TRAINER.LR
    run_config['batch size'] = cfg.TRAINER.BATCH_SIZE
    table = {'run config name': run_config.keys(),
             ' ': run_config.values(),
             }
    print(tabulate(table, headers='keys', tablefmt="fancy_grid", ))

def cross_entropy_loss(pred, y):
    y = y.argmax(dim=1).long()
    return F.cross_entropy(pred, y)

def setup(args):
    cfg = new_config()
    cfg.merge_from_file(f'configs/damage_detection/{args.config_file}.yaml')
    cfg.merge_from_list(args.opts)
    cfg.NAME = args.config_file

    if args.log_dir: # Override Output dir
        cfg.OUTPUT_DIR = path.join(args.log_dir, args.config_file)
    else:
        cfg.OUTPUT_DIR = path.join(cfg.OUTPUT_BASE_DIR, args.config_file)
    os.makedirs(cfg.OUTPUT_DIR, exist_ok=True)

    if args.data_dir:
        cfg.DATASETS.TRAIN = (args.data_dir,)
    return cfg

if __name__ == '__main__':
    args = default_argument_parser().parse_known_args()[0]
    cfg = setup(args)

    out_channels = cfg.MODEL.OUT_CHANNELS
    # if cfg.MODEL.BACKBONE.ENABLED:
    #     if cfg.MODEL.COMPLEX_ARCHITECTURE.ENABLED:
    #         if cfg.MODEL.COMPLEX_ARCHITECTURE.TYPE == 'pspnet':
    #             net = smp.PSPNet(cfg.MODEL.BACKBONE.TYPE,
    #                            encoder_weights=None,
    #                            encoder_depth=3,
    #                            in_channels=cfg.MODEL.IN_CHANNELS,
    #                            classes=cfg.MODEL.OUT_CHANNELS
    #                            )
    #     else:
    #         net = smp.Unet(cfg.MODEL.BACKBONE.TYPE,
    #                        encoder_weights=None,
    #                        encoder_depth=5,
    #                        decoder_channels = [512,256,128,64,32],
    #                        in_channels= cfg.MODEL.IN_CHANNELS,
    #                        classes=cfg.MODEL.OUT_CHANNELS
    #         )
    if cfg.MODEL.SIAMESE.ENABLED:
        use_pretrained = cfg.MODEL.SIAMESE.PRETRAINED
        if cfg.MODEL.SIAMESE.TYPE == 'SENET152':
            net = SeNet154_Unet_Double(use_pretrained)
        elif cfg.MODEL.SIAMESE.TYPE == 'RESNEXT50':
            net = SeResNext50_Unet_Double(use_pretrained)
        else:
            net = Dpn92_Unet_Double(use_pretrained)
    else:
        net = UNet(cfg)

    if args.resume_from:
        full_model_path = path.join(cfg.OUTPUT_DIR, args.resume_from)
        # Removing the module.** in front of keys
        filtered_dict = {}
        state_dict = torch.load(full_model_path)

        # For winning model, quick hack to remove the state_dict
        if 'state_dict' in state_dict.keys():
            state_dict = state_dict['state_dict']
        for k, v in state_dict.items(): # ['state_dict']
            k = '.'.join(k.split('.')[1:])
            filtered_dict[k] = v
        net.load_state_dict(filtered_dict)
        print('Model loaded from {}'.format(full_model_path))

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    # cudnn.benchmark = True # faster convolutions, but more memory

    print('=== Runnning on device: p', device)

    torch.manual_seed(cfg.SEED)
    np.random.seed(cfg.SEED)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    try:

        if args.eval_only:
            wandb.init(
                name=cfg.NAME,
                project='urban_dl_final',
                tags=['eval', 'dmg'],
            )
            dmg_model_eval(net, cfg, device, run_type='TEST', max_samples=2000,  use_confusion_matrix=True, include_disaster_type_breakdown=True)
            dmg_model_eval(net, cfg, device, run_type='TRAIN', max_samples=1000, use_confusion_matrix=True, include_disaster_type_breakdown=True)
        else:
            # Dynamically adjust the batch size to fit on GPU
            orginal_batch_size = cfg.TRAINER.BATCH_SIZE
            for i in range(orginal_batch_size):
                try:
                    wandb.init(
                        name=cfg.NAME,
                        project='urban_dl_final',
                        tags=['run', 'dmg'],
                        reinit=True
                    )
                    train_net(net, cfg)
                    break
                except RuntimeError as runerr:
                    # Check if runtime error is CUDA error
                    if 'CUDA out of memory' not in str(runerr):
                        raise runerr
                    new_batch_size = orginal_batch_size - i * 2
                    print("Run Time Error:" + str(runerr), file=sys.stderr)
                    print('!!!! ---> Original batch size too large, trying batch size:' + str(new_batch_size), file=sys.stderr)
                    cfg.TRAINER.BATCH_SIZE = new_batch_size
    except KeyboardInterrupt:
        torch.save(net.state_dict(), 'INTERRUPTED.pth')
        print('Saved interrupt')
        try:
            sys.exit(0)
        except SystemExit:
            os._exit(0)


from collections import defaultdict
import csv
import os.path
import tempfile
from typing import Dict, Set, Tuple, List, Iterable, TypeVar, Optional, Union
from uuid import UUID
import zipfile

import attr
from django.db import connection
import pandas as pd

from .models import Participant, ContextGroup, Context

def export_results(
        folder_name: str,
        only_complete: bool,
        participant_ids: Optional[Set[UUID]] = None,
        skip_fillers: bool = True,
        ) -> Tuple[bytes, str]:
    groups = {}  # participant_id -> context_set -> [{ctx}]
    named_contexts = get_named_contexts(skip_fillers=skip_fillers)
    with connection.cursor() as cursor:
        ctx_ids_by_cg = defaultdict(set)
        cursor.execute('select contextgroup_id, context_id '
                       'from survey_contextgroup_contexts')
        for cg_id, ctx_id in cursor.fetchall():
            ctx_ids_by_cg[cg_id].add(ctx_id)
    cg_qs = ContextGroup.objects.all()
    if skip_fillers:
        cg_qs = cg_qs.filter(context_set__is_filler=False)
    for cg in cg_qs:
        (groups
         .setdefault(cg.participant_id, {})
         .setdefault(cg.context_set_id, [])
         .append(ctx_ids_by_cg[cg.id])
         )
    n_to_group = {
        p.id: p.cs_to_group.count()
        for p in Participant.objects.prefetch_related('cs_to_group').all()}
    with tempfile.TemporaryDirectory() as dirname:
        archive_name = '{}.zip'.format(folder_name)
        archive_path = os.path.join(dirname, archive_name)
        with zipfile.ZipFile(archive_path, 'w') as archive:
            participants = set()
            write_csv = lambda sheet, name: _write_csv(
                sheet, archive, dirname, folder_name, name=name)
            for participant_id, p_groups in groups.items():
                if participant_ids is not None and \
                        participant_id not in participant_ids:
                    continue
                if only_complete and \
                        len(p_groups) != n_to_group[participant_id]:
                    continue  # not all grouped
                participants.add(participant_id)
                write_csv(p_groups_sheet(named_contexts, p_groups),
                          name='individual/{}'.format(participant_id))
            write_csv(participants_df(participants), name='personal')
            write_csv(words_df(named_contexts), name='words')
            true_groups = get_true_groups(named_contexts)
            write_csv(p_groups_sheet(named_contexts, true_groups),
                      name='model_classification')
            write_csv(all_pairs_df(named_contexts, true_groups),
                      name='pairs_full')
        with open(archive_path, 'rb') as f:
            return f.read(), archive_name

@attr.s
class NamedContext:
    idx = attr.ib()
    name = attr.ib()
    is_filler = attr.ib()

def get_named_contexts(
        skip_fillers: bool = True,
        ) -> Dict[int, NamedContext]:
    named_contexts = {}
    cs_n = ctx_n = 0
    prev_cs = None
    qs = Context.objects.select_related('context_set')
    for idx, ctx in enumerate(qs.order_by('context_set__id', 'order')):
        is_filler = ctx.context_set.is_filler
        if skip_fillers and is_filler:
            continue
        if prev_cs is None or ctx.context_set_id != prev_cs:
            cs_n += 1
            ctx_n = 1
        else:
            ctx_n += 1
        named_contexts[ctx.id] = NamedContext(
            idx=idx,
            name='set{}_stim{}'.format(cs_n, ctx_n),
            is_filler=is_filler,
        )
        prev_cs = ctx.context_set_id
    return named_contexts

class Sheet:
    def __init__(self):
        self.data = {}

    def write(self, row, col, value):
        self.data[row, col] = value

    def write_csv(self, f):
        writer = csv.writer(f)
        n_cols = max(col for _, col in self.data)
        n_rows = max(row for row, _ in self.data)
        for row in range(n_rows + 1):
            writer.writerow([
                self.data.get((row, col), '') for col in range(n_cols + 1)])

def p_groups_sheet(
        named_contexts: Dict[int, NamedContext],
        p_groups: Dict[int, List[Set[int]]]) -> Sheet:
    sheet = Sheet()
    # write header
    for named in sorted(
            named_contexts.values(), key=lambda x: x.idx):
        sheet.write(named.idx + 1, 0, named.name)
        sheet.write(0, named.idx + 1, named.name)

    def indices(x, y):
        return named_contexts[x].idx + 1, named_contexts[y].idx + 1

    # write cells
    for cs_groups in p_groups.values():
        all_ids = set()
        written = set()  # type: Set[Tuple[int, int]]
        for group in cs_groups:
            all_ids.update(group)
            for a, b in power(group):
                idx1, idx2 = indices(a, b)
                if idx1 >= idx2:
                    sheet.write(idx1, idx2, 1)
                    written.add((a, b))
            for a, b in power(all_ids):
                idx1, idx2 = indices(a, b)
                if idx1 > idx2 and (a, b) not in written:
                    sheet.write(idx1, idx2, 0)

    return sheet

def get_true_groups(
        named_contexts: Dict[int, NamedContext]) -> Dict[int, List[Set[int]]]:
    true_groups = {}
    for context in Context.objects.all():
        if context.id not in named_contexts:
            continue
        key = context.derivation.strip()
        (true_groups
         .setdefault(context.context_set_id, {})
         .setdefault(key, set()).add(context.id))
    return {cs_id: list(p_groups.values())
            for cs_id, p_groups in true_groups.items()}

def all_pairs_df(named_contexts: Dict[int, NamedContext],
                 true_groups: Dict[int, List[Set[int]]]) -> pd.DataFrame:
    data = []
    ctx_by_id = {ctx.id: ctx
                 for ctx in Context.objects.select_related('context_set')}
    for group in true_groups.values():
        group_ids = {i for g in group for i in g}
        group_contexts = [(i, named_contexts[i]) for i in group_ids]
        derivations = {ctx_by_id[i].derivation.strip() for i in group_ids}
        der_to_g = {der: idx for idx, der in enumerate(sorted(derivations), 1)}
        for id_1, ctx_1 in group_contexts:
            set_1, stim_1 = ctx_1.name.split('_')
            for id_2, ctx_2 in group_contexts:
                set_2, stim_2 = ctx_2.name.split('_')
                assert set_1 == set_2
                der_1 = ctx_by_id[id_1].derivation.strip()
                der_2 = ctx_by_id[id_2].derivation.strip()
                g_1 = der_to_g[der_1]
                g_2 = der_to_g[der_2]
                data.append({
                    'X1st.part': ctx_1.name,
                    'X2nd.part': ctx_2.name,
                    'pair': '{}_{}{}'.format(set_1, stim_1, stim_2),
                    'word': ctx_by_id[id_1].context_set.word,
                    'X1st.part.derivation': der_1,
                    'X2st.part.derivation': der_2,
                    'group.1st.part': g_1,
                    'group.2nd.part': g_2,
                    'pair.derivation': '{}.{}'.format(der_1, der_2),
                    'pair.group': '{}.{}'.format(g_1, g_2),
                    'correct': int(der_1 == der_2),
                })
    return pd.DataFrame(data, columns=[
        'X2nd.part', 'X1st.part', 'pair',
        'word',
        'X1st.part.derivation', 'X2st.part.derivation',
        'group.1st.part', 'group.2nd.part',
        'pair.derivation', 'pair.group', 'correct',
    ])

T = TypeVar('T')

def power(lst: List[T]) -> Iterable[Tuple[T, T]]:
    return ((a, b) for a in lst for b in lst)

def participants_df(participant_ids: Set=None) -> pd.DataFrame:
    fields = ['person.id', 'profession', 'age', 'leading_hand', 'sex',
              'languages', 'education']
    data = []
    for p in Participant.objects.select_related('leading_hand', 'education'):
        if participant_ids is None or p.id in participant_ids:
            row = {}
            for col, field in enumerate(fields):
                model_field = {'person.id': 'id'}.get(field, field)
                value = getattr(p, model_field)
                if not isinstance(value, int):
                    value = str(value or '')
                row[field] = value
            data.append(row)
    df = pd.DataFrame(data, columns=fields)
    df['type'] = 'naive'
    return df

def words_df(named_contexts: Dict[int, NamedContext]) -> pd.DataFrame:
    data = {
        (ctx.context_set.word,
         ctx.context_set.group,
         named_contexts[ctx.id].name.split('_')[0] + '_',
         )
        for ctx in Context.objects.select_related('context_set')
        if ctx.id in named_contexts}
    data = sorted(data, key=lambda x: int(x[2][3:-1]))
    return pd.DataFrame(data, columns=['word', 'group', 'set'])

def _write_csv(
        data: Union[Sheet, pd.DataFrame],
        archive: zipfile.ZipFile,
        dirname, folder_name, name):
    filename = '{}.csv'.format(name)
    full_path = os.path.join(dirname, filename)
    parent = os.path.dirname(full_path)
    if not os.path.exists(parent):
        os.mkdir(parent)
    with open(full_path, 'wt') as f:
        if isinstance(data, Sheet):
            data.write_csv(f)
        else:
            data.to_csv(f, sep=';', index=None)
    archive.write(
        full_path, arcname='{}/{}'.format(folder_name, filename))


class Solution:
    def validPath(self, n: int, edges: List[List[int]], source: int, destination: int) -> bool:
# undirectional graph => cycles => recursion limit extended due to infinite loop!

        # simply apply dfs/bfs
        # if node is found: return True
        # else return False

        # creating adjacency list:
        graph = {}

        # create list for each node
        for edge in edges:
            if edge[0] not in graph:
                graph[edge[0]] = []
            if edge[1] not in graph:
                graph[edge[1]] = []
            # append adjacent node to the lists of each node
            graph[edge[0]].append(edge[1])
            graph[edge[1]].append(edge[0])
        print(graph)

        visited = [False for i in range(n)]     # list
        # instead pf list hashset could be used would improve the time and space complexity!

        curr = source

        def dfs(edges,curr,destination):
            if curr == destination:
                return True

            # if curr is visited
            if visited[curr] == True:
                return False

            # If curr is not visited
            visited[curr] = True

            # recursive function
            for neighbour in graph[curr]:
                if dfs(edges, neighbour,destination) == True:
                    return True
            return False

        return dfs(edges,curr,destination)


from xivo_dao.resources.queue import dao as queue_dao
from xivo_dao.resources.user import dao as user_dao

from .middleware import QueueMemberMiddleWare
from .resource import (
    QueueMemberAgentItem,
    QueueMemberUserItem,
)
from .service import build_service

class Plugin:
    def load(self, dependencies):
        api = dependencies['api']
        middleware_handle = dependencies['middleware_handle']

        service = build_service()

        queue_member_middleware = QueueMemberMiddleWare(service)
        middleware_handle.register('queue_member', queue_member_middleware)

        api.add_resource(
            QueueMemberAgentItem,
            '/queues/<int:queue_id>/members/agents/<int:agent_id>',
            endpoint='queue_member_agents',
            resource_class_args=(queue_member_middleware,),
        )

        api.add_resource(
            QueueMemberUserItem,
            '/queues/<int:queue_id>/members/users/<uuid:user_id>',
            '/queues/<int:queue_id>/members/users/<int:user_id>',
            endpoint='queue_member_users',
            resource_class_args=(service, queue_dao, user_dao),
        )


import argparse
import pytorch_lightning as pl

class PHyperLogger(pl.callbacks.base.Callback):
    """

    Args:
        args:
        **log_kwargs:
    """

    # More information about the parameters are available in the pytorch-lightning documentation
    # https://pytorch-lightning.readthedocs.io/en/0.7.1/trainer.html
    trainer_keys = ["max_epochs", "min_epochs", "min_steps", "max_steps", "auto_lr_find", "auto_scale_batch_size",
                    "fast_dev_run", "limit_train_batches", "limit_val_batches", "limit_test_batches", "gradient_clip_val",
                    "limit_predict_batches", "overfit_batches", "accumulate_grad_batches", "truncated_bptt_steps"]

    def __init__(self, args: argparse.Namespace, **log_kwargs):
        self.log_kwargs = log_kwargs
        self.log_kwargs.update(vars(args))

    def on_train_start(self, trainer: pl.Trainer, model: pl.LightningModule):
        # Log trainer hyper-parameters as pre-defined in the callback header.
        trainer_parameters = trainer.__dict__.copy()
        log_parameters = {}
        for key in sorted(self.trainer_keys):
            log_parameters[key] = trainer_parameters.get(key, None)
        model.logger.log_hyperparams(log_parameters)

        # Log number of prediction classes (either from arguments or model).
        num_classes = len(self.log_kwargs.get("classes", []))
        if num_classes == 0:
            num_classes = getattr(model, "num_classes", 0)
        self.log_kwargs["num_classes"] = num_classes

        # Log number of model parameters.
        self.log_kwargs["num_parameters"] = sum(p.numel() for p in model.parameters())
        self.log_kwargs["num_trainable_parameters"] = sum(p.numel() for p in model.parameters() if p.requires_grad)

        # Log additional logging kwargs from initialization.
        model.logger.log_hyperparams(self.log_kwargs)


import pandas as pd

SHOW_PENDING_COMMANDS = "show pending commands"
PRICE_OF_SYMBOL = "price of"
allowed_commands = [
    SHOW_PENDING_COMMANDS,
    PRICE_OF_SYMBOL
]

class Messenger:
    def __init__(self, growth_status_receiver_user_id: list, interactive_mode=True):
        self.all_symbols = self.get_all_symbol_names_from_file()
        self.received_command_queue = []
        self.received_ticker_queue = []
        self.interactive_mode = interactive_mode
        self.msg_sender = None
        self.growth_status_receiver_user_id = growth_status_receiver_user_id

    def addpadding(self, text, pad_char=" ", maximum=7):
        req = maximum - len(text)
        if req > 0:
            return text + (pad_char * req)
        else:
            return text

    def get_pending_commands(self):
        res = "Pending Commands\n"
        for command in self.received_command_queue:
            symbol_data = f"{command['ticker']}"
            price_data = f"{command['target_price']}"

            res = res + self.addpadding(symbol_data) + "  ::::  " + price_data + "\n"
        return res

    def get_all_symbol_names_from_file(self):
        """
        Initialize all ticker symbol names from the csv file into all_symbols as list
        :return:
        """
        df = pd.read_csv("../app_data/company_tickers.csv")
        return df["symbol"].to_numpy()

    def received_msg(self, from_id=0, body=""):
        """
        This function is called when we receive a message from allowed id

        :param from_id:
        :param body:
        :return:
        """
        print(from_id)
        print(body)
        msg = body.strip().split(" ")
        if len(msg) == 2:
            try:
                ticker = msg[0].upper()
                if ticker in self.all_symbols:
                    target_price = float(msg[1])
                    self.received_command_queue.append(
                        {"from_id": from_id, "ticker": ticker, "target_price": target_price})
                    self.received_ticker_queue.append(ticker)
                    print("NEW: " + str({"from_id": from_id, "ticker": ticker, "target_price": target_price}))
                    if self.interactive_mode:
                        self.send_msg(to_id=from_id, body="COMMAND RECEIVED")
                else:
                    print("Wrong ticker name received")
                    if self.interactive_mode:
                        self.send_msg(to_id=from_id, body="TICKER NAME IS NOT VALID")
            except:
                print("PRICE MUST BE A NUMBER")
                if self.interactive_mode:
                    self.send_msg(to_id=from_id, body="PRICE MUST BE A NUMBER")
        elif body.lower() in allowed_commands:
            command = body.lower()
            if command == SHOW_PENDING_COMMANDS:
                body = self.get_pending_commands()
                print(f"SENDING PENDING COMMANDS\n{body}")
                self.send_msg(to_id=from_id, body=body)
        elif body.lower().startswith(PRICE_OF_SYMBOL):
            try:
                _, _, symbol = body.lower().split()
                if symbol.upper() in self.all_symbols:
                    # Fetch current price of that symbol and send it
                    pass

            except Exception as e:
                self.send_msg(to_id=from_id, body="Wrong command")

        else:
            print("MESSAGE FORMAT ERROR")
            if self.interactive_mode:
                self.send_msg(to_id=from_id, body="MESSAGE FORMAT ERROR")

    def send_msg(self, to_id=None, body=""):
        if to_id is None:
            for user_id in self.growth_status_receiver_user_id:
                print(f"Sending msg to {user_id} with body = {body}")
                self.msg_sender(chat_id=user_id, body=body)
        else:
            print(f"Sending msg to {to_id} with body = {body}")
            self.msg_sender(chat_id=to_id, body=body)

    def connect_msg_sender(self, msg_sender=None):
        """
        Must connect a sms sender before calling send_msg()
        :param msg_sender:
        :return:
        """
        self.msg_sender = msg_sender


#implmenetations of various sorting algorithms
import random

UNSORTED_ARRAY1 =  map((lambda x: random.randint(1, 400)), range(1,1000) )

def genray(n):
  return map((lambda x: random.randint(1, 400)), range(1,n) )

def test_sorted(array):
  i = 0
  size = len(array)
  while(i < size):
    if(i >= 1):
        if(array[i] < array[i-1]):
          return False
    i = i + 1
  return True

def cmp(a, b):
    if (a == b):
        return 0
    elif (a > b):
        return 1
    else:
        return -1

def swap(i, j, array):
    swapv = array[i]
    array[i] = array[j]
    array[j] = swapv
    return array

def biggest(array):
    i = 0
    biggest_val = array[0]
    biggest_index = i
    while(i < len(array)):
        if (array[i] > biggest_val):
            biggest_val = array[i]
            biggest_index = i
        i = i + 1
    return (biggest_index, biggest_val)

def minv(array, start_index):
    i = start_index
    min_val = array[start_index]
    min_index = i
    while(i < len(array)):
        if (array[i] < min_val):
            min_val = array[i]
            min_index = i
        i = i + 1
    return (min_index, min_val)

def min(array):
    return minv(array, 0)

def selectionSort(array):
    i = 0
    array1 = array
    while (i < len(array1)):
        array1 = swap(i, minv(array1, i)[0], array1)
        i = i + 1
    return array1

def gapSort(array, gap):
    """helper function to aid insertion sort and shell sort"""
    i = 0
    array1 = array
    while (i < len(array)):
        if (i > 0):
            j = i
            while ((cmp(array1[j], array1[j-gap]) < 0) and (j != 0 ) ):
                #while object at i is less than the one before it
                swap(j, j-gap, array1)
                j = j - 1
        i = i + 1
    return array1

def insertionSort(array):
    return gapSort(array, 1)

def shellSort(array):
    """sorts using the shellsort algorithm"""
    vals = [3*h+1 for h in range(len(array)/3)][::-1]
    for val in vals:
        array = gapSort(array, val)
    return array

#merge sort
def merge(array, p, q, r):
    if ((r - p) > 1):
        left = array[p:q+1]
        loggr("left" + str(left))
        right = array[q+1:r+1]
        loggr("right"+ str(right))
        left.append('c')
        right.append('c')
        i = 0
        j = 0
        for k in range(p, r+1):
            if left[i] <= right[j]:
                array[k] = left[i]
                i = i + 1
            else:
                array[k] = right[j]
                j = j + 1
    elif ((r - p) == 1 ):
        if (array[r] < array[p]):
            i = array[p]
            j = array[r]
            array[p] = j
            array[r] = i

def mergeSort(array):
    def sort(p, r, msg):
        #loggr(array)
        #loggr("p = "+ str(p) + ' r = '+ str(r) + " array[p]: " + str(array[p])+ " array[r] " + str(array[r]))
        #loggr(msg)
        if p < r:
            q = (p+r)/2
            if (r - p) > 1:
                #loggr("q = " + str(q) + " array[q] =  " + str(array[q]))
                sort(p, q, "in left array")
                #loggr([p, q])
                sort(q+1, r, "right array")
            #loggr([q, r])
            #loggr("merging")
            merge(array, p, q, r)
            #loggr(array)
    sort(0, len(array)-1, "root")
    return array

def loggr(data):
    f = open('./log.dat', 'a')
    f.write(str(data) + '\n')
    f.close()


import telebot
import random
from text import hello_list, bye_list, what_you_can_request, speak_guid, How_are_you_request_list, \
    What_are_you_doing_list, Question_list, Update, News_list

bot = telebot.TeleBot('1398287849:AAFO3qMjXYycrHNXzxH3LjAfKZZnk7bCSvs')

@bot.message_handler(commands=['start'])
def start_message(message):
    bot.send_message(message.chat.id,
                     'я Боб.  Я рад с Тобой познакомится!  Напиши мне  "Что ты умеешь?" ,  что бы узнать что я умею!')

@bot.message_handler(content_types=['text'])
def send_text(message):
    if message.text in hello_list:
        sti = open('C:\Inf\Sticker\hello_sticker.webp', 'rb')
        bot.send_sticker(message.chat.id, sti)

    elif message.text in bye_list:
        sti = open('C:\Inf\Sticker\good_bye_sticker.webp', 'rb')
        bot.send_sticker(message.chat.id, sti)

    elif message.text == 'Что ты умеешь?':
        bot.send_message(message.chat.id, what_you_can_request)

    elif message.text == 'Как с тобой общаться?':
        bot.send_message(message.chat.id, speak_guid)

    elif message.text in Question_list:
        bot.send_message(message.chat.id, random.choice(How_are_you_request_list))

    elif message.text == 'Что делаешь?':
        bot.send_message(message.chat.id, random.choice(What_are_you_doing_list))
    elif message.text == 'Версия':
        bot.send_message(message.chat.id, Update)
    elif message.text == 'Что нового?':
        bot.send_message(message.chat.id, random.choice(News_list))
    elif message.text == 'Ты веселый':
        sti = open('C:\Inf\Sticker\Тывеселый.webp', 'rb')
        bot.send_sticker(message.chat.id, sti)
    elif message.text == 'Ты крутой':
        sti = open('C:\Inf\Sticker\Тыкрутой.webp', 'rb')
        bot.send_sticker(message.chat.id, sti)
    elif message.text == 'Ты грустный':
        sti = open('C:\Inf\Sticker\Тыгрустный.webp', 'rb')
        bot.send_sticker(message.chat.id, sti)
    elif message.text == 'Скинь свою фотку':
        pic = 'https://lumiere-a.akamaihd.net/v1/images/c2-b5-main_b2481990.jpeg?region=0%2C0%2C1560%2C878&width=768'
        bot.send_photo(message.chat.id, pic)

bot.polling(none_stop=True, interval=0)


import numpy as np
from tqdm import tqdm

from lib import spec_utils

def mixup_generator(X, y, alpha):
    perm = np.random.permutation(len(X))[:len(X) // 2]
    if len(perm) % 2 != 0:
        perm = perm[:-1]
    for i in range(0, len(perm), 2):
        lam = np.random.beta(alpha, alpha)
        X[perm[i]] = lam * X[perm[i]] + (1 - lam) * X[perm[i + 1]]
        y[perm[i]] = lam * y[perm[i]] + (1 - lam) * y[perm[i + 1]]

    return X, y

def get_oracle_data(X, y, instance_loss, oracle_rate, oracle_drop_rate):
    k = int(len(X) * oracle_rate * (1 / (1 - oracle_drop_rate)))
    n = int(len(X) * oracle_rate)
    idx = np.argsort(instance_loss)[::-1][:k]
    idx = np.random.choice(idx, n, replace=False)
    oracle_X = X[idx].copy()
    oracle_y = y[idx].copy()
    return oracle_X, oracle_y, idx

def create_dataset(filelist, cropsize, patches, sr, hop_length,
                   validation=False):
    len_dataset = patches * len(filelist)
    X_dataset = np.zeros(
        (len_dataset, 2, hop_length, cropsize), dtype=np.float32)
    y_dataset = np.zeros(
        (len_dataset, 2, hop_length, cropsize), dtype=np.float32)
    for i, (X_path, y_path) in enumerate(tqdm(filelist)):
        X, y = spec_utils.cache_or_load(X_path, y_path, sr, hop_length)
        for j in range(patches):
            idx = i * patches + j
            start = np.random.randint(0, X.shape[2] - cropsize)
            X_dataset[idx] = X[:, :, start:start + cropsize]
            y_dataset[idx] = y[:, :, start:start + cropsize]
            if not validation:
                if np.random.uniform() < 0.5:
                    # swap channel
                    X_dataset[idx] = X_dataset[idx, ::-1]
                    y_dataset[idx] = y_dataset[idx, ::-1]
                # if np.random.uniform() < 0.5:
                #     # spec aug
                #     f = np.random.randint(0, hop_length // 4)
                #     f0 = np.random.randint(0, hop_length - f)
                #     X_dataset[idx, :, f0:f0 + f, :] = 0
                #     y_dataset[idx, :, f0:f0 + f, :] = 0
                # if np.random.uniform() < 0.5:
                #     # spec aug
                #     t = np.random.randint(0, hop_length // 8)
                #     t0 = np.random.randint(0, cropsize - t)
                #     X_dataset[idx, :, :, t0:t0 + t] = 0
                #     y_dataset[idx, :, :, t0:t0 + t] = 0

    return X_dataset, y_dataset


import os
import torch
import numpy as np
import torch.nn as nn
import random
import string
from pathlib import Path
import torch.nn.functional as F
DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'

def log_comet_metric(exp, name, val, step):
    exp.log_metric(name=name, value=val, step=step)

def save_model(model, path):
    torch.save(model.cpu(), path)

def load_model(path):
    model = torch.load(path)
    return model

def save_task_model_by_policy(model, task, policy, exp_dir):
    # task = 0 is the initilization
    if task == 0 or policy == 'init':
        save_model(model, '{}/init.pth'.format(exp_dir))

    # the first task model is the same for all
    if task == 1:
        save_model(model, '{}/t_{}_seq.pth'.format(exp_dir, task))
        save_model(model, '{}/t_{}_lmc.pth'.format(exp_dir, task))
        save_model(model, '{}/t_{}_mtl.pth'.format(exp_dir, task))
    else:
        save_model(model, '{}/t_{}_{}.pth'.format(exp_dir, task, policy))

def load_task_model_by_policy(task, policy, exp_dir):
    if task == 0 or policy == 'init':
        return load_model('{}/init.pth'.format(exp_dir))
    return load_model('{}/t_{}_{}.pth'.format(exp_dir, task, policy))

def flatten_params(m, numpy_output=True):
    total_params = []
    for param in m.parameters():
            total_params.append(param.view(-1))
    total_params = torch.cat(total_params)
    if numpy_output:
        return total_params.cpu().detach().numpy()
    return total_params

def assign_weights(m, weights):
    state_dict = m.state_dict(keep_vars=True)
    index = 0
    with torch.no_grad():
        for param in state_dict.keys():
            if 'running_mean' in param or 'running_var' in param or 'num_batches_tracked' in param:
                continue
            # print(param, index)
            param_count = state_dict[param].numel()
            param_shape = state_dict[param].shape
            state_dict[param] =  nn.Parameter(torch.from_numpy(weights[index:index+param_count].reshape(param_shape)))
            index += param_count
    m.load_state_dict(state_dict)
    return m

def assign_grads(m, grads):
    state_dict = m.state_dict(keep_vars=True)
    index = 0
    for param in state_dict.keys():
        if 'running_mean' in param or 'running_var' in param or 'num_batches_tracked' in param:
                continue
        param_count = state_dict[param].numel()
        param_shape = state_dict[param].shape
        state_dict[param].grad =  grads[index:index+param_count].view(param_shape).clone()
        index += param_count
    m.load_state_dict(state_dict)
    return m

def get_norm_distance(m1, m2):
    m1 = flatten_params(m1, numpy_output=False)
    m2 = flatten_params(m2, numpy_output=False)
    return torch.norm(m1-m2, 2).item()

def get_cosine_similarity(m1, m2):
    m1 = flatten_params(m1)
    m2 = flatten_params(m2)
    cosine = torch.nn.CosineSimilarity(dim=0, eps=1e-6)
    return cosine(m1, m2)

def save_np_arrays(data, path):
    np.savez(file=path, **data)

class ContinualMeter:
    def __init__(self, name, n_tasks):
        self.name = name
        self.data = np.zeros((n_tasks, n_tasks))

    def update(self, current_task, target_task, metric):
        self.data[current_task-1][target_task-1] = round(metric, 3)

    def save(self, config):
        path = '{}/{}.csv'.format(config['exp_dir'], self.name)
        np.savetxt(path, self.data, delimiter=",")

def get_latex_str_for_minima(policy, task):
    if policy == 'seq':
        return r"$\hat{{w}}_{{{}}}".format(task)
    elif policy == 'lmc':
        return r"$\bar{{w}}_{{{}}}".format(task)
    elif policy == 'mtl':
        return r"$w^*_{{{}}}".format(task)
    else:
        raise Exception("unknown policy")

def get_latex_str_for_path(p1, t1, p2, t2):
    start = get_latex_str_for_minima(p1, t1)
    end = get_latex_str_for_minima(p2, t2)
    return start + r" \rightarrow " + end

def get_model_grads(model, loader):
    grads = []
    criterion = nn.CrossEntropyLoss().to(DEVICE)
    count = 0
    test_loss = 0
    for data, target, task_id in loader:
            data = data.to(DEVICE)
            target = target.to(DEVICE)
            count += len(target)
            output = model(data, task_id)
            curr_loss = criterion(output, target)
            curr_loss.backward()
    for param in model.parameters():
            grads.append(param.grad.view(-1))
    grads = torch.cat(grads)
    # print("Norm grad >> ", torch.norm(grads))
    return grads

from torch.autograd import Variable, Function
import scipy as sp

# Inherit from Function
class OLELoss(Function):
    @staticmethod
    def forward(self, X, y):
        X = X.cpu().numpy()
        y = y.cpu().numpy()

        classes = np.unique(y)
        C = classes.size

        N, D = X.shape

        DELTA = 1.

        # gradients initialization
        Obj_c = 0
        dX_c = np.zeros((N, D))
        Obj_all = 0;
        dX_all = np.zeros((N,D))

        eigThd = 1e-6 # threshold small eigenvalues for a better subgradient

        # compute objective and gradient for first term \sum ||TX_c||*
        for c in classes:
            A = X[y==c,:]

            # SVD
            U, S, V = sp.linalg.svd(A, full_matrices = False, lapack_driver='gesvd')

            V = V.T
            nuclear = np.sum(S);

            ## L_c = max(DELTA, ||TY_c||_*)-DELTA

            if nuclear>DELTA:
              Obj_c += nuclear;

              # discard small singular values
              r = np.sum(S<eigThd)

            #   s = S[0:S.shape[0]-r]/S.max()*0.1 + 0.9

            #   uprod = (U[:,0:U.shape[1]-r].dot(np.diag(s))).dot(V[:,0:V.shape[1]-r].T)
              uprod = U[:,0:U.shape[1]-r].dot(V[:,0:V.shape[1]-r].T)

              dX_c[y==c,:] += uprod
            else:
              Obj_c+= DELTA

        # compute objective and gradient for secon term ||TX||*

        U, S, V = sp.linalg.svd(X, full_matrices = False, lapack_driver='gesvd')  # all classes

        V = V.T

        Obj_all = np.sum(S);

        r = np.sum(S<eigThd)

        # s = 1-S[0:S.shape[0]-r]/S.max()*0.1

        uprod = U[:,0:U.shape[1]-r].dot(V[:,0:V.shape[1]-r].T)
        # uprod = (U[:,0:U.shape[1]-r].dot(np.diag(s))).dot(V[:,0:V.shape[1]-r].T)

        dX_all = uprod

        obj = (Obj_c  - Obj_all)/N*0.325

        # dX = ( - dX_all)/N*np.float(0.25)
        dX = (dX_c  - dX_all)/N*np.float(0.325)
        # assert obj > 0

        self.dX = torch.FloatTensor(dX)
        return torch.FloatTensor([float(obj)]).cuda()

    @staticmethod
    def backward(self, grad_output):
        # print self.dX
        return self.dX.cuda(), None


# 0 , 1 , 1 , 2 , 3 , 5, 8, 13, 21
#def solution(n):
#    answer=[0,1]
#    for i in range(1,n):
#        n=answer[i-1]+answer[i]
#        answer.append(n)
#    return answer[i+1]%1234567

def solution(n):
    a=0
    b=1
    for i in range(1,n):
        c=a+b
        a=b
        b=c
    return c%1234567


import os
import pytest
from hw8.tasks.task_1 import WrapperForStorage

path1 = os.path.join(os.path.abspath(os.path.dirname(__file__)), "bad_data.txt")
path2 = os.path.join(os.path.abspath(os.path.dirname(__file__)), "data.txt")

def test_bad_value():
    with pytest.raises(ValueError):
        WrapperForStorage(path1)

def test_wrapper():
    test1 = WrapperForStorage(path2)

    assert test1.song == "shadilay"
    assert test1["song"] == "shadilay"
    assert isinstance(test1.power, int)
    assert test1.__doc__ != "docstring_that_never_be_used"


from sklearn.feature_selection import RFE

def RFEE(df, df_test):
    cols = list(df.columns)
    cols.remove('quality')
    cols.remove('kfold')
    X = df[cols]
    y = df.quality
    model = LogisticRegression(multi_class = 'multinomial', solver = 'lbfgs')
    rfe = RFE(estimator = model, verbose = 10,n_features_to_select = 210)
    rfe.fit(X,y)
    a = rfe.support_
    df = pd.concat([X.loc[:, a],df[["quality", "kfold"]]], axis = 1)
    df_test.drop("quality", axis = 1)
    t = df_test.drop("quality", axis = 1)
    df_test = pd.concat([t.loc[:, a], df_test[["quality"]]], axis = 1)

    return df, df_test


import glob
import datetime
import os
import pandas as pd
import re
from tkinter import filedialog
from tkinter import *
from tkinter import ttk
from tkinter import messagebox

# pyinstaller --onefile --noconsole --icon GetCSV.ico Arca_GetCSVConverter_1-0-0.py
#for MMW 18-5 spreadsheets

probCol = False
#infer desktop
desktopPath = r'C:\\Users\sharo\Desktop'
filelist=['']
probRecords = []
probColls = []
#filename = r'arms_modsonly_May9.csv'

col_names =  ["BCRDHSimpleObjectPID",'imageLink','filename','directory','childKey','title', 'alternativeTitle', 'creator1', 'creator2','creator3']
col_names += ['corporateCreator1','corporateCreator2','contributor1','contributor2','corporateContributor1','publisher_original','publisher_location']
col_names += ['dateCreated','description','extent','topicalSubject1','topicalSubject2','topicalSubject3','topicalSubject4','topicalSubject5']
col_names += ['geographicSubject1','coordinates','personalSubject1','personalSubject2','corporateSubject1','corporateSubject2', 'dateIssued_start']
col_names += ['dateIssued_end','dateRange', 'frequency','genre','genreAuthority','type','internetMediaType','language1','language2','notes']
col_names += ['accessIdentifier','localIdentifier','ISBN','classification','URI']
col_names += ['source','rights','creativeCommons_URI','rightsStatement_URI','relatedItem_title','relatedItem_PID','recordCreationDate','recordOrigin']

pattern1 = r'^[A-Z][a-z]{2}-\d{2}$' #%b-%Y date (e.g. Jun-17)
pattern2 = r'^\d{2}-\d{2}-[1-2]\d{3}$'

def browse_button():
    # Allow user to select a directory and store it in global var
    # called folder_path1
    lbl1['text'] = ""
    csvname =  filedialog.askopenfilename(initialdir = desktopPath,title = "Select file",filetypes = (("csv files","*.csv"),("all files","*.*")))
    if ".csv" not in csvname:
        lbl1['text'] = "**Please choose a file with a .csv extension!"
    else:
        filelist[0] = csvname
        lbl1['text'] = csvname

def splitMultiHdgs(hdgs):
    if pd.notna(hdgs):
        hdgs = hdgs.replace("\\,",";")
        hdgs = hdgs.split(",")
        newhdgs = []
        for hdg in hdgs:
            newhdg = hdg.replace(";", ",")
            newhdgs.append(newhdg)
        return newhdgs
    else:
        return None

def getPersonalRoles(item, df, pd):
    roleTerm1 = None
    roleTerm2 = None
    roles = None
    if 'mods_name_personal_role_roleTerm_ms' in df.columns:
        roleTerm1 = item.mods_name_personal_role_roleTerm_ms
    if 'mods_name_personal_role_roleTerm_text_ms' in df.columns:
        roleTerm2 = item.mods_name_personal_role_roleTerm_text_ms
    if pd.notna(roleTerm1):
        roles = roleTerm1.split(",")
    elif pd.notna(roleTerm2):
        roles = roleTerm2.split(",")
    return roles

def combineNameParts(subject, item, df):
    wholeNames = []
    if subject:
        givenNames = item.mods_subject_name_personal_namePart_given_ms
        familyNames = item.mods_subject_name_personal_namePart_family_ms
        if len(givenNames) != len(familyNames):
            probRecords.append(item.PID)
            probCol = True
    else:
        givenNames = item.mods_name_personal_namePart_given_ms
        familyNames = item.mods_name_personal_namePart_family_ms
    givenNames = splitMultiHdgs(givenNames)

    familyNames = splitMultiHdgs(familyNames)

    for n in range (len(givenNames)):
        wholeNames.append(familyNames[n].strip() + ", " + givenNames[n].strip())
    return wholeNames

def convert_date(dt_str, letter_date):
    """
    Converts an invalid formatted date into a proper date for ARCA Mods
    Correct format:  Y-m-d
    Fixes:
    Incorrect format: m-d-Y
    Incorrect format (letter date): m-d e.g. Jun-17
    :param dt_str: the date string
    :param letter_date: whether the string is a letter date. Letter date is something like Jun-17
    :return: the correctly formatted date
    """
    if letter_date:
        rev_date = datetime.datetime.strptime(dt_str, '%b-%y').strftime('%Y-%m')  # convert date to yymm string format
        rev_date_pts = rev_date.split("-")
        year_num = int(rev_date_pts[0])
        if year_num > 1999:
            year_num = year_num - 100
        year_str = str(year_num)
        rev_date_pts[0] = year_str
        revised = "-".join(rev_date_pts)

    else:
        revised = datetime.datetime.strptime(dt_str, '%d-%m-%Y').strftime(
            '%Y-%m-%d')  # convert date to YY-mm string format

    return revised

def sortValues(lst):
    for item in lst:
        if pd.isna(item):
            lst.remove(item)
    lst = set(lst)
    lst = list(lst)
    return lst

def dropNullCols(df):
    nullcols = []
    for col in df.columns:
        notNull = df[col].notna().sum()
        if notNull < 1:
            nullcols.append(col)
            print(col + " is an empty column")
    return nullcols

def convert():
    probCol = False
    df2  = pd.DataFrame(columns = col_names)
    df2.append(pd.Series(), ignore_index=True)
    f=filelist[0]
#     if not os.path.exists(savePath):    #if folder does not exist
#         os.makedirs(savePath)
    try:
        df = pd.read_csv(f,dtype = "string", encoding = 'utf_7')
    except UnicodeDecodeError:
        df = pd.read_csv(f,dtype = "string", encoding = 'utf_8')
    print("# of columns before dropping " + str(len(df.columns)))
    print(dropNullCols(df))
    nullcols = dropNullCols(df)
    df.drop(nullcols, axis=1, inplace=True)

    print("# of columns after dropping " + str(len(df.columns)))
    #df = df.where((pd.notna(df)), None) #convert Pandas NaNs to None
    i = 1
    for item in df.itertuples():
        #PID
        df2.at[i, 'BCRDHSimpleObjectPID'] = item.PID
        #print(item.mods_languageTerm_text_ms)
        gNames = None
        fNames = None
        subGN = None
        subFN = None
        if 'mods_name_personal_namePart_given_ms' in df:
            gNames = item.mods_name_personal_namePart_given_ms
        if 'mods_name_personal_namePart_family_ms' in df:
            fNames = item.mods_name_personal_namePart_family_ms

        if 'mods_subject_name_personal_namePart_given_ms' in df:
            subGN = item.mods_subject_name_personal_namePart_given_ms
        if 'mods_subject_name_personal_namePart_family_ms' in df:
            subFN = item.mods_subject_name_personal_namePart_family_ms

        if pd.notna(gNames):
            if pd.isna(fNames):
                probRecords.append(item.PID)
                probCol = True
                print("problem: " + item.PID)
                continue
            gNames = splitMultiHdgs(gNames)
            fNames = splitMultiHdgs(fNames)
            if len(gNames) != len(fNames):
                probRecords.append(item.PID)
                print("problem: " + item.PID)
                continue

        if pd.notna(subGN):
            if pd.isna(subFN):
                probRecords.append(item.PID)
                probCol = True
                print("problem: " + item.PID)
                continue
            gNames = splitMultiHdgs(subGN)
            fNames = splitMultiHdgs(subFN)
            if len(gNames) != len(fNames):
                probRecords.append(item.PID)
                print("problem: " + item.PID)
                probCol = True
                continue

        #Local Identifier
        if 'mods_identifier_local_ms' in df.columns:
            localID = item.mods_identifier_local_ms
            if pd.notna(localID) and localID != "None":
                df2.at[i,'localIdentifier'] = localID

        #Access Identifer
        if 'mods_identifier_access_ms' in df.columns:
            accessID = item.mods_identifier_access_ms
            if pd.notna(accessID):
                df2.at[i,'accessIdentifier'] = accessID
        #Image Link
        # Link to Image

        PIDparts = item.PID.split(":")
        repo = PIDparts[0] #repository code
        num = PIDparts[1] #auto-generated accession number
        imageLink = "https://bcrdh.ca/islandora/object/" + repo + "%3A" + num
        df2.at[i, 'imageLink'] = imageLink

        #Title
        if 'mods_titleInfo_title_ms' in df.columns:
            title = item.mods_titleInfo_title_ms
            if pd.notna(title):
                df2.at[i,'title'] = title.replace("\,",",")

        #Alternative Title
        if "mods_titleInfo_alternative_title_ms" in df.columns:
            altTitle = item.mods_titleInfo_alternative_title_ms
            if pd.notna(altTitle):
                df2.at[i, 'alternativeTitle'] = altTitle.replace("\,",",")

        #Date
        if "mods_originInfo_dateIssued_ms" in df.columns:

            dt = item.mods_originInfo_dateIssued_ms
            if pd.notna(dt):
                if (re.match(pattern1, dt)): #letter date, i.e. Jun-17
                    dt = convert_date(dt, True)
                elif (re.match(pattern2, dt)): #reverse date
                    dt = convert_date(dt, False)
                df2.at[i,'dateCreated'] = dt

        #Date Issued Start
        if 'mods_originInfo_encoding_w3cdtf_keyDate_yes_point_start_dateIssued_ms' in df.columns:
            startDt = item.mods_originInfo_encoding_w3cdtf_keyDate_yes_point_start_dateIssued_ms
            if pd.notna(startDt):
                df2.at[i,'dateIssued_start'] = startDt

        #Date Issued End
        if 'mods_originInfo_encoding_w3cdtf_keyDate_yes_point_end_dateIssued_ms' in df.columns:
            endDt = item.mods_originInfo_encoding_w3cdtf_keyDate_yes_point_end_dateIssued_ms
            if pd.notna(endDt):
                df2.at[i,'dateIssued_end'] = startDt

        #Publisher
        if 'mods_originInfo_publisher_ms' in df.columns:
            pub = item.mods_originInfo_publisher_ms
            if pd.notna(pub):
                df2.at[i, 'publisher_original'] = pub

        #Publisher Location
        if 'mods_originInfo_place_placeTerm_text_ms' in df.columns:
            place = item.mods_originInfo_place_placeTerm_text_ms
            if pd.notna(place):
                df2.at[i,'publisher_location'] = place

        #Frequency (serials only)
        if 'mods_originInfo_frequency_ms' in df.columns:
            freq = item.mods_originInfo_frequency_ms
            if pd.notna(freq):
                df2.at[i,'frequency'] = freq

        #Extent
        if "mods_physicalDescription_extent_ms" in df.columns:
            extent = item.mods_physicalDescription_extent_ms
            if pd.notna(extent):
                extent = extent.replace("\,",",")
                df2.at[i, 'extent'] = extent

        #Notes
        if 'mods_note_ms' in df.columns:
            notes = item.mods_note_ms
            if pd.notna(notes):
                notes = notes.replace("\,",",")
                df2.at[i, 'notes'] = notes

        #Description/Abstract
        if "mods_abstract_ms" in df.columns:
            descr = item.mods_abstract_ms
            #if type(descr) != 'pandas._libs.missing.NAType':
            if pd.notna(descr):
            #if descr is not None:
                df2.at[i, 'description'] = descr.replace("\,",",")

        #Personal Creators & Contributors
        if 'mods_name_personal_namePart_given_ms' in df.columns:
            if pd.notna(item.mods_name_personal_namePart_given_ms):
                wholeNames = combineNameParts(False,item,df)
                roles = getPersonalRoles(item,df,pd)
                creatorCount = 0
                contribCount = 0
                for x in range(len(wholeNames)):
                    if roles[x] == "creator":
                        creatorCount = creatorCount + 1
                        hdg = "creator" + str(creatorCount)
                        df2.at[i,hdg] = wholeNames[x].replace("*", ",")

                    else:
                        contribCount = contribCount + 1
                        hdg = "contributor" + str(contribCount)

                    df2.at[i,hdg] = wholeNames[x]

        #Corporate Creators and Contributors
        if 'mods_name_corporate_namePart_ms' in df.columns:
            corpNames = item.mods_name_corporate_namePart_ms
            if pd.notna(corpNames):
                creatorCount = 0
                contribCount = 0
                corpRoles = item.mods_name_corporate_role_roleTerm_ms.split(",")
                corpNames = splitMultiHdgs(corpNames)
#                 if len(corpNames) > 3:
#                     print(item.PID + " has more than two corporate names ... please review!")
                count = 0
                for corpName in corpNames:
                    if corpRoles[count] == 'creator':
                        creatorCount = creatorCount + 1
                        hdg = "corporateCreator" + str(creatorCount)
                        df2.at[i,hdg] = corpName
                    else:
                        contribCount = contribCount + 1
                        hdg = "corporateContributor" + str(contribCount)
                        df2.at[i,hdg] = corpName

        #topical subjects
        if 'mods_subject_topic_ms' in df.columns:
            topics = item.mods_subject_topic_ms
            if pd.notna(topics):
                topics = splitMultiHdgs(topics)
                for x in range(len(topics)):
                    hdg = "topicalSubject" + str(x+1)
                    df2.at[i, hdg] = topics[x]

        #corporate subjects
        if 'mods_subject_name_corporate_namePart_ms' in df.columns:
            corpSubs = item.mods_subject_name_corporate_namePart_ms
            if pd.notna(corpSubs):
                corpSubs = splitMultiHdgs(corpSubs)
                corpSubs = list(set(corpSubs)) #remove duplicates
                for x in range(len(corpSubs)):
                    hdg = "corporateSubject" + str(x+1)
                    df2.at[i, hdg] = corpSubs[x]

#        #personal subjects
        if 'mods_subject_name_personal_namePart_given_ms' in df.columns:
            if pd.notna(item.mods_subject_name_personal_namePart_given_ms):
                wholeNames = combineNameParts(True,item,df)
                for x in range(len(wholeNames)):
                    hdg = "personalSubject" + str(x+1)
                    df2.at[i,hdg] = wholeNames[x].replace("*", ",")

        #temporal subject (date range)
        if 'mods_subject_temporal_ms' in df.columns:
            tempSub = item.mods_subject_temporal_ms
            if pd.notna(tempSub):
                df2.at[i,'dateRange'] = tempSub

        #geographic subject
        if 'mods_subject_geographic_ms' in df.columns:
            geosub = item.mods_subject_geographic_ms
            if pd.notna(geosub):
                geosubs = splitMultiHdgs(geosub)
                for x in range(len(geosubs)):
                    hdg = "geographicSubject" + str(x+1)
                    df2.at[i, hdg] = geosubs[x]

        #coordinates
        if 'mods_subject_geographic_cartographics_ms' in df.columns:
            coords = item.mods_subject_geographic_cartographics_ms
            if pd.notna(coords):
                df2.at[i,"coordinates"] = coords

        #classification
        if 'mods_classification_authority_lcc_ms' in df.columns:
            lcClass = item.mods_classification_authority_lcc_ms
            if pd.notna(lcClass):
                df2.at[i,'classification'] = lcClass

        #isbn
        if 'mods_identifier_isbn_ms' in df.columns:
            isbn = item.mods_identifier_isbn_ms
            if pd.notna(isbn):
                df2.at[i,'ISBN'] = isbn

        #genre
        if 'mods_genre_authority_aat_ms' in df.columns:
            genre_aat = item.mods_genre_authority_aat_ms
            if pd.notna(genre_aat):
                df2.at[i, 'genre'] = genre_aat
                df2.at[i, 'genreAuthority'] = "aat"
        elif 'mods_genre_authority_marcgt_ms' in df.columns:
            if pd.notna(item.mods_genre_authority_marcgt_ms):
                df2.at[i, 'genre'] = item.mods_genre_authority_marcgt_ms
                df2.at[i, 'genreAuthority'] = "marcgt"
        elif 'mods_originInfo_genre_ms' in df.columns:
            if pd.notna(item.mods_originInfo_genre_ms):
                df2.at[i, 'genre'] = item.mods_originInfo_genre_ms

        #type
        if 'mods_typeOfResource_ms' in df.columns:
            _type = item.mods_typeOfResource_ms
            if pd.notna(_type):
                df2.at[i, 'type'] = _type

        #internet media type
        if 'mods_physicalDescription_internetMediaType_ms' in df.columns:
            mediaType = item.mods_physicalDescription_internetMediaType_ms
            if isinstance (mediaType, str):
                df2.at[i, 'internetMediaType'] = mediaType

        #Languages
        languages = None
        if 'mods_language_languageTerm_ms' in df.columns:
            langs = item.mods_language_languageTerm_ms
            if pd.notna(langs):
                languages = langs
        elif 'mods_language_languageTerm_text_ms' in df.columns:
            langs = item.mods_language_languageTerm_text_ms
            if pd.notna(langs):
                languages = langs
        elif 'mods_languageTerm_text_ms' in df.columns:
            langs = item.mods_languageTerm_text_ms
            if pd.notna(langs):
                languages = langs
        if languages is not None:
            languages = languages.split(",")
            for x in range(len(languages)):
                lang = languages[x]
                hdg = "language" + str(x+1)
                if pd.notna(lang):
                    df2.at[i,hdg] = lang

        #Source
        if 'mods_location_physicalLocation_ms' in df.columns:
            source = item.mods_location_physicalLocation_ms
            if pd.notna(source):
                df2.at[i, 'source'] = source

        #URI
        if 'mods_identifier_uri_ms' in df.columns:
            uri = item.mods_identifier_uri_ms
            if pd.notna(uri):
                df2.at[i, 'URI'] = uri

        #Rights
        if 'mods_accessCondition_use_and_reproduction_ms' in df.columns:
            rights = item.mods_accessCondition_use_and_reproduction_ms
            if isinstance(rights, str):
                rights = splitMultiHdgs(item.mods_accessCondition_use_and_reproduction_ms)

                for stmt in rights:
                    if "Permission to publish" in stmt:
                        df2.at[i, "rights"] = stmt
                    elif "rightsstatements" in stmt:
                        df2.at[i,"rightsStatement_URI"] = stmt
                    else:
                        df2.at[i,"creativeCommons_URI"] = stmt

        #Related Item
        if 'mods_relatedItem_host_titleInfo_title_ms' in df.columns:
            coll_title = item.mods_relatedItem_host_titleInfo_title_ms
            coll_PID = item.mods_relatedItem_host_identifier_PID_ms
            if pd.notna(coll_title):
                df2.at[i, "relatedItem_title"] = coll_title
            if pd.notna(coll_PID):
                df2.at[i, "relatedItem_PID"] = coll_PID

        #Record Origin & Creation Date
        if 'mods_recordInfo_recordOrigin_ms' in df.columns:
            recOrigin = item.mods_recordInfo_recordOrigin_ms
            if pd.notna(recOrigin):
                df2.at[i,'recordOrigin'] = recOrigin
            recDate = item.mods_recordInfo_recordCreationDate_ms
            if pd.notna(recDate):
                recDate = recDate.split(",")
                df2.at[i, 'recordCreationDate'] = recDate[0]
        if (probCol):
            probColls.append(f)
        i = i + 1

    #savePath = r'C:/Users/sharo/Desktop' + '/'

    f = os.path.basename(f).replace(".csv","_Rev1.csv")
    dest = os.path.join(desktopPath,f)
    print("destination is " + dest)
    df2.to_csv(dest, encoding='utf-8', index=False)

    msg = f + " has been written to the your desktop."
    messagebox.showinfo(title = 'Conversion Finished', message = msg)

root = Tk()

root.eval('tk::PlaceWindow %s center' % root.winfo_toplevel())
root.title = "Arca GetCSV -> BCRDH CSV Converter"
root.configure(background='#DAE6F0')
folder_path = StringVar()

intro = ttk.Label(master=root,text="Arca_GetCSV_Converter Version 1.0.0", background='#DAE6F0',font="Arial 15 bold")
intro.grid(row=0,column=1,padx=(10,0),sticky='w')
info = ttk.Label(master=root,text="This app converts an Arca Get Metadata CSV to a MMW 18-6 BCRDH\nmetadata spreadsheet. The revised CSV, with 'Rev1' appended to it,\nwill be written to your desktop. Separated given name and family\nname data are assumed as input, and single-field personal name\nvalues are output to the BCDRH CSV.", background='#DAE6F0',font="Arial 9 bold")
info.grid(row=1,column=1,padx=(10,0),sticky='w')

srclbl = ttk.Label(master=root,text="\nChoose CSV to convert",background='#DAE6F0',font="Arial 10 italic")
srclbl.grid(row=2,column=1,padx=(10,0),sticky='w')

button1 = ttk.Button(text="Browse", command=browse_button)
button1.grid(row=3, column=1,  padx=(10,0),sticky='w')

lbl1 = ttk.Label(master=root,background='#DAE6F0',font="Arial 10")
lbl1.grid(row=4, column=1,padx=(10,5), sticky='w')

compButton = ttk.Button(text="Generate BCRDH CSV",command=convert)
compButton.grid(row=8, column=1,padx=(10,0),pady=(15,15),sticky='w')

mainloop()


import os
import sys
import pythonipsum

try:
    from setuptools import setup
except ImportError:
    from distutils.core import setup

if sys.argv[-1] == 'publish':
    os.system('python setup.py sdist upload')
    sys.exit()

requires = ['loremipsum']

testing_requires = ['konira']

setup(
    name='pythonipsum',
    version=pythonipsum.__version__,
    description='A lorem ipsum generator for the Python world.',
    long_description=open('README.md').read() + '\n\n' +
                     open('HISTORY.md').read(),
    author='Mike Pirnat',
    author_email='mpirnat@gmail.com',
    url='https://github.com/mpirnat/pythonipsum/',
    packages=['pythonipsum'],
    package_dir={'pythonipsum': 'pythonipsum'},
    package_data={'pythonipsum': ['pythonipsum/data/*.txt']},
    include_package_data=True,
    install_requires=requires + testing_requires,
    license=open('LICENSE').read(),
    zip_safe=False,
    classifiers=(
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.5',
        'Programming Language :: Python :: 2.6',
        'Programming Language :: Python :: 2.7',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ),
)


#!/usr/bin/env python3

import pdal

def main():
    with open("pipeline_entwine.json") as pipeline_json:
        json_pipeline = pipeline_json.read()
    pipeline = pdal.Pipeline(json_pipeline)
    pipeline.validate()  # check if our JSON and options were good
    pipeline.loglevel = 8  # really noisy
    pipeline.execute()

if __name__ == "__main__":
    main()


# Definition for singly-linked list.
class ListNode:
    def __init__(self, x):
        self.val = x
        self.next = None

class Solution:
    def hasCycle(self, head: Optional[ListNode]) -> bool:
        # edge case, empty linkedlist [], so none
        if head is None:
            return False

        # set up two pointers
        slow = head
        fast = head

        while fast.next and fast.next.next:
            # fast advances two steps while slow advances one step
            fast = fast.next.next
            slow = slow.next

            if fast == slow:
                return True

        # if it reaches here, linked list has an end node, null
        return False


#! /usr/bin/python3

# @File:   MultiprocessingDemo.py
# @Author: tiannanyihao
# @DATE:   2019-01-18
# @TIME:   16:14
# @Software: PyCharm
# @Production:

from multiprocessing import *

def multiprocessDown(*args):
    arr = [1, 2, 3, 4, 5, 6, 7, 8, 9]

    for data in arr:
        args[0].put(data)

def multiprocessWrit(*args):
    while True:
        data = args[0].get()
        print(data)
        if args[0].empty():
            print('over~')
            break
    pass

def doMain():
    q = Queue()
    multi_down = Process(target=multiprocessDown, args=(q,))
    multi_Write = Process(target=multiprocessWrit, args=(q,))

    multi_down.start()
    multi_Write.start()

if __name__ == '__main__':
    doMain()


#!/usr/bin/python3
"""first endpoint returns status of API"""

from flask import Flask, jsonify, Blueprint
from flask_cors import CORS
from models import storage
from api.v1.views import app_views
from os import getenv

app = Flask(__name__)
app.register_blueprint(app_views)
cors = CORS(app, resources={"/*": {"origins": "0.0.0.0"}})

@app.teardown_appcontext
def teardown_appcontext(code):
    """closes storage"""
    storage.close()

@app.errorhandler(404)
def page_not_found(code):
    """handler for error 404"""
    return (jsonify({
        "error": "Not found"
    }), 404)

if __name__ == "__main__":
    PORT = getenv("HBNB_API_PORT", '5000')
    HOST = getenv("HBNB_API_HOST", '0.0.0.0')
    app.run(host=HOST, port=PORT, threaded=True)


import json

from bindilla.host_http_server import HOST, make
from tornado.testing import AsyncHTTPTestCase

class TestIndex(AsyncHTTPTestCase):
    def get_app(self):
        return make()

    def test_index(self):
        response = self.fetch('/')
        self.assertEqual(response.code, 200)

class TestV0App(AsyncHTTPTestCase):
    def get_app(self):
        return make()

    def test_manifest(self):
        response = self.fetch('/v0/manifest')
        self.assertEqual(response.code, 200)
        self.assertEqual(json.loads(response.body), HOST.manifest())

        environs = [
            'https://github.com/org1/repo1',
            'https://github.com/org2/repo2'
        ]
        url = '/{}/v0/manifest'.format(
            ','.join(environs)
        )
        response = self.fetch(url)
        self.assertEqual(response.code, 200)
        self.assertEqual(json.loads(response.body), HOST.manifest(environs))


H, W = map(int, input().split())
S = [input() for _ in range(H)]

for i in range(H):
    for j in range(W):
        if S[i][j] != '#':
            continue
        di_list = [i, i + 1, i, i -1]
        dj_list = [j + 1, j, j - 1, j]
        ok = False
        for di, dj in zip(di_list, dj_list):
            if di < 0 or H <= di: continue
            if dj < 0 or W <= dj: continue
            if S[di][dj] == '#':
                ok = True
                break
        if not ok:
            print('No')
            exit()

print('Yes')


import config
import pandas as pd

class DBFetch:

    def __init__(self, table_name):
        """
        This class is used to fetch data from mongo db.
        :param table_name: name of the table.
        """
        self.table_name = table_name

    def fetch_data_from_db(self):
        """

        :return: Fetched data
        """
        try:
            table = config.mongo_db.my_db[self.table_name]
            records = []
            for i in table.find():
                records.append(i)
            train_data = pd.DataFrame(data=records)
            train_data['Item_Weight'] = train_data['Item_Weight'].apply(lambda x: x if x != 0 else None)
            train_data['Outlet_Size'] = train_data['Outlet_Size'].apply(lambda x: x if x != '' else None)
            train_data.drop(columns=['_id'], inplace=True)
            message = "Successfully fetched from from db"
            config.logger.log("INFO", message)
            return {"status": "Success", "data": train_data, "message": message}
        except Exception as e:
            message = str(e)
            config.logger.log("ERROR", str(e))
            return {"status": "Failure", "data": None, "message": message}


from django.core.management.base import BaseCommand

from projectmarswar.models import Player

class Command(BaseCommand):
    help = "Adjust ranks of players"

    def add_arguments(self, parser):
        parser.add_argument(
            "--count",
            "-c",
            type=int,
            help="Only adjust players with a minimum number of plays",
            default=0
        )

    def handle(self, *args, **options):
        players = Player.objects.order_by("-rating")
        players.update(rank=-1)
        filtered_players = [
            player for player in players if player.get_total_matches() > options["count"]
        ]
        last_rating_match = (0, 0)
        for rank, player in enumerate(filtered_players, start=1):
            if player.rating == last_rating_match[1]:
                rank = last_rating_match[0]
            else:
                last_rating_match = (rank, player.rating)
            player.rank = rank
            player.save()


import json
from flask import Flask, Request
import escritura
import salones_primero
import threading

app = Flask(__name__)

@app.route('/biblioteca')
def obtenerJson():
    d = ""
    with open('data_biblioteca.json') as json_data:
        d = json.load(json_data)
    return str(d)

@app.route('/escribirb')
def escribirJsonBiblioteca():
    escritura.escribir()
    return "OK"

@app.route('/salones')
def obtenerJsonSalones():
    d = ""
    with open('data_salones.json') as json_data:
        d = json.load(json_data)
    return str(d).replace('1, ', '')

def escSalones():
    salones_primero.principal()

@app.route('/escribirs')
def escribirJsonSalones():
    threads = list()
    t = threading.Thread(target=escSalones)
    threads.append(t)
    t.start()
    return ("OK")

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


# encoding: utf-8

import sys, os
from PyQt5 import QtWidgets, QtCore, QtGui
from PyQt5.Qt import Qt
from datetime import datetime, timedelta
import licence, keygen_rc

class MainWindow(QtWidgets.QDialog):
    def __init__(self, parent = None):
        super(MainWindow, self).__init__(parent)

        self.setWindowFlags(Qt.CustomizeWindowHint | Qt.WindowCloseButtonHint)
        self.setWindowState(Qt.WindowNoState)
        mainLayout = QtWidgets.QVBoxLayout()
        self.createMessageGroupBox(mainLayout)
        self.buttonBox = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok |
                                                    QtWidgets.QDialogButtonBox.Cancel
                                                    , self)
        self.buttonBox.accepted.connect(self.accept)
        self.buttonBox.button(QtWidgets.QDialogButtonBox.Ok).setText(self.tr('生成'))
        self.buttonBox.button(QtWidgets.QDialogButtonBox.Cancel).setText(self.tr('退出'))
        self.buttonBox.rejected.connect(self.reject)
        self.cwd =  os.path.join(os.path.dirname(os.path.abspath(__file__)))

        mainLayout.addWidget(self.buttonBox)
        self.setLayout(mainLayout)
        self.setWindowTitle(self.tr("沃易智信注册机"))
        self.setWindowIcon(QtGui.QIcon(':/images/app.ico'))
        self.resize(360,280)

    def get_lic_expr_date(self, sdate):
        span = int(self.licspanEdit.text())
        currentIndex = self.spanUnit.currentIndex()
        if currentIndex == 2:
            return sdate.replace(year = sdate.year+span)
        elif currentIndex == 1:
            return sdate.replace(year=sdate.year+int(span/12), month=sdate.month+int(span%12))
        return sdate + timedelta(days=span)

    def accept(self):
        edate = self.startdate.dateTime().toPyDateTime()
        licence.key_gen(self.macEdit.text(), self.productEdit.currentText(),
                        self.versionEdit.text(),
                        edate, self.licpathEdit.text(),
                        **{'edition':self.editionEdit.currentText()})
        super(MainWindow, self).accept()

    def reject(self):
        super(MainWindow, self).reject()

    def select_lic_path(self):
        fpath = os.path.join(self.cwd, '%s-%s' %(self.productEdit.currentText(), self.versionEdit.text()))
        fpath, _ = QtWidgets.QFileDialog.getSaveFileName(self, directory=fpath,
                                                         filter=self.tr('Licence Files (*.lic)'))
        self.licpathEdit.setText(fpath)

    def createMessageGroupBox(self, mlayout):
        macLabel = QtWidgets.QLabel(self.tr("机器码:"))
        self.macEdit = QtWidgets.QLineEdit(licence.get_maccode())

        productLabel = QtWidgets.QLabel(self.tr("产品名称:"))
        self.productEdit = QtWidgets.QComboBox()
        self.productEdit.addItems(['otsweb', 'scada'])

        versionLabel = QtWidgets.QLabel(self.tr("产品版本:"))
        self.versionEdit = QtWidgets.QLineEdit('1.0')

        editionLabel = QtWidgets.QLabel(self.tr("产品edition:"))
        self.editionEdit = QtWidgets.QComboBox()
        self.editionEdit.addItems(['标准版', '企业版'])

        sdateLabel = QtWidgets.QLabel(self.tr("许可到期时间:"))
        self.startdate = QtWidgets.QDateTimeEdit(QtCore.QDateTime.currentDateTime())
        self.startdate.setDisplayFormat('yyyy-MM-dd HH:mm:ss')

        licpathLabel = QtWidgets.QLabel(self.tr("许可文件路径:"))
        self.licpathEdit = QtWidgets.QLineEdit()
        self.licpathBtn = QtWidgets.QPushButton()
        self.licpathBtn.clicked.connect(self.select_lic_path)

        layout = QtWidgets.QGridLayout()
        layout.addWidget(macLabel, 0, 0)
        layout.addWidget(self.macEdit, 0, 1, 1, 2)

        layout.addWidget(productLabel, 1, 0)
        layout.addWidget(self.productEdit, 1, 1, 1, 2)

        layout.addWidget(versionLabel, 2, 0)
        layout.addWidget(self.versionEdit, 2, 1, 1, 2)

        layout.addWidget(editionLabel, 3, 0)
        layout.addWidget(self.editionEdit, 3, 1, 1, 2)

        layout.addWidget(sdateLabel, 4, 0)
        layout.addWidget(self.startdate, 4, 1, 1, 2)

        layout.addWidget(licpathLabel, 5, 0)
        layout.addWidget(self.licpathEdit, 5, 1)
        layout.addWidget(self.licpathBtn, 5, 2)

        mlayout.addLayout(layout)

def main(args):
    app = QtWidgets.QApplication(args)
    dlg = MainWindow()
    dlg.show()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main(sys.argv)


# Anonymous user profile.
# Used to debug and test the assistant via shell.
anonymous_profile = {
    "id": "anonymous",
    "name": "anonymous"
}

# Priority to urgency ID mappings.
priorities = {
    "low": 3,
    "medium": 2,
    "high": 1
}

# State key to description mappings.
states = {
    "New": "is currently awaiting triage",
    "In Progress": "is currently in progress",
    "On Hold": "has been put on hold",
    "Closed": "has been closed"
}


"""Runs the rewards search continuously until all accounts in db reach their daily limit"""

from bingBot import BingBot
from dataLink import DataLink

def main():
    data = DataLink()
    loginCount = data.rowCount()
    for i in range(1,loginCount+1):
        bot = BingBot(i)
        bot.desktop()
        bot.mobile()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
import numpy as np
import pickle
import os

def createVector(database):
    vector = []
    dict_database = database.items()
    for file in dict_database : # browse all the files
        words = list(file[1][0]) #  because dictionnary = {file, [[all, words], level SST]}
        for word in words : # browse only all the words uses in the file
            if word not in vector :
                vector.append(word) # add a new word to the vector list
            else :
                pass
    return vector

def tokenize(database, vector):
    """
    This function enable to tokenize all the words in each file.
    """

    dict_database = database.items()
    features = []
    for file in dict_database :
        words = list(file[1][0])
        # Error : ValueError: maximum supported dimension for an ndarray is 32, found 15502
        vectorFile = np.zeros(len(vector)) # create a zeros-vector of the len of the vector word
        for w in vector : # to browse all the words in the general vector of words
            for word in words : # in the file
                if w == word :
                    vectorFile[np.where(vector == w)] = vectorFile[np.where(vector == w)] + 1 # add one more uses of this word in the file
        #print(vectorFile)
        features.append(vectorFile, file[1][1])
    return features

CURR_DIR = os.getcwd()
pickle_in = open("database_dictionary.pickle", "rb")
database = pickle.load(pickle_in)
pickle_in.close()

#len database : 1280 entries
#print(len(database))
vector = createVector(database)
#print(vector)

# len vector : 15502 words
#print(len(vector))
tokenize(database, vector)


#!/usr/bin/env python
# coding: utf-8

# In[2]:

#import numpy as np

#Zadanie 2

def from_adjacency_matrix(M):
    """
    Funkcja, która na podstawie macierzy sasiedztwa M (na wejściu) wyznaczy i zwróci
    liste krotek postaci: (i, j, w_i,j), gdzie i, j oznaczaja wierzchołki grafu, zas w_i,j wage
    łaczacej ich krawedzi.
    """
    lista = [] #pusta lista
    for i in range(1, len(M)):
        for j in range(i): #tylko dla i <j
            lista.append((j, i, np.array(M)[i, j])) #dołączam do pustej listy krotki 3 elementowe (indeks j, indeks i, wartość w itym wierszu jtej kolumnie)
    return lista

def to_adjacency_matrix(x):
    """
    Funkcja, która na podstawie reprezentacji grafu w postaci listy krotek zwróci
    macierz sasiedztwa, czyli listę list.
    """
    n = max([sublist[1] for sublist in x]) + 1 #chcę wymiaru macierzy, a więc biorę maksymalną wartosć spośród drugiej pozycji
                                                #(w pythonie pozycja 1) krotek w liście i dodaję jedynkę, bo numeracja jest od 0
    Matrix = np.zeros((n,n)) #tworzę macierz zer nxn
    for el in x:
        Matrix[el[0], el[1]] = el[2] #uzupełniam macierz, która jest symetryczna, a na diagonali pozostawiam zera, tak jak w naszym grafie
        Matrix[el[1], el[0]] = el[2]
    return Matrix

#pip install disjoint-set
from disjoint_set import DisjointSet #https://pypi.org/project/disjoint-set/

def Kruskal(M):
    """
    Funkcja, która przy uzyciu algorytmu Kruskala wyznaczy i zwróci minimalne drzewo rozpinajace.
    Funkcja przyjmuje na wejściu graf pełny, ważony w postaci macierzy (listy podlist, które
    są tej samej długości, a ich el. są liczbami).
    W funkcji użyto funckji pomocniczej from_adjacency_matrix() w celu przedstawienia macierz(listy podlist)
    w formie listy krotek dla i<j.
    Na wyjściu funkcja zwraca minimalne drzewo rozpinajace w postaci listy podlist
    (za pomocą f. pomocniczej to_adjacency_matrix()).
    """
    assert (type(M) == list), "Macierz musi byćpodana w formie listy list"
    assert all((type(el) == list) for el in M) , "Elementy listy muszą być listami"
    if not all(len(l) == len(next(iter(M))) for l in iter(M)):
        raise ValueError('Podlisty w liście M muszą być tej samej długosci')
    assert all((type(el) == int or type(el) == float) for sublist in M for el in sublist), "Elementy podlist listy muszą być liczbami"
    n = len(M) #wymiar macierzy M, a dokładnie ilość list w liście
    T = [] #deklaruję pustą listę
    x = from_adjacency_matrix(M) #pomocnicza funkcja
    y = sorted(x, key=lambda el: el[2])
    #y = x.sort( key=lambda el: el[2])
    ds = DisjointSet() #https://pypi.org/project/disjoint-set/
    for i in y: #dla krotek z listy
        if ds.find(i[0]) != ds.find(i[1]): #jeżeli w krotce wartość pozycji 0 jest rózna od wart. pozycji 1
            b=T.append((i)) #to rozszerz pustą listę o tą krotkę
            ds.union(i[0], i[1]) #połącz dwa wierzchołki
    return to_adjacency_matrix(T)

# In[3]:

#Przykład 1

M = [[
0, 10.95, 13.98, 18.28, 10.49 ],
[ 10.95, 0, 9.3, 9.68, 4.23 ],
[ 13.98, 9.3, 0, 6.15, 5.31 ],
[ 18.28, 9.68, 6.15, 0, 7.83 ],
[ 10.49, 4.23, 5.31, 7.83, 0 ]
]

# In[4]:

print(from_adjacency_matrix(M))
[
(1, 4, 4.23), (2, 4, 5.31), (2, 3, 6.15), (3, 4, 7.83), (1, 2, 9.3),
(1, 3, 9.68), (0, 4, 10.49), (0, 1, 10.95), (0, 2, 13.98), (0, 3, 18.28)
]

# In[6]:

to_adjacency_matrix(from_adjacency_matrix(M))
[[0, 10.95, 13.98, 18.28, 10.49],
[10.95, 0, 9.3, 9.68, 4.23],
[13.98, 9.3, 0, 6.15, 5.31],
[18.28, 9.68, 6.15, 0, 7.83],
[10.49, 4.23, 5.31, 7.83, 0]]

# In[9]:

mst = Kruskal(M)
print(mst)

# In[10]:

#Przykład 2
M = [
[ 0, 1.3, 4.12, 0.96, 1.87, 3.23, 0.98, 2.3, 2.28, 3.51 ],
[ 1.3, 0, 2.86, 1.57, 2.63, 4.19, 1.92, 1.6, 2.67, 2.55 ],
[ 4.12, 2.86, 0, 4.02, 4.84, 6.41, 4.75, 2.49, 4.45, 1.72 ],
[ 0.96, 1.57, 4.02, 0, 1.07, 2.62, 1.91, 1.76, 1.32, 3.03 ],
[ 1.87, 2.63, 4.84, 1.07, 0, 1.6, 2.66, 2.38, 0.76, 3.57 ],
[ 3.23, 4.19, 6.41, 2.62, 1.6, 0, 3.74, 3.93, 2.02, 5.03 ],
[ 0.98, 1.92, 4.75, 1.91, 2.66, 3.74, 0, 3.23, 3.18, 4.38 ],
[ 2.3, 1.6, 2.49, 1.76, 2.38, 3.93, 3.23, 0, 1.95, 1.27 ],
[ 2.28, 2.67, 4.45, 1.32, 0.76, 2.02, 3.18, 1.95, 0, 3.02 ],
[ 3.51, 2.55, 1.72, 3.03, 3.57, 5.03, 4.38, 1.27, 3.02, 0 ]
]
mst = Kruskal(M)
print(mst)

# In[11]:

#Przykład 3 własny
M = [[
0, 1, 5, 7, 8 ],
[ 1, 0, 2, 4, 12 ],
[ 5, 2, 0, 3, 17 ],
[ 7, 4, 3, 0, 6 ],
[ 8, 12, 17, 6, 0 ]
]
mst=Kruskal(M)
print(mst)

# In[16]:

#Przykład 4 własny
M = [[
0, 10, 13, 18, 10 ],
[ 10, 0, 9, 9, 4 ],
[ 13, 9, 0, 6, 5],
[ 18, 9, 6, 0, 7 ],
[ 10, 4, 5, 7, 0 ]
]
mst=Kruskal(M)
print(mst)

# In[18]:

# test dla niepoprawnych danych -  metoda Kruskal(M)
M = [
0, 10, 13, 18, 10 ],
[ 10, 0, 9, 9, 4 ],
[ 13, 9, 0, 6, 5],
[ 18, 9, 6, 0, 7 ],
[ 10, 4, 5, 7, 0 ]

Kruskal(M)

# In[24]:

M =[5,
[ 10, 0, 9, 9, 4 ],
[ 13, 9, 0, 6, 5],
[ 18, 9, 6, 0, 7 ],
[ 10, 4, 5, 7, 0 ]]

Kruskal(M)

# In[25]:

M = [[
0, 10, 13, 18 ],
[ 10, 0, 9, 9, 4 ],
[ 13, 9, 0, 6, 5],
[ 18, 9, 6, 0, 7 ],
[ 10, 4, 5, 7, 0 ]
]
Kruskal(M)

# In[29]:

M = [[0, 10, 13, 18, "a" ],
[ 10, 0, 9, 9, 4 ],
[ 13, 9, 0, 6, 5],
[ 18, 9, 6, 0, 7 ],
[ 10, 4, 5, 7, 0 ]
]

Kruskal(M)


from embedding.hope import HOPE
from evaluation.metrics import pak
from evaluation.visualise import results_plot
from evaluation.evaluations import link_predict

file = 'data/BlogCatalog.csv'

# for i in range(0, 5):

embedding = HOPE(file, type='undirected')
embedding.embed()

pred = link_predict(embedding)
actual = embedding.get_testing_data()
result = []
for j in range(1, 7):
  result.append(pak(actual, pred, k=j))

x = ['$10^' + str(i) + '$' for i in range(1, 7)]
results_plot(x, result, 'BlogCatalog Dataset')


from asgiref.sync import async_to_sync
from channels.generic.websocket import WebsocketConsumer
import json

from rest_framework.exceptions import ValidationError

from api.serializers import InfoSerializer

class InfoConsumer(WebsocketConsumer):
    def connect(self):
        print("Connect")
        async_to_sync(self.channel_layer.group_add)("info", self.channel_name)
        self.accept()

    def disconnect(self, close_code):
        print("Disconnect")
        async_to_sync(self.channel_layer.group_discard)("info",
                                                        self.channel_name)

    def info_message(self, event):
        print("Send to websocket")
        self.send(text_data=json.dumps(event["data"]))

    def receive(self, text_data=None, bytes_data=None):
        text_data_json = json.loads(text_data)
        info = InfoSerializer(data=text_data_json)
        try:
            info.is_valid()
        except ValidationError:
            self.send(text_data=json.dumps({
                'status': "Incorrect message"
            }))
        else:
            info.save()
            self.send(text_data=json.dumps({
                'status': "Message received"
            }))


#Identical to Queue1_change, but for queue table 3
from robolink import *    # RoboDK API
from robodk import *      # Robot toolbox
from time import sleep
RDK = Robolink()

q3_table = RDK.Item('Queue 3 table')

def change_table():
    q3_rotors = q3_table.Childs()
    for i in range(16):
        q3_rotors[i].Delete()

while 1:
    q3 = RDK.getParam('q3')
    if q3 == 16:
        mbox('Empty q3')
        change_table()
        RDK.setParam('q3', 0)

    sleep(0.5)