# -*- coding: utf8 -*-
import logging
from datetime import datetime

# enum
from console.enum.app import GovernanceModeEnum
from .enum import ActionType
# exception
from console.exception.main import EnvAlreadyExist, InvalidEnvName
# repository
from console.repositories.app_config import port_repo
# service
from console.services.app_config import env_var_service
# model
from www.models.main import (ServiceProbe, TenantServiceConfigurationFile, TenantServiceEnvVar, TenantServicesPort,
                             TenantServiceVolume)
from www.models.label import ServiceLabels
from console.models.main import ComponentGraph, ServiceMonitor
from console.models.main import ServiceSourceInfo, ComponentK8sAttributes
# util
from www.utils.crypt import make_uuid

logger = logging.getLogger("default")

class Component(object):
    def __init__(self,
                 component,
                 component_source: ServiceSourceInfo,
                 envs,
                 ports,
                 volumes,
                 config_files,
                 probes: [ServiceProbe],
                 extend_info,
                 monitors,
                 graphs,
                 plugin_deps,
                 http_rules=None,
                 http_rule_configs=None,
                 tcp_rules=None,
                 service_group_rel=None,
                 labels=None,
                 support_labels=None,
                 k8s_attributes=None):
        self.component = component
        self.component_source = component_source
        self.envs = list(envs)
        self.ports = list(ports)
        self.http_rules = list(http_rules) if http_rules else []
        self.http_rule_configs = list(http_rule_configs) if http_rule_configs else []
        self.tcp_rules = list(tcp_rules) if tcp_rules else []
        self.volumes = list(volumes)
        self.config_files = list(config_files)
        self.probes = list(probes) if probes else []
        self.extend_info = extend_info
        self.monitors = list(monitors)
        self.graphs = list(graphs)
        self.component_deps = []
        self.volume_deps = []
        self.plugin_deps = list(plugin_deps)
        self.app_config_groups = []
        self.labels = list(labels) if labels else []
        self.service_group_rel = service_group_rel
        self.support_labels = {label.label_name: label for label in support_labels}
        self.k8s_attributes = list(k8s_attributes) if k8s_attributes else []
        self.action_type = ActionType.NOTHING.value

    def set_changes(self, tenant, region, changes, governance_mode):
        """
        Set changes to the component
        """
        update_funcs = self._create_update_funcs()
        for key in update_funcs:
            if not changes.get(key):
                continue
            update_func = update_funcs[key]
            update_func(changes.get(key))

        self.ensure_port_envs(governance_mode)

    def ensure_port_envs(self, governance_mode):
        # filter out the old port envs
        envs = [env for env in self.envs if env.container_port == 0]
        # create outer envs for every port
        for port in self.ports:
            envs.extend(self._create_envs_4_ports(port, governance_mode))
        self.envs = envs

    def _create_envs_4_ports(self, port: TenantServicesPort, governance_mode):
        attr_name_prefix = port.port_alias.upper() if port.port_alias else self._create_default_attr_name_prefix(port)
        host_value = "127.0.0.1" if governance_mode == GovernanceModeEnum.BUILD_IN_SERVICE_MESH.name else port.k8s_service_name
        host_env = self._create_port_env(port, "连接地址", attr_name_prefix + "_HOST", host_value)
        port_env = self._create_port_env(port, "端口", attr_name_prefix + "_PORT", str(port.container_port))
        return [host_env, port_env]

    def _create_default_attr_name_prefix(self, port: TenantServicesPort):
        port_alias = self.component.service_alias.upper()
        return port_alias + str(port.container_port)

    def _create_port_env(self, port: TenantServicesPort, name, attr_name, attr_value):
        return TenantServiceEnvVar(
            tenant_id=self.component.tenant_id,
            service_id=self.component.component_id,
            container_port=port.container_port,
            name=name,
            attr_name=attr_name,
            attr_value=attr_value,
            is_change=False,
            scope="outer",
            create_time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        )

    def _create_update_funcs(self):
        return {
            "deploy_version": self._update_deploy_version,
            "app_version": self._update_version,
            "envs": self._update_inner_envs,
            "connect_infos": self._update_outer_envs,
            "ports": self._update_ports,
            "volumes": self._update_volumes,
            "probes": self._update_probe,
            "labels": self._update_labels,
            "component_graphs": self._update_component_graphs,
            "component_monitors": self._update_component_monitors,
            "plugin_deps": self._update_plugin_deps,
            "component_k8s_attributes": self._update_component_k8s_attributes,
        }

    def _update_plugin_deps(self, plugin_deps):
        if not plugin_deps.get("add"):
            return
        self.update_action_type(ActionType.UPDATE.value)

    def _update_deploy_version(self, dv):
        if not dv["is_change"]:
            return
        self.component.deploy_version = dv["new"]
        self.update_action_type(ActionType.BUILD.value)

    def _update_version(self, v):
        if not v["is_change"]:
            return
        self.component.version = v["new"]

    def _update_inner_envs(self, envs):
        self._update_envs(envs, "inner")
        self.update_action_type(ActionType.UPDATE.value)

    def _update_outer_envs(self, envs):
        self._update_envs(envs, "outer")
        self.update_action_type(ActionType.UPDATE.value)

    def _update_envs(self, envs, scope):
        if envs is None:
            return
        # create envs
        add = envs.get("add", [])
        for env in add:
            container_port = env.get("container_port", 0)
            value = env.get("attr_value", "")
            name = env.get("name", "")
            attr_name = env.get("attr_name", "")
            if not attr_name:
                continue
            if container_port == 0 and value == "**None**":
                value = self.component.service_id[:8]
            try:
                env_var_service.check_env(self.component, attr_name, value)
            except (EnvAlreadyExist, InvalidEnvName) as e:
                logger.warning("failed to create env: {}; will ignore this env".format(e))
                continue
            self.envs.append(
                TenantServiceEnvVar(
                    tenant_id=self.component.tenant_id,
                    service_id=self.component.service_id,
                    container_port=container_port,
                    name=name,
                    attr_name=attr_name,
                    attr_value=value,
                    scope=scope,
                ))

    def _update_ports(self, ports):
        if ports is None:
            return

        add = ports.get("add", [])
        for port in add:
            # Optimization: do not update port data iteratively
            self._update_port_data(port)
            new_port = TenantServicesPort(**port)
            new_port.service_id = self.component.component_id
            self.ports.append(new_port)

        old_ports = {port.container_port: port for port in self.ports}
        upd = ports.get("upd", [])
        for port in upd:
            old_port = old_ports.get(port["container_port"])
            old_port.protocol = port["protocol"]
            old_port.port_alias = port["port_alias"]
            if not old_port.is_inner_service:
                old_port.is_inner_service = port["is_inner_service"]
            if not old_port.is_outer_service:
                old_port.is_outer_service = port["is_outer_service"]
        self.update_action_type(ActionType.UPDATE.value)

    def _update_component_graphs(self, component_graphs):
        if not component_graphs:
            return
        graphs = component_graphs.get("add", [])
        for graph in graphs:
            new_graph = ComponentGraph(**graph)
            new_graph.graph_id = make_uuid()
            new_graph.component_id = self.component.component_id
            self.graphs.append(new_graph)

        graphs = component_graphs.get("upd", [])
        old_graphs = {graph.title: graph for graph in self.graphs}
        for graph in graphs:
            old_graph = old_graphs.get(graph.get("title"))
            if not old_graph:
                continue
            old_graph.promql = graph.get("promql", "")
            old_graph.sequence = graph.get("sequence", 99)
        self.update_action_type(ActionType.UPDATE.value)

    def _update_component_monitors(self, component_monitors):
        if not component_monitors:
            return
        monitors = component_monitors.get("add", [])
        for monitor in monitors:
            new_monitor = ServiceMonitor(**monitor)
            new_monitor.service_id = self.component.component_id
            new_monitor.tenant_id = self.component.tenant_id
            self.monitors.append(new_monitor)
        self.update_action_type(ActionType.UPDATE.value)

    def _update_labels(self, labels):
        if not labels:
            return
        labels = labels.get("add", [])
        for key in labels:
            label = self.support_labels.get(key)
            if not label:
                continue
            self.labels.append(
                ServiceLabels(
                    tenant_id=self.component.tenant_id,
                    service_id=self.component.component_id,
                    label_id=label.label_id,
                    region=self.component.service_region,
                    create_time=datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                ))
        self.update_action_type(ActionType.UPDATE.value)

    def _update_port_data(self, port):
        container_port = int(port["container_port"])
        port_alias = self.component.service_alias.upper()
        k8s_service_name = port.get("k8s_service_name", self.component.service_alias)
        if k8s_service_name:
            filter_port = port_repo.get_by_k8s_service_name(self.component.tenant_id, k8s_service_name)
            if filter_port and self.component.service_id:
                k8s_service_name = k8s_service_name + "-" + make_uuid(
                )[-4:] if self.component.service_id != filter_port.service_id else k8s_service_name
            port["k8s_service_name"] = k8s_service_name
        port["tenant_id"] = self.component.tenant_id
        port["service_id"] = self.component.service_id
        port["mapping_port"] = container_port
        port["port_alias"] = port_alias

    def _update_volumes(self, volumes):
        old_volumes = {volume.volume_name: volume for volume in self.volumes}
        for volume in volumes.get("add"):
            volume["service_id"] = self.component.service_id
            host_path = "/grdata/tenant/{0}/service/{1}{2}".format(self.component.tenant_id, self.component.service_id,
                                                                   volume["volume_path"])
            volume["host_path"] = host_path
            file_content = volume.get("file_content")
            if file_content:
                self.config_files.append(
                    TenantServiceConfigurationFile(
                        service_id=self.component.component_id,
                        volume_name=volume["volume_name"],
                        file_content=file_content,
                    ))
            if "file_content" in volume.keys():
                volume.pop("file_content")
            self.volumes.append(TenantServiceVolume(**volume))

        old_config_files = {config_file.volume_name: config_file for config_file in self.config_files}
        for volume in volumes.get("upd"):
            old_volume = old_volumes.get(volume["volume_name"])
            old_volume.mode = volume.get("mode")
            old_config_file = old_config_files.get(volume.get("volume_name"))
            if not old_config_file:
                continue
            old_config_file.file_content = volume.get("file_content")

        self.config_files = old_config_files.values()
        self.update_action_type(ActionType.UPDATE.value)

    def _update_probe(self, probe):
        old_probes = {probe.mode: probe for probe in self.probes}

        new_probes = []
        add = probe.get("add")
        if add:
            new_probes.extend(add)
        upd = probe.get("upd")
        if upd:
            new_probes.extend(upd)
        # There can only be one probe of the same mode
        # Dedup new probes based on mode
        new_probes = {probe["mode"]: probe for probe in new_probes}
        if not new_probes:
            return
        probes = []
        for key in new_probes:
            probe = new_probes[key]
            old_probe = old_probes.get(probe["mode"])
            if not old_probe:
                # create new probe
                probe["probe_id"] = make_uuid()
                probe["service_id"] = self.component.component_id
                probes.append(ServiceProbe(**probe))
                continue
            # update probe
            probe = ServiceProbe(**probe)
            probe.ID = old_probe.ID
            probe.service_id = self.component.component_id
            probes.append(probe)
        self.probes = probes
        self.update_action_type(ActionType.UPDATE.value)

    def _update_component_k8s_attributes(self, component_k8s_attributes):
        if not component_k8s_attributes:
            return
        old_k8s_attributes = {attr.name: attr for attr in self.k8s_attributes}
        new_k8s_attributes = []
        if component_k8s_attributes.get("add", []):
            new_k8s_attributes.extend(component_k8s_attributes.get("add"))
        if component_k8s_attributes.get("upd", []):
            new_k8s_attributes.extend(component_k8s_attributes.get("upd"))

        k8s_attributes = {}
        for attribute in new_k8s_attributes:
            attr_name = attribute.get("name")
            if not attr_name:
                continue
            attr = old_k8s_attributes.get(attr_name)
            if not attr:
                self.k8s_attributes.append(
                    ComponentK8sAttributes(
                        tenant_id=self.component.tenant_id,
                        component_id=self.component.component_id,
                        name=attribute.get("name"),
                        save_type=attribute.get("save_type"),
                        attribute_value=attribute.get("attribute_value"),
                    ))
                continue
            # update attrs
            k8s_attr = ComponentK8sAttributes(**attribute)
            k8s_attr.tenant_id = self.component.tenant_id
            k8s_attr.component_id = self.component.component_id
            k8s_attributes[attr_name] = k8s_attr

        for original_attr in self.k8s_attributes:
            if k8s_attributes.get(original_attr.name):
                original_attr.attribute_value = k8s_attributes[original_attr.name].attribute_value

        self.update_action_type(ActionType.UPDATE.value)

    def update_action_type(self, action_type):
        if action_type > self.action_type:
            self.action_type = action_type


def exponential(a, n):
    """
    Mathematical exponential function, using recursion.

    :param a: The constant/base of the function
    :param n: An integer variable representing the exponent
    :return: The exponential value of a (n times)
    """
    # calculate e
    e = 1 if (n == 0 and a != 0) else \
        (a if n == 1 else (a * exponential(a, n - 1) if n > 1
                           else 1 / (a * exponential(a, (n * -1) - 1))))
    return e


from PySide6.QtWidgets import QWidget, QLabel, QHBoxLayout, QVBoxLayout, QSizePolicy, QLineEdit, QPushButton

class Widget(QWidget):

    def __init__(self):
        super().__init__()

        self.setWindowTitle("Size policies and stretches")

        # Size policy: how the widgets behaves if container space is expanded or separated
        label = QLabel("Some text: ")
        line_edit  = QLineEdit()

        label.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
        line_edit.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)

        h_layout_1 = QHBoxLayout()
        h_layout_1.addWidget(label)
        h_layout_1.addWidget(line_edit)

        button_1 = QPushButton("One")
        button_2 = QPushButton("Two")
        button_3 = QPushButton("Three")

        h_layout_2 = QHBoxLayout()
        h_layout_2.addWidget(button_1, 3)
        h_layout_2.addWidget(button_2, 1)
        h_layout_2.addWidget(button_3, 1)

        v_layout = QVBoxLayout()
        v_layout.addLayout(h_layout_1)
        v_layout.addLayout(h_layout_2)

        self.setLayout(v_layout)


from data import *
from utils.augmentations import SSDAugmentation
from layers.modules import MultiBoxLoss
from ssd import build_ssd
from os.path import join
import os
import sys
import time
import torch
from pathlib import Path
from torch.autograd import Variable
import torch.nn as nn
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torch.nn.init as init
import torch.utils.data as data
import torchvision

import numpy as np
import argparse
import tqdm

sys.path.append(os.path.abspath('../../active_learning'))
from active_learning import ActiveLearning
from active_loss import LossPredictionLoss
from active_learning_utils import *

def str2bool(v):
    return v.lower() in ("yes", "true", "t", "1")

def create_dir_if_doesnt_exist(path):
    # Create working dir if not exists.
    directory=Path(path)
    if not directory.exists():
        directory.mkdir(parents=True)

parser = argparse.ArgumentParser(
    description='Single Shot MultiBox Detector Training With Pytorch'
)
train_set = parser.add_mutually_exclusive_group()
parser.add_argument(
    '--dataset',
    default='VOC',
    choices=['VOC', 'COCO'],
    type=str,
    help='VOC or COCO'
)
parser.add_argument(
    '--dataset_root', default=VOC_ROOT, help='Dataset root directory path'
)
parser.add_argument(
    '--basenet', default='vgg16_reducedfc.pth', help='Pretrained base model'
)
parser.add_argument(
    '--batch_size', default=32, type=int, help='Batch size for training'
)
parser.add_argument(
    '--resume',
    default=None,
    type=str,
    help='Checkpoint state_dict file to resume training from'
)
parser.add_argument(
    '--start_iter', default=0, type=int, help='Resume training at this iter'
)
parser.add_argument(
    '--num_workers',
    default=4,
    type=int,
    help='Number of workers used in dataloading'
)
parser.add_argument(
    '--cuda', default=True, type=str2bool, help='Use CUDA to train model'
)
parser.add_argument(
    '--lr',
    '--learning-rate',
    default=1e-3,
    type=float,
    help='initial learning rate'
)
parser.add_argument(
    '--momentum', default=0.9, type=float, help='Momentum value for optim'
)
parser.add_argument(
    '--weight_decay', default=5e-4, type=float, help='Weight decay for SGD'
)
parser.add_argument(
    '--gamma', default=0.1, type=float, help='Gamma update for SGD'
)
parser.add_argument(
    '--use_active_learning', default=False, type=bool, help='If True, will use active learning'
)
parser.add_argument(
    '--lamda', default=1, type=int, help='Active learning loss weight'
)

parser.add_argument(
    '--visdom',
    default=False,
    type=str2bool,
    help='Use visdom for loss visualization'
)
parser.add_argument(
    '--save_folder',
    default='weights/',
    help='Directory for saving checkpoint models'
)
parser.add_argument(
    '--output_superannotate_csv_file',
    required=False,
    type=str,
    default=None,
    help='Path to the output csv file with the selected indices. Can be uploaded to annotate.online.')

args = parser.parse_args()
COCO_ROOT=''
if torch.cuda.is_available():
    if args.cuda:
        torch.set_default_tensor_type('torch.cuda.FloatTensor')
    if not args.cuda:
        print(
            "WARNING: It looks like you have a CUDA device, but aren't " +
            "using CUDA.\nRun with --cuda for optimal training speed."
        )
        torch.set_default_tensor_type('torch.FloatTensor')
else:
    torch.set_default_tensor_type('torch.FloatTensor')

if not os.path.exists(args.save_folder):
    os.mkdir(args.save_folder)

def train():
    if args.dataset == 'COCO':
        if args.dataset_root == VOC_ROOT:
            if not os.path.exists(COCO_ROOT):
                parser.error('Must specify dataset_root if specifying dataset')
            print(
                "WARNING: Using default COCO dataset_root because " +
                "--dataset_root was not specified."
            )
            args.dataset_root = COCO_ROOT
        cfg = coco
        dataset = COCODetection(
            root=args.dataset_root,
            transform=SSDAugmentation(cfg['min_dim'], MEANS)
        )
    elif args.dataset == 'VOC':
        if args.dataset_root == COCO_ROOT:
            parser.error('Must specify dataset if specifying dataset_root')
        cfg = voc
        dataset = VOCDetection(
            root=args.dataset_root,
            transform=SSDAugmentation(cfg['min_dim'], MEANS)
        )
    IMG_CNT = len(dataset)
    if args.visdom:
        import visdom
        viz = visdom.Visdom()

    criterion = MultiBoxLoss(
        cfg['num_classes'], 0.5, True, 0, True, 3, 0.5, False, args.cuda
    )

    # loss counters
    loc_loss = 0
    conf_loss = 0
    epoch = 0
    print('Loading the dataset...')

    rand_state = np.random.RandomState(1311)
    epoch_size = len(dataset) // args.batch_size
    global pool_idx
    pool_idx = list(range(IMG_CNT))
    print('Training SSD on:', args.dataset)
    print('Using the specified args:')
    print(args)

    step_index = 0

    if args.visdom:
        vis_title = 'SSD.PyTorch on ' + dataset.name
        vis_legend = ['Loc Loss', 'Conf Loss', 'Total Loss']
        iter_plot = create_vis_plot('Iteration', 'Loss', vis_title, vis_legend)
        epoch_plot = create_vis_plot('Epoch', 'Loss', vis_title, vis_legend)

    #random image choice
    train_idx = []
    progress = tqdm.tqdm(range(10))
    for cycle in progress:
        net = build_ssd('train', cfg['min_dim'], cfg['num_classes'])

        if args.use_active_learning:
            net = ActiveLearning(net)

        if args.cuda:
            net = torch.nn.DataParallel(net)
            cudnn.benchmark = True

        if args.resume:
            print('Resuming training, loading {}...'.format(args.resume))
            net.load_weights(args.resume)
        else:
            vgg_weights = torch.load(args.save_folder + args.basenet)
            print('Loading base network...')
            if args.use_active_learning:
                if args.cuda:
                    net.module.base_model.vgg.load_state_dict(vgg_weights)
                else:
                    net.base_model.vgg.load_state_dict(vgg_weights)
            else:
                if args.cuda:
                    net.module.vgg.load_state_dict(vgg_weights)
                else:
                    net.vgg.load_state_dict(vgg_weights)

        if args.cuda:
            net = net.cuda()

        if not args.resume:
            print('Initializing weights...')
            # initialize newly added layers' weights with xavier method
            if args.use_active_learning:
                if args.cuda:
                    model = net.module.base_model
                else:
                   model = net.base_model
            else:
                if args.cuda:
                    model = net.module
                else:
                   model = net
            model.extras.apply(weights_init)
            model.loc.apply(weights_init)
            model.conf.apply(weights_init)

        net.train()

        if args.use_active_learning and not cycle == 0:
            cfg = voc
            dataset = VOCDetection(
                root=args.dataset_root,
                transform=BaseTransform(300, MEANS)
            )
            device = 'cuda' if (torch.cuda.is_available() and args.cuda) else 'cpu'
            indices, losses = choose_indices_loss_prediction_active_learning(
                net, cycle, rand_state, pool_idx, dataset, device)
            train_idx.extend(indices)

            if args.output_superannotate_csv_file is not None:
                # Write image paths to csv file which can be uploaded to annotate.online.
                write_entropies_csv(
                    dataset, indices,losses, args.output_superannotate_csv_file)
        else:
            train_idx.extend(random_indices(pool_idx, rand_state, count=1000))
        cfg = voc
        dataset = data.Subset(VOCDetection(
            root=args.dataset_root,
            transform=SSDAugmentation(cfg['min_dim'], MEANS)
            ), train_idx
        )
        data_loader = data.DataLoader(
            dataset,
            args.batch_size,
            num_workers=args.num_workers,
            shuffle=True,
            collate_fn=detection_collate,
            pin_memory=True
        )
        optimizer = optim.SGD(
            net.parameters(),
            lr=args.lr,
            momentum=args.momentum,
            weight_decay=args.weight_decay
        )
        for epoch in range(300):
            if epoch == 240:
                adjust_learning_rate(optimizer)
            for iteration, (images, targets) in enumerate(data_loader):
                if args.cuda:
                    images = Variable(images.cuda())
                    targets = [Variable(ann.cuda(), requires_grad=False) for ann in targets]
                else:
                    images = Variable(images)
                    targets = [Variable(ann, requires_grad=False) for ann in targets]
                # forward
                t0 = time.time()
                if args.use_active_learning:
                    out, loss_pred = net(images)
                else:
                    out =  net(images)
                # backprop
                optimizer.zero_grad()
                loss_l, loss_c, N = criterion(out, targets)
                # print("loss_l {} loss_c {} N {}".format(loss_l, loss_c, N))
                loss = (loss_l + loss_c).sum() / N
                # print("loss = {}".format(loss))
                if args.use_active_learning:
                    criterion_lp = LossPredictionLoss()
                    loss_prediction_loss = criterion_lp(loss_pred, loss_l + loss_c)
                    loss += args.lamda * loss_prediction_loss
                loss.backward()
                optimizer.step()
                t1 = time.time()
                loc_loss += loss_l.mean().item()
                conf_loss += loss_c.mean().item()

                if iteration % 10 == 0:
                    progress.set_description('timer: %.4f sec, ' % (t1 - t0) +
                        'iter ' + repr(iteration) + (', Loss: %.4f' %
                        (loss.item())))
        folder = join(args.save_folder,
                      'active_learning' if args.use_active_learning else 'random')
        create_dir_if_doesnt_exist(folder)
        torch.save(
            net.state_dict(),
            join(folder, 'cycle_'+str(cycle+1)+'k.pth')
        )

def adjust_learning_rate(optimizer):
    """Sets the learning rate to the initial LR decayed by 10 at every
        specified step
    # Adapted from PyTorch Imagenet example:
    # https://github.com/pytorch/examples/blob/master/imagenet/main.py
    """
    lr = args.lr * 0.1
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

def xavier(param):
    init.xavier_uniform_(param)

def weights_init(m):
    if isinstance(m, nn.Conv2d):
        xavier(m.weight.data)
        m.bias.data.zero_()

def create_vis_plot(_xlabel, _ylabel, _title, _legend):
    return viz.line(
        X=torch.zeros((1, )).cpu(),
        Y=torch.zeros((1, 3)).cpu(),
        opts=dict(xlabel=_xlabel, ylabel=_ylabel, title=_title, legend=_legend)
    )

def update_vis_plot(
    iteration, loc, conf, window1, window2, update_type, epoch_size=1
):
    viz.line(
        X=torch.ones((1, 3)).cpu() * iteration,
        Y=torch.Tensor([loc, conf, loc + conf]).unsqueeze(0).cpu() / epoch_size,
        win=window1,
        update=update_type
    )
    # initialize epoch plot on first iteration
    if iteration == 0:
        viz.line(
            X=torch.zeros((1, 3)).cpu(),
            Y=torch.Tensor([loc, conf, loc + conf]).unsqueeze(0).cpu(),
            win=window2,
            update=True
        )

if __name__ == '__main__':
    train()


#!/usr/bin/env python3

# ---------------------------------------------------------------------------------------------------------------------------------
# This script computes the total amount of block storage used in each compartment in a region or all active regions using OCI Python SDK
# Optionnaly list all boot volumes and block volumes
#
# Note: OCI tenant and region given by an OCI CLI PROFILE
# Author        : Christophe Pauliat
# Platforms     : MacOS / Linux
#
# prerequisites : - Python 3 with OCI Python SDK installed
#                 - OCI config file configured with profiles
# Versions
#    2020-17-12: Initial Version
#    2020-22-12: Add support for all subscribed regions and add optional details
#    2021-07-28: Fix usage() function, no compartment needed
#    2022-01-03: use argparse to parse arguments
#    2022-01-04: add --no_color option
# ---------------------------------------------------------------------------------------------------------------------------------

# -------- import
import oci
import sys
import operator
import argparse

# -------- colors for output
# see https://misc.flogisoft.com/bash/tip_colors_and_formatting to customize
COLOR_TITLE0 = "\033[95m"             # light magenta
COLOR_TITLE1 = "\033[91m"             # light red
COLOR_TITLE2 = "\033[32m"             # green
COLOR_AD     = "\033[94m"             # light blue
COLOR_COMP   = "\033[93m"             # light yellow
COLOR_BREAK  = "\033[91m"             # light red
COLOR_NORMAL = "\033[39m"

# -------- variables
configfile = "~/.oci/config"    # Define config file to be used.

# -------- functions

# ---- usage syntax
def usage():
    print ("Usage: {} [-nc] [-a] [-v] -p OCI_PROFILE".format(sys.argv[0]))
    print ("")
    print ("    By default, only the region provided in the profile is processed")
    print ("    If -a is provided, all subscribed regions are processed (by default, only the region in the profile is processed)")
    print ("    If -v is provided, detailed list of all volumes is displayed")
    print ("")
    print ("note: OCI_PROFILE must exist in {} file (see example below)".format(configfile))
    print ("")
    print ("[EMEAOSCf]")
    print ("tenancy     = ocid1.tenancy.oc1..aaaaaaaaw7e6nkszrry6d5hxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
    print ("user        = ocid1.user.oc1..aaaaaaaayblfepjieoxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
    print ("fingerprint = 19:1d:7b:3a:17:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx:xx")
    print ("key_file    = /Users/cpauliat/.oci/api_key.pem")
    print ("region      = eu-frankfurt-1")
    exit (1)

# ---- Disable colored output
def disable_colored_output():
    global COLOR_TITLE0
    global COLOR_TITLE1
    global COLOR_TITLE2
    global COLOR_AD
    global COLOR_COMP
    global COLOR_BREAK
    global COLOR_NORMAL

    COLOR_TITLE0 = ""
    COLOR_TITLE1 = ""
    COLOR_TITLE2 = ""
    COLOR_AD     = ""
    COLOR_COMP   = ""
    COLOR_BREAK  = ""
    COLOR_NORMAL = ""

# ---- Get the complete name of a compartment from its id, including parent and grand-parent..
def get_cpt_name_from_id(cpt_id):

    if cpt_id == RootCompartmentID:
        return "root"

    name=""
    for c in compartments:
        if (c.id == cpt_id):
            name=c.name

            # if the cpt is a direct child of root compartment, return name
            if c.compartment_id == RootCompartmentID:
                return name
            # otherwise, find name of parent and add it as a prefix to name
            else:
                name = get_cpt_name_from_id(c.compartment_id)+":"+name
                return name

# ---- Build the block storage report for one region then display it
def get_report_for_region():
    print ("--------------------------------------------------------------------------------------------------------------")

    # Query (see https://docs.cloud.oracle.com/en-us/iaas/Content/Search/Concepts/querysyntax.htm)
    query_block_volume = "query volume resources"
    query_boot_volume  = "query bootvolume resources"

    # Clients
    gb_used = {}
    SearchClient = oci.resource_search.ResourceSearchClient(config)
    BlockstorageClient = oci.core.BlockstorageClient(config)
    total_gb_used = 0

    # Run the search query to get list of BLOCK volumes then for each volume, use get_volume() to get size
    # Finally store the result in a dictionary
    if details:
        print (f"REGION {config['region']}: LIST OF BLOCK VOLUMES:")

    response = SearchClient.search_resources(oci.resource_search.models.StructuredSearchDetails(type="Structured", query=query_block_volume))
    for item in response.data.items:
        if item.lifecycle_state != "TERMINATED":
            # as size is not returned by the query search, we need to get the size of each block volume.
            response2 = BlockstorageClient.get_volume(item.identifier)
            vol = response2.data
            if gb_used.get(item.compartment_id) == None:
                gb_used[item.compartment_id] = vol.size_in_gbs
            else:
                gb_used[item.compartment_id] += vol.size_in_gbs
            if details:
                print (f"- {vol.id}, {vol.size_in_gbs:5d} GBs, {vol.display_name}")
            total_gb_used += vol.size_in_gbs

    # Run the search query to get list of BOOT volumes then for each volume, use get_boot_volume() to get size
    # Finally store the result in the same dictionary
    if details:
        print ("")
        print (f"REGION {config['region']}: LIST OF BOOT VOLUMES:")

    response = SearchClient.search_resources(oci.resource_search.models.StructuredSearchDetails(type="Structured", query=query_boot_volume))
    for item in response.data.items:
        if item.lifecycle_state != "TERMINATED":
            # as size is not returned by the query search, we need to get the size of each boot volume.
            response2 = BlockstorageClient.get_boot_volume(item.identifier)
            vol = response2.data
            if gb_used.get(item.compartment_id) == None:
                gb_used[item.compartment_id] = vol.size_in_gbs
            else:
                gb_used[item.compartment_id] += vol.size_in_gbs
            if details:
                print (f"- {vol.id}, {vol.size_in_gbs:5d} GBs, {vol.display_name}")
            total_gb_used += vol.size_in_gbs

    # sort the dictionary by descending total size
    gb_used_sorted = dict(sorted(gb_used.items(), key=operator.itemgetter(1), reverse=True))

    # display the result
    if details:
        print ("")

    print (f"REGION {config['region']}: BLOCK STORAGE CONSUMPTION (boot volumes and block volumes) PER COMPARTMENT ",end="")
    print (f"Total =  {total_gb_used} GBs = {total_gb_used/1024:.1f} TBs")
    for cpt_id in gb_used_sorted.keys():
        cpt_name = get_cpt_name_from_id(cpt_id)
        gb = gb_used_sorted[cpt_id]
        print (f"- {gb:6d} GBs, {cpt_name} ")
    print ("")

# -------- main

# -- parse arguments
parser = argparse.ArgumentParser(description = "List block storage capacity used for each compartment in an OCI tenant")
parser.add_argument("-p", "--profile", help="OCI profile", required=True)
parser.add_argument("-a", "--all_regions", help="Do this for all regions", action="store_true")
parser.add_argument("-v", "--verbose", help="Give more details", action="store_true")
parser.add_argument("-nc", "--no_color", help="Disable colored output", action="store_true")
args = parser.parse_args()

profile     = args.profile
all_regions = args.all_regions
details     = args.verbose
if args.no_color:
  disable_colored_output()

# -- get info from profile
try:
    config = oci.config.from_file(configfile,profile)
except:
    print ("ERROR: profile '{}' not found in config file {} !".format(profile,configfile))
    exit (2)

IdentityClient = oci.identity.IdentityClient(config)
user = IdentityClient.get_user(config["user"]).data
RootCompartmentID = user.compartment_id

# -- get list of subscribed regions
response = oci.pagination.list_call_get_all_results(IdentityClient.list_region_subscriptions, RootCompartmentID)
regions = response.data

# -- Get list of compartments with all sub-compartments
response = oci.pagination.list_call_get_all_results(IdentityClient.list_compartments,RootCompartmentID,compartment_id_in_subtree=True)
compartments = response.data

# -- Build and print block storage reports for regions
if all_regions:
    for region in regions:
        config["region"] = region.region_name
        get_report_for_region()
else:
    get_report_for_region()

# -- the end
exit (0)


#!/usr/bin/env python

def find_Rscript():
    import os

    search_paths = [
        "/opt/local/bin/Rscript",
        "/usr/local/bin/Rscript",
        "/usr/bin/Rscript",
        ]
    path = None
    for spath in search_paths:
        if spath is None or not os.path.exists(spath):
            continue
        path = spath
        break
    assert path is not None, "I could not find: Rscript"
    return path

def run_R(script):
    # Run an R script in the background.  Makes sure nothing prints to
    # the screen.
    import os
    import tempfile
    import subprocess
    from genomicode.dwdnorm import _safe_unlink

    # Bug: assumes that Rscript can be executed.
    assert type(script) is type("")

    rscript = find_Rscript()
    temp_path = "."
    script_file = None
    try:
        x, script_file = tempfile.mkstemp(dir=temp_path); os.close(x)
        open(script_file, 'w').write(script)

        command = [rscript, script_file]
        p = subprocess.Popen(
            command, bufsize=0, stdin=subprocess.PIPE,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
        w, r, e = p.stdin, p.stdout, p.stderr
        w.close()
        output = r.read()
        err = e.read()
        assert not err.strip(), err
    finally:
        _safe_unlink(script_file)
    return output

def parse_indexes(MATRIX, indexes, indexes_include_headers):
    from genomicode import parselib

    max_index = MATRIX.ncol()
    num_headers = len(MATRIX._row_names)
    assert max_index, "empty matrix"

    I = []
    for s, e in parselib.parse_ranges(indexes):
        if indexes_include_headers:
            s, e = s-num_headers, e-num_headers
        assert s >= 1, "Index out of range: %s" % s
        assert e <= max_index, "Index out of range: %s" % e
        s, e = s - 1, min(e, max_index)
        I.extend(range(s, e))
    return I

def make_sample_names(MATRIX):
    num = MATRIX.ncol()
    names = ['Sample%d' % (i+1) for i in range(num)]
    return names

def make_array_names(MATRIX):
    num = MATRIX.ncol()
    names = ['Array%d' % (i+1) for i in range(num)]
    return names

def write_EIF_file(MATRIX, filename):
    # write EIF file
    array_names = make_array_names(MATRIX)

    handle = open(filename, 'w')
    print >>handle, "\t".join(array_names)
    for row in MATRIX._X:
        for i in range(len(row)):
            if row[i] < 1:
                row[i] = 1
        print >>handle, "\t".join(map(str, row))

def write_SIF_file(MATRIX, filename, batches):
    sample_names = make_sample_names(MATRIX)
    array_names = make_array_names(MATRIX)

    assert len(sample_names) == len(array_names)
    assert len(sample_names) == len(batches)

    handle = open(filename, 'w')
    header = 'Array name', 'Sample name', 'Batch'
    print >>handle, "\t".join(header)
    for x in zip(array_names, sample_names, batches):
        print >>handle, "\t".join(map(str, x))

def main():
    import os
    import sys
    import argparse
    import tempfile
    import shutil

    import arrayio
    from genomicode import filelib
    from genomicode import config

    usage = "combatnorm.py [options] expression_file"
    parser = argparse.ArgumentParser(usage=usage)
    parser.add_argument("expression_file", help="Matrix to normalize.")
    #parser.add_argument("normalized_file", help="Normalized matrix.")
    parser.add_argument(
        "--indexes1",
        help="Which columns in batch 1, E.g. 1-5,8 (1-based, "
        "inclusive).  Columns leftover will be in the remaining batches.")
    parser.add_argument(
        "--indexes2",
        help="If three or more batches, then specify the samples for the "
        "second batch.")
    # Allow up to 4 groups for now.
    parser.add_argument("--indexes3")
    parser.add_argument(
        "--indexes_include_headers", "--iih", action="store_true",
        help="If not given (default), then column 1 is the first column "
        "with data.  If given, then column 1 is the very first column in "
        "the file, including the headers.")

    args = parser.parse_args()
    assert os.path.exists(args.expression_file), \
        "File not found: %s" % args.expression_file
    assert args.indexes1
    if args.indexes2:
        assert args.indexes1
    if args.indexes3:
        assert args.indexes2

    MATRIX = arrayio.read(args.expression_file)

    # Figure out the batch for each sample.
    assert MATRIX.ncol(), "empty matrix"
    # Should be 1 to <num_groups>.
    batches = [None] * MATRIX.ncol()
    all_indexes = [args.indexes1, args.indexes2, args.indexes3]
    for i, indexes in enumerate(all_indexes):
        if not indexes:
            continue
        I = parse_indexes(MATRIX, indexes, args.indexes_include_headers)
        for j in I:
            assert j < len(batches)
            assert batches[j] is None, "overlapping: %d" % j
            batches[j] = i+1
    # The remaining samples are in the next batch.
    x = batches
    x = [x for x in batches if x]
    max_batch = max(x)
    for i in range(len(batches)):
        if batches[i] == None:
            batches[i] = max_batch + 1

    temp_path = None
    cwd = os.getcwd()
    try:
        temp_path = tempfile.mkdtemp(dir=".")
        os.chdir(temp_path)

        EIF_file = "EIF.dat"
        SIF_file = "SIF.dat"

        # Write the input files.
        write_EIF_file(MATRIX, EIF_file)
        write_SIF_file(MATRIX, SIF_file, batches)

        # Run combat.
        ComBat_R = config.combat
        assert os.path.exists(ComBat_R), 'File not found: %s' % ComBat_R

        assert os.path.exists(EIF_file)
        assert os.path.exists(SIF_file)
        x = [
            'source("%s")' % ComBat_R,
            'ComBat("%s", "%s", write=T)' % (EIF_file, SIF_file),
            ]
        script = "\n".join(x)
        run_R(script)
        #R = jmath.start_R()
        #R('source("%s")' % ComBat_R)
        #R('ComBat("%s", "%s", write=T)' % (EIF_file, SIF_file))
        outfile = "Adjusted_EIF.dat_.xls"
        assert filelib.exists_nz(outfile), "File not found: %s" % outfile

        x = open(outfile).readlines()
        x = [x.rstrip("\r\n") for x in x]
        x = [x for x in x if x]
        x = [x.split("\t") for x in x]
        #header = x[0]
        norm = x[1:]
        MATRIX_norm = MATRIX.matrix()
        assert len(norm) == MATRIX_norm.nrow()
        for x in norm:
            assert len(x) == MATRIX_norm.ncol()
        MATRIX_norm._X = norm
    finally:
        os.chdir(cwd)
        if temp_path is not None and os.path.exists(temp_path):
            shutil.rmtree(temp_path)

    arrayio.tab_delimited_format.write(MATRIX_norm, sys.stdout)

if __name__=='__main__':
    main()


# django
from django.contrib import admin

class ProductAdmin(admin.ModelAdmin):
    fieldsets = (
        ('General information', {
            'fields': ('uuid', 'name', 'type', 'price')
        }),
        ('Status', {
            'fields': ('status',)
        }),
        ('Audit data', {
            'fields': ('created_at', 'updated_at', 'created_by', 'updated_by')
        })
    )

    list_display = ('name', 'type', 'price', 'status', 'created_at')
    search_fields = ['name', 'price']
    list_filter = ['type']
    ordering = ('name', 'price', 'created_at')

    readonly_fields = ['id', 'uuid', 'created_at', 'updated_at', 'created_by', 'updated_by']


import numpy as np

def enumerate_fragment3(f1f2, dup=False):
    """
    This function enumerates all possible fragment3s based on fragment1 and fragment2.
    Input: f1f2: array (2,2), dup: boolean
    Output: fragment_list: array (numFragments,2)
    """
    f1 = f1f2[0]
    f2 = f1f2[1]

    f3_dict = {}
    for i in range(len(f1)):
        for k in range(3):
            f3_1 = np.copy(f1)
            f3_1[i] = (f3_1[i] + k + 1) % 4
            f3_dict["".join(str(aa) for aa in f3_1)] = f3_1

            f3_2 = np.copy(f2)
            f3_2[i] = (f3_2[i] + k + 1) % 4
            f3_dict["".join(str(aa) for aa in f3_2)] = f3_2

    if dup:
        f3_dict["".join(str(aa) for aa in f1)] = f1
        f3_dict["".join(str(aa) for aa in f2)] = f2

    f3_list = list(f3_dict.values())

    fragment_list = []
    for i in range(len(f3_list)):
        fragment_list.append([f1, f2, f3_list[i]])

    return fragment_list

def enumerate_distance1_genotype(ref_genotype):
    """
    This function enumerates all possible distance 1 genotypes based on given genotype.
    Input: ref_genotype: array (2,1)
    Output: genotype_list: array (numGenotypes,2)
    """
    genotype_list = []

    for i in range(ref_genotype.shape[0]):
        for j in range(ref_genotype.shape[1]):
            for k in range(3):
                temp = np.copy(ref_genotype)
                temp[i, j] = (temp[i, j] + k + 1) % 4
                genotype_list.append(temp)

    return genotype_list

def enumerate_fragments_possible(bulk, z):
    """
    This function enumerates all possible fragments based on bulk and mutation type genotypes.
    Input: Bulk: array (2,2), Z: array (2,2)
    Output: fragment_list_possible: array (numFragments,2)
    """
    fragment_list_possible = []

    fragment_list = enumerate_fragment3(bulk)
    for fragment in fragment_list:
        fragment_list_possible.append(fragment)

    fragment_list = enumerate_fragment3(z)
    for fragment in fragment_list:
        fragment_list_possible.append(fragment)

    return fragment_list_possible

def enumerate_fragments_possible_single_site(bulk, z):
    fragment_list = []

    for i in range(3):
        fragment_list.append([bulk[0], bulk[1], (bulk[0] + i + 1) % 4])

    for i in range(4):
        fragment_list.append([z[0], z[1], (z[0] + i) % 4])

    return fragment_list


from flask import Flask, render_template, request, redirect, url_for
import os
import requests
from forms import UrlSearchForm

import nltk
nltk.data.path.append('./nltk_data/')
from newspaper import Article

from wordcloud import WordCloud

import base64
import io

import datetime

def get_wordcloud(text):
    pil_img = WordCloud().generate(text=text).to_image()
    img = io.BytesIO()
    pil_img.save(img, "PNG")
    img.seek(0)
    img_b64 = base64.b64encode(img.getvalue()).decode()
    return img_b64

app = Flask(__name__)

@app.route('/', methods = ["GET", "POST"])
def index():
    errors = []
    urlsearch = UrlSearchForm(request.form)
    return render_template("index.html", form = urlsearch, errors = errors)

@app.route("/results", methods = ["GET", "POST"])
def search_results(urlsearch):

    if request.method == "POST":

        try:
            search_string = urlsearch.data['search']

            article = Article(search_string)

            article.download()
            article.parse()
            nltk.download("punkt")
            article.nlp()

            data = article.text
            title = article.title
            date = article.publish_date
            published_date = date.strftime("%d %B %Y")
            author = article.authors[0]

            image = article.top_image

            cloud = get_wordcloud(data)

            summary = article.summary

            return render_template("results.html", search_string = search_string, data = data, title=title, published_date=published_date, author = author, image = image, cloud = cloud, summary = summary)

        except:
            errors.append(
                "Unable to get the URL.  Please enter a valid URL for news article."
            )
            return render_template("index.html", errors = errors)
    return "END"

if __name__ == '__main__':
      app.run()


#!/usr/bin/env python
# coding: utf-8

# In[82]:

# import torch
# import torch.nn
# from torch.nn import functional as f
# import pandas as pd
# import json
# import numpy as np
# import os

# DIRECTORY = 'data'
# paths = []

# for root, dirs, files in os.walk(DIRECTORY):
#     for name in files:
#         paths.append(os.path.join(root, name))

# names = [i.split('/')[-1] for i in paths][1:]
# data_dict = dict(zip([i[:-4] for i in names], paths[1:]))

# In[83]:

# import urllib
# from newspaper import Article
# from data import *

# def meta_extract(url):
#     article = Article(url)
#     article.download()
#     article.parse()
#     article.download('punkt')
#     article.nlp()

#     return article.authors, article.publish_date, article.top_image, article.images, article.title, article.summary

# # meta_extract(url)

# # In[84]:

# from nltk.classify import NaiveBayesClassifier
# from nltk.corpus import subjectivity
# from nltk.sentiment import SentimentAnalyzer, SentimentIntensityAnalyzer
# from nltk.sentiment.util import *
# import nltk

# def sentiment(inp_text):
#     sia = SentimentIntensityAnalyzer()
#     return sia.polarity_scores(inp_text)

# In[104]:

from data import *
from meta import *
from net import *

# class PreprocessingDataset(Dataset):
#     def __init__(self, file, root, x_col, y_col, meta_columns, label_idx = -1):
#         self.x_col = x_col
#         self.y_col = y_col
#         self.data = pd.read_csv(file)
#         self.data = self.data.sample(frac=1).reset_index(drop=True)
#         self.data = self.data.drop(meta_columns, axis=1)

#         # self.data, self.base_ref = self.tokenizer(self.data, [x_col])
#         self.x_data = self.data[x_col]
#         self.max_len = max([len(i) for i in self.x_data])
#         self.max_len = 600

#         self.x_data, self.token = self.word_vector(self.x_data)
#         self.data[x_col] = [torch.FloatTensor(i) for i in self.x_data]
#         self.data = self.vectorize(self.data, [y_col])
#         self.df_data = self.data
#         self.data = self.data.to_numpy()

#         self.root = root
#         self.transform = transforms.Compose([transforms.ToTensor()])

#     def format_text(self, token):
#         clean_token = ''.join(chr for chr in token if chr.isalnum() and chr.isalpha())
#         return clean_token

#     def word_vector(self, data):
#         x_data = data
#         x_data = list(x_data)
#         maximum_length = 0
#         max_idx = 0
#         for idx, i in enumerate(x_data):

#             if len(i) > maximum_length:
#                 maximum_length = len(i)
#                 max_idx = idx
#         maximum_length = 600
#         t = Tokenizer(num_words=600, filters='\n.,:!"#$()&@%^()-_`~[];.,{|}')
#         t.fit_on_texts(x_data)
#         sequences = t.texts_to_sequences(x_data)
#         sequences = keras.preprocessing.sequence.pad_sequences(sequences, maxlen=maximum_length)
#         print(x_data[0])
#         print(len(x_data[0]))
#         print(sequences[0])

#         return sequences, t

#     def vectorize(self, data_inp, columns):
#         data = data_inp
#         for column in columns:
#             labels = list(data[column].unique())
#             ref = dict(zip(data[column].unique(), [i for i in range(len(labels))]))
#             print(ref)
#             for idx, val in enumerate(data[column]):
#                 vectorized = ref[data[column][idx]]
#                 data[column][idx] = torch.tensor(vectorized, dtype=float)
#         return data

#     def __len__ (self):
#         return len(self.data)

#     def __getitem__ (self, idx):

#         self.transpose_data = self.data
#         self.transpose_data = self.transpose_data.transpose()
#         x_data = self.transpose_data[0]
#         y_data = self.transpose_data[1]

#         return x_data[idx], y_data[idx]

DIRECTORY = 'data'
data_dict = {'politifact': 'data/truth-detectiondeception-detectionlie-detection/politifact.csv', 'politifact_clean': 'data/truth-detectiondeception-detectionlie-detection/politifact_clean.csv', 'politifact_clean_binarized': 'data/truth-detectiondeception-detectionlie-detection/politifact_clean_binarized.csv'}
clean_truth_data = PreprocessingDataset(data_dict['politifact_clean_binarized'], DIRECTORY, 'statement', 'veracity', ['source', 'link'])

BATCH_SIZE = 64

primary_data = clean_truth_data #secondary option of truth_data

train_len = int(len(primary_data)*0.8)
test_len = len(primary_data) - train_len

train_set, test_set = torch.utils.data.random_split(primary_data, [train_len, test_len])

train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)
test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True)

# print(len(train_set))
# print(len(test_set))

num_feats = np.array([train_set[i][0].numpy() for i in range(len(train_set))])
num_labels = np.array([train_set[i][1] for i in range(len(train_set))])

a = iter(train_loader)
b = next(a)
b = np.asarray(b[0])
# print(b.shape)
inp_size = (b.shape)[1]
# print(inp_size)
# print(inp_size)

# In[87]:

import itertools
ab = list(itertools.chain(*[i[0] for i in clean_truth_data]))
# print(len(ab))
ab = set([int(i) for i in ab])
emb_dim = len(ab)

# In[105]:

# import torch.nn as nn
# import torch.nn.functional as F

# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# class FeedForward(nn.Module):
#     def __init__(self, num_classes, input_size, kernel_size=4):
#         super(FeedForward, self).__init__()
#         self.fc1 = nn.Linear(input_size, 200)
#         self.fc3 = nn.Linear(200, 100)
#         self.fc4 = nn.Linear(100, 100)
#         self.fc5 = nn.Linear(100, 50)
#         self.fc6 = nn.Linear(50, 20)
#         self.fc7 = nn.Linear(20, 1)
#         self.dropout = nn.Dropout(0.3)

#     def forward(self, x):
#         x = self.dropout(F.relu(self.fc1(x)))
#         x = self.dropout(F.relu(self.fc3(x)))
#         x = self.dropout(F.relu(self.fc4(x)))
#         x = self.dropout(F.relu(self.fc5(x)))
#         x = self.dropout(F.relu(self.fc6(x)))
#         x = self.dropout(F.relu(self.fc7(x)))

#         return x

# class RecurrentClassifier(nn.Module):
#     def __init__(self, embedding_dim, input_size, hidden_size, output_size, num_layers, dropout=0.3):
#         super(RecurrentClassifier, self).__init__()

#         self.embedding = nn.Embedding(embedding_dim, input_size)
#         self.rnn = nn.LSTM(input_size,
#                             hidden_size,
#                             num_layers,
#                             batch_first = True,
#                             dropout=dropout)
#         self.fc1 = nn.Linear(hidden_size*2, hidden_size)
#         self.fc2 = nn.Linear(hidden_size, output_size)
#         self.dropout = nn.Dropout(dropout)

#     def forward(self, x):
#         x = self.embedding(x)
#         x, (hidden, cell) = self.rnn(x)
#         print(hidden.shape)
#         hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1, :, :]), dim=1))
#         x = self.fc1(hidden)
#         x = self.dropout(self.fc2(x))

#         return x

max_len = len(train_set[1][0])
ref_check = 1

feedforward = FeedForward(ref_check, inp_size).to(device)
recurrent = RecurrentClassifier(emb_dim, inp_size, 50, ref_check, 2, dropout=0.2).to(device)

# print(feedforward, recurrent)

# In[114]:

def train(net, train_loader, LR, DECAY, EPOCHS):
    optimizer = torch.optim.Adam(net.parameters(), lr=LR, weight_decay=DECAY)
    loss_func = torch.nn.BCEWithLogitsLoss()

    epochs = EPOCHS
    losses = []

    for step in range(epochs):
        running_loss = 0.0
        for i, data in enumerate(train_loader):
            inp, labels = data
            if net == recurrent:
                inp, labels = inp.long().to(device), labels.float().to(device)
                optimizer.zero_grad()
                outputs = net(inp)
                cost = loss_func(torch.squeeze(outputs), torch.squeeze(labels))
            elif net == feedforward:
                inp, labels = inp.float().to(device), labels.float().to(device)
                optimizer.zero_grad()
                outputs = net(inp)
                cost = loss_func(torch.squeeze(outputs), labels)
            cost.backward()
            optimizer.step()

            running_loss += cost.item()
        print(f'Epoch: {step}   Training Loss: {running_loss/len(train_loader)}')
    print('Training Complete')

    return losses

def eval(net, test_loader):
    total = 0
    acc = 0
    optimizer = torch.optim.Adam(net.parameters(), lr=LR, weight_decay=DECAY)

    for i, data in enumerate(test_loader):
        inp, labels = data
        optimizer.zero_grad()
        output = net(inp.float())
        output = output.detach().numpy()
        output = list(output)
        output = [list(i).index(max(i)) for i in output]

        for idx, item in enumerate(torch.tensor(output)):
            total += 1
            if item == labels[idx]:
                acc += 1
    print(f'{acc/total*100}%')

# In[116]:

def model_load(net, PATH, name, export=True):
    if export:
        torch.save(net.state_dict(), PATH+name+'.pth')
        return PATH+name+'.pth'
    else:
        net.torch.load_state_dict(torch.load(PATH + name + '.pth'))
        return net

# train(feedforward, train_loader, 1e-3, 5e-3, 200)
model_load(feedforward, 'model_parameters/', 'linear_politifact')

# train(recurrent, train_loader, 1e-3, 5e-3, 200)
model_load(recurrent, 'model_parameters/', 'lstm_politifact')

# In[138]:

# url = 'https://www.historyofvaccines.org/content/blog/defense-of-common-antivaxxer'

# _, _, _, _, _, summ = meta_extract(url)

token_basis = clean_truth_data.token

def tokenize_sequence(text_inp, tokenizer):
    text_inp = text_inp.lower().split('\n')
    tokenizer.fit_on_texts(text_inp)
    sequences = tokenizer.texts_to_sequences(text_inp)
    sequences = [i if i!=[] else [0] for i in tokenizer.texts_to_sequences(text_inp)]
    sequences = [i[0] for i in sequences]
    pad_len =  [0]*int(inp_size - len(sequences))
    sequences += pad_len
    return torch.FloatTensor(sequences)[:600]

# inp = tokenize_sequence(summ, token_basis)
# inp = inp[None, :]
# print(inp.shape)

def prediction(inp, model):
    output = model(inp)
    return output

# a = prediction(inp, feedforward)

# In[143]:

# feedforward_template = FeedForward(ref_check, inp_size).to(device)
# recurrent_template = RecurrentClassifier(emb_dim, inp_size, 50, ref_check, 2, dropout=0.2).to(device)
# model_load(feedforward_template, '/Users/devpatelio/Downloads/Coding/Global_Politics_EA/model_parameters/', 'linear_politifact')
# model_load(recurrent_template, '/Users/devpatelio/Downloads/Coding/Global_Politics_EA/model_parameters/', 'lstm_politifact')

# # feedforward_template.eval()
# # recurrent_template.eval()

# output_linear = '0'
# output_lstm = '1' #check for error without passing error

# output_linear = prediction(inp, feedforward_template)
# output_lstm = prediction(inp.long(), recurrent_template)

# if output_linear == 0:
#     output_linear = f"No Bias: Prediction = {output_linear}"
# elif output_linear == 1:
#     output_linear = f"Bias: Prediction = {output_linear}"

# if output_lstm == 0:
#     output_lstm = f"Limited Veracity: Prediction = {output_lstm}"
# elif output_lstm == 1:
#     output_lstm = f"Expressive Veracity: Prediction = {output_lstm}"

# In[179]:

from django.shortcuts import render
# from scrape import meta_extract
import flask
from flask import Flask, request, render_template, redirect, url_for
# from sentiment import sentiment
import numpy as np
import pandas as pd
import json
import random
import os
import html
import torch
# from linear_politifact_basis import token_basis, convert_word_to_token
from flask_ngrok import run_with_ngrok
from flask import Flask, render_template, redirect, url_for, request
app = Flask(__name__)
# run_with_ngrok(app)   #starts ngrok when the app is run

from flask import Flask, jsonify, request
from string import punctuation
from collections import Counter
import random

def model_load(net, PATH, name, export=True):
    if export:
        torch.save(net.state_dict(), PATH+name+'.pth')
        return PATH+name+'.pth'
    else:
        net.torch.load_state_dict(torch.load(PATH + name + '.pth'))
        net.eval()

@app.route('/')
def hello():
    return render_template("home.html")

@app.route('/link', methods=["POST", "GET"])
def link():
    if request.method == "POST":
        link_inp = request.form['linker']
        # print(type(link_inp))

        link_inp = link_inp.replace('.com', 'comkey')
        link_inp = link_inp.replace('https://', 'https')
        link_inp = link_inp.replace('www.', 'www')
        link_inp = link_inp.replace('/', 'slash')
        # print(link_inp)
        main = link_inp

        return redirect(url_for("preview_linker", linkage=main, tag='link_url'))
    else:
        return render_template("link.html")

@app.route('/text', methods=["POST", "GET"])
def pure_text():
    if request.method == "POST":
        inp_raw = request.form['raw_text']
        inp_raw = format_raw_text(inp_raw)
        return redirect(url_for('preview_linker', linkage=inp_raw, tag='pure_text'))
    else:
        return render_template("pure_text.html")

@app.route(f"/output/<tag>/<linkage>")
def preview_linker(linkage, tag):
    preview = linkage
    if tag == 'link_url':
        preview = preview.replace('https', 'https://')
        preview = preview.replace('www', 'www.')
        preview = preview.replace('slash', '/')
        preview = preview.replace('comkey', '.com')
        authart, publ, timg, allimg, tit, summ = meta_extract(preview)

    elif tag == 'pure_text':
        preview = preview.replace('uxd', ' ')
        summ = preview
        empty_msg = 'None'
        authart = empty_msg
        publ = empty_msg
        timg = empty_msg
        allimg = empty_msg
        tit = empty_msg

    sent = sentiment(summ)

    inp = tokenize_sequence(summ, token_basis)
    inp = inp[:600]
    inp = inp[None, :]
    # print(inp.shape)

    feedforward_template = FeedForward(ref_check, inp_size).to(device)
    recurrent_template = RecurrentClassifier(emb_dim, inp_size, 50, ref_check, 2, dropout=0.2).to(device)
    model_load(feedforward_template, 'model_parameters/', 'linear_politifact')
    model_load(recurrent_template, 'model_parameters/', 'lstm_politifact')

    # feedforward_template.eval()
    # recurrent_template.eval()

    output_linear = '0 ERROR'
    output_lstm = '1 ERROR' #check for error without passing error

    output_linear = F.sigmoid(prediction(inp, feedforward_template)).round()
    output_lstm = F.sigmoid(prediction(inp.long(), recurrent_template))

    all_types = list(pd.read_csv(data_dict['politifact_clean'])['veracity'].unique())

    if output_linear == 0:
        output_linear = f"Little Bias: Prediction = {output_linear}"
    elif output_linear == 1:
        output_linear = f"Substantial Bias: Prediction = {output_linear}"

    statement_type = ''
    if output_lstm <= 0.25:
        statement_type = 'True'
    elif 0.25 < output_lstm <= 0.5:
        statement_type = 'Mostly True'
    elif 0.5 < output_lstm <= 0.75:
        statement_type = 'Mostly False'
    elif 0.75 < output_lstm <= 1:
        statement_type = 'False'
    elif output_lstm > 1:
        statement_type = 'Pants on Fire!'

    output_lstm = f"Veracity -> {statement_type}: {output_lstm}"

    # if output_lstm == 0:
    #     output_lstm = f"Limited Veracity: Prediction = {output_lstm}"
    # elif output_lstm == 1:
    #     output_lstm = f"Expressive Veracity: Prediction = {output_lstm}"

    return render_template("preview.html", preview_link=preview,
                                            author_article=authart,
                                            published_article=publ,
                                            top_image = timg,
                                            all_image = allimg,
                                            title_article=tit,
                                            summary_article=summ,
                                            sentiment=sent,
                                            bias_point=output_linear,
                                            skew_point=output_lstm)


#!/usr/bin/python
# _*_ coding: utf-8 -*-

import time
import RobotApi

RobotApi.ubtRobotInitialize()
#------------------------------Connect-------------------------------------
gIPAddr = ""

robotinfo = RobotApi.UBTEDU_ROBOTINFO_T()
#The robot name you want to connect
robotinfo.acName="Yanshee_8F83"
ret = RobotApi.ubtRobotDiscovery("SDK", 15, robotinfo)
if (0 != ret):
	print ("Return value: %d" % ret)
	exit(1)

gIPAddr = robotinfo.acIPAddr
ret = RobotApi.ubtRobotConnect("SDK", "1", gIPAddr)
if (0 != ret):
	print ("Can not connect to robot %s" % robotinfo.acName)
	exit(1)

#-----------------------Cheak App Status-------------------------
pcBuf = ["static","slant_forward","slant_backward","slant_left","slant_right","swaying"]
iWaitTime = 100
#ret = RobotApi.ubtCheckAPPStatus(pcBuf,iWaitTime)
for buff in pcBuf:
    ret = RobotApi.ubtCheckAPPStatus(buff,iWaitTime)
    if ret != 0:
        print("Can not get App status. It is possible that the App is not connect to the robot. Error code: %d" % ret)
        exit(3)
    else:
        print("pcBuf: %s" % buff)

#---------------------------Disconnect--------------------------------------
RobotApi.ubtRobotDisconnect("SDK","1",gIPAddr)
RobotApi.ubtRobotDeinitialize()


from flask import request
from flask_restx import Resource

from ...util.dto import CustomerDto
from ..service.customers_service import Customer, CustomerById

api = CustomerDto.api
_customer = CustomerDto.customer

@api.route('/')
class CustomerList(Resource):
    @api.doc('list_of_registered_customer')
    @api.marshal_list_with(_customer)
    def get(self):
        return Customer.get()

    @api.response(201, 'Customer successfully created.')
    @api.doc('create a new customer')
    @api.expect(_customer, validate=True)
    def post(self):
        data = request.json
        return Customer.post(data=data)

    @api.response(201, 'Customer successfully updated.')
    @api.doc('update customer')
    @api.expect(_customer, validate=True)
    def put(self):
        data = request.json
        return Customer.put(data=data)

@api.route('/<public_id>')
@api.param('public_id', 'The Customer identifier')
@api.response(404, 'Customer not found.')
class CustomerByParam(Resource):
    @api.doc('get a customer')
    @api.marshal_with(_customer)
    def get(self, public_id):
        id = int(public_id)
        customer = CustomerById.get(id)
        # return {'message': 'Ok', 'status': 'success'}
        if not customer:
            api.abort(404)
        else:
            return customer

    def delete(self, public_id):
        id = int(public_id)
        customer = CustomerById.delete(id)
        return customer


"""
Scale(parent, options, ...)

basic usage of Scale

get value via Scale object
"""

from tkinter import *

def changeNum(source):
    print("change value 1", var1.get())

def getNum():
    print("print value ", var1.get())

def setNum():
    var1.set(5)
    print("set value as", var1.get())

# main program
root = Tk()
root.title("Python GUI - Scale")
root.geometry("640x480")
# label
label = Label(root, text="Scale binding a variable", bg="lightyellow", width=30)
label.pack()
# var1 = IntVar()
var1 = IntVar()
scale1 = Scale(root, from_=0, to=10, orient=HORIZONTAL, variable=var1, command=changeNum)
scale1.pack()

Button(root, text="Get value", width=20, command=getNum).pack()
Button(root, text="Set value", width=20, command=setNum).pack()
root.mainloop()


#!/usr/bin/env python
import pika
import pymongo
import uuid
import requests
from pymongo import MongoClient

connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()

client = MongoClient('mongodb://localhost:27017/')

db = client['imdb_']

collection = db['imdb_collection2']

channel.queue_declare(queue='names')

def callback(ch, method, properties, body):
    bdy = body.decode()
    bdy = bdy.strip('[]"')
    print(" [x] Received %r" % bdy)
    url = bdy
    new_name = uuid.uuid5(uuid.NAMESPACE_URL,url)

    HEADERS = {
   'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
   'accept-encoding': 'gzip, deflate, br',
   'accept-language': 'en-GB,en-US;q=0.8,en;q=0.6',
   'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
}

    r = requests.get(url,headers=HEADERS)
    response = r.content
    f = open('/home/user/mysys/jb6/dumps/%s.html' %new_name, 'w')
    f.write(str(response))
    f.close
    db.collection.insert({'url':url})

channel.basic_consume(callback,
                      queue='names',
                      no_ack=True)

print(' [*] Waiting for messages. To exit press CTRL+C')
channel.start_consuming()


import math
import numpy as np
import cv2
import glob
import os.path as osp
import matplotlib.pyplot as plt

path_to_GT = "GT/*"

def PSNR(original, compressed):
    mse = np.mean((original - compressed) ** 2)
    if (mse == 0):
        return 100
    max_pixel = 255.0
    psnr = 20 * math.log10(max_pixel / math.sqrt(mse))
    return psnr

def ssim(img1, img2):
    C1 = (0.01 * 255)**2
    C2 = (0.03 * 255)**2

    img1 = img1.astype(np.float64)
    img2 = img2.astype(np.float64)
    kernel = cv2.getGaussianKernel(11, 1.5)
    window = np.outer(kernel, kernel.transpose())

    mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5]
    mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5]
    mu1_sq = mu1**2
    mu2_sq = mu2**2
    mu1_mu2 = mu1 * mu2
    sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq
    sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq
    sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2

    ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) *
                                                            (sigma1_sq + sigma2_sq + C2))
    ssim_out = 1-ssim_map.mean()
    return ssim_out

def calc_ssim(img1, img2):
    if not img1.shape == img2.shape:
        raise ValueError('Input images must have the same dimensions.')
    if img1.ndim == 2:
        return ssim(img1, img2)
    elif img1.ndim == 3:
        if img1.shape[2] == 3:
            ssims = []
            for i in range(3):
                ssims.append(ssim(img1, img2))
            return np.array(ssims).mean()
        elif img1.shape[2] == 1:
            return ssim(np.squeeze(img1), np.squeeze(img2))
    else:
        raise ValueError('Wrong input image dimensions.')

def gt_finder(idx):
    idx1 = -1
    for path in glob.glob(path_to_GT):
        idx1 += 1
        if idx1 == idx:
            return np.array(cv2.imread(path))

path_to_ESRGAN = 'ESRGAN/*'

psnre = []
ssime = []

idx = 0
print("Calculating for ESRGAN")
for path in glob.glob(path_to_ESRGAN):
    idx += 1
    try:
        base = osp.splitext(osp.basename(path))[0]
        image1 = np.array(cv2.imread(path))
        image2 = gt_finder(idx)
        image1.resize(image2.shape)
        psnre.append((PSNR(image1, image2))*1000)
        ssime.append((calc_ssim(image1, image2)))
    except:
        continue
print(psnre)
print(ssime)

path_to_OURMODEL = 'Our_model/*'

psnro = []
ssimo = []

print("Calculating for Our_model")
idx = 0
for path in glob.glob(path_to_OURMODEL):
    idx += 1
    try:
        base = osp.splitext(osp.basename(path))[0]
        image1 = np.array(cv2.imread(path))
        image2 = gt_finder(idx)
        image1.resize(image2.shape)
        psnro.append((PSNR(image1, image2))*1000)
        ssimo.append((calc_ssim(image1, image2)))
    except:
        continue

print(psnro)
print(ssimo)

path_to_SRGAN = 'SRGAN/*'

psnrs = []
ssims = []

print("Calculating for SRGAN")
idx = 0
for path in glob.glob(path_to_SRGAN):
    idx += 1
    try:
        base = osp.splitext(osp.basename(path))[0]
        image1 = np.array(cv2.imread(path))
        image2 = gt_finder(idx)
        image1.resize(image2.shape)
        psnrs.append((PSNR(image1, image2))*1000)
        ssims.append((calc_ssim(image1, image2)))
    except:
        continue

print(psnrs)
print(ssims)

path_to_g = 'Ground/*'
print("Calculating for Ground Truth")

psnrg = []
ssimg = []
idx = 0
for path in glob.glob(path_to_g):
    idx += 1
    try:
        base = osp.splitext(osp.basename(path))[0]
        image1 = np.array(cv2.imread(path))
        image2 = gt_finder(idx)
        image1.resize(image2.shape)
        psnrg.append((PSNR(image1, image2))*1000)
        ssimg.append((calc_ssim(image1, image2)))
    except:
        continue

print(psnrg)
print(ssimg)

X = ['1','2','3','4','5','6','7','8','9']

X_axis = np.arange(len(X))

plt.bar(X_axis - 0.2 , psnrs , 0.2 , label = 'SRGAN')
plt.bar(X_axis  , psnre , 0.2 , label = 'ESRGAN')
plt.bar(X_axis + 0.2 , psnro , 0.2 , label = 'Our Model')

plt.xticks(X_axis, X)
plt.xlabel("Each Test Image")
plt.ylabel("PSNR Value")
plt.legend()
plt.show()

X = ['1','2','3','4','5','6','7','8','9']

X_axis = np.arange(len(X))

plt.bar(X_axis - 0.2 , ssims , 0.2 , label = 'SRGAN')
plt.bar(X_axis  , ssime , 0.2 , label = 'ESRGAN')
plt.bar(X_axis + 0.2 , ssimo , 0.2 , label = 'Our Model')

plt.xticks(X_axis, X)
plt.xlabel("Each Test Image")
plt.ylabel("SSIM Value")
plt.legend()
plt.show()