"""
This module provides helper functions for sw-manager commands
"""

import time

from utils import cli, exceptions
from utils.tis_log import LOG
from utils.clients.ssh import ControllerClient
from consts.stx import OrchestStrategyPhase, OrchStrategyKey, OrchStrategyState
from consts.timeout import OrchestrationPhaseTimeout, HostTimeout
from keywords import common

PHASE_COMPLETION_CHECK_INTERVAL = 20
IGNORED_ALARM_IDS = ['200.001', '700.004', '900.001', '900.005', '900.101']

def create_strategy(orchestration, controller_apply_type=None, storage_apply_type=None,
                    compute_apply_type=None, max_parallel_computes=0, instance_action=None,
                    alarm_restrictions=None, wait_for_completion=True, timeout=None, conn_ssh=None, fail_ok=False):
    """
    Creates a orchestration strategy
    Args:
        orchestration (str): indicates the orchestration strategy type. Choices are  patch or upgrade
        controller_apply_type (str): Valid only for patch orchestration. indicates how the strategy is applied on
            controllers: serial/parallel/ignore.  Default is serial
        storage_apply_type (str): indicates how the strategy is applied on storage hosts. Choices are: serial/parallel/
                                    ignore. Default is serial
        compute_apply_type (str): indicates how the strategy is applied on computes hosts. Choices are: serial/parallel/
                                    ignore.  'parallel' is valid only for patch orchestration. Default is serial.
        max_parallel_computes(int):  indicates the maximum compute hosts to apply strategy in parallel
        instance_action (str): valid only for patch orchestration. Indicates what action to perform on instances.
            Choices are migrate or stop-start. The default is to stop-start
        alarm_restrictions(str): indicates how to handle alarm restrictions based on the management affecting statuses.
            Choices are strict or relaxed. Default is strict.
        wait_for_completion:
        timeout:
        conn_ssh:
        fail_ok:

    Returns: tuple (rc, dict/msg)
        (0, dict) - success
        (1, output) - CLI reject
        (2, err_msg) - strategy build completion timeout
        (3, err_msg) - strategy build completed but with failed state

    """
    if orchestration is None:
        raise ValueError("The orchestration type must be specified: choices are 'patch' or 'upgrade'")

    args_dict = {
        '--storage-apply-type': storage_apply_type,
        '--compute-apply-type': compute_apply_type,
        '--max-parallel-compute-hosts': max_parallel_computes if max_parallel_computes >= 2 else None,
        '--alarm-restrictions': alarm_restrictions,
    }

    cmd = ''
    if orchestration is "patch":
        cmd += "patch-strategy create"
        args_dict['--controller-apply-type'] = controller_apply_type
        args_dict['--instance-action'] = instance_action
    elif orchestration is "upgrade":
        cmd += "upgrade-strategy create"
    else:
        raise exceptions.OrchestrationError("Invalid orchestration type (choices are 'patch' or 'upgrade') specified")

    args = ''
    for key, val in args_dict.items():
        if val is not None and val != '':
            args += ' {} {}'.format(key, val)

    LOG.info("Creating {} orchestration strategy with arguments: {} {}".format(orchestration, cmd, args))
    rc, output = cli.sw_manager(cmd, args, ssh_client=conn_ssh, fail_ok=fail_ok)
    LOG.info("Verifying if the {} orchestration strategy is created".format(orchestration))
    if rc != 0:
        msg = "Create {} strategy failed: {}".format(orchestration, output)
        LOG.warn(msg)
        if fail_ok:
            return 1, msg
        else:
            raise exceptions.OrchestrationError(msg)

    if 'strategy-uuid' not in [tr.strip() for tr in output.split(':')]:
        msg = "The {} strategy not created: {}".format(orchestration, output)
        LOG.warn(msg)
        if fail_ok:
            return 1, msg
        else:
            raise exceptions.OrchestrationError(msg)

    if wait_for_completion:
        if timeout is None:
            timeout = OrchestrationPhaseTimeout.BUILD

        if not wait_strategy_phase_completion(orchestration, OrchestStrategyPhase.BUILD, timeout, conn_ssh=conn_ssh)[0]:
            msg = "The {} strategy created failed build: {}".format(orchestration, output)
            if fail_ok:
                LOG.warning(msg)
                return 2, msg
            else:
                raise exceptions.OrchestrationError(msg)

    # get values of the created strategy
    results = get_current_strategy_info(orchestration, conn_ssh=conn_ssh)

    if OrchestStrategyPhase.BUILD != results['current-phase']:

        msg = "Unexpected {} strategy phase= {} encountered. A 'build' phase was expected. "\
            .format(orchestration, results["current-phase"])
        if fail_ok:
            LOG.warning(msg)
            return 3, msg
        else:
            raise exceptions.OrchestrationError(msg)

    if "failed" in results["state"]:
        msg = "The {} strategy  'failed' in build phase; reason = {}".format(orchestration, results["build-reason"])
        if fail_ok:
            LOG.warning(msg)
            return 3, msg
        else:
            raise exceptions.OrchestrationError(msg)

    LOG.info("Create {} strategy completed successfully: {}".format(orchestration, results))
    return 0, results

def apply_strategy(orchestration, timeout=OrchestrationPhaseTimeout.APPLY, conn_ssh=None, fail_ok=False):
    """
    applies an orchestration strategy
    Args:
        orchestration (str): indicates the orchestration strategy type. Choices are  patch or upgrade
        timeout (int):

        conn_ssh:
        fail_ok:

    Returns (tuple):
        (0, <actual_states>(dict)) - success  strategy applied successfully
        (1, <std_err>) - CLI command rejected
        (2, <actual_states>(dict)) - Strategy apply failed

    """
    if orchestration not in ('patch', 'upgrade'):
        raise ValueError("The orchestration type (choices are 'patch' or 'upgrade') must be specified")

    # strategy_values = get_current_strategy_values(orchestration)
    if orchestration is "patch":
        cmd = "patch-strategy apply"
    else:
        cmd = "upgrade-strategy apply"

    rc, output = cli.sw_manager(cmd, ssh_client=conn_ssh, fail_ok=fail_ok)

    if rc != 0:
        return 1, output

    res, results = wait_strategy_phase_completion(orchestration, OrchestStrategyPhase.APPLY, timeout=timeout,
                                                  conn_ssh=conn_ssh, fail_ok=True)
    if not res:
        c_phase = results[OrchStrategyKey.CURRENT_PHASE]
        c_compl = results['current-phase-completion']

        if c_phase == OrchestStrategyPhase.APPLY and int(c_compl.strip()[:-1]) > 50:
            LOG.warning('current-phase-completion > 50%, extend wait time for {} strategy to apply'.
                        format(orchestration))
            res, results = wait_strategy_phase_completion(orchestration, OrchestStrategyPhase.APPLY, timeout=timeout,
                                                          conn_ssh=conn_ssh, fail_ok=True)
    if not res:
        msg = "{} strategy failed to apply. Current state: {}".format(orchestration, results[OrchStrategyKey.STATE])
        LOG.warn(msg)
        if fail_ok:
            return 2, results
        else:
            raise exceptions.OrchestrationError(msg)

    if results[OrchStrategyKey.STATE] != OrchStrategyState.APPLIED or \
            results[OrchStrategyKey.APPLY_RESULT] != 'success':
        raise exceptions.OrchestrationError('{} strategy not in applied state after completion'.format(orchestration))

    LOG.info("apply {} strategy completed successfully: {}".format(orchestration, results))
    return 0, results

def delete_strategy(orchestration, check_first=True, fail_ok=False, conn_ssh=None):
    """
    Deletes an orchestration strategy
    Args:
        orchestration (str): indicates the orchestration strategy type. Choices are  patch or upgrade
        conn_ssh:
        check_first (bool): Check if strategy exits, if so, check if strategy is still in-progress, if so, abort it
        fail_ok:

    Returns (tuple):
        (-1, "No strategy available. Do nothing.")
        (0, "<orch_type> orchestration strategy deleted successfully.")
        (1, <std_err>)  # CLI command rejected

    """
    if orchestration is None:
        raise ValueError("The orchestration type (choices are 'patch' or 'upgrade') must be specified")
    cmd = ''
    if orchestration is "patch":
        cmd += "patch-strategy "
    elif orchestration is "upgrade":
        cmd += "upgrade-strategy "

    strategy_info = get_current_strategy_info(orchestration, conn_ssh=conn_ssh)
    if check_first:
        if not strategy_info:
            msg = "No strategy available. Do nothing."
            LOG.info(msg)
            return -1, msg

        if strategy_info.get(OrchStrategyKey.INPROGRESS, None) == 'true':
            LOG.info("Strategy in progress. Abort.")
            strategy_state = strategy_info[OrchStrategyKey.STATE]
            if strategy_state in (OrchStrategyState.APPLYING, OrchStrategyState.BUILDING, OrchStrategyState.INITIAL):
                cli.sw_manager(cmd, 'abort', ssh_client=conn_ssh, fail_ok=False)
                wait_strategy_phase_completion(orchestration, OrchestStrategyPhase.ABORT)
            elif strategy_state == OrchStrategyState.ABORTING:
                wait_strategy_phase_completion(orchestration, OrchestStrategyPhase.ABORT)

    rc, output = cli.sw_manager(cmd, 'delete', ssh_client=conn_ssh, fail_ok=fail_ok)

    if rc != 0:
        return 1, output

    post_strategy_info = get_current_strategy_info(orchestration, conn_ssh=conn_ssh)
    if post_strategy_info:
        raise exceptions.OrchestrationError("{} strategy still exists after deletion: {}".
                                            format(orchestration, post_strategy_info))

    msg = "{} orchestration strategy deleted successfully.".format(orchestration)
    LOG.info(msg)
    return 0, msg

def get_current_strategy_info(orchestration, conn_ssh=None):
    """
    Gets orchestration strategy values
    Args:
        orchestration (str):
        conn_ssh:

    Returns: dict of strategy values

    """

    if orchestration not in ("patch", "upgrade"):
        raise ValueError("Invalid orchestration type (choices are 'patch' or 'upgrade') specified")

    if not conn_ssh:
        conn_ssh = ControllerClient.get_active_controller()

    cmd = ''
    if orchestration is "patch":
        cmd += "patch-strategy show"
    else:
        cmd += "upgrade-strategy show"
    try:
        output = cli.sw_manager(cmd, ssh_client=conn_ssh, fail_ok=False)[1]
    except:
        time.sleep(20)
        if not conn_ssh.is_connected():
            conn_ssh.connect(retry=True)
        output = cli.sw_manager(cmd, ssh_client=conn_ssh, fail_ok=False)[1]

    rtn = {}
    if 'No strategy available' in output:
        return rtn

    for line in output.splitlines():
        k, v = line.strip().split(sep=':', maxsplit=1)
        rtn[k.strip()] = v.strip()

    return rtn

def wait_strategy_phase_completion(orchestration, current_phase, timeout=None, conn_ssh=None, fail_ok=False):
    """
    Waits until the orchestration strategy phase is completed
    Args:
        orchestration (str): - indicates the orchestration type. possible values: upgrade or patch
        current_phase (str): - indicates the current phase of the orchestration. Possible values: build, apply or abort
        timeout (int): - indicates the timeout value to wait for the current phase to complete
        conn_ssh:
        fail_ok:

    Returns (tuple):
        (True, <strategy info>(dict))
        (False, <strategy info>(dict))

    """

    if orchestration is None:
        raise ValueError("The orchestration type (choices are 'patch' or 'upgrade') must be specified")
    elif orchestration is not "patch" and orchestration is not "upgrade":
        raise ValueError("Invalid orchestration type (choices are 'patch' or 'upgrade') specified")

    if not validate_current_strategy_phase(orchestration, current_phase):
        raise exceptions.OrchestrationError("Current {} strategy phase does not match the specified phase={}"
                                            .format(orchestration, current_phase))
    check_interval = PHASE_COMPLETION_CHECK_INTERVAL
    if timeout is not None and timeout < PHASE_COMPLETION_CHECK_INTERVAL:
        timeout = PHASE_COMPLETION_CHECK_INTERVAL
    else:
        if timeout is None:
            if current_phase == OrchestStrategyPhase.BUILD:
                timeout = OrchestrationPhaseTimeout.BUILD
            elif current_phase == OrchestStrategyPhase.APPLY:
                timeout = OrchestrationPhaseTimeout.APPLY
                check_interval = 40
            else:
                timeout = OrchestrationPhaseTimeout.ABORT

    end_time = time.time() + timeout

    output = None
    prev_phase_completion = "0%"

    if conn_ssh is None:
        conn_ssh = ControllerClient.get_active_controller()

    while time.time() < end_time:
        if not conn_ssh.is_connected():
            # ssh connection is lost. Controllers may swact in path application.
            time.sleep(30)
            conn_ssh.connect(retry=True, retry_timeout=HostTimeout.SWACT-30)
            time.sleep(60)
            end_time = end_time + HostTimeout.SWACT

        output = get_current_strategy_info(orchestration, conn_ssh=conn_ssh)
        if output:
            if current_phase == OrchestStrategyPhase.ABORT:
                if output[OrchStrategyKey.STATE] == OrchStrategyState.ABORT_TIMEOUT:
                    msg = '{} strategy abort timed out. Stop waiting for completion.'.format(orchestration)
                    if fail_ok:
                        LOG.warning(msg)
                        return False, output
                    else:
                        raise exceptions.OrchestrationError(msg)

            elif output[OrchStrategyKey.CURRENT_PHASE] == OrchestStrategyPhase.ABORT:
                msg = "{} strategy {} phase is aborted. Stop waiting for completion."\
                    .format(orchestration, current_phase)
                if fail_ok:
                    LOG.warn(msg)
                    return False, output
                else:
                    raise exceptions.OrchestrationError(msg)

            phase_completion = output['current-phase-completion']
            if phase_completion != prev_phase_completion:
                LOG.info("Orchestration current phase completion is at {}".format(phase_completion))
                prev_phase_completion = phase_completion

            if phase_completion == '100%':
                return True, output
            else:
                time.sleep(check_interval)
    else:
        msg = "{} strategy {} phase did not complete within automation timeout {}seconds. Current status: {}"\
            .format(orchestration, current_phase, timeout, output)
        if fail_ok:
            LOG.warn(msg)
            return False, output
        else:
            raise exceptions.OrchestrationError(msg)

def get_current_strategy_phase(orchestration, conn_ssh=None):
    """
    Gets the current strategy phase for the specified orchestration ( upgrade or patch)
    Args:
        orchestration:
        conn_ssh:

    Returns (str):  current phase ( build, apply or abort)

    """
    results = get_current_strategy_info(orchestration, conn_ssh=conn_ssh)
    return results[OrchStrategyKey.CURRENT_PHASE]\
        if OrchStrategyKey.CURRENT_PHASE in results else None

def validate_current_strategy_phase(orchestration, expected_phase, conn_ssh=None):
    """
    Validates the current expected phase
    Args:
        orchestration:
        expected_phase:
        conn_ssh:

    Returns (bool):  True if valid otherwise return False

    """

    if not OrchestStrategyPhase.validate(phase=expected_phase):
        LOG.warn("The specified orchestration strategy phase='{}' is not valid phase. Valid phases are: {}"
                 .format(expected_phase, [OrchestStrategyPhase.BUILD,
                                          OrchestStrategyPhase.ABORT,
                                          OrchestStrategyPhase.APPLY]))
        return False

    current_phase = get_current_strategy_phase(orchestration, conn_ssh=conn_ssh)

    if current_phase is not None and current_phase == expected_phase:
        LOG.info("Current orchestration strategy phase is {} as expected".format(current_phase))
        return True
    else:
        LOG.warn("Current orchestration strategy phase='{}' does not match with expected phase='{}'"
                 .format(current_phase, expected_phase))
        return False

def get_current_strategy_uuid(orchestration):
    """
    Gets the current strategy uuid
    Args:
        orchestration (str): - indicates the orchestration type ( upgrade or patch)

    Returns (str): the uuid of the current orchestration strategy or None if the strategy does not exist

    """
    results = get_current_strategy_info(orchestration)

    uuid = results[OrchStrategyKey.STRATEGY_UUID] \
        if OrchStrategyKey.STRATEGY_UUID in results else None
    LOG.info("{} strategy uuid = {}".format(orchestration, uuid))
    return uuid

def get_current_strategy_details(orchestration, conn_ssh=None):
    """
    Gets orchestration strategy details when successfully applied.
    Args:
        orchestration:
        conn_ssh:

    Returns: dict of strategy values

    """

    if orchestration is None:
        raise ValueError("The orchestration type (choices are 'patch' or 'upgrade') must be specified")
    if orchestration is not "patch" and orchestration is not "upgrade":
        raise ValueError("Invalid orchestration type (choices are 'patch' or 'upgrade') specified")

    cmd = ''
    if orchestration is "patch":
        cmd += "patch-strategy show --details"
    else:
        cmd += "upgrade-strategy show --details"
    try:
        rc,  output = cli.sw_manager(cmd, ssh_client=conn_ssh, fail_ok=True)
    except:
        time.sleep(20)
        if not conn_ssh.is_connected():
            conn_ssh.connect(retry=True)
            ControllerClient.set_active_controller(ssh_client=conn_ssh)

        rc,  output = cli.sw_manager(cmd, ssh_client=conn_ssh, fail_ok=True)

    rtn = {}
    if rc == 0 and output is not None and ('strategy-uuid' in [tr.strip() for tr in output.split(':')]):
        lines = output.splitlines()
        build_phase_index = [i for i, word in enumerate(lines) if "build-phase" in word]
        apply_phase_index = [i for i, word in enumerate(lines) if "apply-phase" in word]
        strategy_lines = []
        build_phase_lines = []
        apply_phase_lines = []
        if len(build_phase_index) > 0:
            strategy_lines.extend(lines[1:build_phase_index[0]])
            if len(apply_phase_index) > 0:
                build_phase_lines.extend(lines[build_phase_index[0]:apply_phase_index[0]])
                apply_phase_lines.extend(lines[apply_phase_index[0]:])
            else:
                build_phase_lines.extend(lines[build_phase_index[0]:])
        else:
            strategy_lines.extend(lines[1:])

        strategy_values = {}
        build_phase_values = {}
        apply_phase_values = {}
        if len(strategy_lines) > 0:
            for line in strategy_lines:
                pairs = line.split(':', 1)
                strategy_values[pairs[0].strip()] = pairs[1].strip()
            rtn['strategy'] = strategy_values
        if len(build_phase_lines) > 0:
            for line in build_phase_lines:
                pairs = line.split(':')
                if pairs[0].strip() == "stages":
                    break
                build_phase_values[pairs[0].strip()] = pairs[1].strip()

            rtn['build'] = build_phase_values

        if len(apply_phase_lines) > 0:
            for line in apply_phase_lines:
                pairs = line.split(':', 1)
                if pairs[0].strip() == "stages":
                    break
                apply_phase_values[pairs[0].strip()] = pairs[1].strip()
            rtn['apply'] = apply_phase_values

    return rtn

def get_current_strategy_phase_duration(orchestration, phase, conn_ssh=None):
    """
    Gets the elapsed time in seconds to execute the specified orchestration strategy phase
    Args:
        orchestration:
        phase:
        conn_ssh

    Returns:

    """
    if orchestration not in ("patch", "upgrade"):
        raise ValueError("Invalid orchestration type (choices are 'patch' or 'upgrade') specified")

    if phase not in ("build", "apply"):
        raise ValueError("Invalid orchestration phase type (choices are 'build' or 'apply') specified")

    duration = None
    strategy_details = get_current_strategy_details(orchestration, conn_ssh=conn_ssh)
    if phase in strategy_details.keys():
        start_date_time = strategy_details[phase]["start-date-time"]
        end_date_time = strategy_details[phase]["end-date-time"]
        duration = common.get_timedelta_for_isotimes(start_date_time, end_date_time)
    return duration.seconds


import pygame
from settings import Settings
from ship import Ship
import game_functions as gf
from bullet import Bullet1

def run_game():
    # 初始化
    pygame.init()
    pygame.display.set_caption('Plane Battle')
    game_settings = Settings()
    screen = pygame.display.set_mode(
        (game_settings.screen_width, game_settings.screen_height))

    # 创造飞船
    ship = Ship(game_settings, screen)
    bullet1 = []
    BULLET1_NUM = 1
    for i in range(BULLET1_NUM):
        new_bullet = Bullet1(game_settings, screen, ship)
        bullet1.append(new_bullet)

    # 延迟
    delay = 100
    # 开始游戏主循环
    while True:
        # 监听键盘鼠标事件
        gf.check_events(ship)
        # 刷新飞船
        x = ship.update()

        if delay % 10:
            for bullet in bullet1:
                bullet.update(x)

        delay -= 1
        if not delay:
            delay = 100
        # 刷新页面
        gf.update_screen(game_settings, screen, ship, bullet1)

run_game()


import argparse
import json
import os
import shutil
import time
import torchvision.transforms as transforms
import torch.nn as nn
import torch.optim.lr_scheduler as lr_scheduler
import torch.nn.functional as F
import numpy as np
import logging
import time
import datetime
import sys
import torch.optim as optim
import yaml

from torch.utils import data
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
from utils.lane_eval.tusimple_eval import LaneEval
from utils.lane_eval import getLane
from utils.lane_eval.intersection import SegmentationMetric
from utils.transforms import *
from utils.scheduler.lr_scheduler import get_scheduler
from dataset.tusimple import tuSimple
from models.enet import ENet

best_mIoU = 0
best_val_loss = 1e6
original_stdout = sys.stdout
def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--resume", "-r", action="store_true")
    parser.add_argument("--eval", action="store_true")
    parser.add_argument("--val", action="store_true")
    parser.add_argument("s_exp_name", help="name of student exp")
    parser.add_argument("t_exp_name", help="name of teacher exp")
    args = parser.parse_args()
    return args
args = parse_args()

class Trainer(object):
    def __init__(self, s_exp_name, t_exp_name):
        cfg_path = os.path.join(os.getcwd(), 'config/tusimple_config.yaml')
        self.s_exp_name = s_exp_name
        self.t_exp_name = t_exp_name
        self.writer = SummaryWriter('tensorboard/' + self.s_exp_name)
        self.metric = SegmentationMetric(7)
        with open(cfg_path) as cfg:
            config = yaml.load(cfg, Loader=yaml.FullLoader)
        self.device = torch.device(config['DEVICE'])
        self.max_epochs = config['TRAIN']['MAX_EPOCHS']
        self.dataset_path = config['DATASET']['PATH']
        self.mean = config['DATASET']['MEAN']
        self.std = config['DATASET']['STD']
        '''
        self.input_transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize(self.mean, self.std),
            ])
        '''
        self.train_transform = Compose(
            Resize(size=(645,373)),
            RandomCrop(size=(640,368)),
            RandomFlip(0.5),
            Rotation(2),
            ToTensor(),
            Normalize(mean=self.mean,std=self.std))
        self.val_transform = Compose(
            Resize(size=(640,368)),
            ToTensor(),
            Normalize(mean=self.mean,std=self.std)
            )
        self.train_dataset = tuSimple(
            path=config['DATASET']['PATH'],
            image_set='train',
            transforms=self.train_transform
            )
        self.val_dataset = tuSimple(
            path=config['DATASET']['PATH'],
            image_set='val',
            transforms=self.val_transform,
            )
        self.train_loader = data.DataLoader(
            dataset=self.train_dataset,
            batch_size=config['TRAIN']['BATCH_SIZE'],
            shuffle=True,
            num_workers=0,
            pin_memory=True,
            drop_last=True,
            )
        self.val_loader = data.DataLoader(
            dataset=self.val_dataset,
            batch_size=config['TRAIN']['BATCH_SIZE'],
            shuffle=False,
            num_workers=0,
            pin_memory=True,
            drop_last=False,
            )
        self.iters_per_epoch = len(self.train_dataset) // config['TRAIN']['BATCH_SIZE']
        self.max_iters = self.max_epochs * self.iters_per_epoch

        # ------------network------------
        self.s_model = ENet(num_classes=7).to(self.device)
        self.t_model = ENet(num_classes=7).to(self.device)
        self.optimizer = optim.SGD(
            self.s_model.parameters(),
            lr=config['OPTIM']['LR'],
            weight_decay=config['OPTIM']['DECAY'],
            momentum=0.9,
            )
        self.lr_scheduler = get_scheduler(
            self.optimizer,
            max_iters=self.max_iters,
            iters_per_epoch=self.iters_per_epoch,
            )
        self.ce = nn.CrossEntropyLoss(weight=torch.tensor([0.4,1,1,1,1,1,1])).cuda() #background weight 0.4
        self.bce = nn.BCELoss().cuda()
        self.kl = nn.KLDivLoss().cuda()#reduction='batchmean' gives NaN
        self.mse = nn.MSELoss().cuda()

    def train(self, epoch, start_time):
        running_loss = 0.0
        is_better = True
        prev_loss = float('inf')
        logging.info('Start training, Total Epochs: {:d}, Total Iterations: {:d}'.format(self.max_epochs, self.max_iters))
        print("Train Epoch: {}".format(epoch))
        self.s_model.train()
        self.t_model.eval()
        epoch_loss = 0
        iteration = epoch*self.iters_per_epoch if epoch>0 else 0
        start_time = start_time
        for batch_idx, sample in enumerate(self.train_loader):
            iteration+=1
            img = sample['img'].to(self.device)
            segLabel = sample['segLabel'].to(self.device)
            exist = sample['exist'].to(self.device)
            with torch.no_grad():
                t_outputs, t_sig = self.t_model(img)
            s_outputs, s_sig = self.s_model(img)
            ce = self.ce(s_outputs, segLabel)
            bce = self.bce(s_sig, exist)
            kl = self.kl(
                F.log_softmax(s_outputs, dim=1),
                F.softmax(t_outputs, dim=1),
                )
            mse = self.mse(s_outputs, t_outputs) #/ s_outputs.size(0)
            loss = ce + (0.1*bce) + kl + (0.5*mse)
            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()
            self.lr_scheduler.step()
            epoch_loss += loss.item()
            running_loss += loss.item()
            eta_seconds = ((time.time() - start_time)/iteration) * (self.max_iters - iteration)
            eta_string = str(datetime.timedelta(seconds=int(eta_seconds)))
            iter_idx = epoch*len(self.train_loader) + batch_idx
            if iteration%10==0:
                logging.info(
                    "Epoch: {:d}/{:d} || Iters: {:d}/{:d} || Lr: {:6f} || Loss: {:.4f} || Cost Time: {} || Estimated Time: {}".format(
                    epoch,
                    self.max_epochs,
                    iteration % self.iters_per_epoch,
                    self.iters_per_epoch,
                    self.optimizer.param_groups[0]['lr'],
                    loss.item(),
                    str(datetime.timedelta(seconds=int(time.time() - start_time))),
                    eta_string,
                    ))
            if batch_idx % 10 == 9:
                self.writer.add_scalar(
                    'train_loss',
                    running_loss/10,
                    epoch * len(self.train_loader) + batch_idx + 1)
                running_loss = 0.0
        if epoch % 1 == 0:
            save_dict = {
                "epoch": epoch,
                "model": self.s_model.state_dict(),
                "optim": self.optimizer.state_dict(),
                "best_mIoU": best_mIoU,
                "best_val_loss": best_val_loss,
                }
            save_name = os.path.join(os.getcwd(), 'results', self.s_exp_name, 'run.pth')
            torch.save(save_dict, save_name)
            print("Model is saved: {}".format(save_name))

    def val(self, epoch):
        self.metric.reset()
        global best_val_loss
        global best_mIoU
        print("Val Epoch: {}".format(epoch))
        self.s_model.eval()
        val_loss = 0
        with torch.no_grad():
            for batch_idx, sample in enumerate(self.val_loader):
                img = sample['img'].to(self.device)
                segLabel = sample['segLabel'].to(self.device)
                exist = sample['exist'].to(self.device)
                outputs, sig = self.s_model(img)
                ce = self.ce(outputs, segLabel)
                bce = self.bce(sig, exist)
                loss = ce + (0.1*bce)
                self.metric.update(outputs, segLabel)
                pixAcc, mIoU = self.metric.get()
                logging.info("Sample: {:d}, pixAcc: {:.3f}, mIoU: {:.3f}".format(
                batch_idx + 1,
                pixAcc * 100,
                mIoU * 100
                ))

        pixAcc, mIoU, category_iou = self.metric.get(
            return_category_iou=True
            )
        print(category_iou)
        logging.info("Final pixAcc: {:.3f}, mIoU: {:.3f}".format(
            pixAcc * 100,
            mIoU * 100,
            ))
        iter_idx = (epoch+1)*len(self.train_loader)
        if (mIoU*100) > best_mIoU:
            best_mIoU = mIoU*100
            save_dict = {
                "epoch": epoch,
                "model": self.s_model.state_dict(),
                "optim": self.optimizer.state_dict(),
                "best_val_loss": best_val_loss,
                "best_mIoU": best_mIoU,
                }
            save_name = os.path.join(os.getcwd(), 'results', self.s_exp_name, 'best_mIoU.pth')
            torch.save(save_dict, save_name)
            print("mIoU is higher than best mIoU! Model saved to {}".format(save_name))

if __name__ == '__main__':
    t = Trainer(args.s_exp_name, args.t_exp_name)
    os.makedirs(os.path.join(os.getcwd(), 'results', t.s_exp_name),exist_ok=True)
    logging.basicConfig(filename=str(os.path.join(os.getcwd(), 'results', t.s_exp_name) + '/log.txt'), level=logging.INFO)
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    logging.getLogger('').addHandler(console)
    logger = logging.getLogger(__name__)
    start_epoch = 0
    start_time = time.time()
    epoch = 0
    save_name = os.path.join(os.getcwd(), 'results', t.t_exp_name, '268.pth')
    save_dict = torch.load(save_name, map_location='cpu')
    print("Loading", save_name, "from Epoch {}:".format(save_dict['epoch']))
    t.t_model.load_state_dict(save_dict['model'])
    t.t_model = t.t_model.to(t.device)
    for epoch in range(start_epoch, t.max_epochs):
    # Can remove epoch_train_loss
        t.train(epoch, start_time)
        t.val(epoch)


items = input().split("|")

while True:
    line = input()
    if line == "Yohoho!":
        break
    command = line.split()
    if "Loot" in command:
        for i in range(1, len(command)):
            if command[i] not in items:
                items.insert(0, command[i])
    elif "Drop" in command:
        idx = int(command[1])
        if 0 <= idx < len(items):
            loot = items[idx]
            items.pop(idx)
            items.append(loot)
    elif "Steal" in command:
        count = int(command[1])
        if count <= len(items):
            last_loots = items[len(items) - count:len(items)]
            print(", ".join(last_loots))
            del items[len(items) - count:]
        elif count > len(items):
            last_loots = items[0::]
            print(", ".join(last_loots))
            del items[0::]

if len(items) > 0:
    average = 0
    for i in items:
        average += len(i)
    print(f"Average treasure gain: {(average / len(items)):.2f} pirate credits.")
else:
    print(f"Failed treasure hunt.")

2. Python Fundamentals/MID EXAMS/06. Programming Fundamentals Mid Exam Retake/02_treasure_hunt.py


#warning: The algorithm takes 36 seconds before reaching the result
def longestCollatzSequence(limit):
    i = 1
    temp_i = 1
    longest_chain = 1
    chain_lenght = 1
    starting_number = 1

    while(i<=limit):
        if(temp_i == 1):
            if(chain_lenght > longest_chain):
                longest_chain = chain_lenght
                starting_number = i
            i += 1
            temp_i = i
            chain_lenght = 1
        else:
            if(temp_i % 2 == 0):
                temp_i = int(temp_i/2)
                chain_lenght += 1
            else:
                temp_i = 3*temp_i +1
                chain_lenght += 1

    print("The length of the longest chain: %d" % longest_chain)

    return starting_number

def printProblemStatement():
    print("Problem 14:")
    print("Large sum")
    print("==================================")
    print("The following iterative sequence is defined for the set of positive integers:")
    print("n -> n/2 (n is even)\nn -> 3n + 1 (n is odd)\n")
    print("Using the rule above and starting with 13, we generate the following sequence:\n13 -> 40 -> 20 -> 10 -> 5 -> 16 -> 8 -> 4 -> 2 -> 1")
    print("It can be seen that this sequence (starting at 13 and finishing at 1) contains 10 terms. \nAlthough it has not been proved yet (Collatz Problem), it is thought that all starting numbers finish at 1.\n")
    print("Which starting number, under one million, produces the longest chain?\n")
    print("NOTE: Once the chain starts the terms are allowed to go above one million.\n")

def main():
    printProblemStatement()
    print("The solution:")
    print("=============")
    print("#warning: The algorithm takes 36 seconds before reaching the result")
    print("\nThe number that produces the longest chain is: %d" % longestCollatzSequence(1000000))

main()


import pygame
import random
from enum import Enum
from collections import namedtuple

# ----- SETUP -----
pygame.init()
font = pygame.font.SysFont('arial', 25)

class Direction(Enum):
    """
    Represent direction snake is moving using enumeration
    """
    RIGHT = 1
    LEFT = 2
    UP = 3
    DOWN = 4

# represent a block
Block = namedtuple('Block', 'x, y')

# colors
WHITE = (255, 255, 255)
RED = (200, 0, 0)
LIGHT_BLUE = (0, 0, 255)
DARK_BLUE = (0, 100, 255)
BLACK = (0, 0, 0)

# game constants
BLOCK_SIZE = 20
SPEED = 20

# ----- GAME BEHAVIOR -----
class SnakeGame:
    """
    Represent a game of Snake
    """
    def __init__(self, w=640, h=480):
        # display
        self.w = w
        self.h = h
        self.display = pygame.display.set_mode((self.w, self.h))
        pygame.display.set_caption('Snake')
        self.clock = pygame.time.Clock()
        # game state
        self.direction = Direction.RIGHT
        self.head = Block(self.w / 2, self.h / 2)
        self.snake = [self.head,
                      Block(self.head.x - BLOCK_SIZE, self.head.y),
                      Block(self.head.x - (2 * BLOCK_SIZE), self.head.y)]
        self.score = 0
        self.food = None
        self._place_food()

    def _place_food(self):
        """
        Place food block randomly on display
        :return: None
        """
        x = random.randint(0, (self.w - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE
        y = random.randint(0, (self.h - BLOCK_SIZE) // BLOCK_SIZE) * BLOCK_SIZE
        self.food = Block(x, y)
        if self.food in self.snake:
            self._place_food()

    def play_step(self):
        """
        Update game state based upon user input
        :return: <tuple> (<bool> whether game is over, <int> current score)
        """
        # user input
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                quit()
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT:
                    self.direction = Direction.LEFT
                elif event.key == pygame.K_RIGHT:
                    self.direction = Direction.RIGHT
                elif event.key == pygame.K_UP:
                    self.direction = Direction.UP
                elif event.key == pygame.K_DOWN:
                    self.direction = Direction.DOWN
        # move
        self._move(self.direction)
        self.snake.insert(0, self.head)
        game_over = False
        # collision
        if self._is_collision():
            game_over = True
            return game_over, self.score
        # place new food or simply move
        if self.head == self.food:
            self.score += 1
            self._place_food()
        else:
            self.snake.pop()
        # update display
        self._update_ui()
        self.clock.tick(SPEED)
        return game_over, self.score

    def _is_collision(self):
        """
        Check whether there is a collision
        :return: True if collision, False if not
        """
        # hit boundary
        if self.head.x > self.w - BLOCK_SIZE or self.head.x < 0 or self.head.y > self.h - BLOCK_SIZE or self.head.y < 0:
            return True
        # hit self
        if self.head in self.snake[1:]:
            return True
        return False

    def _update_ui(self):
        """
        Update display with current game state
        :return: None
        """
        self.display.fill(BLACK)
        for pt in self.snake:
            pygame.draw.rect(self.display, LIGHT_BLUE, pygame.Rect(pt.x, pt.y, BLOCK_SIZE, BLOCK_SIZE))
            pygame.draw.rect(self.display, DARK_BLUE, pygame.Rect(pt.x + 4, pt.y + 4, 12, 12))
        pygame.draw.rect(self.display, RED, pygame.Rect(self.food.x, self.food.y, BLOCK_SIZE, BLOCK_SIZE))
        text = font.render("Score: " + str(self.score), True, WHITE)
        self.display.blit(text, [0, 0])
        pygame.display.flip()

    def _move(self, direction):
        """
        Move the snake based upon user input
        :param direction: <int> direction to move
        :return: None
        """
        x = self.head.x
        y = self.head.y
        if direction == Direction.RIGHT:
            x += BLOCK_SIZE
        elif direction == Direction.LEFT:
            x -= BLOCK_SIZE
        elif direction == Direction.DOWN:
            y += BLOCK_SIZE
        elif direction == Direction.UP:
            y -= BLOCK_SIZE
        self.head = Block(x, y)

# ----- GAME LOOP -----
if __name__ == '__main__':
    game = SnakeGame()
    while True:
        game_over, score = game.play_step()
        if game_over == True:
            break
    print('Final Score', score)
    pygame.quit()


from django.shortcuts import render
from rest_framework.decorators import api_view
from .models import Students, Score
from .serializers import StudentSerializer, ScoreSerializer
from rest_framework.response import Response
from rest_framework import status
from django.shortcuts import get_object_or_404
# Create your views here.

@api_view(['GET', 'POST'])
def StudentView(request):
    if request.method == 'GET':
        qs = Students.objects.all()
        serializer = StudentSerializer(qs, many=True)
        return Response(serializer.data)
    elif request.method == 'POST':
        serializer = StudentSerializer(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status = status.HTTP_201_CREATED)
        return Response(serializer.errors, status = status.HTTP_400_BAD_REQUEST)

@api_view(['GET',"POST"])
def ScoreView(request):
    qs = Score.objects.all()

    if request.method == "GET":
        serializer = ScoreSerializer(qs, many=True)
        return Response(serializer.data)

    elif request.method == 'POST':
       serializer = ScoreSerializer(data=request.data)
       if serializer.is_valid():
           serializer.save()
           return Response(serializer.data, status=status.HTTP_201_CREATED)
       return Response(serializer.error, status=status.HTTP_400_BAD_REQUEST)

@api_view(["GET","PUT","DELETE"])
def StudentDetailView(request, name):
    qs = get_object_or_404(Students, pk=name)

    if request.method == 'GET':
        serializer = StudentSerializer(qs)
        return Response(serializer.data)

    elif request.method == 'PUT':
        serializer = StudentSerializer(qs, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)

        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        qs.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

@api_view(["GET","PUT","DELETE"])
def ScoreDetailView(request, pk):
    qs = get_object_or_404(Score, pk=pk)
    print(qs)

    if request.method == 'GET':
        serializer = ScoreSerializer(qs)
        return Response(serializer.data)

    elif request.method == 'PUT':
        serializer = ScoreSerializer(qs, data=request.data)
        if serializer.is_valid():
           serializer.save()
           return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    elif request.method == 'DELETE':
        qs.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)


import sys
from Bio.Blast import NCBIXML
if len(sys.argv) !=4:
    print ("Error, wrong number of arguments")
#command line arguments 1, 2 and 3 as indicated in midterm question
file1 = sys.argv[1]
file2 = sys.argv[2]
outfile= sys.argv[3]
"""In command line, input is <this file> <xml document 1> <xml document 2> <target txt file>"""
"""output: putitative orthologs between the two species in target txt file"""
#parse for BLAST on first file, using dictionary
with open (sys.argv[1]) as br1:
    blast_record_1=NCBIXML.parse(br1)
    D1={}
    E_VALUE_THRESH = 1e-20
    for blast_record in blast_record_1:
        for alignment in blast_record.alignments:
            for hsp in alignment.hsps:
                if hsp.expect < E_VALUE_THRESH:#making sure the e value is less than 1e-20
                    D1[hsp.match]=alignment.hit_def #the protein is added as a key to a dict with its matching sequence as a value.

#second xml parse for result, same as above
with open (sys.argv[2]) as br2:
    blast_record_2 = NCBIXML.parse(br2)
    D2={}
    E_VALUE_THRESH = 1e-20#ignores everything larger than 1e-20
    for blast_record in blast_record_2:
        for alignment in blast_record.alignments:
            for hsp in alignment.hsps:
                if hsp.expect < E_VALUE_THRESH: #making sure the e value is less than 1e-20
                    D2[hsp.match]=alignment.hit_def #the protein is added as a key to a dict with its matching sequence as a value.
outfile = open(sys.argv[3],'w')
for k in set(D1.keys()) & set(D2.keys()):#orthologous pairs need to be chosen, using dictionary comparison between values (hsp.match sequences)
    x = (D1[k],"\t",D2[k])
    print >>outfile, x#writes the keys (the names of the proteins) into the file
outfile.close()


from typing import Tuple

import pandas as pd
import numpy as np

import torch

class VectorSimSearch:
  """
    A class for performing vector similarity search on a dataframe.
    Assume embedding vector is available
  """
  def __init__(self, df: pd.DataFrame, embedding_col=None, embedding_model=None, device=torch.device('cpu')):
    self.df = df.copy()    # take a copy to ensure original df is not altered.
    assert embedding_col is not None, 'embedding_col must be provided'
    self.embedding_col = embedding_col
    self.embeddings = np.array(self.df[self.embedding_col].values.tolist())

    self.embedding_model = embedding_model
    self.device = device

  def search(self, query: str, top_n=5) -> Tuple[np.ndarray, np.ndarray]:
    """
    Given a query string, return the top_n most similar.
    Returns: dataframe indices and their cosine similarity scores
    """
    assert self.embedding_model is not None, 'embedding_model must be provided'
    with torch.no_grad():
      query_embedding = self.embedding_model.encode(query, convert_to_tensor=True, device=self.device, normalize_embeddings=True)
      query_embedding = query_embedding.detach().cpu().numpy()

    cos_sim = query_embedding[None] @ self.embeddings.T

    sorted_sim = np.squeeze(np.sort(cos_sim))[::-1]
    rank = np.squeeze(np.argsort(cos_sim))[::-1]

    return rank[:top_n], sorted_sim[:top_n]

  def find_similar_item(self, indice: int, top_n=5) -> Tuple[np.ndarray, np.ndarray]:
    """
    Given an item's indice in the dataframe, return the top_n most similar.
    """
    try:
      query_embedding = self.df.loc[indice, self.embedding_col]
    except:
      raise ValueError(f'indice {indice} is out of range in dataframe which has size {self.df.shape[0]}')

    cos_sim = query_embedding[None] @ self.embeddings.T

    sorted_sim = np.squeeze(np.sort(cos_sim))[::-1]
    rank = np.squeeze(np.argsort(cos_sim))[::-1]

    return rank[:top_n], sorted_sim[:top_n]


#This method can be used to print the Hollow inverted Triangle but it uses 2 loop
"""n=4
j=0
for i in range(n,0,-1):
    print("  "*j,end=" ")
    for k in range(2*i-1):
        if k==0 or k==2*i-2 or i==n:
            print("* ",end="")
        else:
            print("  ",end="")
    print()
    j+=1"""

#This method can be used to print the Hollow inverted Triangle but it uses single loop, so it is well optimised.
n=int(input("Enter the number of lines.:\t"))
k=0
m=2*n-4
for i in range(n):
    print("  "*k,end=" ")
    if i==0:
        print("* "*(2*n-1))
    elif i==n-1:
        print("* ")
    else:
        print("* ","  "*m,"* ")
    m-=2
    k+=1

Programming Using Python/Chapter-3 Back Questions/Q No. 6/i).py


import sys
import urllib.request
import html

if len(sys.argv) < 2:
  print(sys.argv)
  raise "Expected an argument with the URL to fetch"

url = sys.argv[1]
#print(url)

def rt(s, sep):
  if s.find(sep) < 0:
    return s
  ps = s.split(sep)
  return sep.join(ps[1:])

def lt(s, sep):
  if s.find(sep) < 0:
    return s
  ps = s.split(sep)
  return ps[0]

START = '
'
END = '

'

allhtml = urllib.request.urlopen(url).read().decode('utf-8').replace('\r', '')
#allhtml = open('the-input.txt', 'rt').read().replace('\r', '')
#print(allhtml)

middle = lt(rt(allhtml, START), END)
#print(middle)

middle = html.unescape(middle.replace("", ''))
middle = middle.replace('&', '&')

pieces = [x.strip().replace('\n', ' ') for x in middle.split('
\n\n')]

def pwrap(x):
  if x.endswith('>') or x.startswith('<'):
    return x
  return '
' + x + '
'
pieces = [pwrap(x) for x in pieces]

print('\n'.join(pieces))

# #python3 grabber.py https://www.hplovecraft.com/writings/texts/fiction/bws.aspx > bws.xhtml


import numpy as np
import os
import glob
import json
import cv2
from scipy.spatial import distance
from pathlib import Path
import csv
from absl import flags
from absl import app
import pickle
import warnings

warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)
flags.DEFINE_string('video_name', None, 'the name of the test video')
FLAGS = flags.FLAGS

class Preprocesser():
    def __init__(self, video_name):
        super().__init__()
        self.video_name = video_name
        self.root_dir = "/home/lin10/projects/SkatingApp/cache/jump/data/alphapose/alpha_pose_{}".format(video_name)
        self.output_file = "/home/lin10/projects/SkatingApp/cache/jump/data/pkl/{}.pkl".format(video_name)
        self.sample_list = []
        self.SKELETON_CONF_THRESHOLD = 0.0

    def writePickle(self):
        with open(self.output_file, "wb") as f:
            pickle.dump(self.sample_list, f)

    def get_main_skeleton(self):
        path_to_json = "{}/alphapose-results.json".format(self.root_dir)
        root_dir = self.root_dir
        frames = list(Path(os.path.join(self.root_dir, 'vis')).glob("*.jpg"))
        last_frame = sorted([int(os.path.split(frame)[1].replace('.jpg','')) for frame in frames])[-1]
        with open(path_to_json) as f:
            json_skeleton = json.load(f)

        # Get first image id
        prev_frame = int(json_skeleton[0]['image_id'].rstrip('.jpg'))
        score_list = []
        main_skeleton_list = []
        for skeleton in json_skeleton:
            # Get current image id
            cur_frame = int(skeleton['image_id'].rstrip('.jpg'))
            # Get all skeleton in current frame
            if cur_frame == prev_frame:
                score_list.append([skeleton['score'], np.reshape(skeleton['keypoints'], (17,3))])
            # Get the main skeleton in previous frame by score or distance
            else:
                if main_skeleton_list == []:
                    main_skeleton_list.append([prev_frame, np.array(max(score_list))])
                else:
                    dist_list = []
                    # Pick a skeleton that is the closest to the previous one
                    for score in score_list:
                        ske_distance = distance.euclidean(np.delete(score[1], (2), axis=1).ravel(), np.delete(main_skeleton_list[-1][1][1], (2), axis=1).ravel())
                        dist_list.append([ske_distance, score])
                    main_skeleton_list.append([prev_frame, np.array(min(dist_list)[1])])
                # Clear score list and append first skeleton for current frame
                score_list.clear()
                score_list.append([skeleton['score'], np.reshape(skeleton['keypoints'], (17,3))])
            prev_frame = cur_frame
        # append max score for the last frame
        main_skeleton_list.append([cur_frame, np.array(max(score_list))])
        main_skeleton_list = np.reshape(main_skeleton_list, (-1,2))

        keypoints_info = []
        for main_skeleton in main_skeleton_list:
            delete_pt = []
            keypoints = main_skeleton[1][1]
            for row in range(len(keypoints)):
                if keypoints[row][2] < self.SKELETON_CONF_THRESHOLD:
                    delete_pt.append(row)
            keypoints = np.delete(keypoints, (delete_pt), axis=0)
            keypoints_info.append([main_skeleton[0], keypoints])
        keypoints_info = np.array(keypoints_info)

        # Insert missing frame info
        for i in range(last_frame+1):
            if i not in keypoints_info[:,0]:
                insert_row = np.concatenate([[i],keypoints_info[i-1,1:]], axis=0)
                keypoints_info = np.insert(keypoints_info, i, insert_row, axis=0)
        return keypoints_info

    def subtract_features(self, keypoints):
        subtracted_keypoints = []
        for i in range(1, 17, 2):
            subtraction = keypoints[i][0] - keypoints[i+1][0]
            subtracted_keypoints.append(subtraction)
        return np.array(subtracted_keypoints)

    def load_data(self):
        frames = list(Path(os.path.join(self.root_dir, 'vis')).glob("*.jpg"))
        alphaposeResults = self.get_main_skeleton()
        assert len(frames) == len(alphaposeResults), "{} error".format(self.video_name)
        subtractions_list = [self.subtract_features(alphaposeResult[1]) for alphaposeResult in alphaposeResults]
        keypoints_list = [np.delete(alphaposeResult[1], 2, axis=1).reshape(-1) for alphaposeResult in alphaposeResults]
        subtraction_features_list = np.append(keypoints_list, subtractions_list, axis=1)
        features_list = keypoints_list
        sample = {"subtraction_features": subtraction_features_list, "features": features_list, "video_name": self.video_name}
        self.sample_list.append(sample)
        self.writePickle()

def main(_argv):
    preprocesser = Preprocesser(FLAGS.video_name)
    preprocesser.load_data()

if __name__ == '__main__':
    try:
        app.run(main)
    except SystemExit:
        pass


from bs4 import BeautifulSoup

import requests

from openpyxl import Workbook

wb = Workbook()

sheet = wb.active

page_source = requests.get("https://www.totallyuselesswebsites.com/")

soup =  BeautifulSoup(page_source.text,'html.parser')

# print(soup.prettify())

all_links = soup.find_all('a')

for link in all_links:
    print(link.get('href'))

wb.save("demo.xlsx")


from . import main
from ..models import User, Admin, Machine, Order, MachineStatus
from flask import render_template, redirect, url_for, flash, abort, request, current_app, jsonify
from flask_login import login_required, current_user
import os

# 未登录转向登录界面。已登录主页为管理员个人页面的概览。
@main.route('/', methods=['GET', 'POST'])
def index():
    if current_user.is_anonymous:
        return redirect(url_for('auth.login'))
    else:
        a = current_user
        return render_template('index.html', a=a)

# 管理员所拥有的全部机器。
@main.route('/machines', methods=['GET'])
@login_required
def machines():
    m_list = current_user.machines
    return render_template('machines.html', m_list=m_list)

# 单个机器页面。
@main.route('/machine/<int:m_id>', methods=['GET'])
@login_required
def machine(m_id):
    m = Machine.query.get_or_404(m_id)
    if m.admin_id == current_user.id or current_user.is_super():
        return render_template('machine.html', m=m)
    else:
        abort(403)

# 机器的二维码。
@main.route('/machine/<int:m_id>/qrcode', methods=['GET', 'POST'])
@login_required
def machine_qrcode(m_id):
    m = Machine.query.get_or_404(m_id)
    if m.admin_id == current_user.id or current_user.is_super():
        if request.method == 'POST':
            if 'file' not in request.files:
                flash('未选择图片。')
                return redirect(request.url)
            file = request.files['file']
            if file:
                file_path = os.path.join(current_app.config['MACHINE_QRCODE_PATH'] + 'mid_' + str(m.id) + '.png')
                try:
                    os.remove(file_path)
                except OSError:
                    pass
                file.save(file_path)
                flash('二维码上传成功。如果图片没有更换，是浏览器缓存问题。')
        qrcode_s = gen_qrcode_string(m)
        return render_template('machine_qrcode.html', m=m, qrcode_s=qrcode_s)
    else:
        abort(403)

# 单个机器订单。
@main.route('/machine/<int:m_id>/orders', methods=['GET'])
@login_required
def machine_orders(m_id):
    m = Machine.query.get_or_404(m_id)
    if m.admin_id == current_user.id or current_user.is_super():
        o_list = m.orders
        return render_template('orders.html', m=m, o_list=o_list)
    else:
        abort(403)

# 当前商家的订单。
@main.route('/orders', methods=['GET'])
@login_required
def orders():
    a = current_user
    return render_template('orders.html', o_list=a.orders)

def gen_qrcode_string(m):
    import time
    from hashlib import md5

    api_key = current_app.config['WXPAY_API_KEY']
    qrcode_dict = {
        'appid': current_app.config['WXPAY_APPID'],
        'mch_id': current_app.config['WXPAY_MCH_ID'],
        'nonce_str': current_app.config['WXPAY_NONCE_STR'],
        'product_id': str(m.id),
        'time_stamp': str(int(time.time())),
    }
    sorted_s = ''
    for k in sorted(qrcode_dict.keys()):
        sorted_s += '{k}={v}&'.format(k=k, v=qrcode_dict[k])
    sign = md5((sorted_s + 'key=' + api_key).encode(encoding='utf-8')).hexdigest().upper()
    qrcode_s = 'weixin://wxpay/bizpayurl?sign={sign}&{sorted_s}'.format(sign=sign, sorted_s=sorted_s)

    return qrcode_s


import thingy52
import sqlite3
import time
import binascii

class DBConnection:

  dataBase = 'environment'
  fileName = dataBase + '.db'

  def __init__(self):
    self.con = sqlite3.connect(self.fileName)
    self.cur = self.con.cursor()

  def saveData(self, measurement, time):

    self.addData(measurement['temperature'],
                 measurement['humidity'],
                 measurement['pressure'],
                 measurement['eco2'],
                 measurement['tvoc'],
                 measurement['battery'])

  def addData(self, temperature, humidity, pressure, eco2, tvoc, battery, timestamp = None):
    #add date/time
    id = 'F6:CD:CD:6D:62:09'
    if timestamp is None:
      timestamp = int(time.time())
    self.cur.execute(f"INSERT into {self.dataBase} VALUES ({timestamp},'{id}', {temperature},{humidity}, {pressure}, {eco2}, {tvoc}, {battery})")
    self.con.commit()

  def __del__(self):
    self.con.close()

class MyDelegate(thingy52.MyDelegate):

    def __init__(self):
      self.measurements = dict()
      self.measurements['eco2'] = 0
      self.measurements['tvoc'] = 0
      self.measurements['battery'] = 100
      self.measurements['temperature'] = 23.0
      self.measurements['pressure'] = 1000.0
      self.measurements['humidity'] = 60

    def addBatteryMeasurement(self, battery_level):
      self.measurements['battery'] = battery_level

    def getCurrentMeasurement(self):
      return self.measurements

    def handleNotification(self, handle, data):

        if (handle == thingy52.e_temperature_handle):
            teptep = binascii.b2a_hex(data)
            temperature = self._str_to_int(teptep[:-2]) + int(teptep[-2:], 16) * 0.01
            self.measurements['temperature'] = temperature

            print(f'Notification: Temp received:  {temperature} degCelsius')

        elif (handle == thingy52.e_pressure_handle):
            pressure_int, pressure_dec = self._extract_pressure_data(data)
            pressure = pressure_int + pressure_dec * 0.01
            self.measurements['pressure'] = pressure

            print(f'Notification: Pressure received: {pressure} hPa')

        elif (handle == thingy52.e_humidity_handle):
            humidity = int(binascii.b2a_hex(data),16)
            self.measurements['humidity'] = humidity

            print(f'Notification: Humidity received: {humidity} %')

        elif (handle == thingy52.e_gas_handle):
            eco2, tvoc = self._extract_gas_data(data)
            self.measurements['eco2'] = eco2
            self.measurements['tvoc'] = tvoc

            print(f'Notification: Gas received: eCO2 ppm: {eco2}, TVOC ppb: {tvoc} %')

        else:
            print(f'Notification: UNKOWN: handle {handle}')

def main():
    thingy = thingy52.Thingy52('F6:CD:CD:6D:62:09')
    delegate = MyDelegate()
    thingy.setDelegate(delegate)

    db = DBConnection()

    try:

      thingy.ui.enable()
      thingy.ui.set_led_mode_off()

      thingy.environment.enable()

      second_to_ms = 1000
      thingy.environment.configure(temp_int =  5* 60 * second_to_ms,
                                  press_int = 5 * 60 * second_to_ms,
                                  humid_int = 5 * 60 * second_to_ms,
                                  gas_mode_int = 3)
      thingy.environment.set_temperature_notification(True)
      thingy.environment.set_gas_notification(True)
      thingy.environment.set_humidity_notification(True)
      thingy.environment.set_pressure_notification(True)

      thingy.battery.enable()

      last_snapshot = time.time()
      while True:

        now = time.time()

        interval = 20
        if int(now - last_snapshot) > interval :
          delegate.addBatteryMeasurement(thingy.battery.read())
          print("saving data to DB")
          last_snapshot = now
          db.saveData(delegate.getCurrentMeasurement(), now)

        thingy.waitForNotifications(10)

    except Exception as e:
      print (e)
      thingy.disconnect()

def test():
  db = DBConnection()

  db.addData(1,2,3,4, int(time.time()-60))
  db.addData(2,1,3,10, int(time.time()-120))
  db.addData(3,1,3,10, int(time.time()-240))
  db.addData(1,0,3,4, int(time.time()-300))

main()


from rest_framework import serializers
from order_app.models import Order,OrderInformation
from table_app.models import Table
from paymentmode_app.models import Payment
from cusine_app.models import Cusine
from dish_app.models import DishType,DishModel
from table_app.models import Table

class CusineSerializer(serializers.ModelSerializer):
    cusine_name = serializers.CharField(max_length=255)
    date_added = serializers.DateTimeField(read_only=True)

    class Meta:
        model = Cusine
        fields = ['cusine_name','date_added']

class TableSerializer(serializers.ModelSerializer):

    table_name = serializers.CharField(max_length=255)

    class Meta:
        model = Table
        fields = ["id","table_name","table_status"]

class DishTypeSerializer(serializers.ModelSerializer):
    dish_typename = serializers.CharField(max_length=255)
    dish_type_added = serializers.DateTimeField(read_only=True)

    class Meta:
        model = DishType
        fields = '__all__'

class AvailableDishSerializer(serializers.ModelSerializer):
    dish_id = serializers.IntegerField()
    cusine = CusineSerializer(source='cusine_id', read_only=True)
    dish_name = serializers.CharField(max_length=255)
    dish_image = serializers.ImageField(allow_empty_file=False, use_url=True)
    dish_type = DishTypeSerializer(read_only=True)
    dish_description = serializers.CharField(max_length=255)
    dish_price = serializers.DecimalField(decimal_places=2, max_digits=20)
    date_added = serializers.DateTimeField()

    class Meta:
        model = DishModel
        fields = '__all__'

class OrderInformationSerializer(serializers.ModelSerializer):
    order_info_id = serializers.PrimaryKeyRelatedField(queryset=Order.objects.all())
    customer_name = serializers.CharField(max_length=255)
    table = serializers.PrimaryKeyRelatedField(queryset=Table.objects.all())
    payment_mode = serializers.PrimaryKeyRelatedField(queryset=Payment.objects.all())

    class Meta:
        model = OrderInformation
        fields = ["order_info_id","customer_name","table","payment_mode","grand_total"]

class CreateOrderSerializer(serializers.ModelSerializer):
    order_dishname = serializers.PrimaryKeyRelatedField(queryset=DishModel.objects.all())
    order_quantity = serializers.IntegerField(default=0)
    order_status = serializers.CharField(max_length=50)

    class Meta:
        model = Order
        fields = ['order_dishname','order_quantity','order_status']

class OrderSerializer(serializers.ModelSerializer):
    order_dishname = serializers.SerializerMethodField()
    order_quantity = serializers.IntegerField(default=0)

    class Meta:
        model = Order
        fields = ['order_id','order_dishname','order_quantity','order_status']

    def get_order_dishname(self, obj):
        dish = obj.order_dishname
        serializer = AvailableDishSerializer(dish)
        return serializer.data

class PaymentSerializer(serializers.ModelSerializer):
    payment_mode = serializers.CharField(max_length=50)

    class Meta:
        model = Payment
        fields = ['id','payment_mode']


#test
import scipy.ndimage as ndimage
from scipy.ndimage.measurements import center_of_mass, maximum_position
from scipy.signal import argrelmax, savgol_filter
import sys
import numpy as np
import new_functions as fn
import pylab as pl
#% matplotlib inline

class SmartCentroider(object):
    '''A class for smart centroiding TimepixCam data.
    Gets smart x,y,t centroids from a fileset. A ToF spectrum is generated
    using a subset of the files (evenly sampled from the set)
    to set the band boundaries, though they can be manually specified as well.
    If use_CoM_as_centroid==False, the earliest pixel in the cluster is used,
    otherwise the centre of mass of the cluster is taken as the centroid.'''

    @classmethod
    def DataFrom(cls, data_source):
        self = SmartCentroider(data_source.filelist)
        for k, v in data_source.__dict__.items():
            self.__dict__[k] = v
            print (k)
            print (v)
        return self

    def __init__(self, filelist, bands=None):
        #print (filelist)
        assert len(filelist)>0
        # default setup values go here, all can be overridden

        self.filelist = filelist
        self.n_tof_files = 50

        self.bands = bands
        self.use_CoM_as_centroid = True
        self.inc_diagnoal_joins = True
        self.peak_range = 5
        self.skiplines = 0
        self.files_have_bunchIDs = False
        self.ToF_noise_threshold = 2
        self.savgol_window_length = 5
        self.sample_TOF_raw = None
        self.npix_per_cluster_cut = (1,1e9)
        self.gaussian_size = 1.5
        self.sample_TOF_smoothed = None
        self.peaks = []
        self.peak_indices = []
        self.ret = {}

        self.main_TOF = None

        self.DEBUG = 1

    def MakeSampleTOF(self):
        '''Build a set of files to use for making the ToF.
        The ToF can chage during delay scans etc, so it is
        good to pull from across the range of files rather
        than simply taking the first n files'''
        n_files = len(self.filelist)
        tof_fileset = []

#        tof_fileset = self.filelist

        if self.n_tof_files >= n_files: #if there fewer files that the nominal, take them all
            tof_fileset = self.filelist
        else:
            indices = [int(np.ceil(i * n_files / self.n_tof_files)) for i in xrange(self.n_tof_files)] #evenly sample the files
            if self.DEBUG >2:print ('Selecting file nums for ToF: %s'%indices)
            tof_fileset = [self.filelist[i] for i in indices]
            if self.DEBUG >1: print ('selected %s files for ToF'%len(tof_fileset));
            sys.stdout.flush()

        tof_imgs = []
        for filename in tof_fileset: # load images
            tof_imgs.append(fn.TimepixFileToImage(filename, skiplines=self.skiplines))
        maxval = int(np.max(tof_imgs)) # get ranges for histogram
        minval = int(np.min([np.min(_[_>0]) for _ in tof_imgs])) # minumum which is >0 over all images

        ys = np.zeros(((maxval-minval)+1,), dtype=np.int64)
        xs = np.linspace(minval,maxval,(maxval-minval+1)) # make x points for ToF plot
        assert xs[1]-xs[0]==1 #ensure x-axis space is exactly 1

        for img in tof_imgs: # histogram each imgage
            ys += ndimage.histogram(img[img>0],minval,maxval,bins = (maxval-minval)+1) #much faster than pl.hist

        self.sample_TOF_raw = np.zeros((maxval-minval+1,2), dtype=np.int64)
        self.sample_TOF_raw[:,0] = xs
        self.sample_TOF_raw[:,1] = ys

        ys[ys<self.ToF_noise_threshold] = 0 # filter bins with small values

        # redo the ranges after filtering out the small values:
        # TODO: replace this with the bisect function
        first_non_zero, last_non_zero = 0, maxval-minval
        while ys[first_non_zero]<=0:
            first_non_zero += 1
        while ys[last_non_zero]<=0:
            last_non_zero -= 1

        xs = xs[first_non_zero:last_non_zero+1] # trim array
        ys = ys[first_non_zero:last_non_zero+1]

        new_ys = savgol_filter(ys,self.savgol_window_length,3) # smooth the ToF spectrum

        self.sample_TOF_smoothed = np.zeros((len(xs),2), dtype=np.int64)
        self.sample_TOF_smoothed[:,0] = xs
        self.sample_TOF_smoothed[:,1] = new_ys

    def CalculateBands(self):
        self.peak_indices = argrelmax(self.sample_TOF_smoothed[:,1], axis=0, order=self.peak_range) # find local maxima, range of 5 each side
        if self.DEBUG>=3: print ('Found %s peaks at %s'%(len(self.peaks), self.peaks));
        sys.stdout.flush()
        self.peaks = [self.sample_TOF_smoothed[_,0] for _ in self.peak_indices[0]] # Get peak location from indices
        self.bands = [] # generate banks from peaks - need to take the midpoints though!
        minval = np.min(self.sample_TOF_smoothed[:,0])
        maxval = np.max(self.sample_TOF_smoothed[:,0])
        self.bands.append((minval,(self.peaks[0]+self.peaks[1])//2)) #first point to midpoint of first peaks
        for i in xrange(1,len(self.peaks)-1):
            self.bands.append((self.bands[-1][1],(self.peaks[i]+self.peaks[i+1])//2)) # loop through
        self.bands.append((self.bands[-1][1],maxval))# add last midpoint to last value

    def ShowTOF(self, data):
        f = pl.figure(figsize=[12,4]) # Make the figure an appropriate shape
        ax = pl.subplot(111)
        pl.plot(data[:,0],data[:,1],'b') #plot the original ToF
        pl.show()

    def ShowBands(self):
        f = pl.figure(figsize=[12,4]) # Make the figure an appropriate shape
        ax = pl.subplot(111)

        pl.plot(self.sample_TOF_raw[:,0],self.sample_TOF_raw[:,1],'b') #plot the original ToF
        pl.plot(self.sample_TOF_smoothed[:,0],self.sample_TOF_smoothed[:,1],'r')#plot the new, smoothed ToF
        if self.peak_indices!=[]: pl.plot(self.peaks,[-10 for _ in self.peak_indices[0]],'bo') # plot peaks in blue
        pl.plot([_[0] for _ in self.bands],[10 for _ in self.bands],'ro') # plot left band boundaries in red
        pl.plot([_[1] for _ in self.bands],[10 for _ in self.bands],'ro') # plot right band boundaries in red too

        ticks = [] # use ticks to display band edges
        for (a, b) in self.bands: # TODO: This could be more elegant
            if a not in ticks: ticks.append(a)
            if b not in ticks: ticks.append(b)

        ticks = sorted(ticks) # don't know if this is necessary but doesn't hurt
        ax.set_xticks(ticks, minor=False)
        ax.xaxis.grid(True, which='major')

        # set plots limits
        plot_range = (np.min(self.sample_TOF_smoothed[:,0]),np.max(self.sample_TOF_smoothed[:,0]))
        pl.xlim(plot_range[0]-10,plot_range[1]+10)

        # TODO: add a legend saying that colours are what
        if self.DEBUG>=1:
            print ('Peaks at %s'%self.peaks)
            print ('Bands set at %s'%self.bands);
            sys.stdout.flush()
        pl.show()

    def FindClusters(self):
        if self.DEBUG>=2: print ('Using %s bands...'%len(self.bands));
        sys.stdout.flush()

        if self.inc_diagnoal_joins:
            struct_el=[[1,1,1],[1,1,1],[1,1,1]] # for including diagonal connections as well
        else:
            struct_el=[[0,1,0],[1,1,1],[0,1,0]] # for vertical/horizontal connections only

        #     xs, ys, ts = [], [], [] #return lists
        for filenum, filename in enumerate(self.filelist):
            if filenum%500==0: print ('Smart centroided %s of %s files...'%(filenum, len(self.filelist)));
            sys.stdout.flush()
            if self.files_have_bunchIDs: # if we're using bunchIDs then use these as keys in return dict
                fileID = fn.GetBunchIDFromFile(filename)
            else: #otherwise use the filenames
                fileID = filename
            self.ret[fileID]={'xs':[],'ys':[],'ts':[],'npixs':[]}

            img = fn.TimepixFileToImage(filename, skiplines=self.skiplines)

            segmentation, segments = ndimage.label(img, struct_el) # find clusters
            if self.DEBUG>2: print ('Found %s clusters without using band information'%segments);
            sys.stdout.flush()
            if self.DEBUG>3: self.DebugPlot(segmentation, 'Segmented image not using bands:')

            seg_sum = 0
            for bandnum,(tmin, tmax) in enumerate(self.bands): # Process each band in turn
                band_img = img.copy() # make a copy
                # TODO: Check that the correct edge is being included/excluded so you're not taking one band edge
                # twice and the other not at all!
                band_img[band_img > tmax] = 0 # threshold the new image
                band_img[band_img <= tmin] = 0
                if self.DEBUG>=3: title = self.DebugPlot(band_img, 'Band %s (%s - %s) image:'%(bandnum, tmin, tmax))

                # find clusters
                segmentation, segments = ndimage.label(band_img, struct_el)
                seg_sum += segments
                if self.DEBUG>=3: print ('Found %s segs in band %s (%s - %s)'%(segments, bandnum, tmin, tmax));
                sys.stdout.flush()

                # Get centroids from clusters using specified method:
                if self.use_CoM_as_centroid: # use center of mass weighting
                    # TODO: check whether cluster 0 is included in CoMs
                    # TODO: check order of these, i.e. if x,y,t will be coherent
                    CoMs = center_of_mass(band_img, segmentation, [_ for _ in xrange(1,segments+1)]) #TODO: replace with range(1,segments+1) ??
                    # NB do not replace with enumerate, you need to know the actual number for each cluster - think about it.
                    for clust_num in xrange(1, segments): # cluster 0 = background so skip it
                        clust_pix_index = np.where(segmentation==clust_num) # find pixels associated with cluster
                        self.ret[fileID]['ts'].append(np.max(img[clust_pix_index]))
                        self.ret[fileID]['xs'].append(CoMs[clust_num][0])
                        self.ret[fileID]['ys'].append(CoMs[clust_num][1])
                        self.ret[fileID]['npixs'].append(len(clust_pix_index[0]))

                else: # Take the earliest timecode in the cluster as the centroid
                    max_positions = maximum_position(band_img, segmentation, [_ for _ in xrange(1,segments+1)])
                    for max_pos in max_positions: # Stepping through together, so def. coherent
                        self.ret[fileID]['ts'].append(img[max_pos]) # TODO XXXX: I think this is wrong, should be same as logic above
                        self.ret[fileID]['xs'].append(max_pos[0])
                        self.ret[fileID]['ys'].append(max_pos[1])
                        self.ret[fileID]['npixs'].append('????')

                if self.DEBUG>=2: print ('CoMs for band %s: %s'%(bandnum, CoMs));
                sys.stdout.flush()
            #     index = (np.asarray([_[0] for _ in CoMs]),np.asarray([[_[1] for _ in CoMs]]))
            #     codes = img[CoMs]

            if self.DEBUG>=2: print ('Found %s clusters when using bands'%seg_sum);
            sys.stdout.flush()
        print ('Finished smart centroiding %s files...'%(len(self.filelist)));
        sys.stdout.flush()

    def MakeVMIsFromBands(self, custom_bands=None, use_gaussians=False, round_centroid_coords=False, only_use_n_files=1e15):
        '''Produce VMI images from centroided clusters for each band defined.
        Custom bands can be provided here, and will not overwrite the main band definitions.
        If using Gaussians, exact centroids are always used.
        If not using Gaussians, if round_centroid_coords==True the nearest pixel is used, otherwise, it is'''
        # TODO: put in a time estimator for this stage as it can be very slow. Might need to loop over band inside files instead of the other way around.
#         import time

        if custom_bands is None: custom_bands = self.bands # default to the defined bands if they're not provided
        self.VMI_images = [np.zeros((256,256), dtype=np.float64) for _ in custom_bands] # create images

#         now = time.time()
        for band_num, t_range in enumerate(custom_bands): # loop over bands
            if self.DEBUG>=3:print ('Processing band %s of %s'%(band_num+1, len(bands)));
            sys.stdout.flush()
            #loop over clusters, file by file, using only first n files if specified.
            for filenum, datafilename in enumerate(sorted(self.ret.keys())[:min(only_use_n_files,len(self.ret.keys()))]):
#                 if filenum == 50:
#                     dt = time.time() - now
#                     print 'Estimated time for completion is another %s seconds...'%
                for x,y,t,npix in zip(self.ret[datafilename]['xs'],
                                      self.ret[datafilename]['ys'],
                                      self.ret[datafilename]['ts'],
                                      self.ret[datafilename]['npixs']):
                    if (npix<self.npix_per_cluster_cut[0]) or (npix>self.npix_per_cluster_cut[1]): continue # apply cluster threshold
                    if t >= t_range[0] and t<t_range[1]: # check cluster is within band
                        if use_gaussians: # add a gaussian if using them
                            self.VMI_images[band_num] += fn.makeGaussian(256,1,self.gaussian_size,[x,y])
                        else: # otherwise, make coordinate integers in the correct way and increment that pixel
                            if round_centroid_coords:
                                x = int(np.round(x,0))
                                y = int(np.round(y,0))
                            else:
                                x = int(x)
                                y = int(y)
                            self.VMI_images[band_num][x][y] += 1

    def BuildMainTOF(self):
        all_ts = []
        for k, v in self.ret.items():
            all_ts.extend(v['ts'])
        print ('Found %s timecodes after centroiding'%len(all_ts))
        all_ts = np.asarray(all_ts)
        minval = int(np.min(all_ts))
        maxval = int(np.max(all_ts))

        ys = np.zeros(((maxval-minval)+1,), dtype=np.int64)
        xs = np.linspace(minval,maxval,(maxval-minval+1)) # make x points for ToF plot
        assert xs[1]-xs[0]==1 #ensure x-axis space is exactly 1

        ys += ndimage.histogram(all_ts,minval,maxval,bins = (maxval-minval)+1)

        self.main_TOF = np.zeros((len(xs),2), dtype=np.int64)
        self.main_TOF[:,0] = xs
        self.main_TOF[:,1] = ys

        #         print sum(ys)
#         f = pl.figure(figsize=[12,4])

    def PrintBandsForEditing(self, one_band_per_line=True):
        '''Print (out the band definitions in a format useful for editing.)'''
        if one_band_per_line:
            print ('bands = [' + str(self.bands[0]) + ',')
            for band in self.bands[1:-1]:
                print ('         ' + str(band) + ',')
            print ('         ' + str(self.bands[-1]) + ']')
        else:
            print ('bands = %s'%self.bands)

    def DebugPlot(self, data, title=''):
        print (title);
        sys.stdout.flush()
        f = pl.figure(figsize=[8,8])
        pl.imshow(data)
        pl.show()

    def ShowAllVMIs(self):
        for image in self.VMI_images:
            self.ShowVMIimage(image)

    def ShowVMIimage(self, image, vmin=None, vmax=None, cmap='jet', title = '', savefig=''):
        import numpy as np
        from mpl_toolkits.axes_grid1 import make_axes_locatable

        fig = pl.figure(figsize = [10,10])
        ax = fig.add_subplot(111)
#         ax.set_xlim([YMIN,YMAX])
#         ax.set_ylim([XMIN,XMAX])

        if vmax == 'auto':
            element = (256*256) - 200
            tmp = image.flatten()
            tmp.sort()
            vmax = tmp[element]
            vmin = tmp[200]
            print ('Auto vmax = %s, real max = %s'%(vmax, np.max(image)))

        if vmin == 'auto':
            tmp = image.flatten()
            vmin = min(_ for _ in tmp if _ > 0)
            print ('Auto vmin = %s'%vmin)

        im = ax.imshow(image, vmin=vmin, vmax=vmax, cmap=cmap, interpolation='nearest')
        ax.set_title(title)
        divider = make_axes_locatable(ax)
        cax = divider.append_axes("right", size="5%", pad=0.05)
        pl.colorbar(im, cax=cax)

        if savefig: fig.savefig(savefig)
        return im

#     def ShowVMIimage(self, image, title=''):
#         print title; sys.stdout.flush()
#         f = pl.figure(figsize=[8,8])
#         pl.imshow(image)
#         pl.show()

    def run(self):
        print (type(self))
        self.MakeSampleTOF()
        if self.DEBUG>=2:
            print ('Raw ToF spectrum for sample files')
            self.ShowTOF(self.sample_TOF_raw)
        if self.DEBUG>=1:
            print ('Smoothed, noise suppressed and truncated ToF spectrum for sample files')
            self.ShowTOF(self.sample_TOF_smoothed)

        if not self.bands:
            self.CalculateBands()
        if self.DEBUG>=0: self.ShowBands()

        self.FindClusters()
        self.MakeVMIsFromBands()
        self.ShowAllVMIs()
        return self.ret.copy()


#!/usr/bin/env python

from __future__ import print_function

# ------------------------------------------------------------------------
# Test client for the pyserv project. Encrypt test.

import  os, sys, getopt, signal, select, socket, time, struct
import  random, stat

#sys.path.append('../common')
#import support, pycrypt, pyservsup, pyclisup, syslog, comline, pypacker

# Set parent as module include path
current = os.path.dirname(os.path.realpath(__file__))
parent = os.path.dirname(current)
sys.path.append(parent)

from common import support, pycrypt, pyservsup, pyclisup
from common import pysyslog, comline, pypacker

version = "1.0"

optarr =  comline.optarr
optarr.append ( ["p:",  "port",  6666,   None, "Port to use (default: 9999)"] )
#                ["c:",  "comm",     "",     None],      \

conf = comline.Config(optarr)

# ------------------------------------------------------------------------

if __name__ == '__main__':

    args = conf.comline(sys.argv[1:])

    if len(args) == 0:
        ip = '127.0.0.1'
    else:
        ip = args[0]

    hand = pyclisup.CliSup()
    hand.verbose = conf.verbose
    hand.pgdebug = conf.pgdebug
    #hand.comm  = conf.comm

    #print("ip:", ip, "port", conf.port)
    try:
        resp2 = hand.connect(ip, conf.port)
    except:
        print( "Cannot connect to:", ip + ":" + str(conf.port), sys.exc_info()[1])
        sys.exit(1)

    #print ("Server initial:", resp2[1])

    resp3 = hand.client(["hello",] , "", False)
    if conf.verbose:
        print("Hello Response:", resp3)

    #sys.exit(0)

    cresp = hand.client(["crap", "this com"])
    print ("Server crap response:", cresp[1])

    cresp = hand.client(["ls", "out of order"])
    print ("Server ls response:", cresp[1])

    cresp = hand.client(["sess", "out of order"])
    print ("Server sess response:", cresp[1])

    cresp = hand.client(["user", "peter"])
    print ("Server user response:", cresp[1])

    cresp = hand.client(["pass", "12345"])
    print ("Server pass response:", cresp[1])

    qresp = hand.client(["quit"])
    #hand.close()
    print ("Server quit response:", qresp[1])

    sys.exit(0)

# EOF


from django import forms
from .models import *
from django.utils import timezone
from django.forms import ModelForm, Textarea, TextInput
from ckeditor.widgets import CKEditorWidget
from django.core.files.images import get_image_dimensions
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from django.forms.widgets import FileInput, EmailInput

class TopicoForm(ModelForm):
    class Meta:

        fields = '__all__'
        exclude = ['data_postagem']
        widgets = {
            'text': CKEditorWidget()
        }

    def clean_titulo(self):
        titulo = self.cleaned_data['titulo']
        if not titulo:
            raise ValidationError(_("Это поле является обязательным."), code='invalid')
        return titulo

    def clean_texto(self):
        texto = self.cleaned_data['texto']
        if not texto:
            raise ValidationError(_("Это поле является обязательным."), code='invalid')
        return texto

class RespostaForm(ModelForm):
    class Meta:
        model = Resposta
        fields = '__all__'
        exclude = ['data_postagem']
        widgets = {
            'texto': CKEditorWidget()
        }

    def clean_texto(self):
        texto = self.cleaned_data['texto']
        if not texto:
            raise ValidationError(_("Это поле является обязательным."), code='invalid')
        return texto

class UserForm(ModelForm):
    class Meta:
        model = User
        fields = ['avatar', 'username', 'password', 'first_name', 'last_name', 'email']

    def clean_avatar(self):
        avatar = self.cleaned_data['avatar']
        if avatar:
            try:
                w, h = get_image_dimensions(avatar)

                max_width = 800
                max_height = 600
                if w > max_width or h > max_height:

                    raise ValidationError(_('Максимальное разрешение изображения 800x600.'), code='invalid')

                main, sub = avatar.content_type.split('/')
                if not (main == 'image' and sub in ['jpeg', 'pjpeg', 'gif', 'png']):
                    raise ValidationError(_('Доступные форматы изображения JPEG, GIF и PNG.'), code='invalid')

                #valida o tamanho da imagem
                if len(avatar) > (40 * 1024):
                    raise ValidationError(_('Максимальный размер изображения 40k.'), code='invalid')

            except AttributeError:
                pass

            return avatar

    def clean_username(self):
        username = self.cleaned_data['username']
        #verifica se o usuario já existe
        if User.objects.filter(username=username).exists():
            raise ValidationError(_("Имя пользователя занято, выберите другой!"), code='invalid')

        return username

    def clean_email(self):
        email = self.cleaned_data['email']

        if not email:
            raise ValidationError(_("Это поле является обязательным."), code='invalid')

        #verifica se o email já foi usado
        if User.objects.filter(email=email).exists():
            raise ValidationError(_("Этот адрес электронной почты уже связан с учетной записью."), code='invalid')
        return email

class UserUpdateForm(ModelForm):
    class Meta:
        model = User
        fields = ['avatar', 'first_name', 'last_name', 'email']

    def clean_email(self):
        email = self.cleaned_data['email']
        if not email:
            raise ValidationError(_("Это поле является обязательным."), code='invalid')
        return email

    def clean_avatar(self):
        avatar = self.cleaned_data['avatar']
        if avatar:
            try:
                w, h = get_image_dimensions(avatar)

                #verifica a resolucao da imagem
                max_width = 800
                max_height = 600
                if w > max_width or h > max_height:

                    raise ValidationError(_('Максимальное разрешение изображения 800x600.'), code='invalid')

                #valida o formato da imagem
                main, sub = avatar.content_type.split('/')
                if not (main == 'image' and sub in ['jpeg', 'pjpeg', 'gif', 'png']):
                    raise ValidationError(_('Доступные форматы изображения  JPEG, GIF и PNG.'), code='invalid')

                #valida o tamanho da imagem
                if len(avatar) > (40 * 1024):
                    raise ValidationError(_('Максимальный размер изображения 40k.'), code='invalid')

            except AttributeError:
                pass
        return avatar


#! /usr/bin/env python3

import numpy as np
from sigmoid import sigmoid_

def vec_log_gradient_(x, y_true, y_pred):
    """
    Compute the gradient.
    Args:
        x: a 1d or 2d numpy ndarray for the samples
        y_true: a scalar or a numpy ndarray for the correct labels
        y_pred: a scalar or a numpy ndarray for the predicted labels
    Returns:
        The gradient as a scalar or a numpy ndarray of the width of x.
        None on any error.
    Raises:
        This function should not raise any Exception.
    """
    x = np.array(x)
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)
    if len(x.shape) > 1:
        return (y_pred - y_true).dot(x)
    return (y_pred - y_true) * x

if __name__ == "__main__":
    x = np.array([1, 4.2])  # x[0] represent the intercept
    y_true = 1
    theta = np.array([0.5, -0.5])
    y_pred = sigmoid_(np.dot(x, theta))
    print(vec_log_gradient_(x, y_pred, y_true))
    # [0.83201839 3.49447722]

    x = np.array([1, -0.5, 2.3, -1.5, 3.2])  # x[0] represent the intercept
    y_true = 0
    theta = np.array([0.5, -0.5, 1.2, -1.2, 2.3])
    y_pred = sigmoid_(np.dot(x, theta))
    print(vec_log_gradient_(x, y_true, y_pred))
    # [ 0.99999686 -0.49999843  2.29999277 -1.49999528  3.19998994]

    x_new = np.arange(2, 14).reshape((3, 4))
    x_new = np.insert(x_new, 0, 1, axis=1)
    # first column of x_new are now intercept values initialized to 1
    y_true = np.array([1, 0, 1])
    theta = np.array([0.5, -0.5, 1.2, -1.2, 2.3])
    y_pred = sigmoid_(np.dot(x_new, theta))
    print(vec_log_gradient_(x_new, y_true, y_pred))
    # [0.99994451 5.99988885 6.99983336 7.99977787 8.99972238]


# Import dependencies
from bs4 import BeautifulSoup
import requests
import selenium
from splinter import Browser
import pandas as pd
import re
import time

def init_browser():
    executable_path = {'executable_path': r'C:\webdriver\chromedriver'}
    return Browser('chrome', **executable_path, headless=False)

def scrape():
    browser = init_browser()
    mars_info = {}

    # visit nasa website
    url = 'https://mars.nasa.gov/news/'
    browser.visit(url)
    time.sleep(5)
    html = browser.html
    soup = BeautifulSoup(html, 'html.parser')

    # retrieve title and paragraph for first news title on NASA site
    news_title = soup.find("li", class_="slide").find("div", class_="content_title").text
    news_p = soup.find("li", class_="slide").find("div", class_="article_teaser_body").text
    mars_info["news_title"] = news_title
    mars_info["news_p"] = news_p

    #JPL Mars Space Images - Featured Image

    url2 = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'
    browser.visit(url2)

    # scrape page into soup
    html = browser.html
    soup2 = BeautifulSoup(html, 'html.parser')

    # retrieve the text from the style which includes background image url
    div_style = soup2.find('article')['style']

    # extract image location from style and combine with jpl index url
    short_url = div_style.replace("background-image: url('","").replace("');","")
    featured_image_url = "https://www.jpl.nasa.gov"+short_url
    mars_info["featured_image_url"] = featured_image_url

    # Mars Weather

    url3 = 'https://twitter.com/marswxreport?lang=en'
    browser.visit(url3)
    time.sleep(5)
    html = browser.html
    soup3 = BeautifulSoup(html, 'html.parser')

    # retrieve Mars weather tweet
    mars_tweet = soup3.find('div', class_='css-1dbjc4n')
    weather_tweets = mars_tweet(text=re.compile(r' sol'))
    mars_weather = weather_tweets[0]
    mars_info["mars_weather"] = mars_weather

    # Mars Facts

    url4 = 'https://space-facts.com/mars/'
    browser.visit(url4)
    html = browser.html
    soup4 = BeautifulSoup(html, 'html.parser')

    #retrieve Mars Facts
    table = soup4.find('table', class_='tablepress')
    table = pd.read_html(url4)
    mars_df = table[0]
    html_table = mars_df.to_html()
    html_table = html_table.replace('\n', '')
    mars_info["html_table"] = html_table

    # Mars Hemispheres

    url5 = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
    browser.visit(url5)

    # use a loop to loop through the 4 hemispheres, click through to their page and pull high res image
    html = browser.html
    soup5 = BeautifulSoup(html, 'html.parser')

    hemi_list = []
    hemispheres = soup5.find_all('div', class_="item")
    for hemi in hemispheres:
        link = hemi.a['href']
        link_long = "https://astrogeology.usgs.gov"+link
        browser.visit(link_long)
        html2 = browser.html
        soup6 = BeautifulSoup(html2, 'html.parser')
        raw_title = soup6.find('title').text
        title = raw_title.replace("Enhanced | USGS Astrogeology Science Center","").replace("e ","e")
        hemi_dict = {}
        hemi_dict.update({'title': title})
        short_img_url = soup6.find('img', class_="wide-image")["src"]
        img_url = "https://astrogeology.usgs.gov"+short_img_url
        hemi_dict.update({'img_url': img_url})
        hemi_list.append(hemi_dict)

    mars_info["hemi_list"] = hemi_list

    # Close the browser after scraping
    browser.quit()

    return mars_info


import sys
from PyQt5 import QtCore, QtGui, QtWidgets
from weather_gui import Ui_MainWindow

app = QtWidgets.QApplication([])

class MainApplication(QtWidgets.QMainWindow, Ui_MainWindow):

    def __init__(self, *args, **kwargs):
        super(MainApplication, self).__init__(*args, **kwargs)
        self.setupUi(self)

        self.show()

window = MainApplication()
window.show()

sys.exit(app.exec_())


from .models import Event
from django import forms
from datetime import date, datetime

class DateInput(forms.DateInput):
    input_type = "date"

class EventForm(forms.ModelForm):
    class Meta:
        model = Event
        fields = [
            'title',
            'event_date',
            'time',
            'content',
            'featured_image',
            'price',
        ]
        widgets = {
            "event_date": DateInput(),
            'time': forms.TimeInput(attrs={'type': 'time'})
        }


from hana.errors import HanaPluginError
import pathspec

# TODO: consider splitting prev/next to a 'navigation' plugin, rather than here. Leave this to only
#       deal with tags.
#
class Tags():
    def __init__(self, config={}, metadata_key='tags', default_tag=None, previous_key='previous', next_key='next'):
        # default_denotes which collection is default and applied to the actual files. Unless
        # articles in each collection are generated dynamically or in separate files, there can
        # only be one real collection, and the default one is it.

        #TODO: validate config input. can't start with _
        self.config = config
        self.metadata_key = metadata_key
        self.default_tag = default_tag
        self.previous_key = previous_key
        self.next_key = next_key

        # Initialize empty tags
        self.tags = {}
        self.patterns = {}

        # Generate patterns
        for tag, tag_def in self.config.items():
            if 'pattern' in tag_def:
                patterns = tag_def['pattern']

                if isinstance(patterns, str):
                    patterns = [patterns]

                self.patterns[tag] = pathspec.PathSpec.from_lines('gitwildmatch', patterns)

                #Initialize tags for all the patterns
                if tag not in self.tags:
                    self.tags[tag] = {}

    def __call__(self, files, hana):

        for filename, f in files:
            #TODO: this should probably be taken care of globally
            f['path'] = filename

            # Add tags metadata
            if not self.metadata_key in f:
                f[self.metadata_key] = []

            # Add pattern tags
            for tag, tag_pat in self.patterns.items():
                if tag_pat.match_file(filename):
                    f[self.metadata_key].append(tag)

            # Make set and grab the tags
            file_tags = set(f[self.metadata_key])
            f[self.metadata_key] = file_tags

            # Enumerate tags and keep track of files
            for tag in file_tags:
                if tag not in self.tags:
                    self.tags[tag] = {}

                # Synthetize the metadata, as we don't want it to update from other plugins
                # TODO: rethink this. We do need content in certain format (transformed, but not
                #       full templates applied), but other metadata should be updated?
                self.tags[tag][filename] = dict(f)

        # Initialize main tag list in hana
        hana.metadata[self.metadata_key] = {}

        for tag, tag_def in self.tags.items():
            tag_def = tag_def.values()

            # Sort
            if tag in self.config and self.config[tag].get('sort_by'):
                sort_key = self.config[tag].get('sort_by')
                tag_def = sorted(tag_def, key=lambda f_meta: f_meta[sort_key])

            # Reverse
            if tag in self.config and self.config[tag].get('reverse', False):
                tag_def = reversed(tag_def)

            # Synthetize a list, as Jinja doesn't seem to be dealing well with iterator here
            # Set prev/next links for primary tag
            coll = []

            # Set prev/next links for tag
            for idx, post in enumerate(tag_def):
                coll.append(post)

                if idx < 1:
                    continue

                prev_post = coll[idx-1]
                curr_post = coll[idx]

                #next_item = {'path': post['path'], 'title': post.get('title')}
                #prev_item = {'path': prev_post['path'], 'title': prev_post.get('title')}

                # set it within tag
                prev_post[self.next_key] = curr_post
                coll[idx][self.previous_key] = prev_post

                if tag == self.default_tag:
                    # Set prev/next on the default tag. This is necessary for the global blog
                    # prev/next to work.
                    hana.files[prev_post['path']][self.next_key] = curr_post
                    hana.files[post['path']][self.previous_key] = prev_post

            hana.metadata[self.metadata_key][tag] = coll


# Package imports
import pandas as pd
import glob
import pickle
from tqdm import tqdm

# Create data
files = glob.glob("data/processed/Graphs/*")
cols = ['time', 'time_to_reservation', 'park_location_lat', 'park_location_long', 'leave_location_lat', 'leave_location_long','park_zone', 'leave_zone', 'park_fuel', 'leave_fuel', 'engine', 'moved','prev_customer', 'next_customer',
        'movedTF', 'action', 'weekend', 'Time_Cos', 'Time_Sin', 'mean_temp', 'mean_wind_speed', 'acc_precip', 'bright_sunshine', 'mean_pressure', 'mean_relative_hum', 'mean_cloud_cover', 'dist_to_station', 'degree']
df = pd.DataFrame(columns = cols)

i = 0
for f in tqdm(files):
    with open(f, 'rb') as day_file:
        day = pickle.load(day_file)

    for attr, adj in day.values():
        df.loc[i] = list(attr.iloc[-1].values)+[adj.getrow(-1).sum()]
        i += 1

df['timeh'] = df.time.round('h').dt.hour
trafic_index = dict(df.groupby('timeh').mean()['time_to_reservation'])
df.drop(columns = ['timeh'], inplace=True)
df['hour_index'] = df.time.dt.hour.map(trafic_index)

# Only to customers
df = df[df.next_customer]
df.drop(columns=['prev_customer', 'next_customer'], inplace = True)

# No more than 2 days
df = df[df.time_to_reservation < 48]

# Remove zones with too little support
df = df[~df.leave_zone.isin((df.leave_zone.value_counts() < 30).index[df.leave_zone.value_counts() < 30])]

# One hot encoding
df = pd.get_dummies(df, columns = ['engine','leave_zone'], prefix=['eng','lz'])

# Drop those far away
df = df[df['dist_to_station'] <= 7000]

# Drop some columns
df.drop(columns=['park_location_lat', 'park_location_long', 'leave_location_lat', 'leave_location_long', 'park_fuel', 'park_zone', 'moved', 'movedTF', 'action'], inplace = True)

# Reset index
df = df.reset_index(drop=True)

# Save
df.to_csv('data/processed/SimpleNNData.csv')