import math
def quad_root(a,b,c):
 # finds the roots of the quadratic ax^2+bx+c,
 # assuming a != 0

  if a == 0:
    print("This is not a quadratic.")
  else:
    disc = b**2 - 4*a*c
    if disc < 0:
      print("This has no real roots.")
    elif disc == 0:
      print("The quadratic has a double root at:", - b/(2*a))
    else:
      if b > 0:
        t = -b - math.sqrt(disc)
        print("The quadratic has two distinct roots at:", (2*c)/t, t/(2*a))
      else:
        t = -b + math.sqrt(disc)
        print("The quadratic had two distinct roots at:", (2*c)/t, t/(2*a))

#quad_root(-1,-99,-1)

def bisect_for(a,b,nmax,eps):
  # finds the root of the external function f(x)
    #in the interval [a,b]

  fa = f(a)
  fb = f(b)

  if fa*fb > 0:
    print("There is an error")
    return
  error = b-a
  for i in range(1, nmax):
    error = error/2
    m = a + error
    fm = f(m)
    if abs(error) < eps:
      print("The solution for m is", m, "and the number of steps is" ,nmax)
      return
    if fa*fm > 0:
      a = m
      fa = fm
    else:
      b = m
      fb =fm
  print("The root estimate for m is", m, "and the function took too many steps", nmax)

def bisect_while(a,b,nmax,eps):
  #finds the root of the external function f(x)
    #in the interval [a,b] if there is a sign change

  fa = f(a)
  fb = f(b)

  if fa*fb > 0:
    print("an error message")
  else:
    error = b-a
    count = 0
    while count < nmax and abs(error) >= eps:
      count = count + 1
      error = error/2
      m = a + error
      fm = f(m)
      if fa*fm > 0:
        a = m
        fa = fm
      else:
        b = m
        fb =fm
    if count == nmax:
      print("root estimate", m,"and it took too many steps", nmax)
    if abs(error) < eps:
        print("the solution", m, "and the number of steps is" , nmax)

def f(x):
  #return x**3*math.cos(x)
 #return math.exp(-.01*x)*math.sin(0.2*x)
  #return x**10-8*x**3+2*x-.25
  #return x**5-8*x**4+24*x**3-32*x**2+16*x
  return x**6-10*x**5+40*x**4-80*x**3+80*x**2-32*x

bisect_for(1, 4, 100, 10**-10)
bisect_while(1, 4, 100, 10**-10)


from lesson23.lib.interface import cabeçalho

def archiveExist(name):
    try:
        a = open(name, 'rt')
        a.close()
    except FileNotFoundError:
        return False
    else:
        return True

def creatArchive(name):
    try:
        a = open(name, 'wt+')
        a.close()
    except:
        print('There was an ERROR in creating the file.')
    else:
        print(f'{name} file created successfully!')

def readArchive(name):
    try:
        a = open(name, 'rt')
    except:
        print('ERRO in read the archive!')
    else:
        cabeçalho('PESSOAS CADASTRADAS')
        # print(a.read())
        for linha in a:
            dado = linha.split(';')
            dado[1] = dado[1].replace('\n', '')
            print(f'{dado[0]:<30}{dado[1]:>3} anos')
    finally:
        a.close()

def cadastrar(arq, name='Desconhecido', age=0):
    try:
        a = open(arq, 'at')
    except:
        print('Houve um ERRO na abertura do arquivo!')
    else:
        try:
            a.write(f'{name};{age}\n')
        except:
            print('There was an ERROR na hora de escrever os dados!')
        else:
            print(f'Novo registro de {name} adicionado.')
            a.close()


#程序文件Pex16_6.py
import numpy as np
import matplotlib.pyplot as plt
plt.rc('font',size=16); N=10000;
x,y=np.random.uniform(-1,1,size=(2,N))
inside=(x**2+y**2)<=1
mpi=inside.sum()*4/N  #求pi的近似值
error=abs((mpi-np.pi)/np.pi)*100
outside=np.invert(inside)
plt.plot(x[inside],y[inside],'b.')
plt.plot(x[outside],y[outside],'r.')
plt.plot(0,0,label='$\hat\pi$={:4.3f}\nerror={:4.3f}%'.
         format(mpi,error),alpha=0)
plt.axis('square'); plt.legend(); plt.show()


# -*- coding: utf-8 -*-
"""
/////
"""
import os
import time
import numpy as np
from tqdm import tqdm
from random import choice
import random
import json
import logging
import torch
import torch.utils.data as data
from copy import deepcopy
from utils import init_session, init_goal, state_vectorize, action_vectorize, domain_vectorize, action_seq, pad_to
import itertools

def expand_da(meta):
    for k, v in meta.items():
        domain, intent = k.split('-')
        if intent.lower() == "request":
            for pair in v:
                pair.insert(1, '?')
        else:
            counter = {}
            for pair in v:
                if pair[0] == 'none':
                    pair.insert(1, 'none')
                else:
                    if pair[0] in counter:
                        counter[pair[0]] += 1
                    else:
                        counter[pair[0]] = 1
                    pair.insert(1, str(counter[pair[0]]))

class DataManager:
    """Offline data manager"""
    def __init__(self, data_dir, cfg):
        self.data = {}
        self.goal = {}
        self.cfg = cfg
        self.pol_dataset = dict()
        self.data_dir = data_dir
        self.data_dir_new = data_dir + '/processed_data'
        self.a_seq_lst = None
        self.loaded_data = None
        if os.path.exists(self.data_dir_new):
            logging.info('Load processed data file')
            for part in ['train', 'valid', 'test']:
                with open(self.data_dir_new + '/' + part + '.json', 'r') as f:
                    self.data[part] = json.load(f)
                with open(self.data_dir_new + '/' + part + '_goal.json', 'r') as f:
                    self.goal[part] = json.load(f)
        else:
            from .dbquery import DBQuery
            db = DBQuery(data_dir)
            logging.info('Start pre-processing the dataset')
            self._build_data(data_dir, self.data_dir_new, cfg, db)

    def _build_data(self, data_dir, data_dir_new, cfg, db, do_check=True):
        data_filename = data_dir + '/' + cfg.data_file
        with open(data_filename, 'r') as f:
            origin_data = json.load(f)
        for part in ['train', 'valid', 'test']:
            self.data[part] = []
            self.goal[part] = {}
        valList = []
        with open(data_dir + '/' + cfg.val_file) as f:
            for line in f:
                valList.append(line.split('.')[0])
        testList = []
        with open(data_dir + '/' + cfg.test_file) as f:
            for line in f:
                testList.append(line.split('.')[0])
        for k_sess in tqdm(origin_data):
            sess = origin_data[k_sess]
            if k_sess in valList:
                part = 'valid'
            elif k_sess in testList:
                part = 'test'
            else:
                part = 'train'
            turn_data, session_data = init_session(k_sess, cfg)
            init_goal(session_data, sess['goal'], cfg)
            self.goal[part][k_sess] = session_data
            belief_state = turn_data['belief_state']
            for i, turn in enumerate(sess['log']):
                book_status = turn['metadata']
                print(turn['dialog_act'])
                # if len(book_status) == 0: continue
                # for domain in self.cfg.belief_domains:
                #     if book_status[domain]['book']['booked']:
                #         print(turn['metadata'])

        import ipdb;ipdb.set_trace()
        for k_sess in tqdm(origin_data):
            sess = origin_data[k_sess]
            if k_sess in valList:
                part = 'valid'
            elif k_sess in testList:
                part = 'test'
            else:
                part = 'train'

            turn_data, session_data = init_session(k_sess, cfg)
            init_goal(session_data, sess['goal'], cfg)
            self.goal[part][k_sess] = session_data
            belief_state = turn_data['belief_state']

            for i, turn in enumerate(sess['log']):
                turn_data['others']['turn'] = i
                turn_data['others']['terminal'] = i + 2 >= len(sess['log'])
                da_origin = turn['dialog_act']
                expand_da(da_origin)
                turn_data['belief_state'] = deepcopy(belief_state)  # from previous turn

                if i % 2 == 0:  # user
                    if 'last_sys_action' in turn_data:
                        turn_data['history']['sys'] = dict(turn_data['history']['sys'], **turn_data['last_sys_action'])
                        del(turn_data['last_sys_action'])
                    turn_data['last_user_action'] = deepcopy(turn_data['user_action'])
                    turn_data['user_action'] = dict()
                    for domint in da_origin:
                        domain_intent = da_origin[domint]
                        _domint = domint.lower()
                        _domain, _intent = _domint.split('-')
                        if _intent == 'thank':
                            _intent = 'welcome'
                            _domint = _domain+'-'+_intent
                        for slot, p, value in domain_intent:
                            _slot = slot.lower()
                            _value = value.strip()
                            _da = '-'.join((_domint, _slot, p))
                            if _da in cfg.da_usr:
                                turn_data['user_action'][_da] = _value
                                if _intent == 'inform':
                                    inform_da = _domain+'-'+_slot+'-1'
                                    if inform_da in cfg.inform_da:
                                        belief_state['inform'][_domain][_slot] = _value
                                elif _intent == 'request':
                                    request_da = _domain+'-'+_slot
                                    if request_da in cfg.request_da:
                                        belief_state['request'][_domain].add(_slot)
                else:  # sys
                    if 'last_user_action' in turn_data:
                        turn_data['history']['user'] = dict(turn_data['history']['user'], **turn_data['last_user_action'])
                        del(turn_data['last_user_action'])
                    turn_data['last_sys_action'] = deepcopy(turn_data['sys_action'])
                    turn_data['sys_action'] = dict()
                    for domint in da_origin:
                        domain_intent = da_origin[domint]
                        _domint = domint.lower()
                        _domain, _intent = _domint.split('-')
                        for slot, p, value in domain_intent:
                            _slot = slot.lower()
                            _value = value.strip()
                            _da = '-'.join((_domint, _slot, p))
                            if _da in cfg.da:
                                turn_data['sys_action'][_da] = _value
                                if _intent == 'inform' and _domain in belief_state['request']:
                                    belief_state['request'][_domain].discard(_slot)
                                elif _intent == 'book' and _slot == 'ref':
                                    for domain in belief_state['request']:
                                        if _slot in belief_state['request'][domain]:
                                            belief_state['request'][domain].remove(_slot)
                                            break

                    book_status = turn['metadata']
                    for domain in cfg.belief_domains:
                        if book_status[domain]['book']['booked']:
                            entity = book_status[domain]['book']['booked'][0]
                            if domain == 'taxi':
                                belief_state['booked'][domain] = 'booked'
                            elif domain == 'train':
                                found = db.query(domain, [('trainID', entity['trainID'])])
                                belief_state['booked'][domain] = found[0]['ref']
                            else:
                                found = db.query(domain, [('name', entity['name'])])
                                belief_state['booked'][domain] = found[0]['ref']
                if i + 1 == len(sess['log']):
                    turn_data['next_belief_state'] = belief_state
                self.data[part].append(deepcopy(turn_data))

        def _set_default(obj):
            if isinstance(obj, set):
                return list(obj)
            raise TypeError
        os.makedirs(data_dir_new)
        for part in ['train', 'valid', 'test']:
            with open(data_dir_new + '/' + part + '.json', 'w') as f:
                self.data[part] = json.dumps(self.data[part], default=_set_default)
                f.write(self.data[part])
                self.data[part] = json.loads(self.data[part])
            with open(data_dir_new + '/' + part + '_goal.json', 'w') as f:
                self.goal[part] = json.dumps(self.goal[part], default=_set_default)
                f.write(self.goal[part])
                self.goal[part] = json.loads(self.goal[part])

    def create_dataset(self, part, file_dir, cfg, db):
        datas = self.data[part]
        goals = self.goal[part]
        s, a, next_s, a_seq = [], [], [], []
        d_m = []
        logging.info('total {} samples'.format(len(datas)))
        sa2label = dict()
        for idx, turn_data in tqdm(enumerate(datas)):
            if turn_data['others']['turn'] % 2 == 0:
                continue
            turn_data['user_goal'] = goals[turn_data['others']['session_id']]
            aug_action = None
            state_tsr = torch.Tensor(state_vectorize(turn_data, cfg, db, True))
            s.append(state_tsr)
            act_tsr = torch.Tensor(action_vectorize(turn_data, cfg, provided_act=aug_action))
            a.append(act_tsr)
            d_m.append(torch.Tensor(domain_vectorize(turn_data, cfg)))
            a_seq.append(torch.Tensor(action_seq(turn_data, cfg, provided_act=aug_action)))
            if not int(turn_data['others']['terminal']):
                next_s.append(torch.Tensor(state_vectorize(datas[idx+2], cfg, db, True)))
            else:
                next_turn_data = deepcopy(turn_data)
                next_turn_data['others']['turn'] = -1
                next_turn_data['user_action'] = {}
                next_turn_data['last_sys_action'] = next_turn_data['sys_action']
                next_turn_data['sys_action'] = {}
                next_turn_data['belief_state'] = next_turn_data['next_belief_state']
                next_s.append(torch.Tensor(state_vectorize(next_turn_data, cfg, db, True)))
        torch.save((s, a, next_s, d_m, a_seq), file_dir)

    def create_dataset_PEDP(self, part, batchsz, cfg, db, full_data=False, other_data=None):
        t0 = time.time()
        if other_data is None:
            logging.info('=' * 100)
            load_file_dir = self.data_dir_new + '/' + part + '.pt'
            if not os.path.exists(load_file_dir):
                logging.info('MultiWOZ | Create Dataset: {}'.format(load_file_dir))
                self.create_dataset(part, load_file_dir, cfg, db)

            s, a, next_s, _ = torch.load(load_file_dir)
            logging.info('Source Loaded: Total {} samples'.format(len(a)))
            n_s, n_next_s, a_seq, s_pos_seq = [], [], [], []
            a_seq_next, a_pos_seq = [], []
            dist_dict = dict()
            cur_id = 0

            turn_id_list = []
            for i, turn_data in enumerate(self.data[part]):
                if i % 2 == 1:
                    turn_id_list.append(int((turn_data['others']['turn'] - 1) / 2))
            # import ipdb; ipdb.set_trace()
            for s_, a_, next_s_ in tqdm(zip(s, a, next_s)):
                if a_.nonzero().shape[0] == 0:
                    continue

                # print([cfg.idx2da[x] for x in a_.nonzero().squeeze(1).tolist()])
                if turn_id_list[cur_id + 1] == 0 or cur_id + 1 == len(turn_id_list):
                    a_next_ = torch.zeros_like(a_)
                else:
                    a_next_ = a[cur_id + 1]

                a_seq_next.append(torch.Tensor(a_next_.nonzero().squeeze(1).tolist()))
                # import ipdb;ipdb.set_trace()
                a_pos_seq.append(torch.Tensor([a_.nonzero().shape[0] + a_next_.nonzero().shape[0]]))
                # if not final_turn:
                # import ipdb; ipdb.set_trace()
                len_now = a_.nonzero().shape[0]
                if len_now not in dist_dict:
                    dist_dict[len_now] = 1
                else:
                    dist_dict[len_now] += 1

                n_s.append(s_)
                n_next_s.append(next_s_)
                s_pos_seq.append(torch.Tensor([a_.nonzero().shape[0]]))
                a_seq.append(torch.Tensor(a_.nonzero().squeeze(1).tolist()))
                cur_id += 1
            dataset = DatasetPEDPnew(n_s, a_seq, n_next_s, s_pos_seq, a_seq_next, a_pos_seq)
        else:
            print(other_data + '{}.pt'.format(part))
            n_s, a_seq, n_next_s, s_pos_seq, a_seq_next, a_pos_seq, _ = torch.load(other_data + '{}.pt'.format(part))
            dataset = DatasetPEDPnew(n_s, a_seq, n_next_s, s_pos_seq, a_seq_next, a_pos_seq)
            print(other_data + '{}.pt  total of {} samples'.format(part, len(dataset)))

        def col_fn(batch):
            """
            prepare batch
            --- for TEXT -> max_len * batch
            - cat stc in the doc
            - sort by doc_text len & store max_len
            - build doc_text order index
            --- for target -> catted 1D longTensor
            """
            b_s, b_a_seq, b_s_next, b_s_pos_seq, b_a_seq_aug = [], [], [], [], []
            b_a_seq_full, b_a_pos_seq = [], []
            for cb_s, cb_a_seq, cb_s_next, cb_s_pos_seq, cb_a_seq_next, cb_a_pos_seq in batch:
                # import ipdb;ipdb.set_trace()
                b_s.append(cb_s)
                cp_cb_a_seq = deepcopy(cb_a_seq.tolist())
                cp_cb_a_seq.sort()
                # cp_cb_a_seq.insert(0, 166)
                # import ipdb;ipdb.set_trace()
                cp_cb_a_seq.append(cfg.a_dim + 1)
                b_a_seq_aug.append(torch.Tensor(pad_to(cfg.max_len,
                                                       cp_cb_a_seq, True, pad_id=cfg.a_dim + 2)))

                cb_a_seq = cb_a_seq.tolist()
                random.shuffle(cb_a_seq)
                b_a_seq.append(torch.Tensor(pad_to(cfg.max_len, cb_a_seq, True)))

                cb_a_seq_next = cb_a_seq_next.tolist()
                random.shuffle(cb_a_seq_next)
                cb_a_seq.extend(cb_a_seq_next)
                b_a_seq_full.append(torch.Tensor(pad_to(cfg.max_len, cb_a_seq, True)))

                b_a_pos_seq.append(cb_a_pos_seq)

                b_s_next.append(cb_s_next)
                b_s_pos_seq.append(cb_s_pos_seq)
            return [torch.stack(b_s), torch.stack(b_a_seq), torch.stack(b_s_next),
                    torch.stack(b_s_pos_seq), torch.stack(b_a_seq_aug), torch.stack(b_a_seq_full), torch.stack(b_a_pos_seq)]

        if part == 'train':
            # dataloader = data.DataLoader(dataset, batchsz, shuffle=True)
            dataloader = data.DataLoader(dataset, batchsz, shuffle=True, collate_fn=col_fn, drop_last=True)
        else:
            dataloader = data.DataLoader(dataset, batchsz, shuffle=False, collate_fn=col_fn, drop_last=True)
        logging.info('MultiWOZ | DataLoader {} | Total Batches {} | Batch Size {} | time {:.3}'.format(part, len(dataloader), batchsz, time.time()-t0))
        logging.info('=' * 100)
        return dataloader
    #
    # def create_dataset_PEDP(self, part, batchsz, cfg, db, full_data=False):
    #     logging.info('=' * 100)
    #     load_file_dir = self.data_dir_new + '/' + part + '.pt'
    #     aug_file_dir = self.data_dir_new + '/' + part + '_aug_wozero_seq.pt'
    #     # aug_file_dir = self.data_dir_new + '/' + part + '_full.pt'
    #     if not os.path.exists(load_file_dir):
    #         logging.info('MultiWOZ | Create Dataset: {}'.format(load_file_dir))
    #         self.create_dataset(part, load_file_dir, cfg, db)
    #     t0 = time.time()
    #     # if not os.path.exists(aug_file_dir):
    #     s, a, next_s, _ = torch.load(load_file_dir)
    #     logging.info('Source Loaded: Total {} samples'.format(len(a)))
    #     n_s, n_next_s, a_seq, s_pos_seq = [], [], [], []
    #     a_next_seq = []
    #     dist_dict = dict()
    #     for s_, a_, next_s_ in zip(s, a, next_s):
    #         len_now = a_.nonzero().shape[0]
    #         if len_now not in dist_dict:
    #             dist_dict[len_now] = 1
    #         else:
    #             dist_dict[len_now] += 1
    #         if not full_data:
    #             if a_.nonzero().shape[0] == 0:
    #                 continue
    #         n_s.append(s_)
    #         n_next_s.append(next_s_)
    #         s_pos_seq.append(torch.Tensor([a_.nonzero().shape[0]]))
    #         a_seq.append(torch.Tensor(a_.nonzero().squeeze(1).tolist()))
    #     dataset = DatasetPEDP(n_s, a_seq, n_next_s, s_pos_seq)
    #         # torch.save(dataset, aug_file_dir)
    #     # else:
    #     #     dataset = torch.load(aug_file_dir)
    #
    #     def col_fn(batch):
    #         """
    #         prepare batch
    #         --- for TEXT -> max_len * batch
    #         - cat stc in the doc
    #         - sort by doc_text len & store max_len
    #         - build doc_text order index
    #         --- for target -> catted 1D longTensor
    #         """
    #         b_s, b_a_seq, b_s_next, b_s_pos_seq, b_a_seq_aug = [], [], [], [], []
    #         b_a_seq_sorted = []
    #         for cb_s, cb_a_seq, cb_s_next, cb_s_pos_seq in batch:
    #             b_s.append(cb_s)
    #             cp_cb_a_seq = deepcopy(cb_a_seq.tolist())
    #             cp_cb_a_seq.sort()
    #             # cp_cb_a_seq.insert(0, 166)
    #             cp_cb_a_seq.append(167)
    #             b_a_seq_aug.append(torch.Tensor(pad_to(cfg.max_len, cp_cb_a_seq, True, pad_id=168)))
    #
    #             cb_a_seq = cb_a_seq.tolist()
    #             random.shuffle(cb_a_seq)
    #
    #             b_a_seq.append(torch.Tensor(pad_to(cfg.max_len, cb_a_seq, True)))
    #
    #             b_s_next.append(cb_s_next)
    #             b_s_pos_seq.append(cb_s_pos_seq)
    #         return [torch.stack(b_s), torch.stack(b_a_seq), torch.stack(b_s_next), torch.stack(b_s_pos_seq), torch.stack(b_a_seq_aug)]
    #
    #     if part == 'train':
    #         # dataloader = data.DataLoader(dataset, batchsz, shuffle=True)
    #         dataloader = data.DataLoader(dataset, batchsz, shuffle=True, collate_fn=col_fn)
    #     else:
    #         dataloader = data.DataLoader(dataset, batchsz, shuffle=False, collate_fn=col_fn)
    #     logging.info('MultiWOZ | DataLoader {} | Total Batches {} | Batch Size {} | time {:.3}'.format(part, len(dataloader), batchsz, time.time()-t0))
    #     logging.info('=' * 100)
    #     return dataloader

    def create_dataset_gcas(self, part, batchsz, cfg, db):
        # logging.info('=' * 100)
        # load_file_dir = self.data_dir_new + '/' + part + '.pt'
        # if not os.path.exists(load_file_dir):
        #     logging.info('MultiWOZ | Create Dataset: {}'.format(load_file_dir))
        #     self.create_dataset(part, load_file_dir, cfg, db)
        t0 = time.time()
        # s, a, next_s, _ = torch.load(load_file_dir)
        s_load, a_load, _, _, _, _, _ = torch.load(self.data_dir[:-4] + '/sgd_data/processed/{}.pt'.format(part))
        logging.info('Source Loaded: Total {} samples'.format(len(a_load)))
        n_s, n_gcas_c, n_gcas_a, n_gcas_s, n_atgt = [], [], [], [], []
        for s_, a_ in tqdm(zip(s_load, a_load)):
            tmp = torch.zeros(cfg.a_dim)
            for a_idx in a_.long().tolist():
                tmp[a_idx] = 1
            a_ = tmp
            if a_.nonzero().shape[0] == 0:
                continue
            n_s.append(s_)
            n_atgt.append(a_)
            a_seq_lst = a_.nonzero().squeeze(1).tolist()
            a_ori_lst = [cfg.idx2da[x] for x in a_seq_lst]

            di_lst, sv_lst, continue_lst = [], [], []
            di2sv = dict()
            for ori_act in a_ori_lst:
                # di_ori = '-'.join(ori_act.split('-')[:2])
                # sv_ori = '-'.join(ori_act.split('-')[2:])
                di_ori = ori_act.split('-')[1]
                sv_ori = ori_act.split('-')[0] + '-' + '-'.join(ori_act.split('-')[2:])
                # import ipdb; ipdb.set_trace()
                di_idx = cfg.di2idx[di_ori]
                sv_idx = cfg.sv2idx[sv_ori]
                if di_idx not in di2sv:
                    di2sv[di_idx] = []
                di2sv[di_idx].append(sv_idx)

            for di_idx_, sv_idx_ in di2sv.items():
                di_lst.append(di_idx_)
                sv_lst.append(sv_idx_)
                continue_lst.append(1)
            continue_lst.append(0)

            n_gcas_c.append(continue_lst)
            n_gcas_a.append(di_lst)
            n_gcas_s.append(sv_lst)

        dataset = DatasetGCAS(n_s, n_gcas_c, n_gcas_a, n_gcas_s, n_atgt)

        def col_fn(batch):
            """
            prepare batch
            --- for TEXT -> max_len * batch
            - cat stc in the doc
            - sort by doc_text len & store max_len
            - build doc_text order index
            --- for target -> catted 1D longTensor
            """
            b_s, b_c, b_a, b_ss, b_a_tgt = [], [], [], [], []
            for cb_s, cb_c, cb_a, cb_ss, cb_a_tgt in batch:
                b_s.append(cb_s)
                b_a_tgt.append(cb_a_tgt)
                # 13, 14 intent
                b_a.append(torch.Tensor(pad_to(cfg.max_len, cb_a, True, pad_id=cfg.gcas_a_dim)))
                # 111, 112, 113 dsv
                # cb_ss.insert(0, 111)
                # b_ss.append(torch.Tensor(pad_to(cfg.max_len, cb_ss, True, pad_id=112)))
                slot_cb = []
                for slot_set in cb_ss:
                    slot_vec = torch.zeros(cfg.gcas_s_dim)
                    for slot_ in slot_set:
                        slot_vec[slot_] = 1
                    slot_cb.append(slot_vec)
                # -- pad and concat
                if len(slot_cb) < cfg.max_len:
                    slot_cb.extend([torch.zeros(cfg.gcas_s_dim)] * (cfg.max_len - len(slot_cb)))
                b_ss.append(torch.stack(slot_cb))

                # 2, 3, 4 continue
                b_c.append(torch.Tensor(pad_to(cfg.max_len, cb_c, True, pad_id=cfg.gcas_c_dim)))

            return [torch.stack(b_s), torch.stack(b_c), torch.stack(b_a), torch.stack(b_ss), torch.stack(b_a_tgt)]

        if part == 'train':
            # dataloader = data.DataLoader(dataset, batchsz, shuffle=True)
            dataloader = data.DataLoader(dataset, batchsz, shuffle=True, collate_fn=col_fn)
        else:
            dataloader = data.DataLoader(dataset, batchsz, shuffle=False, collate_fn=col_fn)
        logging.info('MultiWOZ | DataLoader {} | Total Batches {} | Batch Size {} | time {:.3}'.format(part, len(dataloader), batchsz, time.time()-t0))
        logging.info('=' * 100)
        return dataloader

    def create_dataset_seq_pro(self, part, batchsz, cfg, db, curriculum=0, fetch=False, just_build=False, advance=False, world=False):
        do_aug = False if cfg.aug_type == 'none' else True
        logging.info('=' * 100)
        self.loaded_data = torch.load(self.data_dir_new + '/' + part + '.pt')
        s, a, next_s, term = self.loaded_data
        import ipdb; ipdb.set_trace()
        if do_aug:
            seq_file_dir = self.data_dir_new + '/' + part + '_aug_c{}.pt'.format(cfg.aug_type, curriculum)
        else:
            seq_file_dir = self.data_dir_new + '/' + part + '_c{}.pt'.format(curriculum)
        if not os.path.exists(seq_file_dir) or just_build or advance:
            logging.info('MultiWOZ | Build: {}'.format(seq_file_dir))
            file_dir = self.data_dir_new + '/' + part + '_source.pt'
            if not os.path.exists(file_dir) or just_build:
                logging.info('MultiWOZ | Create Dataset: {}'.format(file_dir))
                self.create_dataset(part, file_dir, cfg, db)
                if just_build:
                    return
            start_time = time.time()
            if part == 'train':
                if self.loaded_data is None:
                    self.loaded_data = torch.load(file_dir)
                s, a, next_s, d, a_seq = self.loaded_data
            else:
                s, a, next_s, d, a_seq = torch.load(file_dir)
            ds_all = len(s)
            logging.info('MultiWOZ | Load Augmented Dataset: {} | Samples: {} | Time Cost: {:.1} s'.format(file_dir, ds_all, time.time()-start_time))
            if part != 'train':
                turn_id_list_val = []
                for i, turn_data in enumerate(self.data[part]):
                    if i % 2 == 1:
                        turn_id_list_val.append(int((turn_data['others']['turn'] - 1) / 2))
                new_a_seq = []
                for act_seq_padded in a_seq:
                    sel_index = torch.arange(1, (act_seq_padded == 2).nonzero(as_tuple=True)[0][-1].item())
                    act_seq = torch.index_select(act_seq_padded, -1, sel_index).tolist()
                    new_a_seq.append(act_seq)
                a_seq = new_a_seq
                new_a_seq = []
                new_s_seq = []
                s_pos_seq = []
                ori_s_seq = []
                for cur_s, cur_id, act_seq, next_s_sample, turn_id_val in zip(s, list(range(len(a_seq))), a_seq, next_s, turn_id_list_val):
                    if len(act_seq) == 0:
                        continue
                    ori_s_seq.append(cur_s)
                    default_seq = act_seq
                    # if cur_id != len(a_seq) - 1:
                    #     if turn_id_val < turn_id_list_val[cur_id + 1]:
                    #         default_seq.extend(a_seq[cur_id + 1])
                    # default_seq.append(170)
                    default_seq = pad_to(20, default_seq, True)
                    tmp_seq = deepcopy(act_seq)
                    tmp_seq = sorted(tmp_seq)
                    s_pos_seq.append(torch.Tensor([len(tmp_seq)]))
                    default_seq.extend(tmp_seq)
                    default_seq.append(2)
                    default_seq = pad_to(40, default_seq, True)
                    new_a_seq.append(torch.Tensor(default_seq))
                    new_s_seq.append(next_s_sample)
                s = ori_s_seq
                a_seq = new_a_seq
                if fetch:
                    return s, a_seq

                dataset = DatasetSeqPro(s, a_seq, new_s_seq, s_pos_seq)
                dataloader = data.DataLoader(dataset, batchsz, shuffle=True, drop_last=True)
                logging.info('MultiWOZ | DataLoader {} | Total Batches {} | Batch Size {} | Shuffle: {}'.format(part,
                                                                                len(dataloader),
                                                                                batchsz,
                                                                                True))
                logging.info('=' * 100)
                return dataloader

            # -- turn id list
            turn_id_list = []
            turn_cnt, session_cnt = 0, 0
            for i, turn_data in enumerate(self.data[part]):
                if i % 2 == 1:
                    cur_id = int((turn_data['others']['turn'] - 1)/2)
                    turn_cnt += 1
                    if cur_id == 0:
                        session_cnt += 1
                    turn_id_list += [cur_id] * len(turn_data['sys_action_aug'])
            logging.info('MultiWOZ | {} | total sessions: {} | total turns: {}'.format(part, session_cnt, turn_cnt))

            # -- un-pad
            new_a_seq = []
            for act_seq_padded in a_seq:
                sel_index = torch.arange(1, (act_seq_padded == 2).nonzero(as_tuple=True)[0][-1].item())
                act_seq = torch.index_select(act_seq_padded, -1, sel_index).tolist()
                new_a_seq.append(act_seq)
            a_seq = new_a_seq
            smp_num_counter = []
            sample_cnt = 0

            # -- store act seq of len curriculum (with curriculum <= session length) for every state
            # -- here 0 is 2 turns >
            for data_curriculum in range(1,2):
                max_act_len, len_dist_dict = 0, dict()
                s_data, a_seq_data, s_seq_data, s_pos_data = [], [], [], []
                last_turn_id = -1

                for start_idx, start_turn_id in tqdm(zip(list(range(len(s))), turn_id_list)):
                    # -- filter out repeating ones
                    if start_turn_id == last_turn_id:
                        smp_num_counter[-1] += 1
                    else:
                        smp_num_counter.append(1)
                    last_turn_id = start_turn_id
                if cfg.aug_type == 'mean':
                    aug_num = int(np.mean(smp_num_counter)) + 1
                elif cfg.aug_type == 'max':
                    aug_num = max(smp_num_counter) // 2
                elif cfg.aug_type == 'base':
                    # -- base
                    aug_num = 1
                else:
                    aug_num = 0

                for start_idx, s_sample, a_seq_sample, start_turn_id, next_s_sample \
                        in tqdm(zip(list(range(len(s))), s, a_seq, turn_id_list, next_s)):
                    # -- filter out repeating ones
                    if start_turn_id == last_turn_id and not do_aug:
                        continue
                    if world:
                        if len(a_seq_sample) != 1:
                            continue
                    last_turn_id = start_turn_id
                    # -- turn_id_list: [0, 0, 1, 2, 2, 2]
                    # -- target_idx_seqs: get next idx seqs like  [[2], [3, 4, 5]] for 0 and 1
                    target_idx_seqs = []
                    end_turn_id = start_turn_id + data_curriculum + 1
                    end_idx = start_idx
                    last_note_turn_id = start_turn_id
                    for next_turn_id in turn_id_list[start_idx + 1:]:
                        # -- current index
                        end_idx += 1
                        # -- break if step into further turns or next session
                        if next_turn_id > end_turn_id or next_turn_id < start_turn_id:
                            break
                        # -- skip repeat items in current turn
                        if next_turn_id == start_turn_id:
                            continue
                        # -- if first step into target turn
                        if next_turn_id > last_note_turn_id:
                            target_idx_seqs.append([end_idx])
                            last_note_turn_id = next_turn_id
                        else:
                            target_idx_seqs[-1].append(end_idx)
                    # -- skip if not enough for curriculum
                    # if len(target_idx_seqs) < data_curriculum:
                    #     continue

                    # -- add data
                    # -- get possible idx paths
                    num_paths = np.prod([len(x) for x in target_idx_seqs])
                    possible_idx_paths = []
                    while len(possible_idx_paths) < num_paths:
                        cur_path_sample = [choice(idx_lst) for idx_lst in target_idx_seqs]
                        if cur_path_sample not in possible_idx_paths:
                            possible_idx_paths.append(cur_path_sample)

                    # -- add data for each path
                    # import ipdb;
                    # ipdb.set_trace()
                    if not do_aug:
                        possible_idx_paths = [choice(possible_idx_paths)]

                    for idx_path_ in possible_idx_paths:
                        cur_a_seq = []
                        cur_a_seq.extend(a_seq_sample)
                        # for act_idx in idx_path_:
                        #     cur_a_seq.extend(a_seq[act_idx])

                        if len(a_seq_sample) * np.prod([len(a_seq[x]) for x in idx_path_]) == 0:
                            continue
                        # cur_a_seq.append(170)
                        cur_a_seq = pad_to(20, cur_a_seq, True)
                        tmp_seq = deepcopy(a_seq_sample)
                        tmp_seq = sorted(tmp_seq)
                        s_pos_data.append(torch.Tensor([len(tmp_seq)]))
                        cur_a_seq.extend(tmp_seq)
                        cur_a_seq.append(2)
                        cur_a_seq = pad_to(40, cur_a_seq, True)
                        s_data.append(s_sample)
                        s_seq_data.append(next_s_sample)
                        a_seq_data.append(torch.Tensor(cur_a_seq))
                        sample_cnt += 1

                    if aug_num > 1:
                        for i in range(aug_num - 1):
                            s_data.append(s_data[-1])
                            a_seq_data.append(a_seq_data[-1])
                            s_seq_data.append(s_seq_data[-1])
                            s_pos_data.append(s_pos_data[-1])
                            sample_cnt += 1

                logging.info('MultiWOZ | Curriculum: {} | Samples: {} | Aug Type: {} | mean_max_aug: {} vs {} | Act Length Distribution: {}'
                             .format(data_curriculum, len(s_data), cfg.aug_type, np.mean(smp_num_counter), max(smp_num_counter), [(k, len_dist_dict[k]) for k in sorted(len_dist_dict.keys())]))
                logging.info('MultiWOZ | Save as: {}'.format(seq_file_dir))
                torch.save((s_data, a_seq_data, s_seq_data, s_pos_data), seq_file_dir)
        s_seq, a_tot_seq, s_tot_seq, s_pos_tot_seq = torch.load(seq_file_dir)
        logging.info('MultiWOZ | Load: {} | Samples: {}'.format(seq_file_dir, len(s_seq)))
        if fetch:
            return s_seq, a_tot_seq, s_tot_seq
        dataset = DatasetSeqPro(s_seq, a_tot_seq, s_tot_seq, s_pos_tot_seq)
        dataloader = data.DataLoader(dataset, batchsz, shuffle=True, drop_last=True)
        logging.info('MultiWOZ | DataLoader {} | Total Batches {} | Batch Size {} | Shuffle: {}'.format(part, len(dataloader), batchsz, True))
        logging.info('=' * 100)
        return dataloader

class DatasetPol(data.Dataset):
    def __init__(self, s_s, a_s):
        self.s_s = s_s
        self.a_s = a_s
        self.num_total = len(s_s)

    def __getitem__(self, index):
        s = self.s_s[index]
        a = self.a_s[index]
        return s, a

    def __len__(self):
        return self.num_total

class DatasetGCAS(data.Dataset):
    def __init__(self, s, c, a, ss, n_atgt):
        self.s = s
        self.c = c
        self.a = a
        self.ss = ss
        self.n_atgt = n_atgt
        self.num_total = len(s)

    def __getitem__(self, index):
        s = self.s[index]
        c = self.c[index]
        a = self.a[index]
        ss = self.ss[index]
        n_atgt = self.n_atgt[index]
        return s, c, a, ss, n_atgt

    def __len__(self):
        return self.num_total

class DatasetPEDPnew(data.Dataset):
    def __init__(self, s, a_seq, s_next, s_pos_seq, a_seq_next, a_pos_seq):
        self.s = s
        self.a_seq = a_seq
        self.s_next = s_next
        self.s_pos_seq = s_pos_seq
        self.a_seq_next = a_seq_next
        self.a_pos_seq = a_pos_seq
        self.num_total = len(s)

    def __getitem__(self, index):
        s = self.s[index]
        a_seq = self.a_seq[index]
        s_next = self.s_next[index]
        s_pos_seq = self.s_pos_seq[index]
        a_pos_seq = self.a_pos_seq[index]
        a_seq_next = self.a_seq_next[index]
        return s, a_seq, s_next, s_pos_seq, a_seq_next, a_pos_seq

    def __len__(self):
        return self.num_total

class DatasetPEDP(data.Dataset):
    def __init__(self, s, a_seq, s_next, s_pos_seq):
        self.s = s
        self.a_seq = a_seq
        self.s_next = s_next
        self.s_pos_seq = s_pos_seq
        self.num_total = len(s)

    def __getitem__(self, index):
        s = self.s[index]
        a_seq = self.a_seq[index]
        s_next = self.s_next[index]
        s_pos_seq = self.s_pos_seq[index]
        return s, a_seq, s_next, s_pos_seq

    def __len__(self):
        return self.num_total

class DatasetPEDP_old(data.Dataset):
    def __init__(self, s, a_seq, s_next, term, label, s_pos_seq):
        self.s = s
        self.a_seq = a_seq
        self.s_next = s_next
        self.term = term
        self.label = label
        self.s_pos_seq = s_pos_seq
        self.num_total = len(s)

    def __getitem__(self, index):
        s = self.s[index]
        a_seq = self.a_seq[index]
        s_next = self.s_next[index]
        term = self.term[index]
        label = self.label[index]
        s_pos_seq = self.s_pos_seq[index]
        return s, a_seq, s_next, s_pos_seq, term, label

    def __len__(self):
        return self.num_total

class DatasetPlanner(data.Dataset):
    def __init__(self, s_s, a_s, cur):
        self.s_s = s_s
        self.a_s = a_s
        self.cur = cur
        self.num_total = len(s_s)

    def __getitem__(self, index):
        s = self.s_s[index]
        a = self.a_s[index]
        cur = self.cur[index]
        return s, a, cur

    def __len__(self):
        return self.num_total

class DatasetWorld(data.Dataset):
    def __init__(self, s_s, a_s, next_s_s):
        self.s_s = s_s
        self.a_s = a_s
        self.next_s_s = next_s_s
        #self.t = t
        self.num_total = len(s_s)

    def __getitem__(self, index):
        s = self.s_s[index]
        a = self.a_s[index]
        next_s = self.next_s_s[index]
       # t = self.t[index]
        return s, a, next_s

    def __len__(self):
        return self.num_total

class DatasetSeq(data.Dataset):
    def __init__(self, s_s, a_tot_seq, s_tot_seq):
        self.s_s = s_s
        self.a_tot_seq = a_tot_seq
        self.s_tot_seq = s_tot_seq
        self.num_total = len(s_s)

    def __getitem__(self, index):
        s = self.s_s[index]
        a_tot_seq = self.a_tot_seq[index]
        s_tot_seq = self.s_tot_seq[index]
        return s, a_tot_seq, s_tot_seq

    def __len__(self):
        return self.num_total

class DatasetSeqPro(data.Dataset):
    def __init__(self, s_s, a_tot_seq, s_tot_seq, s_pos):
        self.s_s = s_s
        self.a_tot_seq = a_tot_seq
        self.s_tot_seq = s_tot_seq
        self.s_pos = s_pos
        self.num_total = len(s_s)

    def __getitem__(self, index):
        s = self.s_s[index]
        a_tot_seq = self.a_tot_seq[index]
        s_tot_seq = self.s_tot_seq[index]
        s_pos = self.s_pos[index]
        return s, a_tot_seq, s_tot_seq, s_pos

    def __len__(self):
        return self.num_total


#  coding:utf:8
#  file:findtat2.py
# 创建tkinter窗口和菜单
# 初步添加菜单功能，但是菜单没有对应函数响应

import tkinter
import tkinter.messagebox

class Windows:
    def __init__(self):
        self.root = tkinter.Tk()

        # 创建菜单

        menu = tkinter.Menu(self.root)

        # 创建“系统”子菜单

        submenu = tkinter.Menu(menu, tearoff=0)
        submenu.add_command(label="关于...", command=self.MenuAbout)
        submenu.add_separator()
        submenu.add_command(label="退出", command=self.MenuExit)
        menu.add_cascade(label="系统", menu=submenu)

        #  创建“清理”子菜单

        submenu = tkinter.Menu(menu, tearoff=1)
        submenu.add_command(label="扫描垃圾文件", command=self.MenuScanRubbish)
        submenu.add_separator()
        submenu.add_command(label="删除垃圾文件", command=self.MenuDelRubbish)
        menu.add_cascade(label="清理", menu=submenu)

        # 创建“查找”子菜单

        submenu = tkinter.Menu(menu, tearoff=0)
        submenu.add_command(label="搜索大文件", command=self.MenuScanBigFile)
        submenu.add_separator()
        submenu.add_command(label="按名称搜索文件", command=self.MenuSearchFile)
        menu.add_cascade(label="搜索", menu=submenu)

        self.root.config(menu=menu)

        # 创建标签，用于显示状态信息

        self.progress = tkinter.Label(self.root, anchor=tkinter.W,
                                      text='状态', bitmap='hourglass', compound='left')
        self.progress.place(x=10, y=10, width=480, height=15)

        #  创建文本框，显示文件列表

        self.flist = tkinter.Text(self.root)
        self.flist.place(x=10, y=10, width=480, height=350)

        #  为文本添加垂直滚动条

        self.vscroll = tkinter.Scrollbar(self.flist)
        self.vscroll.pack(side='right', fill='y')
        self.flist['yscrollcommand'] = self.vscroll.set
        self.vscroll['command'] = self.flist.yview

    def Mainloop(self):
        self.root.title("Findfat")
        self.root.minsize(500, 400)
        self.root.maxsize(500, 400)
        self.root.mainloop()

    #  “关于”菜单

    def MenuAbout(self):
        tkinter.messagebox.showinfo("FindFas",
                                    "这是使用Python编写的Windows优化程序。\n")

    #  “退出”菜单

    def MenuExit(self):
        self.root.quit()

    #  “扫描垃圾文件”菜单

    def MenuScanRubbish(self):
        result = tkinter.messagebox.askquestion("FindFat", "扫描垃圾文件将需要较长时间，是否继续?")
        if result == 'no':
            return
        tkinter.messagebox.showinfo("FindFat", "马上开始扫描垃圾文件")

    #  “删除垃圾文件”菜单

    def MenuDelRubbish(self):
        result = tkinter.messagebox.askquestion("FindFat", "删除垃圾文件将需要较长时间，是否继续？")
        if result == 'no':
            return
        tkinter.messagebox.showinfo("FindFat", "马上开始删除垃圾文件")

    #  “扫描大文件”菜单

    def MenuScanBigFile(self):
        result = tkinter.messagebox.askquestion("FindFat", "扫描大文件将需要较长时间，是否继续？")
        if result == 'no':
            return
        tkinter.messagebox.showinfo("FindFat", "马上开始扫描大文件！")

    #  “按名称搜索文件”菜单

    def MenuSearchFile(self):
        result = tkinter.messagebox.askquestion("FindFat", "按名称搜索问及那需要较长的时间，是否继续？")
        if result == 'no':
            return
        tkinter.messagebox.showinfo("FindFat", "马上开始按名称搜索文件！")

if __name__ == '__main__':
    window = Windows()
    window.Mainloop()


from random import *
def is_valid_reround(reround):
    if reround in ("y","n","Y","N"):
        return True
    else:
        return False
def is_valid(integer):
    if integer.isdigit() == True and int(integer) in range(1,int(till)+1):
        return True
    else:
        return False
print("Добро пожаловать в числовую угадайку")
reround = "y"
while reround == "y":
    counter = 0
    print("В каком диапазоне будем угадывать число?:)")
    till = input()
    while till.isdigit() == False:
        print("Введите число")
        till = input()
    a = randint(1, int(till)+1)
    while True:
        print("Введите число от 1 до",int(till))
        b = input()
        if is_valid(b) == False:
            print('А может быть все-таки введем целое число от 1 до',int(till),"?")
            continue
        else:
            counter += 1
            b = int(b)
            if b < a:
                print('Ваше число меньше загаданного, попробуйте еще разок')
                continue
            elif b > a:
                print('Ваше число больше загаданного, попробуйте еще разок')
                continue
            else:
                print('Вы угадали, поздравляем!')
                print('Вам понадобилось',counter,'попыток!:)')
                print('Спасибо, что играли в числовую угадайку.')
                break
    print('Хотите еще раз сыграть? Если да - введите "y",нет - введите "n"')
    c = input()
    while True:
        if is_valid_reround(c) == False:
            print('Повторите ввод.Если да - введите "y",нет - введите "n"')
            c = input()
        else:
            if c in ("n","N"):
                reround = "n"
                print("Еще увидимся...:)")
                break
            else:
                reround = "y"
                break


# 搜索旋转数组。给定一个排序后的数组，包含n个整数，但这个数组已被旋转过很多次了，次数不详。请编写代码找出数组中的某个元素，假设数组元素原先是按升序排列的。若有多个相同元素，返回索引值最小的一个。

#这个题主要是说返回index最小的，所以当nums[mid]==target的时候，不能直接返回，要让right=mid,还有就是当nums[left]==target的时候直接返回left
class Solution:
    def search(self, arr: List[int], target: int) -> int:
        if not arr:
            return -1
        left=0
        right=len(arr)-1
        while left < right:
            # 当left符合时直接返回, 因为找的是最小的索引
            if arr[left]==target:
                return left
            mid=left+(right-left)//2
            #当中间值等于目标值，将右边界移到中间，因为左边可能还有相等的值
            if arr[mid]==target:
                right=mid
            elif arr[left]<arr[mid]:
                #这里不要自己画个数组mid就直接和nums[0]比了，nums[mid]要和nums[left]比
                if arr[left]<=target<=arr[mid]:
                    right=mid
                else:
                    left=mid+1
            elif arr[left]>arr[mid]:
                if arr[mid]<=target<=arr[right]:
                    left=mid
                else:
                    right=mid-1
            else:
                    left+=1
        return left if arr[left] == target else -1


import numpy as np
import os
import isce
import isceobj
import datetime
import logging
import argparse
from isceobj.Util.ImageUtil import ImageLib as IML
from isceobj.Util.decorators import use_api
import s1a_isce_utils as ut
import glob
def createParser():
    '''
    Create command line parser.
    '''

    parser = argparse.ArgumentParser( description='Generate offset field between two Sentinel swaths')
    parser.add_argument('-i', '--reference', type=str, dest='reference', required=True, help='Path to folder')
    parser.add_argument('-k', '--kml', type=str, dest='shapefile', default=None, help='Path to kml')
    parser.add_argument('-f', '--figure', type=str, dest='figure', default=None, help='Path to output PDF')

    return parser

def cmdLineParse(iargs = None):
    '''
    Command line parser.
    '''

    parser = createParser()
    inps = parser.parse_args(args=iargs)

    return inps

def main(iargs=None):
    '''
    Merge burst products to make it look like stripmap.
    Currently will merge interferogram, lat, lon, z and los.
    '''

    inps=cmdLineParse(iargs)
    from osgeo import gdal, ogr, osr
    import matplotlib
    if inps.shapefile is not None:
        matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    import matplotlib.patches as patches

    swathList = ut.getSwathList(inps.reference)

    swathColors = ['r', 'g', 'b']
    shapeColors = ['FF0000','00FF00','0000FF']

    fig = plt.figure('Burst map')
    ax = fig.add_subplot(111,aspect='equal')

    tmin = None
    rmin = None

    xmin = 1e10
    ymin = 1e10
    xmax = -1e10
    ymax = -1e10

    if inps.shapefile is not None:
        ds = ogr.GetDriverByName('KML').CreateDataSource(inps.shapefile)
        srs = osr.SpatialReference()
        srs.SetWellKnownGeogCS('WGS84')
        layer = ds.CreateLayer('bursts', srs=srs)
        field_name = ogr.FieldDefn("Name", ogr.OFTString)
        field_name.SetWidth(16)
        layer.CreateField(field_name)
        field_name = ogr.FieldDefn("OGR_STYLE", ogr.OFTString)
        layer.CreateField(field_name)

    for swath in swathList:
        ifg = ut.loadProduct(os.path.join(inps.reference , 'IW{0}.xml'.format(swath)))
        minBurst = ifg.bursts[0].burstNumber
        maxBurst = ifg.bursts[-1].burstNumber

        if tmin is None:
            tmin = ifg.bursts[0].sensingStart
            dtime = ifg.bursts[0].azimuthTimeInterval
            rmin = ifg.bursts[0].startingRange
            drange = ifg.bursts[0].rangePixelSize

        if minBurst==maxBurst:
            print('Skipping processing of swath {0}'.format(swath))
            continue

        if not inps.shapefile:
            for ii in range(minBurst, maxBurst + 1):
                burst = ifg.bursts[ii-minBurst]
                x0 = np.round( (burst.startingRange - rmin)/drange)
                y0 = np.round( (burst.sensingStart - tmin).total_seconds()/ dtime)
                if ii %2 == 0:
                    style = 'solid'
                else:
                    style = 'dashdot'

                ax.add_patch( patches.Rectangle(
                    (x0,y0),
                    burst.numValidSamples,
                    burst.numValidLines,
                    edgecolor=swathColors[swath-1],
                    facecolor=swathColors[swath-1],
                    alpha=0.2,
                linestyle=style))

                xmin = min(xmin, x0)
                xmax = max(xmax, x0 + burst.numValidSamples)
                ymin = min(ymin, y0)
                ymax = max(ymax, y0 + burst.numValidLines)
        else:
            for ii in range(minBurst, maxBurst+1):
                burst = ifg.bursts[ii-minBurst]
                t0 = burst.sensingStart + datetime.timedelta(seconds = burst.firstValidLine * burst.azimuthTimeInterval)
                t1 = t0 + datetime.timedelta(seconds = burst.numValidLines * burst.azimuthTimeInterval)
                r0 = burst.startingRange + burst.firstValidSample * burst.rangePixelSize
                r1 = r0 + burst.numValidSamples * burst.rangePixelSize

                earlyNear = burst.orbit.rdr2geo(t0,r0)
                earlyFar = burst.orbit.rdr2geo(t0,r1)
                lateFar = burst.orbit.rdr2geo(t1,r1)
                lateNear = burst.orbit.rdr2geo(t1,r0)

                ring = ogr.Geometry(ogr.wkbLinearRing)
                from distutils.version import StrictVersion
                if StrictVersion(gdal.__version__) >= StrictVersion("3.0"):
                    ring.AddPoint(earlyNear[0], earlyNear[1])
                    ring.AddPoint(earlyFar[0], earlyFar[1])
                    ring.AddPoint(lateFar[0], lateFar[1])
                    ring.AddPoint(lateNear[0], lateNear[1])
                    ring.AddPoint(earlyNear[0], earlyNear[1])
                else:
                    ring.AddPoint(earlyNear[1], earlyNear[0])
                    ring.AddPoint(earlyFar[1], earlyFar[0])
                    ring.AddPoint(lateFar[1], lateFar[0])
                    ring.AddPoint(lateNear[1], lateNear[0])
                    ring.AddPoint(earlyNear[1], earlyNear[0])

                feature = ogr.Feature(layer.GetLayerDefn())
                feature.SetField('Name', 'IW{0}-{1}'.format(swath, ii))
                feature.SetField('OGR_STYLE', "PEN(c:#{0},w:8px)".format(shapeColors[swath-1]))
                feature.SetGeometry(ring)
                layer.CreateFeature(feature)
                feature = None

    if not inps.shapefile:
        plt.ylim([ymin, ymax])
        plt.xlim([xmin, xmax])

        if inps.figure is not None:
            plt.savefig(inps.figure, format='pdf')
        else:
            plt.show()

    else:
        ds = None
        print('Wrote KML file: ', inps.shapefile)

if __name__ == '__main__' :
    '''
    Merge products burst-by-burst.
    '''

    main()


## MC European Call

S0 = 100; r = 0.05; sigma = 0.2; T = 1; M = 10000; K = 90

def eu_call_payoff(x):
    return max(x - K, 0)

def Eu_Option_BS_MC (S0, r, sigma, T, K, M, f):

    S = np.zeros((M+1))
    V = np.zeros((M+1))
    X = np.random.normal(0, 1, M+1)

    for i in range(0, M+1):
        S[i] = S0 * math.exp((r-sigma**2/2)*T + sigma*math.sqrt(T)*X[i])
        V[i] = eu_call_payoff(S[i])

    V0 = math.exp(-r*T) * np.mean(V)

    epsilon = 1.96 * math.sqrt(np.var(V) / M)
    c1 = V[0] - epsilon
    c2 = V[0] + epsilon

    return V0, c1, c2

V = Eu_Option_BS_MC(S0, r, sigma, T, K, M, eu_call_payoff)

def bs_formula(S0, T, K, r, sigma):

    d1 = (math.log(S0/K) + r*T + 0.5*sigma**2*T)/sigma*T
    d2 = (math.log(S0/K) + r*T - 0.5*sigma**2*T)/sigma*T
    V = S0 * scipy.stats.norm.cdf(d1) - K*math.exp(-r*T)*scipy.stats.norm.cdf(d2)

    return V

V_bs = bs_formula(S0, T, K, r, sigma)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.clients_index, name='clients_home'),
    path('settings', views.clients_set, name='clients_set'),
    path('add', views.client_add, name='client_add'),
    # path('<client_operations>', views.get_client_operation),
    # path('<client_name>', views.get_new_client, name='get_a_new_client'),
]


""" Handles collection of trial and testing data.

Has a caching layer as to speed up future runs
"""
import json
import shelve
import os
import hashlib
import collections

import yaml
import pymatgen as pmg
from pymatgen.io.cif import CifParser
from pymatgen.io.vasp import Poscar

from .schema import TrainingSchema
from .io.mattoolkit import MTKReader
from .io.siesta import SiestaReader
from . import utils

class Training:
    def __init__(self, schema, cache_filename=None):
        schema_load, errors = TrainingSchema().load(schema)
        self.schema = schema_load
        self._gather_calculations(cache_filename=cache_filename)
        self._gather_material_properties()

    def _gather_calculations(self, cache_filename=None):
        self._calculations = []
        for calculation in self.schema['spec']:
            if calculation['type'] == 'mattoolkit':
                self._calculations.extend(self.download_mattoolkit_calculations(calculation['selector'], cache_filename=cache_filename))
            elif calculation['type'] == 'Siesta':
                self._calculations.extend(SiestaReader.from_selector(calculation['selector']))

    def _gather_material_properties(self):
        self._material_properties_reference_ground_state = None
        self._material_properties = collections.defaultdict(list)
        for material_property in self.schema['spec']:
            if material_property['type'] == 'ground_state':
                if self._material_properties_reference_ground_state:
                    raise ValueError('current cannot have more than one reference ground state')
                self._material_properties_reference_ground_state = self._parse_structure(material_property)
            elif material_property['type'] == 'lattice_constants':
                self._material_properties['lattice_constants'].append(material_property['data'])
            elif material_property['type'] == 'elastic_constants':
                self._material_properties['elastic_constants'].append(material_property['data'])
            elif material_property['type'] == 'bulk_modulus':
                self._material_properties['bulk_modulus'].append(material_property['data'])
            elif material_property['type'] == 'shear_modulus':
                self._material_properties['shear_modulus'].append(material_property['data'])

        if self._material_properties and self._material_properties_reference_ground_state is None:
            raise ValueError('calculating material properties requires a reference ground state structure')

    def _parse_structure(self, structure_schema):
        data = structure_schema['data']
        format = structure_schema['format']
        if format == 'cif':
            # cif lattice can be weird non standard shape
            structure = (CifParser.from_string(data)).get_structures()[0]
            lattice = pmg.Lattice.from_parameters(*structure.lattice.abc, *structure.lattice.angles)
            structure = pmg.Structure(lattice, structure.species, structure.frac_coords, coords_are_cartesian=False)
        elif format == 'POSCAR':
            structure = (Poscar.from_string(data)).structure
        return structure

    @property
    def calculations(self):
        return self._calculations

    @property
    def material_properties(self):
        return self._material_properties

    @property
    def reference_ground_state(self):
        return self._material_properties_reference_ground_state

    def __iter__(self):
        return iter(self._calculations)

    def __len__(self):
        return len(self._calculations)

    def download_mattoolkit_calculations(self, selector, cache_filename=None):
        if cache_filename:
            cache_directory, filename = os.path.split(cache_filename)
            os.makedirs(cache_directory, exist_ok=True)
            key = f'mattoolkit.calculation.' + '.'.join(selector['labels'])
            with shelve.open(cache_filename) as cache:
                if key in cache:
                    calc_ids = cache[key]
                else:
                    from mattoolkit.api import CalculationResourceList

                    calculations = CalculationResourceList()
                    calculations.get(params={'labels': selector['labels']})
                    calc_ids = [c.id for c in calculations.items]
                    cache[key] = calc_ids
        else:
            from mattoolkit.api import CalculationResourceList

            calculations = CalculationResourceList()
            calculations.get(params={'labels': selector['labels']})
            calc_ids = [c.id for c in calculations.items]
        return [MTKReader(calc_id, cache_filename=cache_filename) for calc_id in calc_ids]

    @classmethod
    def from_file(cls, filename, format=None, **kwargs):
        if format not in {'json', 'yaml'}:
            if filename.endswith('json'):
                format = 'json'
            elif filename.endswith('yaml') or filename.endswith('yml'):
                format = 'yaml'
            else:
                raise ValueError('unrecognized filetype from filename %s' % filename)

        if format == 'json':
            with open(filename) as f:
                return cls(json.load(f))
        elif format in {'yaml', 'yml'}:
            with open(filename) as f:
                return cls(yaml.safe_load(f), **kwargs)

    def __str__(self):
        return json.dumps(self.schema, sort_keys=True, cls=utils.NumpyEncoder)

    @property
    def md5hash(self):
        return hashlib.md5(str(self).encode('utf-8')).hexdigest()


import asyncio
import json
import random
import re
import shutil
import string
import tempfile
from enum import Enum
from pathlib import Path
from typing import Dict, List, Optional

import pandas
import zstandard as zstd
from pydantic import BaseModel

class WarpError(Exception):
    pass

class WarpNotFoundError(WarpError):
    pass

class WarpOperationResult(BaseModel):
    name: str
    mbps: float
    objps: float

class WarpResult(BaseModel):
    put: WarpOperationResult
    get: WarpOperationResult
    delete: WarpOperationResult
    stat: WarpOperationResult

class WarpBenchmark:
    objsize: str
    numobjs: int
    duration: str

    class State(Enum):
        NONE = 0
        PREPARING = 1
        RUNNING = 2
        DONE = 3

    state: State
    progress: float

    def __init__(self, objsize: str, numobjs: int, duration: str) -> None:

        found = shutil.which("warp") is not None
        if not found:
            raise WarpNotFoundError()

        self.objsize = objsize
        self.numobjs = numobjs
        self.duration = duration
        self.state = WarpBenchmark.State.NONE
        self.progress = 0.0

    async def run(self, host: str, access_key: str, secret_key: str) -> str:

        bucketname = "".join(
            random.choice(string.ascii_lowercase) for _ in range(16)
        )
        print(f"run warp on bucket {bucketname}")
        tmp_benchdata_dir = Path(tempfile.mkdtemp())
        tmp_benchdata_file = tmp_benchdata_dir.joinpath("warp-result")
        cmd: List[str] = [
            "warp",
            "mixed",
            "--quiet",
            "--json",
            "--no-color",
            "--benchdata",
            tmp_benchdata_file.as_posix(),
            "--host",
            host,
            "--access-key",
            access_key,
            "--secret-key",
            secret_key,
            "--obj.size",
            self.objsize,
            "--objects",
            str(self.numobjs),
            "--duration",
            self.duration,
            "--noclear",
            "--bucket",
            bucketname,
        ]
        print(f"run warp cmd: {cmd}")
        proc = await asyncio.subprocess.create_subprocess_exec(
            *cmd,
            stdout=asyncio.subprocess.DEVNULL,
            stderr=asyncio.subprocess.DEVNULL,
        )
        retcode = await proc.wait()
        if retcode != 0:
            raise WarpError()

        result = self.parse_csv(tmp_benchdata_file)
        for dirent in tmp_benchdata_dir.iterdir():
            dirent.unlink()
        tmp_benchdata_dir.rmdir()
        return result

        assert proc.stdout
        assert proc.stderr
        res = await asyncio.gather(self._process_output(proc.stdout))
        stdout: List[str] = res[0]

        retcode = await proc.wait()
        if retcode != 0:
            assert proc.stderr is not None
            print((await proc.stderr.read()).decode("utf-8"))
            raise WarpError()

        json_raw_lst: List[str] = []
        in_json = False
        for l in stdout:
            if not in_json:
                if l.strip() == "{":
                    in_json = True
                else:
                    continue
            json_raw_lst.append(l)

        json_raw = "".join(json_raw_lst)
        json_dict = json.loads(json_raw)
        with Path("./warp-result.json").open("w+") as fd:
            json.dump(json_dict, fd)

        print(json_dict)
        raise WarpError()

        print("finished warp")
        res = self._parse(stdout)
        assert "DELETE" in res
        assert "GET" in res
        assert "PUT" in res
        assert "STAT" in res
        return WarpResult(
            put=res["PUT"],
            get=res["GET"],
            delete=res["DELETE"],
            stat=res["STAT"],
        )

    def parse_csv(self, datafile: Path) -> str:
        zstd_file = datafile.with_suffix(".csv.zst")
        csv_file = datafile.with_suffix(".csv")
        assert zstd_file.exists()
        with zstd_file.open("rb") as source:
            decomp = zstd.ZstdDecompressor()
            with csv_file.open("wb") as dest:
                decomp.copy_stream(source, dest)

        assert csv_file.exists()
        data = pandas.read_csv(csv_file)  # type: ignore
        data["duration_ms"] = data.apply(  # type: ignore
            lambda r: r["duration_ns"] * 1e-6, axis=1  # type: ignore
        )
        return data.to_json()  # type: ignore

    async def _process_output(self, reader: asyncio.StreamReader) -> List[str]:
        def _handle_progress(line: str) -> bool:
            res = re.match("Preparing:.*\\s([\\d+.]+)%.*", line)
            if res is not None:
                assert len(res.groups()) == 1
                self.state = WarpBenchmark.State.PREPARING
                self.progress = float(res.group(1))
                if self.progress % 10 == 0:
                    print(f"preparing: {self.progress}%")
                return True
            res = re.match("Benchmarking:.*\\s([\\d+.]+)%.*", line)
            if res is not None:
                assert len(res.groups()) == 1
                self.state = WarpBenchmark.State.RUNNING
                self.progress = float(res.group(1))
                if self.progress % 10 == 0:
                    print(f"benchmarking: {self.progress}%")
                return True
            return False

        def _is_warp_output(line: str) -> bool:
            res = re.match("warp:.*", line)
            if res is not None:
                return True
            return False

        lines: List[str] = []
        cur_line: bytearray = bytearray()
        while not reader.at_eof():
            b = await reader.read(1)
            if len(b) == 0:
                # at eof
                break
            if b == b"\r" or b == b"\n":
                # end of line
                line = cur_line.decode("utf-8")
                cur_line = bytearray()
                if len(line.strip()) == 0:
                    continue
                if _is_warp_output(line):
                    continue
                if _handle_progress(line):
                    continue
                lines.append(line)
                print(line)
            else:
                cur_line.extend(b)
        return lines

    def _parse(self, lines: List[str]) -> Dict[str, WarpOperationResult]:
        def _parse_objps(v: str) -> float:
            m = re.match("([\\d.]+) obj/s", v)
            assert m is not None
            assert len(m.groups()) == 1
            return float(m.group(1))

        def _parse_mbps(v: str) -> float:
            m = re.match("([\\d.]+) ([\\w]+)/s", v)
            assert m is not None
            assert len(m.groups()) == 2
            value = float(m.group(1))
            units = m.group(2)
            if units == "MiB":
                return value
            elif units == "KiB":
                return value / 1024
            elif units == "B":
                return value / 1024**2
            elif units == "GiB":
                return value * 1024
            raise WarpError()

        in_operation = False
        operations: Dict[str, WarpOperationResult] = {}
        cur_operation: Optional[str] = None
        for line in lines:
            if line.startswith("Operation:"):
                res = re.match("Operation:[\\s]+([\\w]+),.*", line)
                assert res is not None
                assert len(res.groups()) == 1
                in_operation = True
                cur_operation = res.group(1)
                continue
            elif in_operation:
                objps = 0.0
                mbps = 0.0

                res = re.match(".*Throughput: (.*)", line)
                assert res is not None
                parts = res.group(1).split(",")
                assert len(parts) > 0
                if len(parts) == 1:
                    # only got obj/s
                    objps = _parse_objps(parts[0].strip())
                else:
                    mbps = _parse_mbps(parts[0].strip())
                    objps = _parse_objps(parts[1].strip())
                in_operation = False
                assert cur_operation is not None
                operations[cur_operation] = WarpOperationResult(
                    name=cur_operation, mbps=mbps, objps=objps
                )
                continue

        return operations


import falcon
import re
import requests
import json
import datetime
import os
import jinja2
import shutil
from configparser import ConfigParser
from time import sleep
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session
from sqlalchemy.orm import sessionmaker

from scrapy.utils.project import get_project_settings
from sparepart.models import ScrapingJob
from crawler_queue import CrawlerQueue

SCRAPY_SETTINGS = get_project_settings()

CRAWLER_DELAY = {
    'isuzu': os.environ.get('DELAY_ISUZU', 1),
    'suzuki': os.environ.get('DELAY_SUZUKI', 1),
    'daihatsu': os.environ.get('DELAY_DAIHATSU', 1),
    'megazip': os.environ.get('DELAY_MEGAZIP', 15),
    'parts.com': os.environ.get('DELAY_PARTS', 15),
}

scrapyd_config = ConfigParser()
scrapyd_config.read('scrapyd.conf')

SCRAPYD_HOST = 'http://{}:{}'.format('localhost',
                                     scrapyd_config['scrapyd']['http_port'])

# create database session
CONNECTION_STRING = os.environ.get('CONNECTION_STRING')
if CONNECTION_STRING:
    engine = create_engine(CONNECTION_STRING, echo=True)
    session_factory = sessionmaker(bind=engine)
    Session = scoped_session(session_factory)

import logging
logger = logging.getLogger(__name__)

class SQLAlchemySessionManager:
    """
    Create a scoped session for every request and close it when the request
    ends.
    """

    def __init__(self, Session):
        self.Session = Session

    def process_resource(self, req, resp, resource, params):
        resource.session = self.Session()

    def process_response(self, req, resp, resource, req_succeeded):
        if hasattr(resource, 'session'):
            if not req_succeeded:
                resource.session.rollback()
            Session.remove()

class AuthMiddleware(object):

    def process_request(self, req, resp):
        auth_header = req.get_header('Authorization')
        if auth_header:
            req.context['authenticated'] = auth_header.split(' ')[-1] == 'dXNlcm5hbWU6cGFzc3dvcmQ='

def validate_auth(req, resp, resource, params):
    description = 'Please provide valid authentication header.'
    challenges = ['Authorization: Basic ...']
    if 'authenticated' not in req.context or not req.context['authenticated']:
        raise falcon.HTTPUnauthorized('Authentication required',
                                      description,
                                      challenges,
                                      href='http://docs.example.com/auth')

def save_job(session, job):
    """
    save new job to database
    """
    try:
        session.add(job)
        session.commit()

    except Exception as e:
        logger.error("{} - {}".format(type(e), str(e)))
        session.rollback()
        raise

    finally:
        session.close()

def get_job(session, job_id):
    """
    get job from database
    """
    try:
        return session.query(ScrapingJob.jobdir).filter_by(id=job_id).first()[0]
    except Exception as e:
        logger.error("{} - {}".format(type(e), str(e)))
        raise

def is_job_slot_available(spider, job_list=None):
    """
    check if job slot is available, max 1 job for each spider
    """
    if not job_list:
        response = requests.get('{}/listjobs.json?project=default'.format(SCRAPYD_HOST))
        data = response.json()
        job_list = data['pending'] + data['running']

    for job in job_list:
        for k, v in job.items():
            if k == 'spider' and v == spider:
                break
        else:
            continue

        # job slot for spider not available
        # response message here
        print('job slot not available')
        return False

    else:  # job slot available
        print('job slot available')
        return True

def schedule_job(spider, params, session):
    """
    schedule crawling job
    """
    scheduler_url = '{}/schedule.json'.format(SCRAPYD_HOST)
    if isinstance(params, bytes):
        params = params.decode()

    if not isinstance(params, dict):
        params = json.loads(params)

    options = params.copy()
    options.update({'spider': spider, 'project': 'default'})
    response = requests.post(scheduler_url, options)
    data = response.json()
    if 'status' in data:
        status = data['status']
    if status == 'ok':
        success = True
        msg = 'Scheduled crawling for spider {}. JobID ({})'.format(spider, data['jobid'])
        index = None
        for k, v in enumerate(params['setting']):
            if 'JOBDIR' in v:
                index = k
                break
        jobdir = params['setting'][index].split('=')[-1]

        job = ScrapingJob(**{'id': data['jobid'], 'spider': spider, 'start': datetime.datetime.now(),
                             'status': 'started', 'input_param': json.dumps(params), 'jobdir': jobdir})
        if session:
            save_job(session, job)
    else:
        status = data['status']
        msg = data['message']

    return success, status, msg

def get_valid_filename(s):
    str_ = str(s).strip().replace(' ', '_')
    return re.sub(r'(?u)[^-\w.]', '', str_)

class Crawler(object):
    job_directory = 'data/crawljobs'
    date_format = '%Y_%m_%d_%H_%M_%S'
    max_jobdir_length = int(os.environ.get('MAX_JOBDIR_LENGTH', 10))

    def __init__(self):
        self.queue = CrawlerQueue()

    @falcon.before(validate_auth)
    def on_get(self, req, resp):

        params = req.params

        spider = params.get('spider', None)
        merk = params.get('merk', None)

        if not spider or not merk:
            resp.body = json.dumps({'error': 'parameter `spider` and `merk` must not be empty'}, indent=4)
            resp.status = falcon.HTTP_400
            return

        if merk.lower() == 'suzuki' and isinstance(params.get('model', None), list):
            params['model'] = " ".join([a for a in params.get('model', [])])

        delay = params.get('delay', None)
        model = params.get('model', None)
        if not spider:
            raise falcon.HTTPBadRequest('Missing Spider', 'Missing spider in query string parameters.')
        if not merk:
            raise falcon.HTTPBadRequest('Missing Merk', 'Missing merk in query string parameters.')
        if not model:
            raise falcon.HTTPBadRequest('Missing Model', 'Missing model in query string parameters.')

        # cleanse params
        param_list = ['spider', 'merk', 'model', 'tahun', 'year', 'mesin', 'engine', 'varian', 'type', 'tipe', 'delay']
        param_list += ['region']
        for k in list(params.keys()):
            if k not in param_list:
                del params[k]

        # add delay parameter
        if not delay:
            delay = CRAWLER_DELAY[spider]
        params['setting'] = ['DOWNLOAD_DELAY={}'.format(delay)]
        job_dir = '{}/{}_{}_{}_{}'.format(self.job_directory, spider, merk, get_valid_filename(model),
                                          datetime.datetime.now().strftime(self.date_format))

        params['jobdir'] = job_dir
        params['setting'] += ['JOBDIR={}'.format(job_dir)]

        def sort_by_datetime(datestring):
            splitup = datestring.split('_')[-6:]
            dttm = '_'.join(splitup)
            return datetime.datetime.strptime(dttm, self.date_format)

        # housekeeping check for CRAWLING JOBDIR
        spiderjoblist = []
        for itm in os.listdir(self.job_directory):
            if itm.split('_')[0] == spider:
                spiderjoblist.append(itm)
        if len(spiderjoblist) > self.max_jobdir_length:
            spiderjoblist.sort(key=sort_by_datetime, reverse=True)
            while len(spiderjoblist) > self.max_jobdir_length:  # max jobdir length as settings
                shutil.rmtree('{}/{}'.format(self.job_directory, spiderjoblist.pop()))

        # add job for this spider to queue db
        self.queue.push(spider, params)

        # check if there is available job slot for current spider
        response = requests.get('{}/listjobs.json?project=default'.format(SCRAPYD_HOST))
        data = response.json()
        pending_running_jobs = data['pending'] + data['running']
        if not is_job_slot_available(spider, pending_running_jobs):  # queue crawling job
            success = True
            status = 'ok'
            msg = 'Crawling job has been added to queue list'

        else:  # schedule crawling job

            try:
                params, date_added = self.queue.pop(spider)  # get next job params
                session = self.session if hasattr(self, 'session') else None
                success, status, msg = schedule_job(spider, params, session)

            except Exception as e:
                success = False
                status = 'not ok'
                msg = str('{} - {}'.format(type(e), str(e)))

        resp.status = falcon.HTTP_200
        resp.body = json.dumps({'success': success, 'status': status, 'message': msg}, indent=4)

class CrawlJob(object):

    def on_get(self, req, resp):

        # limit scheduled crawler process
        response = requests.get('{}/jobs'.format(SCRAPYD_HOST))
        resp.status = falcon.HTTP_200
        resp.content_type = 'text/html'
        text = response.text
        text = re.sub('<title>Scrapyd</title>', '<title>Crawling Jobs</title>', text)
        text = re.sub('
Jobs
', '
Crawling Jobs
', text)
        text = re.sub('
.*Go back.*
', '', text)
        text = text.replace("
Pending
", '')
        resp.body = text

class Log(object):

    def on_get(self, req, resp, spider, log_file):
        response = requests.get('{}/logs/default/{}/{}'.format(SCRAPYD_HOST, spider, log_file))
        resp.status = falcon.HTTP_200
        text = response.text
        resp.body = text

def load_template(name):
    path = os.path.join('templates', name)
    with open(os.path.abspath(path), 'r') as fp:
        return jinja2.Template(fp.read())

class QueueList(object):
    """Crawl Queue Resource"""
    def __init__(self):
        self.queue = CrawlerQueue()

    def on_get(self, req, resp):
        # render template
        template = load_template('crawler_queue.html')

        resp.status = falcon.HTTP_200
        resp.content_type = 'text/html'
        resp.body = template.render(spiders=self.queue.get_all_queue())

class CancelQueue(object):
    """Cancel Crawling Queue"""
    def __init__(self):
        self.queue = CrawlerQueue()

    def on_get(self, req, resp, spider=None, queue_id=None):
        success, status, msg = False, 'ok', '{} queue with id:{} '.format(spider, queue_id)
        try:
            # remove queue from db
            if self.queue.delete(spider, queue_id):
                success = True
                msg += 'deleted'
            else:
                msg += 'not found'

        except Exception as e:
            success = False
            status = 'exception occured.'
            msg = str('{} - {}'.format(type(e), str(e)))
            print('{} - {}'.format(type(e), str(e)))

        resp.status = falcon.HTTP_200
        resp.body = json.dumps({'success': success, 'status': status, 'message': msg}, indent=4)

class CancelJob(object):
    """Cancel Crawling Job"""
    def on_get(self, req, resp, job_id=None):
        success, status, msg = False, 'ok', 'job with id:{} '.format(job_id)
        try:
            # stop running crawling
            cancel_url = '{}/cancel.json'.format(SCRAPYD_HOST)
            options = {'project': 'default', 'job': job_id}
            response = requests.post(cancel_url, options)
            data = response.json()
            if data['status'] == 'ok' and data['prevstate']:
                success = True
                msg += ' is being canceled. please wait a moment.'
            else:
                msg = 'cancelling job failed.'

        except Exception as e:
            success = False
            status = 'exception occured.'
            msg = str('{} - {}'.format(type(e), str(e)))
            print('{} - {}'.format(type(e), str(e)))

        resp.status = falcon.HTTP_200
        resp.body = json.dumps({'success': success, 'status': status, 'message': msg}, indent=4)

class CrawlerQueueCheck(object):
    def __init__(self):
        self.queue = CrawlerQueue()

    def on_get(self, req, resp, spider=None, job_id=None):

        delay = 5
        sleep(delay)  # add delay
        print('checking queue for {}...'.format(spider))
        success, status, msg = False, 'ok', 'crawler queue checked'
        try:
            if is_job_slot_available(spider):  # schedule new crawling job
                params, date_added = self.queue.pop(spider)  # get next job params
                session = self.session if hasattr(self, 'session') else None
                success, status, msg = schedule_job(spider, params, session)

        except Exception as e:
            success = False
            status = 'exception occured.'
            msg = str('{} - {}'.format(type(e), str(e)))

        resp.status = falcon.HTTP_200
        resp.body = json.dumps({'success': success, 'status': status, 'message': msg}, indent=4)

class TestResource(object):
    def on_get(self, req, resp):
        """Handles GET requests"""
        resp.status = falcon.HTTP_200  # This is the default status
        resp.body = ('\nTwo things awe me most, the starry sky '
                     'above me and the moral law within me.\n'
                     '\n'
                     '    ~ Immanuel Kant\n\n')

# falcon.API instances are callable WSGI apps
middleware_to_use = [AuthMiddleware()]
if CONNECTION_STRING:
    middleware_to_use.append(SQLAlchemySessionManager(Session))

app = falcon.API(middleware=middleware_to_use)

# Resources are represented by long-lived class instances
crawler = Crawler()
crawljob = CrawlJob()
log = Log()
crawler_queue_check = CrawlerQueueCheck()
queue_list = QueueList()
cancel_queue = CancelQueue()
cancel_job = CancelJob()

# will handle all requests to the '/crawl' URL path
app.add_route('/crawl', crawler)

# will handle all requests to the '/jobs' URL path
app.add_route('/jobs', crawljob)

# will handle all requests to the '/log' URL path
app.add_route('/logs/default/{spider}/{log_file}', log)

# check and update crawling queue
app.add_route('/crawler_queue_check/{spider}/{job_id}', crawler_queue_check)
app.add_route('/queue_list', queue_list)

# cancel crawl job
app.add_route('/cancel_job/{job_id}', cancel_job)

# cancel queue
app.add_route('/cancel_queue/{spider}/{queue_id}', cancel_queue)

# Resources are represented by long-lived class instances
test = TestResource()

# things will handle all requests to the '/test' URL path
app.add_route('/test', test)


import asyncio
from pathlib import Path

from nurses_2.app import App
from nurses_2.io import MouseButton
from nurses_2.widgets.widget_data_structures import Anchor
from nurses_2.widgets.animation import Animation
from nurses_2.widgets.parallax import Parallax

from .trajectory import Trajectory

ASSETS = Path("assets")
PARALLAX_IMAGES = ASSETS / "parallax_frames"
POWER = .3
SCROLL_FPS = .05
SHOOT_TIMEOUT = .5

class AutoScrollParallax(Parallax):
    def on_add(self):
        super().on_add()
        self._scroll_task = asyncio.create_task(self._scroll())

    def on_remove(self):
        super().on_remove()
        self._scroll_task.cancel()

    async def _scroll(self):
        while True:
            self.horizontal_offset += 1

            for child in self.root.children:
                if isinstance(child, Trajectory):
                    child.left -= 1

            await asyncio.sleep(SCROLL_FPS)

class Submarine(Animation):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._can_shoot = True

    async def _shoot_timeout(self):
        await asyncio.sleep(SHOOT_TIMEOUT)
        self._can_shoot = True

    def on_mouse(self, mouse_event):
        if (
            mouse_event.button == MouseButton.NO_BUTTON
            or not self._can_shoot
        ):
            return False

        self._can_shoot = False
        asyncio.create_task(self._shoot_timeout())

        cy, cx = self.center
        cy *= 2

        y, x = self.to_local(mouse_event.position)
        y *= 2

        dx = int(POWER * (x - cx))
        dy = int(POWER * (y - cy))

        self.parent.add_widget(Trajectory(dx, dy, size_hint=(1.0, 2.0)))
        self.pull_to_front()

class TrickShot(App):
    async def on_start(self):
        background = AutoScrollParallax(path=PARALLAX_IMAGES, size_hint=(1.0, 1.0))

        submarine = Submarine(
            path=ASSETS / "submarine",
            pos_hint=(.5, .4),
            anchor=Anchor.CENTER,
            size_hint=(.4, .4),
        )

        self.add_widgets(background, submarine)

        submarine.play()

TrickShot(title="Trick Shot!").run()


instructions = [
 {
  "instruction": "mov",
  "destination_register": "edi",
  "data": 0x23
}, {
  "instruction": "mov",
  "source_register": "edi",
  "destination_register": "eax",
  "source_type": "register",
  "destination_type": "register"
},
{
  "instruction": "ret"
}]

instruction_lookup = {
  "mov": {"immediate": 0x8b, "register": 0x8B, "by_register": {
    "edi": 0xbf
  }},
  "ret": {"register": 0xc3, "immediate": 0xc3}
}

registers = ["eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi"]
# EAX
# 000	ECX
# 001	EDX
# 010	EBX
# 011	ESP
# 100	EBP
# 101	ESI
# 110	EDI

def assemble(instructions):

  outputs = []

  for instruction in instructions:
    print("#####")
    data = []
    second_byte = 0x0000
    instruction_data = instruction_lookup[instruction["instruction"]]
    if "data" in instruction:
      if "by_register" in instruction_data:
        opcode = instruction_data["by_register"][instruction["destination_register"]]
      else:
        opcode = instruction_data["immediate"]
      second_byte = instruction["data"]

    else:
      opcode = instruction_data["register"]
    data.append(opcode)
    data.append(second_byte)
    print(data)
    reg = 0
    mod = 0
    rm = 0

    print(instruction_data)
    print(instruction)

    if "data" not in instruction:
      if "source_type" in instruction and instruction["source_type"] == "register":
        print("source type is a register")
        mod = 3
        print("mode is {}".format(mod))
        if "source_register" in instruction and instruction["source_register"] in registers:
          rm = registers.index(instruction["source_register"])
          print("rm is {}".format(rm))
      if "destination_register" in instruction and instruction["destination_register"] in registers:

        reg = registers.index(instruction["destination_register"])
        print("destination register is {}".format(reg))

    components = [opcode, second_byte, mod, reg, rm]
    byte_1 = "{:8b}".format(opcode)
    byte_2 = "{:8b}".format(second_byte)

    byte_3 = "{:8b}".format(0x00)
    byte_7 = "{:8b}".format(0x00)

    byte_5 = "{:8b}".format(0x00)
    byte_6 = "{:8b}".format(0x00)

    byte_8 = "{:8b}".format(0x00)

    byte_4 = "{:8b}".format(0x00)

    byte_9 = "{:8b}".format(0x00)
    binary = '{:8b}{:03b}{:02b}{:03b}{:b}'.format(*components)
    output = []
    if "data" in instruction:
      bytes = [byte_1, byte_2, byte_3, byte_4, byte_5]
    else:
      byte_2 = "{:02b}{:03b}{:03b}".format(mod, reg, rm)
      bytes = [byte_1, byte_2]

    # if "data" in instruction:
    #   bytes.append("{:8b}".format(second_byte))
    #   bytes.append("{:8b}".format(0x00))
    #   bytes.append("{:8b}".format(0x00))
    #   bytes.append("{:8b}".format(0x00))

    for byte in bytes:
      hexdata = hex(int(byte, 2))
      if hexdata == "0x0":
        output.append("0x00")
      else:
        output.append(hexdata)

    print(hex(int(binary, 2)))
    print(output)

    outputs.append(output)
  print("0x")
  for output in outputs:
    for suboutput in output:

      print(suboutput.replace("0x", ""), end="")

  print("")

assemble(instructions)


import sys

# This could use the n 2n rule and other ideas about primes
# Reinstate p[] to use only those as candidate multipliers for new composites?...
# ...no that won't work beyond a certain point: You don't know them yet

def NthPrime(nth):
    if nth == 1: print('first prime is 2'); return 2
    if nth == 2: print('second prime is 3'); return 3

    # p = [2, 3]      # list of primes is not necessary with a sieve
    n = 2             # n is how many primes found so far
    c = []            # list of composites
    lp = 3            # lp is short for 'last prime' (largest so far)
                      #   lp starts out as 3
    while n < nth:
        # Go through all of the odd multipliers of the current prime
        #   Even multipliers will never be checked because they are
        #   automatically composite.
        endpoint = int(1000000/lp)
        for i in range(lp, endpoint, 2): c.append(lp*i)
            # lc = lp*i                     # 'largest composite'
            # c.append(lp*i)
            # the next line makes it incredibly slow
            # if not lc in c: c.append(lc)  # sieve approach
            # if i > 1000: print(i, lp, endpoint, lc, len(c))
        for j in range(lp+2, 1000000, 2):
            if not j in c:
                lp = j
                # p.append(lp)
                n = n+1
                break
    return lp

# There are 78,498 primes less than one million
nth = int(input('Enter which prime you want: '))
if nth > 78498:
    print('Sorry this program only handles up to the 78498th prime...')
    sys.exit()

nthprime = NthPrime(nth)
print('prime ', nth, ' = ', nthprime)


import mxnet as mx
import os, time, shutil
import logging
from mxnet import gluon, image, init, nd
from mxnet import autograd as ag
from mxnet.gluon import nn
from mxnet.gluon.data.vision import transforms
from gluoncv.model_zoo import get_model
import numpy as np
import gluoncv
from gluoncv.utils import export_block,viz,makedirs
from tools.f1_metric import f1_score

model_name = 'xception'
root_dir = '/home/lin/sheldon/my_mxnet'
# weights_name = 'gluoncv-xception'
num_classes = 25
num_gpus = 2
batch_per_gpu = 8
train_epoch = 10

batch_size = batch_per_gpu * num_gpus
rec_path = os.path.join(root_dir, 'img_file')

predict_path = os.path.join(root_dir, 'predict_img')

prefix = os.path.join(root_dir, 'weights/xception')
prefix_epoch = 10

def get_iterators(batch_size = batch_size, data_shape=(3, 299, 299),shuffle=True,root_path = rec_path):
    train_data = mx.io.ImageRecordIter(
        path_imgrec=os.path.join(root_path, 'train.rec'),
        path_imgidx=os.path.join(root_path, 'train.idx'),
        data_shape=data_shape,
        batch_size=batch_size,
        resize = 299,
        saturation = 0.2,
        contrast = 0.2,
        shuffle=shuffle,
        rand_mirror = True,
        brightness = 0.2,
        data_name='data',
        label_name='softmax_label',
        rotate = 180
    )
    val_data = mx.io.ImageRecordIter(
        path_imgrec=os.path.join(root_path, 'val.rec'),
        path_imgidx=os.path.join(root_path, 'val.idx'),
        data_shape=data_shape,
        data_name='data',
        label_name='softmax_label',
        batch_size=batch_size,
        resize = 299,
    )
    return (train_data, val_data)

def get_eval_metrics(metrics_list):
    eval_metrics = mx.metric.CompositeEvalMetric()
    eval_metrics_1 = mx.metric.Accuracy()
    eval_metrics_2 = mx.metric.TopKAccuracy(5)
    metrics_list.append([eval_metrics_1,eval_metrics_2])
    for child_metric in metrics_list:
        eval_metrics.add(child_metric)
    # (train, val) = get_iterators(batch_size)
    # a = mod.score(eval_data = val,eval_metric = eval_metrics)
    # a = dict(a)
    return eval_metrics

def predict(data,prefix = prefix,prefix_epoch = prefix_epoch,num_batch = None):
    mod = get_model(prefix=prefix, prefix_epoch=prefix_epoch,batch_size = 4)
    pred_result = mod.predict(data, num_batch = num_batch).asnumpy()
    pred_result = np.squeeze(pred_result)
    result = np.argsort(pred_result, axis=1)[:, -1]
    return result
    # return pred_result

def val_model():
    eval_metrics = get_eval_metrics([f1_score()])
    mod = get_model()
    (train, val) = get_iterators(batch_size)
    eval_result = dict(mod.score(eval_data=val, eval_metric=eval_metrics,num_batch = 1))
    return eval_result

def get_model(prefix = prefix,prefix_epoch = prefix_epoch,batch_size=batch_size):
    sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, prefix_epoch)
    devs = [mx.gpu(i) for i in range(num_gpus)]
    mod = mx.mod.Module(symbol=sym, context=devs,
                        data_names=["data"], label_names=None)
    mod.bind(for_training=False, data_shapes=[("data", (batch_size, 3, 299, 299))],
             label_shapes=[("softmax_label", (batch_size,))])
    # 设定模型参数
    mod.set_params(arg_params, aux_params, allow_missing=True)
    return mod

if __name__ == '__main__':
    predict_data = mx.io.ImageRecordIter(
        path_imgrec=os.path.join(predict_path, 'predict.rec'),
        path_imgidx=os.path.join(predict_path, 'predict.idx'),
        data_shape=(3,299,299),
        data_name='data',
        label_name='softmax_label',
        batch_size=1,
        resize=299,
    )
    print (predict(data = predict_data))


from discord.ext import commands
import discord
from typing import Tuple, Dict, List
import re

class CustomHelp(commands.Cog):
    __slots__ = 'bot',
    def __init__(self, bot):
        self.bot = bot
        self.bot.remove_command('help')

    def commands_dict(self) -> Dict[str, List[commands.Command]]:
        final = {}
        for cogname, cog in self.bot.cogs.items():
            if hasattr(cog, 'overwrite_name'):
                cogname = cog.overwrite_name
            for cmd in cog.get_commands():
                if not cmd.hidden:
                    if cogname not in final:
                        final[cogname] = []
                    final[cogname].append(cmd)
        final = dict(sorted(final.items(), key=lambda i: len(i[1]), reverse=True))
        return final

    @commands.command(name='help', hidden=True)
    async def _help(self, ctx, lookup=None, *subcommands):
        """
        Shows help for given command or
        lists all/category commands
        Examples::
        >
        Sends an embed listing all commands
        > money
        Sends help and info for command *money*
        """
        # Show all cogs and their commands WITHOUT short doc
        if lookup is None:
            cmds = self.commands_dict()

            # Sort by name
            for category in cmds:
                cmds[category].sort(key=lambda c: c.name)

            embed = discord.Embed(title='**Commands**',colour=0x3498db)
            for cog in cmds:
                formatted_cmds = ' '.join(map(lambda cmd: f'`{cmd.name}`', cmds[cog]))
                embed.add_field(name=cog, value=formatted_cmds, inline=False)
            await ctx.send(embed=embed)

        # Look up command
        else:
            cmd = self.bot.get_command(' '.join([lookup.lower(), *map(str.lower, subcommands)]))
            if cmd:
                cmd_help = cmd.help
                examples = None
                if cmd_help:
                    examples = ''
                    final_help = ''
                    in_example = False
                    for line in cmd_help.splitlines():
                        if not in_example:
                            if line == 'Examples::':
                                in_example = True
                                final_help = final_help[:-1]
                            else:
                                final_help += line + '\n'
                        else:
                            if line[0] == '>':
                                examples += f'`> {ctx.prefix}{cmd.qualified_name} {line[2:]}`\n'
                            else:
                                examples += line + '\n'
                    cmd_help = final_help
                    examples = examples[:-1]

                embed = discord.Embed(
                    title=cmd.qualified_name,
                    description=cmd_help,
                    colour=0x87ef73
                    )
                embed.add_field(name='Syntax', value=f'`{ctx.prefix}{cmd.qualified_name} {cmd.signature}`')
                if cmd.aliases:
                    embed.add_field(name='Aliases', value=' '.join(map(lambda x: f'`{x}`', cmd.aliases)))
                if isinstance(cmd, commands.Group):
                    embed.add_field(name='Subcommands', value=' '.join(f'`{i.name}`' for i in cmd.commands))
                if examples:
                    embed.add_field(name='Examples', value=examples, inline=False)
                await ctx.send(embed=embed)
            else:
                return await ctx.send('Command not found')

async def setup(bot):
    await bot.add_cog(CustomHelp(bot))