#!/usr/bin/env python
"""Selection procedures (local and global)"""

import numpy as np
import multiprocessing as mp
import time
import random
import anneal
import itertools
from multiprocessing import pool, Process  # Redundant?

import simulate_data as sim
import summary_stats as sum_stat
import estimators as estim
import assess_sv_estimation as a

theta_initial = []
stats_initial = []
sample_dim = []
theta_out = np.matrix([])
theta_out_sigma = np.matrix([])
theta_in_sigma = np.matrix([])
stats_out = []
local_knn_estimate = []
obs_stats_standard = []
criterion = ""

class BinaryAnnealer(anneal.Annealer):

    def move(self):
        i = 0
        while i == 0 or sum(self.state) == 0:
            n = random.randrange(len(self.state))
            self.state[n] = 1 - self.state[n]
            i += 1

    def energy(self):
        return obj_f(self.state, criterion)

def select_sv(y, s, r, ar, criterion_f):
    """Statistic selection procedure for SV process"""
    # s: In sample size.
    # r: Out sample size.
    # ar:

    # Start timer.
    start = time.time()
    print("Started at:", time.strftime("%H:%M:%S on %d-%m-%Y"))

    t = len(y)

    delta = np.array([1]*22)

    # Calculate statistics from observed sample.
    obs_stats = sum_stat.sv_stats(delta, y)

    # Generate out theta.
    global theta_out
    theta_out = [generate_theta_sv(ar) for i in range(r)]
    theta_out = np.matrix(theta_out).T

    # Generate out sample of time series.
    out_y = [sim.sim_sv(t, theta_out[0, i], theta_out[1, i], theta_out[2, i],
                        theta_out[3, i], 1) for i in range(r)]

    # Generate out sample statistics.
    global stats_out
    stats_out = [sum_stat.sv_stats(delta, out_y[i]) for i
                 in range(len(out_y))]

    # Standardize the out-of-sample parameters.
    theta_out_mean = sum(theta_out.T)/r
    u = sum([np.square(theta_out[:, i]-theta_out_mean.T) for i in range(r)])/r
    theta_out = np.hstack([(theta_out[:, i] - theta_out_mean.T)/np.sqrt(u)
                           for i in range(r)])
    global theta_out_sigma
    theta_out_sigma = np.sqrt(u)

    # Standardize the out-of-sample statistics.
    stats_out_mean = sum(stats_out)/r
    u = sum([np.square(stats_out[i]-stats_out_mean) for i in range(r)])/r
    stats_out = [(stats_out[i] - stats_out_mean)/np.sqrt(u) for i in range(r)]

    # Generate in theta.
    global theta_initial
    theta_initial = [generate_theta_sv(ar) for i in range(s)]
    theta_initial = np.matrix(theta_initial).T

    # Generate in sample of time series.
    in_y = [sim.sim_sv(t, theta_initial[0, i], theta_initial[1, i],
                       theta_initial[2, i], theta_initial[3, i], 1)
            for i in range(s)]

    # Generate in sample statistics.
    global stats_initial
    stats_initial = [sum_stat.sv_stats(delta, in_y[i]) for i
                     in range(s)]

    # Standardize parameters used for estimation.
    theta_initial_mean = sum(theta_initial.T)/s
    u = sum([np.square(theta_initial[:, i]-theta_initial_mean.T)
             for i in range(s)])/s
    theta_initial = np.hstack([(theta_initial[:, i]
                                - theta_initial_mean.T)/np.sqrt(u)
                               for i in range(s)])

    global theta_in_sigma
    global theta_in_mean
    theta_in_sigma = np.sqrt(u)
    theta_in_mean = theta_initial_mean

    # Standardize statistics for estimation.
    stats_initial_mean = sum(stats_initial)/s
    u = sum([np.square(stats_initial[i]-stats_initial_mean)
             for i in range(s)])/s
    stats_initial = [(stats_initial[i] - stats_initial_mean)/np.sqrt(u) for
                     i in range(s)]

    # Standardize statistics calculated from observed sample.
    global obs_stats_standard
    obs_stats_standard = (obs_stats - stats_initial_mean)/np.sqrt(u)

    # Guess delta.
    delta = np.around(np.random.rand(1, 22)).flatten().astype(int)

    global criterion
    criterion = criterion_f

    #  Summary of sample sizes.
    print("Out sample size:", r)
    print("In sample size:", s)
    print("Observed sample size:", len(y))
    print("Number of persistence parameters:", ar)

    # Initiate binary annealing algorithm and assign tuning parameters.
    print("Initial guess for delta is:", delta)
    print("Criterion function is:", criterion)
    opt = BinaryAnnealer(delta)
    opt.Tmax = 0.10  # 0.25
    opt.Tmin = 0.0001  # 0.001
    opt.steps = len(delta)*10
    opt.updates = len(delta)
    print("Simmulated annealing parameters are:")
    print("Tmax:", opt.Tmax)
    print("Tmin:", opt.Tmin)
    print("Beginning simulated annealing at:",
          time.strftime("%H:%M:%S on %d-%m-%Y"))
    results = opt.anneal()

    results_file = open("annealing_results.txt", "a")
    results_file.write("### Results from annealing run finished at: " +
                       time.strftime("%H:%M:%S on %d-%m-%Y") + " ###\n")
    results_file.write(str(results) + "\n")
    results_file.close()

    print("Final selection and minimum are: ", results)
    print("Finished at :", time.strftime("%H:%M:%S on %d-%m-%Y"))
    print("Total time to run (in minutes):", (time.time()-start)/60, "\n")

    return results

def select_lin_reg(obs_y, obs_x, s, r, q, criterion_f):
    """Selection procedure for linear regression"""
    # r: out of sample simulation size.
    # s: within sample simulation size.
    # q: size of initial importance sampler draw.

    # Start timer.
    t = time.time()

    # Sample dimensions.
    global sample_dim
    sample_dim.append(obs_x.shape[0])
    sample_dim.append(obs_x.shape[1])

    # Candidate statistic vector:
    # (6*sample_dim beta) (3*2 intercepts and error term variances)
    # (5 noise random variables)
    n_candidates = 6*sample_dim[1] + 6 + 5
    delta = np.array([0]*n_candidates)
    delta[0:len(delta)] = 1  # Inititalize delta.

    # Print starting message.
    print("### Beginning summary statistic selection from", n_candidates,
          "candidate summary statistics. ###")

    # Calculate full set of summary statistics from observed data.
    obs_stats = sum_stat.lin_reg_stats(delta, obs_y, obs_x)

    # Generate simulated samples from the linear model.
    generate_lin_reg_sample(delta, s, r, sample_dim, obs_stats)

    # Initial guess for delta.
    delta = np.around(np.random.rand(1, n_candidates)).flatten().astype(int)

    #  Summary of sample sizes.
    print("Out sample size:", r)
    print("In sample size:", s)
    print("Observed sample size:", sample_dim[0])
    print("Number of regressors:", sample_dim[1])

    global criterion
    criterion = criterion_f

    # Initiate binary annealing algorithm and assign tuning parameters.
    print("Initial guess for delta is:", delta)
    print("Criterion function is:", criterion)
    opt = BinaryAnnealer(delta)
    opt.Tmax = 0.25  # 0.25
    opt.Tmin = 0.001  # 0.001
    opt.steps = len(delta)*10
    opt.updates = len(delta)
    print("Simmulated annealing parameters are:")
    print("Tmax:", opt.Tmax)
    print("Tmin:", opt.Tmin)
    print("Beginning simulated annealing at:",
          time.strftime("%H:%M:%S on %d-%m-%Y"))
    results = opt.anneal()

    results_file = open("annealing_results.txt", "a")
    results_file.write("### Results from annealing run finished at: " +
                       time.strftime("%H:%M:%S on %d-%m-%Y") + " ###\n")
    results_file.write(str(results) + "\n")
    results_file.close()

    print("Final selection and minimum are: ", results)
    print("Finished at :", time.strftime("%H:%M:%S on %d-%m-%Y"))
    print("Total time to run (in minutes):", (time.time()-t)/60, "\n")
    return

def obj_f(delta, criterion_type):
    """Objective functions for the different types of selection procedures"""
    if criterion_type == 'integrated_kl':
        # Integrated Kullback-Leibler distance criterion.
        criterion = -criterion_f(delta, criterion_type)

    if criterion_type == 'abs_dev_knn':
        # Integrated Bayesian loss of posterior mean estimate
        # (estimated by k-nn).
        a = 0
        criterion = (1+a*sum(delta))*criterion_f(
            delta, criterion_type)

    if criterion_type == 'local_abs_dev':
        # Bayesian loss of posterior mean estimate.
        a = 0  # sum(delta)/len(delta)
        sim_stats = [stats_initial[i][delta > 0]
                     for i in range(len(stats_initial))]
        global local_knn_estimate
        local_knn_estimate = estim.sbil_knn(obs_stats_standard[delta > 0],
                                            sim_stats, theta_initial)
        criterion = (1+a*sum(delta))*criterion_f(
            delta, criterion_type)

    if criterion_type == 'local_kl':
        criterion = -criterion_f(delta, criterion_type)

    return criterion

def criterion_f(delta, criterion_type='abs_dev_knn'):
    """Function calls to the different criteria associated with the different
    selection procedures (with the spawning of multiple processes)"""
    stats_out_filtered = [stats_out[i][delta > 0]
                          for i in range(len(stats_out))]

    if criterion_type == 'local_kl':
        # Find k nearest statistics from R sample (full candidate set).
        norms = [np.linalg.norm(stats_out[i] - obs_stats_standard)
                 for i in range(len(stats_out))]
        k_nearest_index = np.argsort(norms)[0:round(len(  # 0.01
            stats_out_filtered)**.25)]
        k_nearest_out_theta = [theta_out.T[i] for i in k_nearest_index]
        obs_stats_standard_filtered = obs_stats_standard[delta > 0]
        # Estimate posterior density for each of the k nearest thetas.
        p = pool.Pool()

        results = p.map_async(local_kl_star, zip(itertools.repeat(delta),
                                                 k_nearest_out_theta,
                                                 itertools.repeat(
                                                     obs_stats_standard_filtered)))

        p.close()
        p.join()
        results.wait()
        posterior_densities = results.get()
        numerator = sum(np.log(posterior_densities))
        denominator = len(k_nearest_out_theta)

        return numerator/denominator

    if criterion_type == 'local_abs_dev':
        norms = [np.linalg.norm(stats_out_filtered[i] -
                                obs_stats_standard[delta > 0])
                 for i in range(len(stats_out_filtered))]
        k_nearest_stats = np.argsort(norms)[0:round(len(  # 0.01
            stats_out_filtered)**0.25)]
        k_nearest_theta = [theta_out.T[i] for i in k_nearest_stats]
        k_nearest_norms_std = [1 for i in k_nearest_stats]  #[norms[i]/largest_norm for i in k_nearest_stats]
        p = pool.Pool()

        results = p.map_async(local_abs_dev_star, zip(k_nearest_theta,
                                                      itertools.repeat(delta),
                                                      k_nearest_norms_std))
        p.close()
        p.join()
        results.wait()
        num_denom = results.get()
        numerator = [num_denom[i][0] for i in range(len(num_denom))]
        denominator = [num_denom[i][1] for i in range(len(num_denom))]
        criterion = sum(numerator)/sum(denominator)

    if criterion_type == 'integrated_kl':
        p = pool.Pool()
        results = p.map_async(integrated_kl_star, zip(theta_out.T,
                                                      itertools.repeat(delta),
                                                      stats_out_filtered))
        p.close()
        p.join()
        results.wait()
        posterior_densities = results.get()
        criterion = sum(np.log(posterior_densities))/len(posterior_densities)

    if criterion_type == 'abs_dev_knn':
        p = pool.Pool()
        results = p.map_async(abs_dev_knn_star, zip(theta_out.T,  # theta_out.T
                                                    itertools.repeat(delta),
                                                    stats_out_filtered))
        p.close()
        p.join()

        results.wait()
        norms = results.get()
        criterion = sum(norms)/len(norms)

    if criterion_type == '':
        print("Loss function is L2-norm.")
        manager = mp.Manager()
        norms = manager.list()
        procs = []
        chunk_size = int(len(stats_out)/p_n)
        print("Chunk size is: ", chunk_size)
        for i in range(4):
            fargs = (theta_out[:, i*chunk_size:(i+1)*chunk_size], delta,
                     stats_out[i*chunk_size:(i+1)*chunk_size], s,
                     q, i, norms)
            p = Process(target=norm_L2_mp, args=fargs)
            procs.append(p)

        print("Spawned these processes: ", procs)

        for p in procs:
            p.start()

        for p in procs:
            p.join()

    return criterion

def local_kl_star(args):
    """Intermediate function call to expand arguments
    (required for map_async())"""
    return estim.sbil_pd_accept_reject(*args)

def local_abs_dev_star(args):
    """Intermediate function call to expand arguments
    (required for map_async())"""
    return local_abs_dev(*args)

def integrated_kl_star(args):
    """Intermediate function call to expand arguments
    (required for map_async())"""
    return integrated_kl(*args)

def abs_dev_knn_star(args):
    """Intermediate function call to expand arguments
    (required for map_async())"""
    return abs_dev_knn(*args)

def local_abs_dev(theta_r, delta, norm):
    abs_dev = (1/theta_r.shape[1])*sum(abs(
        theta_r.T-local_knn_estimate))
    #u = np.linalg.norm(stat_r - obs_stats_standard[delta > 0])#/0.01
    #kernel = (1/math.sqrt((2*math.pi)))*math.exp(-u/2)
    #kernel = int(u < 2)
    kernel = 1  # (3/4)*(1-norm**2)
    return abs_dev*kernel, kernel

def integrated_kl(theta_r, delta, stat_r):
    posterior_density = estim.sbil_pd_accept_reject(delta, theta_r.T, stat_r)
    return posterior_density

def abs_dev_knn(theta_out, delta, obs_stat):

    sim_stats = [stats_initial[i][delta > 0]
                 for i in range(len(stats_initial))]
    knn_estimate = estim.sbil_knn(obs_stat, sim_stats, theta_initial)
    abs_dev = (1/theta_out.shape[1])*sum(theta_in_sigma.T*abs(
        theta_out.T-knn_estimate))
    return abs_dev

def norm_L2_mp(theta_out, delta, stats_out, s, q, i, norms):
    L2_norm = []
    for i in range(len(stats_out)):
        L2_norm.append(np.linalg.norm(theta_out[:, i] - np.matrix(estim.sbil_kernel(
        delta, stats_out[i], s, q, i)).T))
    print("Finished one set!")
    print(L2_norm)
    norms.append(sum(L2_norm))
    return

def theta_and_stats(delta, s, sample_size, n):
    """Draw parameter vectors and simulate associated statistics"""
    # Generate draws from the prior.
    theta = sim.prior_draw(n,  s)

    # Generate data for regressors.
    sim_x = np.matrix(np.random.randn(sample_size, s*n))
    sim_x_iter = [sim_x[:, i*n:(i+1)*n] for i in range(s)]

    # Simulate data from the linear model.
    p = pool.Pool()
    theta_iter = [theta[0:n+1, i] for i in range(s)]
    sigma = [theta[n+1, i] for i in range(s)]
    sim_y = p.map(sim_lin_reg_star, zip(theta_iter, sigma, sim_x_iter,
                                        itertools.repeat(1)))
    p.close()
    p.join()

    # Calculate summary statistics from simulated data.
    p = pool.Pool()
    sim_stats = p.map(lin_reg_stats_star, zip(itertools.repeat(delta),
                                              sim_y, sim_x_iter))
    p.close()
    p.join()

    return [theta, sim_stats]

def sim_lin_reg_star(args):
    return sim.sim_lin_reg(*args)

def lin_reg_stats_star(args):
    return sum_stat.lin_reg_stats(*args)

def generate_lin_reg_sample(delta, s, r, sample_dim, obs_stats):
    """Draw from the prior and simulate associated summary statistics for
    cross validation"""
    global theta_out
    global stats_out
    result = theta_and_stats(delta, r, sample_dim[0], sample_dim[1])
    theta_out = result[0]
    stats_out = result[1]

    # Standardize the out-of-sample parameters.
    theta_out_mean = sum(theta_out.T)/r
    u = sum([np.square(theta_out[:, i]-theta_out_mean.T) for i in range(r)])/r
    theta_out = np.hstack([(theta_out[:, i] - theta_out_mean.T)/np.sqrt(u)
                           for i in range(r)])
    global theta_out_sigma
    theta_out_sigma = np.sqrt(u)

    # Standardize the out-of-sample statistics.
    stats_out_mean = sum(stats_out)/r
    u = sum([np.square(stats_out[i]-stats_out_mean) for i in range(r)])/r
    stats_out = [(stats_out[i] - stats_out_mean)/np.sqrt(u) for i in range(r)]

    # Draw from the prior and caclulate summary statistics for estimation.
    result = theta_and_stats(delta, s, sample_dim[0], sample_dim[1])
    global theta_initial
    global stats_initial
    theta_initial = result[0]
    stats_initial = result[1]

    # Standardize parameters used for estimation.
    theta_initial_mean = sum(theta_initial.T)/s
    u = sum([np.square(theta_initial[:, i]-theta_initial_mean.T)
             for i in range(s)])/s
    global theta_in_sigma
    theta_in_sigma = np.sqrt(u)
    theta_initial = np.hstack([(theta_initial[:, i]
                                - theta_initial_mean.T)/np.sqrt(u)
                               for i in range(s)])

    # Standardize statistics for estimation.
    stats_initial_mean = sum(stats_initial)/s
    u = sum([np.square(stats_initial[i]-stats_initial_mean)
             for i in range(s)])/s
    stats_initial = [(stats_initial[i] - stats_initial_mean)/np.sqrt(u) for
                     i in range(s)]

    # Standardize statistics calculated from observed sample.
    global obs_stats_standard
    obs_stats_standard = (obs_stats - stats_initial_mean)/np.sqrt(u)

def generate_theta_sv(ar):
    """Return a parameter vector draw from the prior for the SV process"""
    theta_out = []

    mu_y = 0
    while abs(mu_y) < 0.5:
        mu_y = np.random.uniform(-6, 6)

    theta_out.append(mu_y)  # Expected return.
    theta_out.append(np.random.uniform(0, 3))  # Mu_s.
    theta_out.append(np.random.uniform(0, 0.7))  # Phi_1
    theta_out.append(np.random.uniform(0, 0.7))  # Phi_2

    # Initialize autoregressive coefficients in lag polynomial.
    #p = [-1, -1, 1]

    #while sum(abs(np.roots(p)) < 1.2) != 0:
        #p = [np.random.randn() for i in range(ar)]
        #p = [np.random.uniform(0, 1)*-1 for i in range(ar)]
        #p.append(1)

    #theta_out.append(-1*p[1])  # Phi_1
    #theta_out.append(-1*p[0])  # Phi_2
    #theta_out.append(-1*p[1])
    #theta_out.append(-1*p[0])
    #theta_out.append(np.random.rand()*2)  # Sigma_eta.

    return theta_out


import os

import deeplenstronomy.deeplenstronomy as dl

import hydra

@hydra.main(version_base=None, config_path='hydra', config_name='config.yaml')
def main(config):
    data_dir = config.machine.data_dir
    repo_dir = config.machine.repo_dir

    dl_config_file = os.path.join(repo_dir, 'dl_config', 'example.yaml')
    dataset = dl.make_dataset(dl_config_file)

if __name__ == '__main__':
    main()


from app.container import city_service, user_service

def check_is_admin(potential_admin):
    if potential_admin.username is None:
        user = user_service.get_one(potential_admin.full_name)
    else:
        user = user_service.get_one(potential_admin.username)
    if user.role in ('admin', 'superuser'):
        return True
    else:
        return False

def check_is_in_db(message):
    lst = []
    for row in city_service.get_all().all():
        lst.append(row[-1].capitalize())

    return message.capitalize() in lst


from aeternity.signing import Account
from aeternity import defaults, identifiers, hashing, utils
import pytest
import random
# from aeternity.exceptions import TransactionNotFoundException

blind_auth_contract = """contract BlindAuth =
      record state = { owner : address }

      entrypoint init(owner' : address) = { owner = owner' }

      stateful entrypoint authorize(r: int) : bool =
        // r is a random number only used to make tx hashes unique
        switch(Auth.tx_hash)
          None          => abort("Not in Auth context")
          Some(tx_hash) => true

      entrypoint to_sign(h : hash, n : int) : hash =
        Crypto.blake2b((h, n))
    """

def test_node_spend_native(chain_fixture):
    node_cli = chain_fixture.NODE_CLI
    sender_account = chain_fixture.ALICE
    recipient_id = Account.generate().get_address()
    # with numbers
    tx = node_cli.spend(sender_account, recipient_id, 100)
    print("DATA", tx)
    assert recipient_id == tx.data.tx.data.recipient_id
    assert sender_account.get_address() == tx.data.tx.data.sender_id
    account_balance = node_cli.get_account_by_pubkey(pubkey=recipient_id).balance
    assert account_balance == 100
    # spend some string
    tx = node_cli.spend(sender_account, recipient_id, "0.5ae")

    assert recipient_id == tx.data.tx.data.recipient_id
    assert sender_account.get_address() == tx.data.tx.data.sender_id

    account_balance = node_cli.get_account_by_pubkey(pubkey=recipient_id).balance
    assert account_balance == 500000000000000100

def test_node_spend_burst(chain_fixture):
    sender_account = chain_fixture.ALICE
    # make a new non blocking client
    ae_cli = chain_fixture.NODE_CLI
    ae_cli.config.blocking_mode = False
    # recipient account
    recipient_account = Account.generate().get_address()
    # send 50 consecutive spend
    ths = []
    print(">>"*20, "spend burst start")
    for i in range(50):
        tx = ae_cli.spend(sender_account, recipient_account, "1AE")
        ths.append(tx.hash)
    print("<<"*20, "spend burst end")

    for th in ths:
        ae_cli.wait_for_transaction(th)

    assert(ae_cli.get_balance(recipient_account) == utils.amount_to_aettos("50ae"))

@pytest.mark.parametrize("height,protocol_version", [(0, 1), (1, 1), (2, 2), (3, 2), (4, 3), (5, 3)])
def test_node_get_protocol_version(chain_fixture, height, protocol_version):
    # this test assumes that the configuration of the node bein tested has the follwing configuration:
    #   hard_forks:
    #      "1": 0
    #      "2": 2
    #      "3": 4
    assert(chain_fixture.NODE_CLI.get_consensus_protocol_version(height)) == protocol_version

def test_node_ga_attach(chain_fixture, compiler_fixture):
    ae_cli = chain_fixture.NODE_CLI
    account = chain_fixture.ALICE
    c_cli = compiler_fixture.COMPILER
    # test that the account is not already generalized
    poa_account = ae_cli.get_account_by_pubkey(pubkey=account.get_address())
    assert poa_account.kind == identifiers.ACCOUNT_KIND_BASIC
    # transform the account
    # compile the contract
    ga_contract = c_cli.compile(blind_auth_contract).bytecode
    # now encode the call data
    init_calldata = c_cli.encode_calldata(blind_auth_contract, "init", account.get_address()).calldata
    # this will return an object
    # init_calldata.calldata
    # now we can execute the transaction
    tx = ae_cli.account_basic_to_ga(account, ga_contract, calldata=init_calldata)

    # now check if it is a ga
    ga_account = ae_cli.get_account_by_pubkey(pubkey=account.get_address())
    assert ga_account.kind == identifiers.ACCOUNT_KIND_GENERALIZED

def test_node_ga_meta_spend(chain_fixture, compiler_fixture):

    ae_cli = chain_fixture.NODE_CLI
    account = chain_fixture.ALICE
    c_cli = compiler_fixture.COMPILER
    # make the account poa

    # transform the account
    # compile the contract
    ga_contract = c_cli.compile(blind_auth_contract).bytecode
    # now encode the call data
    init_calldata = c_cli.encode_calldata(blind_auth_contract, "init", account.get_address()).calldata
    # this will return an object
    # init_calldata.calldata
    # now we can execute the transaction
    tx = ae_cli.account_basic_to_ga(account, ga_contract, calldata=init_calldata)

    print("ACCOUNT is now GA", account.get_address())

    # retrieve the account data
    ga_account = ae_cli.get_account(account.get_address())
    # create a dummy account
    recipient_id = Account.generate().get_address()

    # generate the spend transactions
    amount = 54321
    payload = "ga spend tx"
    fee = defaults.FEE
    ttl = defaults.TX_TTL

    spend_tx = ae_cli.tx_builder.tx_spend(account.get_address(), recipient_id, amount, payload, fee, ttl, 0)

    # encode the call data for the transaction
    calldata = c_cli.encode_calldata(blind_auth_contract, ga_account.auth_fun, hashing.randint()).calldata
    # now we can sign the transaction (it will use the auth for do that)
    spend_tx = ae_cli.sign_transaction(ga_account, spend_tx, auth_data=calldata)
    # broadcast
    tx_hash = ae_cli.broadcast_transaction(spend_tx)
    print(f"GA_META_TX {tx_hash}")
    # check that the account received the tokens
    assert ae_cli.get_account_by_pubkey(pubkey=recipient_id).balance == amount


import os
import numpy as np
from PIL import Image
import torch
from net import Net
import json, base64

import isolation_util

BATCH_SIZE = 32

class Isolation(object):

    def __init__(self, checkpoint, gpu_id=0):
        assert os.path.exists(checkpoint)
        self.net = Net(num_classes=5)
        self.net.load_state_dict(torch.load(checkpoint, map_location=lambda storage, loc: storage))

        if torch.cuda.is_available():
            self.net.cuda(device=gpu_id)
            self.dtype_float = torch.cuda.FloatTensor
        else:
            self.dtype_float = torch.FloatTensor
        self.net.eval()
        self.softmax_func = torch.nn.Softmax()

    def eval(self, image):
        image_var = self._image_preprocess(image)
        scores = self.net(image_var)
        scores = self.softmax_func(scores)
        scores = scores.data.numpy()[0].tolist()
        rsp = {
            "Isolated on a white background": scores[0],
            "Isolated on a non-white background": scores[1],
            "Isolated on a contextual background": scores[2],
            "Partial isolation": scores[3],
            "Non-isolation": scores[4]
        }
        return rsp

    def model(self, image_batch):
        self.image_var = torch.autograd.Variable(torch.Tensor(len(image_batch), 3, 224, 224)).type(self.dtype_float).cuda()
        self.image_var.data.copy_(image_batch)
        scores = self.net(self.image_var)
        scores = self.softmax_func(scores)
        scores = scores.cpu().data.numpy()
        return scores

    def preprocess(self, pil_image):
        #pil_image = isolation_util.bytes_to_pil_image(raw_image_bytes)
        image = isolation_util.preprocess_image(pil_image)
        return image

if __name__ == '__main__':
    image_path = 'test.jpg'
    checkpoint = 'isolation_20171204.pth'
    isolation = Isolation(checkpoint=checkpoint)
    with open('test-chunk.json', 'r') as f:
        items = list(f)

    image_batch = []
    for item in items:
        item = json.loads(item)
        raw_image_bytes = base64.b64decode(item['image'])
        image_batch.append(isolation.preprocess(raw_image_bytes))

    images = torch.squeeze(torch.stack(image_batch))
    for i in range(0, images.shape[0], BATCH_SIZE):
        print('processing batch {}'.format(i // 32))
        batch = images[i:i+BATCH_SIZE]
        scores = isolation.model(batch)
        print(scores.shape, scores[0])


import datetime
import glob
import itertools
import os
import logging
import bottleneck as bn
import click
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from configobj import ConfigObj
from obspy import read, UTCDateTime, Stream
from obspy.signal.util import next_pow_2
from obspy.signal.filter import envelope as obspy_envelope

logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)

# from click_plugins import with_plugins
def to_idds(net, sta, loc, chan, year, jday, hour):
    SDS="YEAR/NET/STA/CHAN.TYPE/JDAY/NET.STA.LOC.CHAN.TYPE.YEAR.JDAY.HOUR"
    file=SDS.replace('YEAR', "%04i"%year)
    file=file.replace('NET', net)
    file=file.replace('STA', sta)
    file=file.replace('LOC', loc)
    file=file.replace('CHAN', chan)
    file=file.replace('JDAY', jday)
    file = file.replace('HOUR', hour)
    file=file.replace('TYPE', "D")
    return file

def to_agung(net, sta, loc, chan, year, jday, hour):
    SDS="YEAR/NET/STA/CHAN.TYPE/JDAY/"
    file=SDS.replace('YEAR', "%04i"%year)
    file=file.replace('NET', net)
    file=file.replace('STA', sta)
    file=file.replace('LOC', loc)
    file=file.replace('CHAN', chan)
    file=file.replace('JDAY', jday)
    file = file.replace('HOUR', hour)
    file=file.replace('TYPE', "D")
    file += "NET.STA.*.*.*.JDAY.HOUR"
    file=file.replace('NET', net)
    file = file.replace('STA', sta)
    file = file.replace('YEAR', "%04i"%year)
    file = file.replace('JDAY', jday)
    file = file.replace('HOUR', hour)
    return file

def to_sds(net, sta, loc,chan, year, jday):
    SDS="YEAR/NET/STA/CHAN.TYPE/NET.STA.LOC.CHAN.TYPE.YEAR.JDAY"
    file=SDS.replace('YEAR', "%04i"%year)
    file=file.replace('NET', net)
    file=file.replace('STA', sta)
    file=file.replace('LOC', loc)
    file=file.replace('CHAN', chan)
    file=file.replace('JDAY', jday)
    file=file.replace('TYPE', "D")
    return file

@click.group()
@click.pass_context
def cli(ctx):
    config_file = os.path.join(os.getcwd(), "config.ini")
    config = ConfigObj(config_file)
    ctx.obj['config'] = config

    pass

@click.command()
@click.pass_context
def init(ctx):
    filelist = os.path.join(os.getcwd(), "filelist.csv")
    if os.path.isfile(filelist):
        logging.debug("Init seems to be already done")
        return
    df = pd.DataFrame([["0","0",datetime.date(2010,1,1),"0","1"],],
                      columns=["net","sta","date","size","process_step"],
                      index=["nopath",])

    df.to_csv(filelist, index_label="path")
    pass

@click.command()
@click.option('--path',  help='Scan all files in specific folder, overrides the'
                              ' default workflow step.')
@click.pass_context
def scan_archive(ctx, path):
    config = ctx.obj["config"]["msara"]
    filelist = os.path.join(os.getcwd(), "filelist.csv")
    filedb = pd.read_csv(filelist, index_col="path")

    startdate = UTCDateTime(config["startdate"])
    enddate = UTCDateTime(config["enddate"])
    data_folder = config["data_folder"]
    data_format = config["data_format"]
    network = config["network"]
    stations = config["stations"]
    location = config["location"]
    channel = config["channel"]
    bandpass = config["bandpass"]
    new_data = False

    if path:
        for root, dirs, _ in os.walk(path):
            for d in dirs:
                tmppath = os.path.join(root, d)
                _ = os.listdir(tmppath)
                if not len(_):
                    continue
                for file in sorted(_):
                    path = os.path.join(tmppath, file)
                    if os.path.isfile(path):
                        print(path)
                        st = read(path)
                        tr = st[0]
                        if tr.stats.station in stations:
                            if tr.stats.sampling_rate < (2* float(bandpass[1])):
                                print("Trace sps too small, skiping")
                                continue
                            if (tr.stats.endtime <= startdate) or (tr.stats.starttime >= enddate):
                                print("Data outside startdate-enddate")
                                continue
                            filesize = os.stat(path).st_size
                            if path in filedb.index:
                                tmp = filedb.loc[path]
                                if tmp["size"] != filesize:
                                    logging.debug(tmp["size"], filesize)
                                    logging.debug("file changed")
                                    filedb.loc[path, "size"] = filesize
                                    filedb.loc[path, "process_step"] = "E"
                                    new_data = True
                                else:
                                    logging.debug(
                                        "path is in the df already, and no size change")
                                    new_data = False
                            else:
                                logging.debug(
                                    "Adding file to the process list")
                                new = pd.Series({
                                    "net": tr.stats.network,
                                    "sta": tr.stats.station,
                                    "date": tr.stats.starttime.date,
                                    "size": filesize,
                                    "process_step": "E"
                                }, name=path)
                                filedb = filedb.append(new)
                                new_data = True
        if new_data:
            filedb.to_csv(filelist)

        return

    if data_format == "SDS":
        dates = pd.date_range(UTCDateTime(config["startdate"]).datetime,
                              UTCDateTime(config["enddate"]).datetime, freq="D")
    elif data_format in ["IDDS", "AGUNG"]:
        dates = pd.date_range(UTCDateTime(config["startdate"]).datetime,
                              UTCDateTime(config["enddate"]).datetime, freq="H")
    else:
        logging.debug("data_format %s not supported" % data_format)
        return

    for station in stations:
        for date in dates:

            if data_format == "SDS":
                path = to_sds(network, station, location, channel, date.year, "%03i"% date.dayofyear)
                path = os.path.join(data_folder, path)
            elif data_format == "IDDS":
                path = to_idds(network, station, location, channel, date.year, "%03i"% date.dayofyear, "%02i" % date.hour)
                path = os.path.join(data_folder, path)
            elif data_format == "AGUNG":
                path = to_agung(network, station, location, channel, date.year, "%03i"% date.dayofyear, "%02i" % date.hour)
                path = os.path.join(data_folder, path)
                path = glob.glob(path)
                if len(path) != 0:
                    path = path[0]
                else:
                    continue

            if not os.path.isfile(path):
                continue
            filesize = os.stat(path).st_size
            if path in filedb.index:
                tmp = filedb.loc[path]
                if tmp["size"] != filesize:
                    logging.debug(tmp["size"], filesize)
                    logging.debug("file changed")
                    filedb.loc[path, "size"] = filesize
                    filedb.loc[path, "process_step"] = "E"
                    new_data = True
                else:
                    logging.debug("path is in the df already, and no size change")
                    new_data = False
            else:
                logging.debug("Adding file to the process list")
                new = pd.Series({
                    "net": network,
                    "sta": station,
                    "date": date.date(),
                    "size": filesize,
                    "process_step": "E"
                }, name=path)
                filedb = filedb.append(new)
                new_data = True

    if new_data:
        filedb.to_csv(filelist)

@click.command()
@click.pass_context
def envelope(ctx):
    config = ctx.obj["config"]["msara"]
    data_folder = config["data_folder"]
    filelist = os.path.join(os.getcwd(), "filelist.csv")
    filedb = pd.read_csv(filelist, index_col="path")
    tmp = filedb[filedb.process_step == "E"]
    if len(tmp) == 0:
        logging.debug("No files to process")
        return

    logging.debug("Will process envelope for: %s files "% len(tmp))
    for file in tmp.index:
        if not os.path.isfile(file):
            logging.debug("%s NOT FOUND")
            continue

        outfile = file.replace(data_folder, os.path.join(os.getcwd(),"ENV"))
        logging.debug("will output to %s"%outfile)
        if not os.path.isdir(os.path.split(outfile)[0]):
            os.makedirs(os.path.split(outfile)[0])

        # PREPROCESS
        logging.debug("Loading")
        st = read(file)
        logging.debug("Detrend")
        st.detrend("demean")
        logging.debug("Merge")
        st.merge(method=1, fill_value="interpolate")
        logging.debug("Taper")
        st.taper(max_percentage=None, max_length=1.0)
        logging.debug("Filter")
        freqmin, freqmax = config["bandpass"]
        st.filter("bandpass", freqmin=float(freqmin), freqmax=float(freqmax),corners=8)
        logging.debug(st)
        trace = st[0]
        n = int(config["env_sampling_rate"])
        sps = int(trace.stats.sampling_rate)

        logging.debug("Envelope")
        from scipy.signal import hilbert
        N = next_pow_2(trace.stats.npts)
        trace.data = np.abs(hilbert(trace.data, N=N)[:trace.stats.npts])
        logging.debug("Moving Median")
        trace.data = bn.move_median(trace.data, sps * n)
        trace.data = trace.data[n * sps - 1::sps * n]
        trace.stats.sampling_rate = 1. / float(n)
        trace.data = np.require(trace.data, np.float32)
        logging.debug("Writing")
        trace.write(outfile, format="MSEED")
        del st, trace
        filedb.loc[file, "process_step"] = "R"

    filedb.to_csv(filelist)

@click.command()
@click.pass_context
def ratio(ctx):
    config = ctx.obj["config"]["msara"]
    data_folder = config["data_folder"]
    data_format = config["data_format"]
    filelist = os.path.join(os.getcwd(), "filelist.csv")
    filedb = pd.read_csv(filelist, index_col="path", parse_dates=["date",])
    tmp = filedb[filedb.process_step == "R"]
    if len(tmp) == 0:
        logging.debug("No files to process")
        return
    groups = filedb.groupby("date")
    for date, group in groups:
        # date = pd.datetime(date)
        if "R" not in group["process_step"].values:
            logging.debug("Nothing do to for %s"%date)
            continue

        logging.debug(date)
        traces = []
        stations = np.unique(sorted(group["sta"]))
        for station in stations:
            logging.debug("Will load %s" % station)
            if data_format == "SDS":
                path = to_sds("*", station, "*", "*", date.year,
                              "%03i" % date.dayofyear)
            elif data_format == "IDDS":
                path = to_idds("*", station, "*", "*", date.year,
                               "%03i" % date.dayofyear, "*")
            elif data_format == "AGUNG":
                path = to_agung("*", station, "*", "*", date.year,
                                "%03i" % date.dayofyear, "*")
            data_folder = os.path.join(os.getcwd(), "ENV")
            path = os.path.join(data_folder, path)
            logging.debug(path)
            for file in glob.glob(path):
                traces.append(read(file)[0])

        st = Stream(traces)
        stations = np.unique([tr.stats.station for tr in st])
        for sta1, sta2 in itertools.combinations(stations,2):
            st1 = st.select(station=sta1)
            st1.merge(method=1, fill_value="interpolate")

            st2 = st.select(station=sta2)
            st2.merge(method=1, fill_value="interpolate")

            max_start = max([tr.stats.starttime for tr in st1],[tr.stats.starttime for tr in st2])[0]
            min_end = min([tr.stats.endtime for tr in st1],
                            [tr.stats.endtime for tr in st2])[0]

            st1 = st1.trim(max_start, min_end)
            st2 = st2.trim(max_start, min_end)

            stR = st1[0].copy()
            stR.data = st1[0].data / st2[0].data
            stR.stats.station = "RATIO"
            pair = "%s_%s" % (sta1, sta2)
            env_output_dir = os.path.join(os.getcwd(), 'RATIO', pair)
            if not os.path.isdir(env_output_dir):
                os.makedirs(env_output_dir)
            stR.write(os.path.join(env_output_dir, str(date.date()) + '.MSEED'),
                        format="MSEED", encoding="FLOAT32")
            del stR
        for file in group.index:
            filedb.loc[file, "process_step"] = "P"
    filedb.to_csv(filelist)

@click.command()
@click.pass_context
def plot(ctx):
    config = ctx.obj["config"]["msara"]
    stations = config["stations"]
    smoothing = config["smoothing"]
    if type(smoothing) in [float, int, str]:
        smoothing = [smoothing,]
    print(smoothing)
    # plt.figure()
    # for sta1, sta2 in itertools.combinations(stations, 2):
    #     pair = "%s_%s" % (sta1, sta2)
    #     path = os.path.join(os.getcwd(), 'RATIO', pair, "*")
    #     st = read(path, format="MSEED")
    #     st.merge(method=1, fill_value="interpolate")
    #     tr = st[0]
    #     times = pd.date_range(tr.stats.starttime.datetime, tr.stats.endtime.datetime, freq="%ims"%(tr.stats.delta*1e3))
    #     plt.plot(times, tr.data, label=pair)
    # plt.legend()
    # plt.title("Raw Ratios")
    # plt.grid(True)

    for smooth in smoothing:
        plt.figure()
        for sta1, sta2 in itertools.combinations(stations, 2):
            pair = "%s_%s" % (sta1, sta2)
            path = os.path.join(os.getcwd(), 'RATIO', pair, "*")
            st = read(path, format="MSEED")
            st.merge(method=1, fill_value="interpolate")
            tr = st[0]
            sps_ratio = int(int(smooth) / tr.stats.delta)

            # tr.data = pd.rolling_median(tr.data, window=sps_ratio)
            idx = pd.date_range(tr.stats.starttime.datetime,
                                  tr.stats.endtime.datetime,
                                  freq="%ims" % (tr.stats.delta * 1e3))
            df = pd.Series(tr.data, index=idx)
            rs = df.resample("%iS" % int(smooth))
            xmedian = rs.median()
            xmean = rs.mean()
            xmean.to_csv("%s_%s_mean.csv" % (pair, smooth))
            xmedian.to_csv("%s_%s_median.csv" % (pair, smooth))
            plt.plot(xmedian.index, xmedian, label="%s - median" % pair)
            plt.plot(xmean.index, xmean, label="%s - mean" % pair)

        plt.legend()
        plt.title("Smoothed ratios (smoothing=%is)"%int(smooth))
        plt.grid(True)
    plt.show()

cli.add_command(init)
cli.add_command(scan_archive)
cli.add_command(envelope)
cli.add_command(ratio)
cli.add_command(plot)

def main():
    cli(obj={})


from datetime import datetime
import boto3
import cfnresponse
import json

client = boto3.client("cloudwatch")

def log(stack, metric, value):
    # Do it for the stack
    client.put_metric_data(
        Namespace="CloudFormation",
        MetricData=[
            {
                "MetricName": metric,
                "Unit": "Count",
                "Value": value,
                "Timestamp": datetime.now(),
                "Dimensions": [
                    {
                        "Name": "By Stack Name",
                        "Value": stack,
                    },
                ],
            },
        ],
    )

    client.put_metric_data(
        Namespace="CloudFormation",
        MetricData=[
            {
                "MetricName": metric,
                "Unit": "Count",
                "Value": value,
                "Timestamp": datetime.now(),
            },
        ],
    )

def handler(event, context):
    print("Received request:", json.dumps(event, indent=4))

    action = event["RequestType"]

    stack = event["ResourceProperties"]["StackName"]
    resources = int(event["ResourceProperties"]["ResourceCount"])

    try:
        log(stack, action, 1)

        if action == "Create":
            log(stack, "ResourceCount", resources)

        cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, "{} metrics".format(stack))
    except Exception as e:
        cfnresponse.send(event, context, cfnresponse.FAILED, {
            "Data": str(e),
        }, "{} metrics".format(stack))


def deserialize(d, **kw):
    vert_section, edge_section = d.split('\n#\n', 1)

    verts = [line.split(None, 1) for line in vert_section.split('\n') if line]

    counts = {}
    edges = []
    for line in edge_section.split('\n'):
        if line:
            fm, to, label = (line.split(None, 2) + [''])[:3]
            c = counts.get(label, 0)
            counts[label] = c + 1
            id = "%s-%d" % (label, c) if c else label
            edges.append((id, label, fm, to))

    return verts, edges


def fun_similar(strs):
    dict1 = count(strs[0])
    dict2 = count(strs[1])
    if dict1 == dict2:
        return True
    else:
        return False

def count(str):
    dic = {}
    for istr in str:
        if istr in dic:
            dic[istr] += 1
        else:
            dic[istr] = 1
    return dic

if __name__ == "__main__":
    while True:
        strs = input("请输入2个英文单词，以空格分隔: ").split()
        if fun_similar(strs):
            print("单词" + strs[0] + "和单词" + strs[1] + "是相似词！")
        else:
            print("单词" + strs[0] + "和单词" + strs[1] + "不是相似词！")


from jobs.search_transcript_job import SearchTranscriptJob
from jobs.clean_video_job import CleanVideoJob
from extensions import aai
from jobs.transcribe_youtube_job import TranscribeYoutubeJob
from jobs.transcribe_upload_job import TranscribeUploadJob
from jobs.yt_download_job import YT_DownloadJob
from werkzeug.utils import secure_filename
import events
from os import getenv
from flask import render_template, request, redirect, url_for, Blueprint, flash
from jobs.filtering_job import FilteringJob
from jobs import rq
import time

http = Blueprint("http", __name__)
ws   = Blueprint("ws", __name__)

default_q = rq.get_queue()

@http.route('/')
def index():
    return render_template("index.html")

@http.route('/webhook', methods=["POST"])
def webhook():
    print('Webhook Received')
    request_json = request.json

    filename = request.args.get("vid_path")

    trans_id = request_json['transcript_id']
    words = request.args.get('words')

    SearchTranscriptJob.queue(trans_id, words, filename)

    return 'Webhook notification received', 200

@http.route('/upload', methods=['POST', 'GET'])
def upload():
    if request.method == 'POST':
        url = request.form['url']
        words = request.form['words']

        fname = str(int(time.time())) + '.mp4'

        if url == "":
            file = request.files['file']
            if file.filename == "":
                flash("No video given")
                return redirect(request.url)
            file.save(secure_filename('/data/' + file.filename))
            job = TranscribeUploadJob.queue('/data/' + file.filename, words)
        else:
            YT_DownloadJob.queue(url, fname)
            job = TranscribeYoutubeJob.queue(url, words, "/data/" + fname)

        # make call to service to get the video
        # job = FilteringJob.queue(url, events.filter_success, meta={"upload_url": url})
        return redirect(url_for("http.show_uploads", job_id=job.id))
    else:
        return render_template("upload/new.html")

@http.route('/uploads/<job_id>')
def show_uploads(job_id):
    job = default_q.fetch_job(job_id)
    if job != None:
        return render_template("upload/show.html", job_id=job_id, job_status=job.get_status(), base_url=getenv("BASE_URL"))
    return "no job", 200


import numpy as np
import rospy
import requests
import time

from sensor_msgs.msg import Image
from nav2d_operator.msg import cmd

API_URL = "http://10.25.111.72:5000/classify"
PARAMS = {}

class Identify():
    """
        Class that will subscribe to the camera topic and will send a request trough an API running
        on a LocalHost to process the image and determine if there is a Tomato or not.
    """
    def __init__(self):
        rospy.on_shutdown(self.stop)
        self.mode = 0
        self.nav2d_publisher = rospy.Publisher('cmd', cmd, queue_size=2)
        self.subscriber = rospy.Subscriber("/usb_cam/image_raw", Image, self.process)
        rate = rospy.Rate(1)
        self.stop_rate = rospy.Rate(1)

        while not rospy.is_shutdown():
            self.nav2d_publisher.publish(cmd(Velocity=0.8, Turn=0.0, Mode=self.mode))
            rate.sleep()

    def process(self, image):
        self.subscriber.unregister()
        img = np.frombuffer(image.data, dtype=np.uint8).reshape(image.height, image.width, -1)
        # Send it to the API here
        PARAMS['Image'] = img.tolist()
        response = requests.get(API_URL, json=PARAMS)
        print(response.text)
        is_tomato = response.text == "Tomato"
        self.mode = 2 if is_tomato else 0
        self.nav2d_publisher.publish(cmd(Velocity=0.8, Turn=0.0, Mode=self.mode))
        self.subscriber = rospy.Subscriber("/usb_cam/image_raw", Image, self.process)

    def stop(self):
        self.nav2d_publisher.publish(cmd(Velocity=0.8, Turn=0.0, Mode=2))
        self.stop_rate.sleep()
        rospy.loginfo("Node Stopped")

if __name__ == "__main__":
    rospy.init_node("tomato_identification", anonymous=True)
    try:
        Identify()
    except Exception as e:
        rospy.logfatal("Couldn't Initialize the Tomato Identification Class")
        rospy.logfatal(e)


import cPickle
from glob import glob
from itertools import permutations
import numpy as np
from numpy import log
from os import path
import pandas as pd
import statsmodels.formula.api as smf
from utils import scale

# **********Processing*********************************
structuredata = []
valuedata = []
taskdata = {}
winning = {}
communities = {0: [0,1,2,3,4],
              2: [5],
              1: [6,7,8,9,10]}
node_lookup = {i:k for k,v in communities.items() for i in v }
f = lambda x: [k for k,v in communities.items() if x in v][0]

datafiles = sorted(glob('Data/RawData/*pkl'))
for exclude in ['/CH', '/GL', '/YH']:
    datafiles = [i for i in datafiles if exclude not in i]

for filey in datafiles:
    subj = path.basename(filey).split('_')[0]
    data = cPickle.load(open(filey,'rb'))
    subj_structuredata = pd.DataFrame(data['structuredata'])
    subj_structuredata.loc[:,'subjid'] = subj
    subj_valuedata = pd.DataFrame(data['valuedata'])
    subj_valuedata.loc[:,'subjid'] = subj

    # add subject specific variables
    # for structuredata
    subj_structuredata.loc[:,'correct_shift'] = subj_structuredata.correct.shift()
    subj_structuredata.loc[:,'congruent_rot'] = subj_structuredata.rotation==subj_structuredata.rotation.shift()
    # add community columns
    subj_structuredata.loc[:,'community'] = subj_structuredata.stim_index.apply(f)
    subj_structuredata.loc[1:,'community_cross'] = (subj_structuredata.community.diff()!=0)[1:]
    # calculate steps within the community
    steps_in_community = []
    steps = 0
    for cross in subj_structuredata.community_cross:
        if cross==False:
            steps+=1
        else:
            steps=1
        steps_in_community.append(steps)
    subj_structuredata.loc[:,'steps_within_community'] = steps_in_community
    # trials since last seen
    steps_since_seen = []
    last_seen = {}
    # ...for value data
    for i, stim in enumerate(subj_structuredata.stim_index):
        if stim in last_seen.keys():
            steps_since = i-last_seen[stim]
        else:
            steps_since = 0
        last_seen[stim] = i
        steps_since_seen.append(steps_since)
    subj_structuredata.loc[:,'steps_since_seen'] = steps_since_seen

    # for value data
    # regress out accuracy
    structure_acc = subj_structuredata[200:].groupby('stim_index').correct.mean()
    stim_freq = subj_structuredata.groupby('stim_index').rt.count()
    subj_valuedata.loc[:,'stim_acc'] = subj_valuedata.stim_index.apply(lambda x: structure_acc[x])
    subj_valuedata.loc[:,'stim_freq'] = subj_valuedata.stim_index.apply(lambda x: stim_freq[x])
    rs = smf.ols('rating ~ stim_acc', subj_valuedata).fit()
    subj_valuedata.loc[:,'reg_rating'] = rs.resid
    # add stim repetitions
    n_stim = len(np.unique(subj_valuedata.stim_index))
    n_repeats = (len(subj_valuedata))//n_stim
    repeat_array = np.hstack([[i]*n_stim for i in range(n_repeats)])
    subj_valuedata.loc[:, 'stim_repetition'] = repeat_array
    # node values
    node_values = data['taskdata']['node_values']
    value_labels = subj_valuedata.stim_index. \
                    apply(lambda x: node_values.get(x,np.nan))
    subj_valuedata.loc[:,'labeled_value'] = value_labels
    # add community
    subj_valuedata.loc[:,'community'] = subj_valuedata.stim_index.apply(f)

    # concatenate with group
    structuredata.append(subj_structuredata)
    valuedata.append(subj_valuedata)
    taskdata[subj] = data['taskdata']
    winning[subj] = data['total_win']

structuredata = pd.concat(structuredata, axis=0)
valuedata = pd.concat(valuedata, axis=0)

# Remove subjects who had no variability on their ratings
rating_variability = valuedata.groupby(['subjid']).rating.std()
drop_subjects = list(rating_variability.index[rating_variability<.1])
valuedata = valuedata.query('subjid not in %s' % drop_subjects)
structuredata = structuredata.query('subjid not in %s' % drop_subjects)

# **********Analysis*********************************
results = {}

regress_data = structuredata.query('rt!=-1').dropna(subset=['correct_shift'])
#regress_data = structuredata.query('rt!=-1 and correct==True').dropna(subset=['correct_shift'])

# structure reaction time data
models = {}
for DV in ['rt']:
    rs = smf.mixedlm("%s ~ community_cross + steps_since_seen" % DV,
                     data=regress_data, groups='subjid',
                     re_formula="~community_cross")
    # larger model
    rs_full = smf.mixedlm("%s ~ community_cross + steps_since_seen \
                          + correct_shift + C(rotation) + congruent_rot" % DV,
                     data=regress_data, groups=regress_data["subjid"],
                     re_formula="~community_cross")
    DV_name = DV[-7:]
    models[DV_name] = rs
    models[DV_name + '_full'] = rs_full

# individual difference variables
rt_raneffects = models['rt_full'].fit().random_effects
structure_coefficients = {k:v.loc['community_cross[T.True]'] for k,v in rt_raneffects.items()}
results['structure_coefficients'] = structure_coefficients

# value effect as a function of community
valuedata.loc[:,'structure_coefficient'] = valuedata.subjid.apply(lambda x: structure_coefficients[x])
value_rs = smf.mixedlm('rating ~ C(community)+ stim_acc + C(stim_file)',
                 data = valuedata.query('community!=2'),
                 groups='subjid', re_formula="~C(community)").fit()

interaction_value_rs = smf.mixedlm('rating ~ C(community)*structure_coefficient + stim_acc + C(stim_file)',
                 data = valuedata.query('community!=2'),
                 groups='subjid').fit()

value_raneffects = value_rs.random_effects
value_coefficients = {k:v.loc['C(community)[T.1]'] for k,v in value_raneffects.items()}
results['value_coefficients'] = value_coefficients

# calculate standard deviation of clusters within community
group_stats = {}
for subj in valuedata.subjid.unique():
    subj_stats = {}
    permutes = list(permutations(range(7),7))
    ratings = valuedata.query('subjid == "%s"' % subj) \
                        .groupby(['subjid','stim_index','community']) \
                        .mean().loc[:,['rating','reg_rating']].reset_index()

    std_community = (ratings.query('community != 2').query('community != 2') \
                    .groupby(['subjid','community']) \
                    .reg_rating.std()).groupby('community').mean().mean()
    subj_stats['community_std'] = std_community
    permute_stds = []
    for permute in permutes:
        n = len(ratings.subjid.unique())
        permute_pop = ratings.copy().reset_index()
        permute_pop.loc[:, 'community'] = permute_pop.community.iloc[list(permute)].reset_index()
        permute_std = (permute_pop.groupby(['subjid','community']).reg_rating.std()).groupby('community').mean().mean()
        permute_stds.append(permute_std)
    subj_stats['permuted_array']  = permute_stds
    subj_stats['p_val'] = np.mean([std_community>=i for i in permute_stds])
    group_stats[subj] = subj_stats

results['clustering'] = {k:v['p_val'] for k,v in group_stats.items()}
results['example_clustering_dist'] = group_stats.values()[0]['permuted_array']

# structure data win stay/lose switch (or just change in attention following error?)
for rot in [0,90]:
    structuredata[(structuredata.rotation.shift()==rot)
                    & (structuredata.rt!=-1)] \
                .groupby(['correct_shift','rotation']).correct.mean()

    structuredata[(structuredata.rotation.shift()==rot)
                & (structuredata.rt!=-1)] \
            .groupby(['correct_shift','rotation']).rt.median()

# save data
save_loc = path.join('Data','ProcessedData')
structuredata.to_csv(path.join(save_loc, 'structuredata.csv'))
valuedata.to_csv(path.join(save_loc, 'valuedata.csv'))
cPickle.dump(taskdata,open(path.join(save_loc,'taskdata.pkl'),'wb'))
cPickle.dump(results, open(path.join('Analysis_Results','analysis.pkl'), 'wb'))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

from werkzeug.contrib.atom import AtomFeed
from flask import request, Blueprint, json

from zhihudaily.cache import cache
from zhihudaily.configs import Config
from zhihudaily.models import Zhihudaily
from zhihudaily.utils import Date
from zhihudaily.crawler import Crawler
from zhihudaily._compat import urljoin

# This is the feeds blueprint, but it is deprecated now.
feeds = Blueprint('feeds', __name__, template_folder='templates')
redis_server = Config.redis_server

@feeds.route('/feeds')
@cache.cached(timeout=1200)
def generate_feed():
    """Code snippet from https://flask.pocoo.org/snippets/10/"""
    day = Date()
    feed = AtomFeed('Zhihudaily', feed_url=request.url, url=request.url_root)
    news = Zhihudaily.get(Zhihudaily.date == int(day.today))

    articles = json.loads(news.json_news)
    for article in articles:
        if redis_server.get(article['url']):
            body = redis_server.get(article['url']).decode('utf-8')
        else:
            body = Crawler().send_request(article['url']).json()['body']
            redis_server.setex(article['url'], (60*60*24), body)
        feed.add(article['title'], body,
                 content_type='html',
                 author='zhihudaily',
                 url=urljoin(request.url_root, article['url']),
                 updated=day.now)
    return feed.get_response()


from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
import torch
import os
from torch import Tensor
import torch.nn as nn
from fairseq import options, utils, checkpoint_utils
from fairseq.dataclass import ChoiceEnum, FairseqDataclass
from fairseq.models import (
    transformer,
    FairseqLanguageModel,
    register_model,
    register_model_architecture,
    FairseqEncoder, FairseqIncrementalDecoder,
    BaseFairseqModel,FairseqEncoderDecoderModel
)
from fairseq.data import Dictionary
from fairseq.models.transformer import (
    Embedding, TransformerDecoder,
    TransformerEncoder,
    TransformerModel
)
from fairseq.models.transformer import base_architecture as transformer_architecture
from fairseq.modules import (
    FairseqDropout,
    LayerNorm,Fp32LayerNorm,
    LayerDropModuleList,
    TransformerEncoderLayer,
    TransformerDecoderLayer
)
from fairseq.modules.checkpoint_activations import checkpoint_wrapper
from fairseq.modules.quant_noise import quant_noise as apply_quant_noise_
from omegaconf import II
from rain.layers import (
    get_available_convs, get_conv,
    PositionalEmbedding,AudioTransformerEncoder,
    UnidirectAudioTransformerEncoder, UnidirectTransoformerEncoder,
    WaitkDecoder
)
from fairseq.dataclass.utils import (
    convert_namespace_to_omegaconf,
    gen_parser_from_dataclass,
)
from .speech_transformer import (
    SpeechTransformerModelConfig, AudioTransformer,audio_transformer_s,randpos_audio_transformer_s
)
from .posemb_transformer import RandposTransformer,randpos_transformer_small

@dataclass
class OnlineTextTransformerConfig(SpeechTransformerModelConfig):
    max_source_positions: Optional[int] = II("task.max_text_positions")
    max_target_positions:Optional[int] = II("task.max_text_positions")
    main_context:int = field(
        default= 1, metadata={"help":"main context frame"}
    )
    right_context :int = field(
        default= 0, metadata={"help":"right context frame"}
    )
    decoder_delay_blocks :int = field(
        default= 4, metadata={"help":"wait-k delay"}
    )
    decoder_blocks_per_token :int = field(
        default= 1, metadata={"help":"wait k token blocks"}
    )
    online_type:ChoiceEnum(["offline", "waitk"])="offline"

@register_model("online_text_transformer", dataclass = OnlineTextTransformerConfig)
class OnlineTextTransformer(TransformerModel):

    @classmethod
    def add_args(cls, parser):
        dc = getattr(cls, "__dataclass", None)
        if dc is not None:
            gen_parser_from_dataclass(parser, dc(), delete_default=True)

    @classmethod
    def build_encoder(cls, args, src_dict, embed_tokens):
        model = UnidirectTransoformerEncoder(args, src_dict, embed_tokens)
        embed_dim= embed_tokens.embedding_dim
        if model.embed_positions is not None and args.rand_pos_encoder >0:
            model.embed_positions= PositionalEmbedding(
                model.max_source_positions,
                embed_dim,
                model.padding_idx,
                rand_max = args.rand_pos_encoder,
                learned=args.decoder_learned_pos,
            )
        return model

    @classmethod
    def build_decoder(cls, args, tgt_dict, embed_tokens):
        if args.online_type == "offline":
            model = transformer.TransformerDecoder(
                args,
                tgt_dict,
                embed_tokens,
                no_encoder_attn=False
            )
        elif args.online_type == "waitk":
            model= WaitkDecoder(
                args,
                tgt_dict,
                embed_tokens
            )
        else:
            raise NotImplementedError(f"unknown online type {args.online_type}")
        if model.embed_positions is not None and args.rand_pos_decoder >0:
            model.embed_positions= PositionalEmbedding(
                model.max_target_positions,
                model.embed_dim,
                model.padding_idx,
                rand_max = args.rand_pos_decoder,
                learned=args.decoder_learned_pos,
            )
        return model

@dataclass
class OnlineAudioTransformerConfig(OnlineTextTransformerConfig):
    max_source_positions: Optional[int] = II("task.max_audio_positions")
    max_target_positions:Optional[int] = II("task.max_text_positions")
    main_context:int = field(
        default= 16, metadata={"help":"main context frame"}
    )
    right_context :int = field(
        default= 16, metadata={"help":"right context frame"}
    )
    decoder_delay_blocks :int = field(
        default= 16, metadata={"help":"wait-k delay"}
    )
    decoder_blocks_per_token :int = field(
        default= 4, metadata={"help":"wait k token blocks"}
    )
    online_type:ChoiceEnum(["offline", "waitk"])="offline"

@register_model("online_audio_transformer", dataclass = OnlineAudioTransformerConfig)
class OnlineAudioTransformer(AudioTransformer):
    @classmethod
    def build_encoder(cls, args):
        return UnidirectAudioTransformerEncoder(args)

    @classmethod
    def add_args(cls, parser):
        dc = getattr(cls, "__dataclass", None)
        if dc is not None:
            gen_parser_from_dataclass(parser, dc(), delete_default=True)

    @classmethod
    def build_decoder(cls, args, tgt_dict, embed_tokens):
        if args.online_type == "offline":
            model = transformer.TransformerDecoder(
                args,
                tgt_dict,
                embed_tokens,
                no_encoder_attn=False
            )
        elif args.online_type == "waitk":
            model= WaitkDecoder(
                args,
                tgt_dict,
                embed_tokens
            )
        else:
            raise NotImplementedError(f"unknown online type {args.online_type}")
        if model.embed_positions is not None and args.rand_pos_decoder >0:
            model.embed_positions= PositionalEmbedding(
                model.max_target_positions,
                model.embed_dim,
                model.padding_idx,
                rand_max = args.rand_pos_decoder,
                learned=args.decoder_learned_pos,
            )
        return model

    def forward(
        self,
        fbank,
        fbk_lengths,
        prev_source=None,
        prev_target= None
    ):

        encoder_out = self.encoder(fbank, fbk_lengths)

        prev_tokens= prev_source if self.task_type=="asr" else prev_target
        assert prev_tokens is not None, f"no prev_tokens for {self.task_type}"
        decoder_out = self.decoder(
            prev_tokens,
            encoder_out=encoder_out,
        )
        logits= decoder_out[0]

        return decoder_out

@register_model_architecture("online_audio_transformer", "online_audio_transformer_offline")
def offline_audio(args):
    args.rand_pos_encoder = getattr(args, "rand_pos_encoder", 300)
    args.rand_pos_decoder= getattr(args, "rand_pos_decoder", 30)
    args.main_context=getattr(args,"main_context",16)
    args.right_context=getattr(args,"right_context",16)
    args.decoder_delay_blocks=getattr(args,"decoder_delay_blocks",32)
    args.decoder_blocks_per_token= getattr(args,"decoder_blocks_per_token",8)
    args.online_type=getattr(args,"online_type","offline")
    randpos_audio_transformer_s(args)

@register_model_architecture("online_audio_transformer", "online_audio_transformer_waitk")
def waitk_audio(args):
    args.online_type=getattr(args,"online_type","waitk")
    offline_audio(args)

@register_model_architecture("online_text_transformer", "online_text_transformer_offline")
def offline_text(args):
    args.rand_pos_encoder = getattr(args, "rand_pos_encoder", 30)
    args.rand_pos_decoder= getattr(args, "rand_pos_decoder", 30)
    args.main_context=getattr(args,"main_context",1)
    args.right_context=getattr(args,"right_context",0)
    args.decoder_delay_blocks=getattr(args,"decoder_delay_blocks", 4)
    args.decoder_blocks_per_token= getattr(args,"decoder_blocks_per_token",1)
    args.online_type=getattr(args,"online_type","offline")
    randpos_transformer_small(args)

@register_model_architecture("online_text_transformer", "online_text_transformer_waitk")
def waitk_text(args):
    args.online_type=getattr(args,"online_type","waitk")
    offline_text(args)


# -*- coding: utf-8 -*-

"""
Title: Carshop class of a car rental system

Authour: Sijia Hou <Sijia at python.org>

Created: 10-Jan-2021
"""
import pandas as pd

class carshop:
    """
    use df_record to record car rental information
    df_record will be used to verfity whether this customer rent a car
    use stock to update the inventory of the car
    """
    def __init__(self,hatchbackNum,sedanNum,suvNum):
        self.df_record=pd.DataFrame(columns=['customer_id', 'cartype', 'days'])
        self.stock={'hatchback':{'in_stock':hatchbackNum},
                    'sedan':{'in_stock':sedanNum},
                    'suv':{'in_stock':suvNum}
                    }

    #List car information sheets to non-VIP customers through df_show
    def show(self):
        df_show= pd.DataFrame([['hatchback',self.stock.get('hatchback').get('in_stock'),'£30','£25'],
                            ['sedan',self.stock.get('sedan').get('in_stock'),'£50','£40'],
                            ['suv',self.stock.get('suv').get('in_stock'),'£100','£90']],
                            columns=['cartype','stock','daily price for less than 7 days',
                                     'daily price for longer period'])
        print(df_show)

    #List car information sheets to VIP customers through df_show
    def show_VIP(self):
        df_showVIP=pd.DataFrame([['hatchback',self.stock.get('hatchback').get('in_stock'),'£20'],
                            ['sedan',self.stock.get('sedan').get('in_stock'),'£35'],
                            ['suv',self.stock.get('suv').get('in_stock'),'£80']],
                            columns=['cartype','stock','Daily price'])
        print(df_showVIP)

    # This method implements the car rental operation
    def rent_car(self,customer_id,rent_cartype,day):
        day=int(day)
        c=str(rent_cartype)
        typelist=['hatchback','sedan','suv']
        d1={'hatchback':'£30','sedan':'£50','suv':'£100'}
        d2={'hatchback':'£25','sedan':'£40','suv':'£90'}

        #Get the customer id in df_record and if the customer has a record then the car cannot be rented
        customer_columns=self.df_record['customer_id'].values.tolist()

        if customer_id in customer_columns:
            print("Sorry,each account can only rent one car at most, so you cannot rent another car"+"\n")
            return False

        elif rent_cartype in typelist:
            #Determine whether the car is still in stock
            if self.stock.get(rent_cartype).get('in_stock')>0:

                #Car has been rented and the inventory of the corresponding model is reduced by one
                self.stock.get(rent_cartype).update({'in_stock':self.stock.get(rent_cartype).get('in_stock')-1})

                #Create a rental car record and add it to df_record
                new=pd.Series({'customer_id':customer_id,'cartype':rent_cartype,'days':day})
                self.df_record=self.df_record.append(new,ignore_index=True)

                if day>0 and day<7:
                    print("You have rented a "+rent_cartype+"car for",day,"days. You will be charged",
                          d1[c],"per day, We hope that you enjoy our service"+"\n")
                    return True
                elif day>=7:
                    print("You have rented a "+rent_cartype+"car for",day,"days. You will be charged",
                          d2[c],"per day, We hope that you enjoy our service"+"\n")
                    return True
                else:
                    return False
            else:
                print("Sorry,this type of car is not in stock now. Please change your choice."+"\n")
                return False
        else:
            d="Unlisted car type."
            return d

    def rent_car_VIP(self,customer_id,rent_cartype,day):
        c=rent_cartype
        typelist=['hatchback','sedan','suv']
        d3={'hatchback':'£20','sedan':'£35','suv':'£80'}

        #List the customer id in df_record and if the customer has a record then the car cannot be rented
        customer_columns=self.df_record['customer_id'].values.tolist()
        if customer_id in customer_columns:
            print("Sorry,each account can only rent one car at most, so you cannot rent another car"+"\n")
            return False

        elif rent_cartype in typelist:

            if self.stock.get(rent_cartype).get('in_stock')>0:
                #Car has been rented and the inventory of the corresponding model is reduced by one
                self.stock.get(rent_cartype).update({'in_stock':self.stock.get(rent_cartype).get('in_stock')-1})

                #Create a rental car record and add it to df_record
                new=pd.Series({'customer_id':customer_id,'cartype':rent_cartype,'days':day})
                self.df_record=self.df_record.append(new,ignore_index=True)

                print("You have rented a "+rent_cartype+"car for",day,"days. You will be charged",
                      d3[c],"per day, We hope that you enjoy our service"+"\n")
                return True

            else:
                print("Sorry,this type of car is not in stock now. Please change your choice."+"\n")
                return False

        else:
            d="Unlisted car type."
            return d

    # This method implements the return operation
    def return_car(self,customer_id):

        #List the customer id in df_record and verfity whether the customer rented a car
        customer_columns=self.df_record['customer_id'].values.tolist()
        if customer_id in customer_columns:

            a=customer_columns.index(customer_id)   #index the row number of customer's row

            #Use row number and column number to index the car type and rent time
            return_type=self.df_record.iloc[int(a),1]
            day=self.df_record.iloc[int(a),2]

            #Car has been returned and the inventory of the corresponding model is increased by one
            self.stock.get(return_type).update({"in_stock":self.stock.get(return_type).get("in_stock")+1})
            return return_type,int(day)

        else:
            print("Sorry,you have no car rental record."+"\n")
            return False


#!/usr/bin/env python
# coding=utf-8
from geyser.item import Item
from geyser.resources_go.types import go_types

class StaticDict( Item ):
    def __init__( self, data, name, seq, default="" ):
        super( StaticDict, self ).__init__( name, data )
        self.seq = seq
        self.default = default

    def getNative( self, go_compiler ):
        return
        gofile = go_compiler.srv_api_package.addGoFile( "{service}.{module}.dict".format( service = go_compiler.service.Name(),
                                                                                module = self.parent.Name().lower() ) )
        self.parent.extensions_file = gofile

        list_name = "{}_list".format( self.Name() )

        seq_list = [ ]
        for i in self.seq:
            seq_list.append( '"{}"'.format( i ) )
        value = "[]string<@{}@>".format( ", ".join( seq_list ) )

        gofile.addVar(list_name, value, go_types.text_array)

        # get_id
        func_get_id = gofile.addFunction( "Get{}{}Id".format( self.parent.Name().capitalize(),
                                                              self.Name().capitalize() ) )
        func_get_id.addArgs( "name string" )
        func_get_id.addResults( "int" )
        body_id = '''    for id, stname := range {list_name} <@
        if stname == name <@
            return id
        @>
    @>
    return -1'''.format( list_name = list_name )
        func_get_id.addBody( body_id )

        # get_name
        func_get_name = gofile.addFunction( "Get{}{}Name".format( self.parent.Name().capitalize(),
                                                                  self.Name().capitalize() ) )
        func_get_name.addArgs( "id int" )
        func_get_name.addResults( "string" )
        body_name = '''    if id < 0 <@
        return "{default}"
    @>
    if id < len({list_name}) <@
    return {list_name}[id]
    @>
    return ""'''.format( list_name = list_name,
                         default = self.default )
        func_get_name.addBody( body_name )


character = input("Enter the desired type of character required : ")
rows = int(input("Enter the number of rows required : "))

# Assigning the space number
space = 30

for i in range(1, rows+1, 1):
    # This loop handles printing the space
    for k in range(space):
        print(" ", end="")

    # This loop handles printing the character
    number = i
    count = -1
    for z in range(number):
        count = count + 2

    for j in range(count):
        print(character, end="")

    print()
    space -=1

print()


from pathlib import Path
from datetime import datetime
import csv

class Inventory():
    def __init__(self):
        self.inventory_file = Path(
            __file__).absolute().parent / "csv-data/inventory.csv"

    def last_current_id(self):
        with open(self.inventory_file, "r") as csv_inventory:
            dictreader = csv.DictReader(csv_inventory)
            last_line = ""
            for line in dictreader:
                last_line = line
            last_id = int(last_line["id"])
            return last_id

    def show(self, report_date="", report_second_date=""):
        with open(self.inventory_file, "r") as csv_inventory:
            dictreader = csv.DictReader(csv_inventory)
            list_of_dics = list(dictreader)
            if report_second_date and report_date:
                format = "%Y-%m-%d"
                first_date = datetime.strptime(report_date, format)
                second_date = datetime.strptime(report_second_date, format)
                filtered_list = []
                for dic in list_of_dics:
                    current_dic_date = datetime.strptime(dic["buy_date"], format)
                    if current_dic_date <= second_date and current_dic_date >= first_date:
                        filtered_list.append(dic)
                return filtered_list
            elif report_date:
                filtered_list = []
                for dic in list_of_dics:
                    if dic["buy_date"] == report_date:
                        filtered_list.append(dic)
                return filtered_list
            return list_of_dics

    def check(self, item_dict):
        item_id = str(item_dict["id"])
        item_quantity = int(item_dict["quantity"])
        with open(self.inventory_file, "r") as csv_inventory:
            dictreader = csv.DictReader(csv_inventory)
            for line in dictreader:
                if line["id"] == item_id:
                    if int(line["quantity"]) >= item_quantity:
                        return {"stock": True,
                                "product": line["product"],
                                "buy_price": line["buy_price"],
                                "exp_date": line["exp_date"]
                                }
        return {"stock": False}

    def add(self, add_dict):
        with open(self.inventory_file, "a", newline="") as csv_inventory:
            fieldnames = ["id", "product", "quantity",
                          "buy_price", "exp_date", "buy_date"]
            dictwriter = csv.DictWriter(csv_inventory, fieldnames=fieldnames)
            add_dict["id"] = str(self.last_current_id()+1)
            dictwriter.writerow(add_dict)

    def remove(self, remove_dict):
        item_id = str(remove_dict["id"])
        item_quantity = int(remove_dict["quantity"])
        with open(self.inventory_file, "r+") as csv_inventory:
            dictreader = csv.DictReader(csv_inventory)

            dicts = []
            for dic in dictreader:
                if dic["id"] == item_id:
                    dic["quantity"] = int(dic["quantity"]) - item_quantity
                if int(dic["quantity"]) > 0:
                    dicts.append(dic)
                else:
                    pass

            with open(self.inventory_file, "w", newline="") as csv_inventory_write:
                fieldnames = ["id", "product", "quantity",
                              "buy_price", "exp_date", "buy_date"]
                dictwriter = csv.DictWriter(
                    csv_inventory_write, fieldnames=fieldnames)
                dictwriter.writeheader()
                dictwriter.writerows(dicts)


#i created this
from django.http import HttpResponse
from django.shortcuts import render
#def index(request):
    #return HttpResponse('''
hello Agnisha
Django coderAgnisha''')
def index(request):
    return render(request, 'index.html')
    #return HttpResponse("
home
")
#personal navigator
def ex1(request):
    s='''
Navigation bar

    video

    mail
'''
    return HttpResponse(s)
def analyse(request):
    djtext=request.POST.get('text','default')
    remove_punc=request.POST.get('analyse','default')
    capfirst=request.POST.get('capfirst','default')
    unique=request.POST.get('unique','default')
    print(djtext)
    print(remove_punc)
    print(capfirst)
    #return HttpResponse("
remove punctuation
")
    #analyse=djtext
    if remove_punc=="on":
        punctuation='''"";:?''></.,{}[]|\+=!@#$%^&*()'''
        analyse=""
        for char in djtext:
            if char not in punctuation:
                analyse=analyse + char
        params={'purpose':"removed punctuation",'analysed_text':analyse}
        djtext=analyse
        #return render(request, 'analyse.html', params)
    if capfirst=="on":
        ana=""
        for i in djtext:
            ana=ana+ i.capitalize()
        params={'purpose':"capitalised",'analysed_text':ana}
        #return render(request, 'analyse.html',params)
        djtext=ana
    if unique=="on":
        def check_unique(str):
            for i in range(len(str)):
                for j in range(i + 1,len(str)):
                    if(str[i] == str[j]):
                        return False
                return True
        x=check_unique(djtext)
        params={'purpose':"whether it is unique or not",'analysed_text':x,"originaltext":djtext,"abc":"yes"}
        #return render(request, 'analyse.html', params)
        return render(request, 'analyse.html', params)
    if (remove_punc!="on" and capfirst!="on" and unique!="on"):
     return HttpResponse("error")
'''def cap_first(request):
    return HttpResponse("
capitalise first
")
def newlineremove(request):
    return HttpResponse("
remove newline
back")
def spaceremove(request):
    return HttpResponse("
spaceremove
")'''


"""This module defines the :class:`QueueFile` class."""

import queue
import io
import itertools

class QueueFile:
    """This class wraps a :class:`queue.Queue` object in the interface
    of a file-like object.

    The :class:`QueueFile` class makes redirecting input and output
    between different parts of a program easier than it would be using
    standard file objects, such as :class:`io.StringIO` or those
    created by :func:`pty.openpty`.
    :class:`QueueFile` is thread-safe, can be read from and written to
    at the same time, and its blocking behaviour can be easily
    controlled.
    (If there is nothing to read, reading from :class:`io.StringIO`
    never blocks, but reading from an object created with
    :func:`pty.openpty` always does.)

    Attributes:
        queue (queue.Queue):
            The wrapped :class:`~queue.Queue` object.
            Strings are saved in the queue one character at a time.

        block (bool):
            A flag indicating whether :meth:`read` and
            :meth:`readline` should block or not.
    """

    def __init__(self, block=False):
        """Intialize a new :class:`QueueFile` with :attr:`block` set
        to the value given as an argument.
        """
        self.queue = queue.Queue()
        self.block = block

    def write(self, string):
        """Write *string* to the queue and return the number of
        characters written.

        This method emulates :meth:`io.TextIOBase.write`.
        """
        # io.TextIOBase.write() doesn't accept keyword arguments,
        # so we can safely change the argument name from *s* to
        # *string* without breaking anything.
        for char in string:
            self.queue.put(char)
        return len(string)

    def read(self, size=-1):
        """Read and return a string of *size* characters from the queue.

        If the queue contains less than *size* characters or *size* is
        ``None`` or less than ``1``, all characters in the queue are
        read.

        This method emulates :meth:`io.TextIOBase.read`.
        """
        return self._read(size)

    def readline(self, size=-1):
        """Like :meth:`read`, but reading characters is stopped if a
        newline character is encountered. If this happens, the string
        returned includes that newline character.

        This method emulates :meth:`io.TextIOBase.readline`.
        """
        return self._read(size, endchar='\n')

    def _read(self, size=-1, endchar=None):
        """This method provides the functionality shared by
        read() and readline().
        """
        buffer = io.StringIO()

        if not size or size < 0:
            # An infinite generator that yields 0,1,2,...
            iterations = itertools.count()
        else:
            iterations = range(size)

        # Read a finite or an infinite number of characters.
        for _ in iterations:
            try:
                char = self.queue.get(block=self.block)
                buffer.write(char)
            except queue.Empty:
                break

            if endchar and char == endchar:
                break

        return buffer.getvalue()

    def flush(self):
        """This method does nothing, but its existence makes it
        possible to :func:`print` to this object with ``flush=True``
        without raising an :class:`AttributeError`."""


# While loops
# The for loop takes a collection of items and executes a block of code once for each item in the collection. A while loop runs as long as, or while, a certain condition is true.
# An example of counting to 5:
# We set a counter, currentNumber to 1 to have a start for our count
# currentNumber = 1

# # While currentNumber is less than or equal to 5 this will loop until it hits 5
# while currentNumber <= 5:
#     # Print the number
#     print(currentNumber)
#     # Increase currentNumber by 1 to increase the count. It's the same as currentNumber = currentNumber + 1
#     currentNumber += 1

# Using continue in a loop
# Rather than breaking out of a loop entirely without executing the rest of its code, you can use the continue statement to return to the beginning of the loop based on the result of a conditional test.

# We set an initial variable so we can enter into the while loop
currentNumber = 0

# While our variable is less than 10, loop
while currentNumber < 10:
    # Increment our variable
    currentNumber += 1

    # If our variable is divisible by 2 with no remainder (2, 4, 6, 8...)
    if currentNumber % 2 == 0:
        # continue with the rest of the program
        continue

    # Print the numbers that aren't divisible by 2 (1, 3, 5...)
    print(currentNumber)

# Avoiding infinite loops
x = 1
while x <= 5:
    print(x)
    # If we omit the line below, it'll cause the program to loop for ever as the counter will never reach 5. In those cases we have to abort in the terminal by using ctrl + c or you can just close it. Test properly and frequently in order to avoid those kinds of issues.
    x += 1

# 7-4. Pizza Toppings: Write a loop that prompts the user to enter a series of
# pizza toppings until they enter a 'quit' value. As they enter each topping,
# print a message saying you’ll add that topping to their pizza.

prompt = "\nPlease enter a topping to add to your pizza"
prompt += "\nEnter quit when you are done. "

while True:
    message = input(prompt)

    if message == "quit":
        break

    message = f"\nAdding {message.title()}"
    print(message)

# 7-5. Movie Tickets: A movie theater charges different ticket prices depending on a person’s age. If a person is under the age of 3, the ticket is free if they are between 3 and 12, the ticket is $10 and if they are over age 12, the ticket is $15. Write a loop in which you ask users their age, and then tell them the cost of their movie ticket.

prompt = "\nPlease enter your age."
prompt += "\nEnter quit to exit. "

while True:
    # Leaving age as string so we can quit the program
    age = input(prompt)

    if age == "quit":
        break

    # If input is no 'quit' then we convert it to an int so we can do comparisons
    age = int(age)
    # comparisons
    if age < 3:
        print("Your ticket is free")
    elif age >= 3 and age < 12:
        print("You're ticket is $10")
    elif age >= 12:
        print("Your ticket is $15")

# 7-6. Three Exits: Write different versions of either Exercise 7-4 or Exercise 7-5 that do each of the following at least once:
# • Use a conditional test in the while statement to stop the loop.

# prompt = "\nPlease enter a topping to add to your pizza"
# prompt += "\nEnter quit when you are done. "

# message = ""
# while message != "quit":
#     message = input(prompt)

#     if message != "quit":
#         print(f"Adding: {message.title()}")

# • Use an active variable to control how long the loop runs.

# prompt = "\nPlease enter a topping to add to your pizza"
# prompt += "\nEnter quit when you are done. "

# active = True
# while active:
#     message = input(prompt)

#     if message == "quit":
#         active = False
#     else:
#         print(f"Adding: {message.title()}")

# • Use a break statement to exit the loop when the user enters a 'quit' value.

prompt = "\nPlease enter a topping to add to your pizza"
prompt += "\nEnter quit when you are done. "

while True:
    message = input(prompt)

    if message == "quit":
        break
    else:
        print(f"Adding: {message.title()}")

# 7-7. Infinity: Write a loop that never ends, and run it. (To end the loop, press ctrl-C or close the window displaying the output.)

# x = 0
# while x <= 100:
#     print(x)


import random
import json
import os

import numpy as np

def get_unique_tokens(list_fin):
    """
    list of json files --> list of unique scene tokens
    """
    list_token_scene = []

    # Open one json file at a time
    for name_fin in list_fin:
        with open(name_fin, 'r') as f:
            dict_fin = json.load(f)

        # Check if the token scene is already in the list and if not add it
        if dict_fin['token_scene'] not in list_token_scene:
            list_token_scene.append(dict_fin['token_scene'])

    return list_token_scene

def split_scenes(list_token_scene, train, val, dir_main, save=False, load=True):
    """
    Split the list according tr, val percentages (test percentage is a consequence) after shuffling the order
    """

    path_split = os.path.join(dir_main, 'scenes', 'split_scenes.json')

    if save:
        random.seed(1)
        random.shuffle(list_token_scene)  # it shuffles in place
        n_scenes = len(list_token_scene)
        n_train = round(n_scenes * train / 100)
        n_val = round(n_scenes * val / 100)
        list_train = list_token_scene[0: n_train]
        list_val = list_token_scene[n_train: n_train + n_val]
        list_test = list_token_scene[n_train + n_val:]

        dic_split = {'train': list_train, 'val': list_val, 'test': list_test}
        with open(path_split, 'w') as f:
            json.dump(dic_split, f)

    if load:
        with open(path_split, 'r') as f:
            dic_split = json.load(f)

    return dic_split

def select_categories(cat):
    """
    Choose the categories to extract annotations from
    """
    assert cat in ['person', 'all', 'car', 'cyclist']

    if cat == 'person':
        categories = ['human.pedestrian']
    elif cat == 'all':
        categories = ['human.pedestrian', 'vehicle.bicycle', 'vehicle.motorcycle']
    elif cat == 'cyclist':
        categories = ['vehicle.bicycle']
    elif cat == 'car':
        categories = ['vehicle']
    return categories

def update_with_tokens(dict_gt, nusc, token_sd):
    """
    Update with tokens corresponding to the token_sd
    """

    table_sample_data = nusc.get('sample_data', token_sd)  # Extract the whole record to get the sample token
    token_sample = table_sample_data['sample_token']  # Extract the sample_token from the table
    table_sample = nusc.get('sample', token_sample)  # Get the record of the sample
    token_scene = table_sample['scene_token']
    dict_gt['token_sample_data'] = token_sd
    dict_gt['token_sample'] = token_sample
    dict_gt['token_scene'] = token_scene
    return dict_gt

def update_with_box(dict_gt, box):

    bbox = np.zeros(7, )
    flag_child = False

    # Save the 3D bbox
    bbox[0:3] = box.center
    bbox[3:6] = box.wlh
    bbox[6] = box.orientation.degrees
    dict_gt['boxes'].append(bbox.tolist()) # Save as list to be serializable by a json file
    if box.name == 'human.pedestrian.child':
        flag_child = True

    return dict_gt, flag_child


#find links (city name + page number)

import requests
from bs4 import BeautifulSoup
from lxml import etree

#find option tags for cities
def cities(type):
    url = f'https://fidilio.com/{type}/'
    reqs = requests.get(url)
    soup = BeautifulSoup(reqs.text, 'html.parser')
    urls = []
    cities=[]
    for link in soup.find_all('option'):
        if(link.get('value')=='0'):
            break
        cities.append(link.get('value'))

    return cities

def cities_page_number(type):
    cities_=cities(type)
    res=dict()
    for city in cities_:
        headers = {'User-Agent': 'Mozilla/5.0'}
        response = requests.get(f"https://fidilio.com/{type}/in/"+city, headers = headers)
        soup = BeautifulSoup(response.text,'html.parser')
        dom = etree.HTML(str(soup))
        pages = dom.xpath('/html/body/div[2]/div/section[1]/section/div[2]/div/div/div[1]/div[2]/div/a')[-1]
        max_number = ((pages.values())[0].split('?')[1].split('=')[1])
        res[city] = max_number
    return res


from collections import deque

def solution(priorities, location):
    result = 1
    q = deque()
    for i in range(len(priorities)):
        q.append((i, priorities[i]))
    while True:
        inx, prio = q.popleft()
        if prio >= max(q, key=lambda x: x[1])[1]:
            if inx == location:
                break
            result += 1
        else:
            q.append((inx, prio))

    return result


'''
Itay dali 204711196
David Toubul 342395563
Chen Azulay 201017159
'''
import pandas as pd
from sklearn import preprocessing

def getColumnTitles(table):
    """

    :param table: pandas DataFrame
    :return: list of columns
    """
    return list(table.columns)

def Discretize(num, table, structureTextFile):
    """

    :param num: number of bins
    :param table: pandas DataFrame
    :param structureTextFile: path of structure file
    :return: pandas DataFrame after discretize
    """
    def numericCol(table, structureTextFile):
        """

        :param table:
        :param structureTextFile: path
        :return: list of column which is numeric by the structure file
        """
        structure = pd.read_csv(structureTextFile, sep=" ", names=['type', 'feature', 'data'])
        column = []
        headers = getColumnTitles(table)
        for i in range(structure.shape[0]):
            if 'NUMERIC' in structure.loc[i]['data']:
                column += [headers[i]]
        return column
    column = numericCol(table, structureTextFile)
    table = table.applymap(lambda s: s.lower() if type(s) == str else s)
    table=table.dropna(subset=['class'])#drop the rows with the NaN values in 'class' column
    for col in column:
        table[col] = pd.qcut(table[col], num, labels=False, duplicates='drop')#Discretize variable into equal-sized buckets
    table = table.fillna(table.mode().iloc[0])#fillna with value that appears most often of Each Column
    # table.fillna(-1, inplace=True)
    # table.apply(lambda x: x.astype(str).str.lower())
    #table = table.applymap(lambda s: s.lower() if type(s) == str else s)
    return table

def p_xy(table,column_x,value_x,column_class,value_class):
    """

    :param table:
    :param column_x:
    :param value_x:
    :param column_class:
    :param value_class:
    :return:
    """
    length=table[column_x].value_counts()[value_x]
    try:
        p=table.loc[table[column_x]==value_x][column_class].value_counts()[value_class]
    except:
        p=1
    p=round(p/length,3)
    return p

def valuesType(table,column):
    """

    :param table:
    :param column:
    :return: return a list with the name of values in column
    """
    columnValues=table[column].unique().tolist()
    if -1 in columnValues:
        columnValues.remove(-1)
    return columnValues

def pArrayByFeature(table,featureCol,classValue,classCol):
    """
    calculate p(x|y=value)
    :param table:
    :param featureCol:
    :param classValue:y
    :param classCol:
    :return: array of probabilities of the values in the column by each value in 'class' column
    """
    array=[]
    for i in valuesType(table,featureCol):
        array+=[p_xy(table,featureCol,i,classCol,classValue)]
    return array

def fit_transforms(table):
    """

    :param table: pandas DataFrame
    :return: turn string values to int
    """
    le=preprocessing.LabelEncoder()
    columns = getColumnTitles(table)
    for col in columns:
        try:
            table[col] = le.fit_transform(table[col])
        except:
            continue
    return table


'''Write a python program to convert snake
case string to camel case string.'''
import re

snake_case = input()
x = re.split("_",snake_case)
camelCase = ""

for i in x:
    camelCase += i.capitalize()
print(camelCase)


import h5py

def get_data(start_idx = 0, end_idx = 699):
    h5_path = r'F:\ASCAD_data\ASCAD_databases\ASCAD.h5'
    with h5py.File(h5_path, "r") as f:
        p_data = {}
        p_data['traces'] = f['Profiling_traces']['traces'][:]
        p_data['labels'] = f['Profiling_traces']['labels'][:]
        p_data['plainx'] = f['Profiling_traces']['metadata']['plaintext']
        p_data['plainx'] = p_data['plainx'][:, 2]
        p_data['traces'] = p_data['traces'][0:45000, :]
        p_data['labels'] = p_data['labels'][0:45000]

        a_data = {}
        a_data['traces'] = f['Attack_traces']['traces'][:]
        a_data['labels'] = f['Attack_traces']['labels'][:]
        a_data['plainx'] = f['Attack_traces']['metadata']['plaintext']
        a_data['plainx'] = a_data['plainx'][:, 2]

    return p_data, a_data


import numpy as np

import sys
sys.path.append("..")

from financepy.utils.date import Date
from financepy.market.discount.interpolator import FinInterpTypes
from financepy.market.discount.curve import DiscountCurve

from FinTestCases import FinTestCases, globalTestCaseMode
testCases = FinTestCases(__file__, globalTestCaseMode)

PLOT_GRAPHS = False

###############################################################################

def test_FinInterpolatedForwards():

    import matplotlib.pyplot as plt

    tValues = np.array([0.0, 3.0, 5.0, 10.0])
    rValues = np.array([0.04, 0.07, 0.08, 0.09])
    df_values = np.exp(-tValues*rValues)
    tInterpValues = np.linspace(0.0, 12.0, 49)

    curveDate = Date(1, 1, 2019)

    tDates = curveDate.addYears(tValues)
    tInterpDates = curveDate.addYears(tInterpValues)

    for interp_type in FinInterpTypes:

        discount_curve = DiscountCurve(curveDate, tDates, df_values, interp_type)
        dfInterpValues = discount_curve.df(tInterpDates)
        fwdInterpValues = discount_curve.fwd(tInterpDates)
        zeroInterpValues = discount_curve.zeroRate(tInterpDates)

        if PLOT_GRAPHS:
            plt.figure(figsize=(8, 6))
            plt.plot(tValues, df_values, 'o', color='g', label="DFS:")
            plt.plot(tInterpValues, dfInterpValues, color='r',
                     label="DF:" + str(interp_type))
            plt.legend()
            plt.figure(figsize=(8, 6))
            plt.plot(tInterpValues, fwdInterpValues, color='r',
                     label="FWD:" + str(interp_type))
            plt.plot(tInterpValues, zeroInterpValues, color='b',
                     label="ZERO:" + str(interp_type))
            plt.plot(tValues, rValues, 'o', color='g',  label="ZERO RATES")
            plt.legend()

###############################################################################

test_FinInterpolatedForwards()
testCases.compareTestCases()