# Code for MultiTask TArC annotation/collection

import os
import re
import sys
import csv
import glob
import torch

from fairseq.tarc_utils import *
from fairseq.data import Dictionary, TarcMultiTaskDataset
from fairseq.tasks import FairseqTask, register_task

from typing import Dict, List

LOSS_INIT_VALUE=999999.9
ER_INIT_VALUE=LOSS_INIT_VALUE

CLS_COL_IDX=1
POS_COL_IDX=4
def set_pos_col_idx(args):
    if args.sub_task == 'tarc-ext':
        return 3
    elif args.sub_task == 'tarc-full':
        return 4
    elif args.sub_task == 'madar-trs-ex':
        return 2
    elif args.sub_task == 'madar-trs-full':
        return 3
    elif args.sub_task == 'tarc-full-npos':
        return 5
    else:
        return 9999999

# Define some utility functions for reading the TArC multi-task data format (tabular format)
def process_morpho(tag):
    res_tags = []
    for c in tag:
        res_tags.append( '@' + c )
    return res_tags

def append_tags( tags, sep ):
    res_tags = []
    for i in range(len(tags)):
        if i == 0:
            res_tags.append( tags[i] )
        else:
            res_tags.append( sep + tags[i] )
    return res_tags

def process_micro_pos(tag):

    res_tags = []
    if tag != '_':
        micro_tags = tag.split('_')
        res_tags.extend( append_tags( micro_tags[:-1], '_' ) )
        if len(micro_tags) >= 2 and (micro_tags[-2] == 'ADJ' or micro_tags[-2] == 'PRON') and micro_tags[-1][0] in ['1', '2', '3']:
            morpho_tags = process_morpho( micro_tags[-1] )
            morpho_tags[0] = morpho_tags[0].replace('@', '_')
            res_tags.extend( morpho_tags )
        else:
            if micro_tags[-1] == 'SG' or micro_tags[-1] == 'PL':
                res_tags.append( '_' + micro_tags[-1] )
            else:
                if micro_tags[-1] != ':':
                    nano_tags = micro_tags[-1].split(':')
                    if len(nano_tags) > 1:
                        if len(nano_tags) != 2:
                            sys.stderr.write(' - Micro POS error: {}\n'.format(micro_tags[-1]))
                            sys.exit(1)
                        res_tags.append( '_' + nano_tags[0] )
                        morpho_tags = process_morpho(nano_tags[1])
                        morpho_tags[0] = morpho_tags[0].replace('@', ':')
                        res_tags.extend( morpho_tags )
                    else:
                        if len(micro_tags) > 1:
                            res_tags.append( '_' + micro_tags[-1] )
                        else:
                            res_tags.append( micro_tags[-1] )
                else:
                    res_tags.append( micro_tags[-1] )
    else:
        res_tags.append( tag )
    return res_tags

def process_suffix_pos(tag):

    res_tags = []
    if tag != '-':
        tag_tokens = tag.split('-')
        if len(tag_tokens) > 1:
            for i in range(len(tag_tokens)):
                t = tag_tokens[i]
                if len(t) > 2 and t[:2] == 'IV' and not (len(t) > 6 and t[:6] == 'IVSUFF'):
                    res_tags.append('IV')
                    morpho_tags = process_morpho( t[2:] )
                    res_tags.extend( morpho_tags )
                else:
                    suffix_tokens = process_micro_pos(t)
                    if i > 0:
                        suffix_tokens[0] = '-' + suffix_tokens[0]
                    res_tags.extend( suffix_tokens )
        else:
            res_tags.extend( process_micro_pos(tag) )
    else:
        res_tags.append(tag)
    return res_tags

def process_macro_pos(tag):

    res_tags = []
    if tag != '+':
        tag_tokens = tag.split('+')
        for i in range(len(tag_tokens)):
            t = tag_tokens[i]
            micro_pos = process_suffix_pos(t)
            if i > 0:
                res_tags.append('+')
            res_tags.extend(micro_pos)
    else:
        res_tags.append(tag)
    return res_tags

def process_pos_tag(tag, args, pad_flag=True, rep_flag=False):
    res_tags = []
    pos_tokens = tag.split(']')
    if len(pos_tokens) > 1:
        if len(pos_tokens) != 2:
            sys.stderr.write(' - POS format error (splitting at ]): {}\n'.format(tag))
            sys.exit(1)
        pos_pos_tokens = pos_tokens[0].split('[')
        if len(pos_pos_tokens) > 1 and len(pos_pos_tokens[0]) > 0:  # Case like 'IV2S-IV+[PREP+PRON_1S]IVSUFF_IO:1S'
            if len(pos_pos_tokens) != 2:
                sys.stderr.write(' - POS format error (splitting at [): {}\n'.format(tag))
                sys.exit(1)

            pref = pos_pos_tokens[0]
            pref_tokens = process_macro_pos( pref )
            infix = pos_pos_tokens[1]
            infix_tokens = process_macro_pos( infix )
            infix_tokens = ['+['] + infix_tokens + [']']
            post = pos_tokens[1]
            post_tokens = process_macro_pos( post )

            res_tags.extend(pref_tokens)
            res_tags.extend( infix_tokens )
            res_tags.extend( post_tokens )
        elif len(pos_pos_tokens) > 1:   # Case like '[NOUN_QUANT]ADV'
            if len(pos_pos_tokens) != 2:
                sys.stderr.write(' - POS format error (splitting at [, with first part empty): {}\n'.format(tag))
                sys.exit(1)

            pref = pos_pos_tokens[1]
            pref_tokens = process_macro_pos( pref )
            pref_tokens = ['['] + pref_tokens + [']']
            post = pos_tokens[1]
            post_tokens = process_macro_pos( post )
            res_tags.extend( pref_tokens )
            res_tags.extend( post_tokens )
        else:
            sys.stderr.write(' - POS format error, possibly unbalanced [] at pos {}\n'.format(tag))
            sys.exit(1)
    else:   # "Normal" case (normal for Arabic people...)
        pos_tokens = process_macro_pos( tag )
        res_tags.extend( pos_tokens )
    if pad_flag:
        return [start_token] + res_tags + [end_token]
    else:
        return res_tags

def create_character_list(token, args, pad_flag=True, rep_flag=False):
    if (not (token in fillers + latin_fillers)) and rep_flag and args.sub_task in ['tarc-ext', 'tarc-full', 'tarc-full-npos']:
        token = replace_all_num(token)
        token = replace_all_pun(token)
        token = replace_all_sym(token)
    elif (not (token in fillers + latin_fillers)) and args.sub_task in ['tarc-ext', 'tarc-full', 'tarc-full-npos']:
        token = replace_all_Lpun(token)
        token = replace_all_Lsym(token)

    tok_lst = []
    tok_idx = 0
    for t in token.split():
        if tok_idx > 0:
            tok_lst.append( space_token )
        for c in t:
            tok_lst.append( c )
        tok_idx += 1

    if args.sub_task in ['tarc-ext', 'tarc-full', 'tarc-full-npos']:
        seq_fillers = detect_fillers(tok_lst, fillers + latin_fillers)
        if len(seq_fillers) > 0:
            new_lst = []
            prev_start = 0
            for t in seq_fillers:
                new_lst = new_lst + tok_lst[prev_start:t[0]] + [''.join(tok_lst[t[0]:t[1]])]
                prev_start = t[1]
            new_lst = new_lst + tok_lst[t[1]:]
            tok_lst = new_lst

    if pad_flag:
        return [start_token] + tok_lst + [end_token]
    else:
        return tok_lst

def process_tokenization( token, args, pad_flag=True, rep_flag=False ):

    res_tok = []
    if token != '+':
        tok_tokens = token.split('+')
        if len(tok_tokens) > 1:
            res_tok.append(start_token)
            for i in range(len(tok_tokens)):
                t = tok_tokens[i]
                if len(t) > 0:
                    if i > 0:
                        res_tok.append( '+' )
                    res_tok.extend( create_character_list( t, args, pad_flag=False, rep_flag=rep_flag ) )
                else:
                    sys.stderr.write('process_tokenization FORMAT ERROR: found empy token splitting on + >{}< (probably you must replace multiple + in your data)\n'.format(token))
                    sys.exit(1)
            res_tok.append(end_token)
        else:
            res_tok.extend( create_character_list( token, args, pad_flag=True, rep_flag=rep_flag ) )
    else:
        res_tok.append(start_token)
        res_tok.append(token)
        res_tok.append(end_token)

    return res_tok

def tiger_label_processing(label, args, pad_flag=True, rep_flag=False):

    if label != '$.':
        label_tokens = label.split('.')
    else:
        label_tokens = [label]

    if pad_flag:
        return [start_token] + label_tokens + [end_token]
    else:
        return label_tokens

def no_processing(token, args, pad_flag=True, rep_flag=False):
    return [token]

madar_translation = (create_character_list, no_processing, create_character_list)
madar_ex_translation = (create_character_list, no_processing, process_pos_tag, create_character_list)
madar_full_translation = (create_character_list, no_processing, create_character_list, process_pos_tag, create_character_list)
tarc_processing_base = (create_character_list, process_tokenization, process_pos_tag)   # Processing functions for raw tunisian, tokenized tunisian and POS
tarc_processing_ext = (create_character_list, no_processing, process_tokenization, process_pos_tag)    # Processing functions for arabish, tunisian, tokenized tunisian and POS
tarc_processing_full = (create_character_list, no_processing, create_character_list, process_tokenization, process_pos_tag)
tarc_processing_full_npos = (create_character_list, no_processing, no_processing, create_character_list, process_tokenization, process_pos_tag)
tarc_substep_1 = (create_character_list, no_processing, create_character_list)
tarc_substep_2 = (create_character_list, process_tokenization)
tarc_substep_3 = (process_tokenization, process_pos_tag)
tiger_mt_processing = (create_character_list, no_processing, no_processing)
tiger_mtext_processing = (create_character_list, no_processing, tiger_label_processing)
tiger4_mt_processing = (create_character_list, no_processing, no_processing, no_processing)

# NOTE: used to decide if tokens and chars representations must be computed and merged when their usage is specified from command line
granularity_merging_flags = {}
granularity_merging_flags['tiger-mt'] = (True, False, False)
granularity_merging_flags['tiger-mt-ext'] = (True, False, True)
granularity_merging_flags['tiger4-mt'] = (True, False, False, False)
granularity_merging_flags['madar-trs'] = (True, False, True)
granularity_merging_flags['madar-trs-ex'] = (True, False, True, True)
granularity_merging_flags['madar-trs-full'] = (True, False, True, True, True)
granularity_merging_flags['tarc-base'] = (True, False, True)
granularity_merging_flags['tarc-ext'] = (True, False, True, True)
granularity_merging_flags['tarc-full'] = (True, False, True, True, True)
granularity_merging_flags['tarc-full-npos'] = (True, False, False, True, True, True)
granularity_merging_flags['tarc-substep1'] = (True, False, True)
granularity_merging_flags['tarc-substep2'] = (True, True)
granularity_merging_flags['tarc-substep3'] = (True, True)

def choose_column_processing(num_columns, args):

    column_processing = None
    if args.sub_task == 'tiger-mt':
        print(' - TArCMultiTask, processing mode set to tiger-mt')
        sys.stdout.flush()
        column_processing = tiger_mt_processing
    elif args.sub_task == 'tiger-mt-ext':
        print(' - TArCMultiTask, processing mode set to tiger-mt-ext')
        sys.stdout.flush()
        column_processing = tiger_mtext_processing
    elif args.sub_task == 'tiger4-mt':
        print(' - TArCMultiTask, processing mode set to tiger4-mt')
        sys.stdout.flush()
        column_processing = tiger4_mt_processing
    elif args.sub_task == 'madar-trs':
        print(' - TArCMultiTask, processing mode set to madar-trs')
        sys.stdout.flush()
        column_processing = madar_translation
    elif args.sub_task == 'madar-trs-ex':
        print(' - TArCMultiTask, processing mode set to madar-trs-ex')
        sys.stdout.flush()
        column_processing = madar_ex_translation
    elif args.sub_task == 'madar-trs-full':
        print(' - TArCMultiTask, processing mode set to madar-trs-full')
        sys.stdout.flush()
        column_processing = madar_full_translation
    elif args.sub_task == 'tarc-base':
        print(' - TArCMultiTask, processing mode set to tarc-base')
        sys.stdout.flush()
        column_processing = tarc_processing_base
    elif args.sub_task == 'tarc-ext':
        print(' - TArCMultiTask, processing mode set to tarc-ext')
        sys.stdout.flush()
        column_processing = tarc_processing_ext
    elif args.sub_task == 'tarc-full':
        print(' - TArCMultiTask, processing mode set to tarc-full')
        sys.stdout.flush()
        column_processing = tarc_processing_full
    elif args.sub_task == 'tarc-full-npos':
        print(' - TArCMultiTask, processing mode set to tarc-full-npos')
        column_processing = tarc_processing_full_npos
    elif args.sub_task == 'tarc-substep1':
        print(' - TArCMultiTask, processing mode set to tarc-substep1')
        sys.stdout.flush()
        column_processing = tarc_substep_1
    elif args.sub_task == 'tarc-substep2':
        print(' - TArCMultiTask, processing mode set to tarc-substep2')
        sys.stdout.flush()
        column_processing = tarc_substep_2
    elif args.sub_task == 'tarc-substep3':
        print(' - TArCMultiTask, processing mode set to tarc-substep3')
        sys.stdout.flush()
        column_processing = tarc_substep_3
    elif args.sub_task == 'base':
        print(' - TArCMultiTask, processing mode set to base')
        sys.stdout.flush()
        column_processing = [no_processing for i in range(num_columns)]
    else:
        print(' - TArCMultiTask, setting default processing mode (no processing)')
        sys.stdout.flush()
        column_processing = [no_processing for i in range(num_columns)]

    return column_processing

def check_column_processing(num_columns, args):

    if num_columns == 3 and not args.sub_task in ['tiger-mt', 'tiger-mt-ext', 'tarc-base', 'base', 'madar-trs', 'tarc-substep1']:
        raise ValueError(' wrong num. of columns in input data for processing mode {}'.format(args.sub_task))
    elif num_columns == 2 and not args.sub_task in ['tarc-substep2', 'tarc-substep3', 'base']:
        raise ValueError(' 2 columns are expected with sub-task tarc-substep2|3')
    elif num_columns == 4 and not args.sub_task in ['tiger4-mt', 'tarc-ext', 'base','madar-trs-ex']:
        raise ValueError(' wrong num. of columns in input data for processing mode {}'.format(args.sub_task))
    elif num_columns == 5 and not args.sub_task in ['tarc-full','madar-trs-full', 'base']:
        raise ValueError(' wrong num. of columns in input data for processing mode {}'.format(args.sub_task))
    elif num_columns == 6 and not args.sub_task in ['tarc-full-npos', 'base']:
        raise ValueError(' wrong num. of columns in input data for processing mode {}'.format(args.sub_task))
    elif (not num_columns in [2, 3, 4, 5, 6]) and (not args.sub_task in ['base']):
        raise ValueError(' Unexpected number of columns in input file, possible values: 2, 3, 4, 5 unless base is specified. Got {}'.format(num_columns))

def apply_global_filling(args, curr_seq, cls_seq, Afillers, Lfillers, OFFcols, idx):

    (fillerFOR, fillerEMO) = Afillers
    (LfillerFOR, LfillerEMO) = Lfillers
    (CLS_COL_IDX, POS_COL_IDX) = OFFcols

    if args.sub_task in ['tarc-ext', 'tarc-full', 'madar-trs', 'madar-trs-ex', 'madar-trs-full', 'tarc-full-npos'] and idx != CLS_COL_IDX and idx != POS_COL_IDX:
        for s_idx in range(len(curr_seq)):
            if cls_seq[s_idx] == 'foreign':
                if (args.sub_task == 'tarc-full' and idx == 0) or (args.sub_task == 'tarc-full-npos' and idx == 0) or (args.sub_task == 'madar-trs' and idx == 2) or (args.sub_task == 'madar-trs-ex' and idx == 3) or (args.sub_task == 'madar-trs-full' and idx == 4):
                    curr_seq[s_idx] = LfillerFOR
                else:
                    curr_seq[s_idx] = fillerFOR
            elif cls_seq[s_idx] == 'emotag':
                if (args.sub_task == 'tarc-full' and idx == 0) or (args.sub_task == 'tarc-full-npos' and idx == 0) or (args.sub_task == 'madar-trs' and idx == 2) or (args.sub_task == 'madar-trs-ex' and idx == 3) or (args.sub_task == 'madar-trs-full' and idx == 4):
                    curr_seq[s_idx] = LfillerEMO
                else:
                    curr_seq[s_idx] = fillerEMO

    return curr_seq

def _add_t2c_entry_safe(D: Dict[str, List[str]], key: str, val: List[str]) -> Dict[str, List[str]]:

    if not (key in D):
        D[key] = val
    else:
        if len(D[key]) != len(val):
            print(' - WARNING: adding ambigous value to entry {}: 1. {} VS. 2. {}'.format(key, D[key], val))
            sys.stdout.flush()
        else:
            check_flag = True
            for i in range(len(val)):
                if val[i] != D[key][i]:
                    check_flag = False
                    break
            if not check_flag:
                print(' - WARNING: adding ambigous value to entry {}: 1. {} VS. 2. {}'.format(key, D[key], val))
                sys.stdout.flush()

    return D

def read_tarc_tabular_data(filename, args):

    with open(filename, 'rb') as f:
        lines = [l.decode('utf-8') for l in f.readlines()]
        f.close()

    POS_COL_IDX = set_pos_col_idx(args)
    test_flag = False
    if filename[-4:] == 'test':
        test_flag = True

    # NOTE: The following are lists to keep token-level and character-level information.
    #       char_seq_lengths and char_seq_lens are used to store start and end indices of tokens in the character-level information,
    #       so that when using both, the model can retrieve the list of embeddings of characters belonging to a given token, and mix them properly.
    char_sequences = []
    char_seq_lengths = []
    tok_sequences = []
    char_seqs = []
    char_seq_lens = []
    tok_seqs = []

    start_idx = 0
    num_columns = 0
    all_sequences = []
    curr_sequences = []
    for line in lines:
        tokens = line.rstrip(' \r\n').split('\t')

        #print(' - Read tokens ({}): {}'.format(len(tokens), tokens))
        #sys.stdout.flush()

        if len(tokens) > 1 and num_columns == 0:
            num_columns = len(tokens)
            all_sequences = [[] for i in range(num_columns)]
            curr_sequences = [[] for i in range(num_columns)]
        if len(tokens) > 1:

            if len(tokens) != num_columns:
                raise IOError(' Wrong data format, found different number of columns at different input lines ({} VS. {})'.format(len(tokens), num_columns))

            for i in range(num_columns):
                curr_sequences[i].append(tokens[i])
        else:
            for i in range(num_columns):
                curr_seq = apply_global_filling(args, curr_sequences[i], curr_sequences[CLS_COL_IDX], [fillerFOR, fillerEMO], [LfillerFOR, LfillerEMO], [CLS_COL_IDX, POS_COL_IDX], i)
                all_sequences[i].append( curr_seq )

            curr_sequences = [[] for i in range(num_columns)]

    print(' *** TarcMultiTask, read {} sequences'.format(len(all_sequences)))
    print(' *** TarcMultiTask, detected {} tasks in data'.format(num_columns-1))
    sys.stdout.flush()

    column_processing = choose_column_processing( num_columns, args )
    check_column_processing( num_columns, args )

    total_tokens = 0
    assert len(all_sequences) == num_columns
    num_sequences = len(all_sequences[0])
    for i in range(num_columns):
        assert len(all_sequences[i]) == num_sequences
    if args.sub_task in ['tarc-ext', 'tarc-full', 'tarc-full-npos']:
        for i in range(num_sequences):
            seq_len = len(all_sequences[0][i])
            for j in range(num_columns):
                assert len(all_sequences[j][i]) == seq_len

    def init_llist(size):
        return [[] for i in range(size)]

    masked_tokens = 0
    token_offsets = []
    token2components = [{} for t_idx in range(num_columns)]
    for ts_idx in range(num_sequences):

        curr_seq_len = len(all_sequences[0][ts_idx])
        for s_idx in range(curr_seq_len):
            tokens = []
            for t_idx in range(num_columns):
                tokens.append( all_sequences[t_idx][ts_idx][s_idx] )

            assert len(tokens) == num_columns

            if len(char_sequences) == 0:
                char_sequences = init_llist( num_columns )
            if len(tok_sequences) == 0:
                tok_sequences = init_llist( num_columns )
            if len(char_seq_lengths) == 0:
                char_seq_lengths = init_llist( num_columns )
            if len(char_seqs) == 0:
                char_seqs = init_llist( num_columns )
            if len(char_seq_lens) == 0:
                char_seq_lens = init_llist( num_columns )
            if len(token_offsets) == 0:
                token_offsets = [0 for i in range(len(tokens))]
            if len(tok_seqs) == 0:
                tok_seqs = init_llist( num_columns )
            for idx in range(num_columns):
                if idx > 0 and args.ignore_test_output and test_flag:
                    process_res = no_processing(tokens[idx], args)
                else:
                    rep_flag = True
                    if (idx == 0 and (args.sub_task == 'tarc-full' or args.sub_task == 'tarc-full-npos')) or (idx == 2 and args.sub_task == 'madar-trs') or (args.sub_task == 'madar-trs-ex' and idx == 3) or (args.sub_task == 'madar-trs-full' and idx == 4):
                        rep_flag = False
                    process_res = column_processing[idx]( tokens[idx], args, rep_flag=rep_flag )

                token2components[idx] = _add_t2c_entry_safe(token2components[idx], tokens[idx], process_res)

                char_seqs[idx].extend( process_res )
                char_seq_lens[idx].append( (token_offsets[idx], token_offsets[idx]+len(process_res)) )
                token_offsets[idx] += len(process_res)
                tok_seqs[idx].append( tokens[idx] )

        total_tokens += len(char_seqs[0])
        for idx in range(len(char_sequences)):
            char_sequences[idx].append( char_seqs[idx] )
            char_seq_lengths[idx].append( char_seq_lens[idx] )
            tok_sequences[idx].append( tok_seqs[idx] )
        char_seqs = []
        char_seq_lens = []
        token_offsets = []
        tok_seqs = []

    print(' - Size of char_sequences: {}'.format(len(char_sequences)))
    sys.stdout.flush()
    for ci in range(len(char_sequences)):
        print('   - Size of dim {}: {}'.format(ci, len(char_sequences[ci])))
        sys.stdout.flush()

    print(' - Loaded {} sequences, {} tokens (masked {}) from {}'.format( len(char_sequences[0]), total_tokens, masked_tokens, filename ))
    sys.stdout.flush()

    return (tok_sequences, char_sequences, char_seq_lengths, token2components)

def read_tarc_parallel_data( file_prefix, args ):

    input_file = file_prefix + '.input'
    output_files = glob.glob(file_prefix + '.output*')

    POS_COL_IDX = set_pos_col_idx(args)
    test_flag = False
    if file_prefix[-4:] == 'test':
        test_flag = True

    input_data = []
    f = open(input_file, encoding='utf-8')
    data = f.readlines()
    f.close()
    total_tokens = 0
    for s in data:
        tokens = s.rstrip(' \r\n').split()
        total_tokens += len(tokens)
        input_data.append(tokens)
    print(' - TArCMultiTask, read {} sequences, {} tokens from input data'.format(len(input_data), total_tokens))
    sys.stdout.flush()

    output_data = [[] for i in range(len(output_files))]
    for i in range( len(output_files) ):
        f = open(output_files[i], encoding='utf-8')
        data = f.readlines()
        f.close()

        total_tokens = 0
        s_idx = 0
        for s in data:
            tokens = s.rstrip(' \r\n').split()
            total_tokens += len(tokens)
            if i > 0:
                i_seq = input_data[s_idx]
                input_data[s_idx] = apply_global_filling(args, i_seq, output_data[CLS_COL_IDX-1][s_idx], [fillerFOR, fillerEMO], [LfillerFOR, LfillerEMO], [CLS_COL_IDX, POS_COL_IDX], 0)
                tokens = apply_global_filling(args, tokens, output_data[CLS_COL_IDX-1][s_idx], [fillerFOR, fillerEMO], [LfillerFOR, LfillerEMO], [CLS_COL_IDX, POS_COL_IDX], i+1)
            output_data[i].append( tokens )
            s_idx += 1
        print(' - TArCMultiTask, read {} sequences, {} tokens from output data {}'.format(len(output_data[i]), total_tokens, i))
        sys.stdout.flush()

    for i in range( len(output_data) ):
        assert len(input_data) == len(output_data[i])
        #for s_idx in range(len(input_data)):
        #    assert len( input_data[s_idx] ) == len( output_data[i][s_idx] )

    num_columns = len(output_data)+1
    column_processing = choose_column_processing(num_columns, args)
    check_column_processing(num_columns, args)

    total_tokens = 0
    # NOTE: The following are lists to keep token-level and character-level information.
    #       char_seq_lengths and char_seq_lens are used to store start and end indices of tokens in the character-level information,
    #       so that when using both, the model can retrieve the list of embeddings of characters belonging to a given token, and mix them properly.
    def init_llist(size):
        return [[] for i in range(size)]
    lsize = len(output_data)+1
    char_sequences = init_llist( lsize )
    char_seq_lengths = init_llist( lsize )
    tok_sequences = init_llist( lsize )
    char_seqs = init_llist( lsize )
    char_seq_lens = init_llist( lsize )
    tok_seqs = init_llist( lsize )
    token_offsets = [0 for i in range( lsize )]
    token2components = [{} for t_idx in range(num_columns)]
    for i in range( len(input_data) ):
        c_idx = 0
        for t in input_data[i]:
            rep_flag = True
            if (c_idx == 0 and (args.sub_task == 'tarc-full' or args.sub_task == 'tarc-full-npos')) or (c_idx == 2 and args.sub_task == 'madar-trs') or (args.sub_task == 'madar-trs-ex' and c_idx == 3) or (args.sub_task == 'madar-trs-full' and c_idx == 4):
                rep_flag = False
            #process_res = column_processing[idx]( tokens[idx], args, rep_flag=rep_flag )
            tok_res = column_processing[c_idx](t, args, rep_flag=rep_flag)
            total_tokens += len(tok_res)
            char_seqs[c_idx].extend( tok_res )
            char_seq_lens[c_idx].append( (token_offsets[c_idx], token_offsets[c_idx]+len(tok_res)) )
            token_offsets[c_idx] += len(tok_res)
            tok_seqs[c_idx].append( t )

            token2components[c_idx] = _add_t2c_entry_safe(token2components[c_idx], t, tok_res)

        char_sequences[c_idx].append( char_seqs[c_idx] )
        char_seq_lengths[c_idx].append( char_seq_lens[c_idx] )
        tok_sequences[c_idx].append( tok_seqs[c_idx] )
        c_idx += 1
        for j in range(len(output_data)):
            for t in output_data[j][i]:
                if args.ignore_test_output and test_flag:
                    process_res = no_processing(t)
                else:
                    rep_flag = True
                    if (c_idx == 0 and (args.sub_task == 'tarc-full' or args.sub_task == 'tarc-full-npos')) or (c_idx == 2 and args.sub_task == 'madar-trs') or (args.sub_task == 'madar-trs-ex' and c_idx == 3) or (args.sub_task == 'madar-trs-full' and c_idx == 4):
                        rep_flag = False
                    process_res = column_processing[c_idx](t, args, rep_flag=rep_flag)
                char_seqs[c_idx].extend( process_res )
                char_seq_lens[c_idx].append( (token_offsets[c_idx], token_offsets[c_idx]+len(process_res)) )
                token_offsets[c_idx] += len(process_res)
                tok_seqs[c_idx].append(t)

                token2components[c_idx] = _add_t2c_entry_safe(token2components[c_idx], t, process_res)

            char_sequences[c_idx].append( char_seqs[c_idx] )
            char_seq_lengths[c_idx].append( char_seq_lens[c_idx] )
            tok_sequences[c_idx].append( tok_seqs[c_idx] )
            c_idx += 1
        char_seqs = init_llist( lsize )
        char_seq_lens = init_llist( lsize )
        tok_seqs = init_llist( lsize )
        token_offsets = [0 for i in range(lsize)]

    print(' - TArCMultiTask, read {} sequences, {} tokens'.format(len(char_sequences[0]), total_tokens))
    sys.stdout.flush()

    return (tok_sequences, char_sequences, char_seq_lengths, token2components)

def map_tokens(data, dict, pad_flag):

    tensors = []
    for s in data:
        tok_lst = []
        for t in s:
            tok_lst.append( dict.add_symbol(t) )
        if pad_flag:
            tok_lst = [dict.bos()] + tok_lst + [dict.eos()]
        tensors.append( torch.LongTensor( tok_lst ) )

    return tensors

@register_task('tarc_multitask')
class TarcMultiTask(FairseqTask):

    @staticmethod
    def add_args(parser):

        # Add some command-line arguments for specifying where the data is
        # located and the maximum supported input length.
        parser.add_argument('data', metavar='FILE',
                            help='Main input file in tabular format. It must contain at least 2 columns; the first column is interpreted as the input, all the other columns are interpreted as outputs, unless --num-of-tasks and --task-indeces specify otherwise')
        parser.add_argument('--num-of-tasks', type=int, default=-1,
                            help='Number of tasks to learn simultaneously. It must correspond at most to the number of columns minus one in the main input file')
        parser.add_argument('--task-indeces', type=str, default='1:',
                            help='Indeces of columns in the main input file corresponding to the learnable tasks. It is represented as an interval string S:E')
        parser.add_argument('--additinal-data', type=str,
                            help='A semicolon separated list of files for multi-task learning')
        parser.add_argument('--data-description', type=str,
                            help='A string describing the content of each additinal input data in the format [N,S:E;]+, where N is the number of tasks, S:E the indeces of the learnable tasks')
        parser.add_argument('--pad-reference', action='store_true', default=False,
                            help='Specify to pad sequences with start and end of sequence markers')
        parser.add_argument('--max-source-positions', type=int, default=2048,
                            help='Maximum number of items in source sequences')
        parser.add_argument('--max-target-positions', type=int, default=2048,
                            help='Maximum number of items in target sequences')
        parser.add_argument('--sub-task', type=str, default='base',
                            help='Which multi-task problem to address: 1) base (default, no pre-processing) 2) tiger-mt, 2) tarc-base 3) tarc-ext')
        parser.add_argument('--data-format', type=str, default='tabular',
                            help='Format of input data: 1) tabular (default), 2) parallel')
        parser.add_argument('--sequence-separator', type=str, default='_SEQ_SEP_',
                            help='Used to separate output sequences from different tasks')
        parser.add_argument('--serialized-data', type=str,
                            help='Load data from a previously created serialization of the whole corpus. If it does not exist, read the whole corpus and serialize it.')
        parser.add_argument('--ignore-test-output', action='store_true', default=False,
                            help='Don\'t apply pre-processing to test output(s)')
        parser.add_argument('--keep-data-order', action='store_true', default=False,
                            help='Keep data in the original order, that is does not sort sequences by length.')
        parser.add_argument('--reverse-sequences', action='store_true', default=False,
                            help='Reverse sequences, that is they will be processed from right to left (e.g. for Arabic processing)')
        parser.add_argument('--reverse-tokens', action='store_true', default=False,
                            help='Reverse tokens individually, that is they will be processed from last to first char (e.g. for Arabic processing)')
        parser.add_argument('--load-madar-model', type=str, default='None',
                            help='Task specific for --sub-task=tarc-full, pre-trained model loading for parameter pre-initialization')
        parser.add_argument('--load-madar-data', type=str, default='None',
                            help='Task specific for --sub-task=tarc-full, serialized data loading for embedding pre-initialization')
        parser.add_argument('--token-sequences', action='store_true', default=False,
                            help='Use token-level information for modelling sequences')
        parser.add_argument('--char-sequences', action='store_true', default=False,
                            help='Use character-level information for modelling sequences')
        parser.add_argument('--double-learning', action='store_true', default=False,
                            help='Learn the model from both token and character representations')

    @classmethod
    def setup_task(cls, args, **kwargs):

        # Here we can perform any setup required for the task. This may include
        # loading Dictionaries, initializing shared Embedding layers, etc.
        # In this case we'll just initialize the label dictionary

        input_vocab = Dictionary(
                                 pad=pad_token,
                                 eos=eos_token,
                                 unk=unk_token,
                                 bos=bos_token,
                                 extra_special_symbols=[start_token, end_token]
        )
        output_vocab = input_vocab

        print('| [token] dictionary: {} types'.format(len(input_vocab)))
        print('| [label] dictionary: {} types'.format(len(output_vocab)))
        if (not args.token_sequences) and (not args.char_sequences):
            args.char_sequences = True

        if args.token_sequences:
            print('  - TArCMultiTask, using token-level information in sequences')
            sys.stdout.flush()
        if args.char_sequences:
            print('  - TArCMultiTask, using character-level information in sequences')
            sys.stdout.flush()

        return TarcMultiTask(args, input_vocab, output_vocab)

    def __init__(self, args, input_vocab, output_vocab):

        super().__init__(args)
        self.args = args

        self.input_vocab = input_vocab
        self.output_vocab = output_vocab

        self.sequence_separator = self.input_vocab.add_symbol(args.sequence_separator)
        self.token2components_tsr = []
        self.granularity_merging_flags = {}
        self.double_learning = args.double_learning
        self.splits = {}
        self.num_of_inputs = 1

    def set_granularity_merging_flags(self, g_flags):
        self.granularity_merging_flags = g_flags

    def get_granularity_merging_flags(self):
        return self.granularity_merging_flags

    def _t2c_to_tsr(self, t2c: Dict[str, List[str]], dict: Dictionary) -> Dict[int, torch.LongTensor]:

        res_dict = {}
        for k in t2c.keys():
            res_dict[dict.add_symbol(k)] = torch.LongTensor( [dict.add_symbol(v) for v in t2c[k]] )
        return res_dict

    def load_dataset(self, split, **kwargs):

        """Load a given dataset split (e.g., train, valid, test)."""

        if len(self.datasets) == 0:
            if self.args.serialized_data is not None and os.path.exists(self.args.serialized_data) and os.path.isfile(self.args.serialized_data):
                print(' - TArCMultiTask, reading serialized data from {}...'.format(self.args.serialized_data))
                sys.stdout.flush()
                self.splits = torch.load( self.args.serialized_data )
                self.input_vocab = self.splits['vocab']
                self.output_vocab = self.input_vocab
                self.token2components_tsr = self.splits['token2components']
            else:
                print(' - TArCMultiTask, creating dictionaries from whole corpus...')
                sys.stdout.flush()

                token2components = []
                for my_split in ['train', 'dev', 'test']:
                    data_sequences = []
                    if self.args.data_format == 'tabular':
                        data_sequences = read_tarc_tabular_data( self.args.data + '.' + my_split, self.args)
                    elif self.args.data_format == 'parallel':
                        data_sequences = read_tarc_parallel_data( self.args.data + '.' + my_split, self.args )
                    else:
                        raise NotImplementedError(' unsupported data format {}'.format(self.args.data_format))

                    if len(token2components) == 0:
                        token2components = [{bos_token : [bos_token], eos_token : [eos_token]} for t_idx in range(len(data_sequences[3]))]
                    for t_idx in range(len(data_sequences[3])):
                        token2components[t_idx].update( data_sequences[3][t_idx] )

                    if self.args.token_sequences and self.args.char_sequences:
                        assert len(data_sequences[0]) == len(data_sequences[1])
                        assert len(data_sequences[1]) == len(data_sequences[2])

                    print(' - load_dataset, read {} data split'.format(my_split))
                    print('   - got {} tasks'.format(len(data_sequences[0])-1))
                    sys.stdout.flush()

                    tensors = []
                    lengths = []
                    tok_tensors = []
                    tok_lengths = []
                    char_seq_lengths = [[] for i in range(len(data_sequences[2]))]
                    for d_idx in range( len(data_sequences[0]) ):
                        tok_tt = map_tokens(data_sequences[0][d_idx], self.input_vocab, self.args.pad_reference)
                        tok_tensors.append( tok_tt )
                        tok_ll = torch.LongTensor( [t.size(0) for t in tok_tt] )
                        tok_lengths.append( tok_ll )

                        seq_tt = []
                        for s in data_sequences[2][d_idx]:
                            if self.args.pad_reference:
                                s = [(p[0]+1, p[1]+1) for p in s]
                            char_seq_lengths[d_idx].append( torch.LongTensor( s ) )    # As many pairs as tokens...

                        tt = map_tokens(data_sequences[1][d_idx], self.input_vocab, self.args.pad_reference)
                        tensors.append( tt )
                        ll = torch.LongTensor([t.size(0) for t in tt])
                        lengths.append(ll)
                    self.splits[my_split] = ([tok_tensors, tensors], [tok_lengths, lengths, char_seq_lengths])

                for t_idx in range(len(token2components)):
                    self.token2components_tsr.append( self._t2c_to_tsr(token2components[t_idx], self.input_vocab) )

                self.splits['vocab'] = self.input_vocab
                self.splits['token2components'] = self.token2components_tsr
                if self.args.serialized_data is not None:
                    print(' - TArCMultiTask, serializing data to file {}...'.format(self.args.serialized_data))
                    sys.stdout.flush()
                    torch.save(self.splits, self.args.serialized_data)

        assert len(self.splits) == 5 and 'train' in self.splits.keys() and 'dev' in self.splits.keys() and 'test' in self.splits.keys()
        print(' - TArCMultiTask, instantiating current split {}'.format(split))
        sys.stdout.flush()
        my_split = split
        if split == 'valid':
            my_split = 'dev'

        tensors, lengths = self.splits[my_split]

        print(' - Split {} data lengths statistics:'.format(split))
        for idx, ll in enumerate(lengths[1]):
            print('\t- Max. length @{}: {}'.format(idx, torch.max(ll).item()))
            print('\t- Min. length @{}: {}'.format(idx, torch.min(ll).item()))
            print('\t-----')
        print(' - Dictionary sizes: {}, {}'.format(len(self.input_vocab), len(self.output_vocab)))
        print(' _______________')

        num_of_tasks = len(tensors[0])-1
        self.num_of_inputs = len(tensors[0]) - num_of_tasks
        self.args.num_of_tasks = num_of_tasks
        bound_idx = len(tensors[0])-num_of_tasks

        if 'base' not in granularity_merging_flags:
            granularity_merging_flags['base'] = tuple([False for i in range(num_of_tasks+1)])
        if self.args.sub_task not in granularity_merging_flags:
            granularity_merging_flags[self.args.sub_task] = tuple([False for i in range(num_of_tasks+1)])
        self.set_granularity_merging_flags(granularity_merging_flags[self.args.sub_task])

        sources = [tensors[0][0:bound_idx], tensors[1][0:bound_idx]]
        src_lengths = [lengths[0][0:bound_idx], lengths[1][0:bound_idx], lengths[2][0:bound_idx]]
        targets = [tensors[0][bound_idx:], tensors[1][bound_idx:]]
        tgt_lengths = [lengths[0][bound_idx:], lengths[1][bound_idx:], lengths[2][bound_idx:]]

        print(' - Tarc MultiTask, learning with {} input(s) (lengths: {}), {} different outputs (num. of tasks: {}, lengths: {})'.format(len(sources), len(src_lengths), len(targets), self.args.num_of_tasks, len(tgt_lengths)))
        sys.stdout.flush()

        input_feed = True
        self.datasets[split] = TarcMultiTaskDataset.TarcMultiTaskDataset(
            src=sources,
            src_sizes=src_lengths,
            src_dict=self.input_vocab,
            tgt=targets,
            tgt_sizes=tgt_lengths,
            sequence_separator=self.sequence_separator,
            tgt_dict=self.output_vocab,
            left_pad_target=False,
            max_source_positions=self.args.max_source_positions,
            max_target_positions=self.args.max_target_positions,
            input_feeding=input_feed,
            keep_data_order=self.args.keep_data_order,
            granularity_flags=(self.args.token_sequences, self.args.char_sequences),
        )

    def max_positions(self):
        """Return the max input length allowed by the task."""
        # The source should be less than *args.max_positions* and the "target"
        # has max length 1.
        return (self.args.max_source_positions, self.args.max_target_positions)

    @property
    def source_dictionary(self):
        """Return the source :class:`~fairseq.data.Dictionary`."""
        return self.input_vocab

    @property
    def target_dictionary(self):
        """Return the target :class:`~fairseq.data.Dictionary`."""
        return self.output_vocab

    # We could override this method if we wanted more control over how batches
    # are constructed, but it's not necessary for this tutorial since we can
    # reuse the batching provided by LanguagePairDataset.
    #
    # def get_batch_iterator(
    #     self, dataset, max_tokens=None, max_sentences=None, max_positions=None,
    #     ignore_invalid_inputs=False, required_batch_size_multiple=1,
    #     seed=1, num_shards=1, shard_id=0,
    # ):
    #     (...)

    def build_generator(self, args):

        from fairseq.tarc_multitask_sequence_generator import TarcSequenceGenerator

        return TarcSequenceGenerator(
            self.target_dictionary,
            max_len_a=getattr(args, "max_len_a", 0),
            max_len_b=getattr(args, "max_len_b", 200),
        )


class Solution:
    def minSubArrayLen(self, target: int, nums: List[int]) -> int:
        start = 0
        result = math.inf
        sum_sofar = 0
        for i in range(len(nums)):
            sum_sofar += nums[i]
            while sum_sofar >= target:
                result = min(result,i-start+1)
                sum_sofar -= nums[start]
                start+=1
        return 0 if result == math.inf else result


class Solution:
    def getRow(self, rowIndex):
        """
        :type rowIndex: int
        :rtype: List[int]
        """
        ret = [1 for _ in range(rowIndex + 1)]
        for i in range(2, rowIndex + 1):
            k = i//2 + 1 if i % 2 == 1 else i // 2
            for j in reversed(range(1, k + 1)):
                ret[j] = ret[j] + ret[j-1]
            if i % 2 == 0:
                ret = ret[0:k + 1] + ret[0:k][::-1]
            else:
                ret = ret[0:k] + ret[0:k][::-1]

        return ret


from edgar_flow.modules.beam_functions import *
from edgar_flow.modules.edgar_utils import *
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that, equal_to

import unittest

class TestBeamFunctions(unittest.TestCase):

    def get_test_dictionary(self):
        return dict(COB='testcob',
                         CUSIP='testcusip',
                         TICKER='testticker',
                         COUNT='tstcount',
                         INDUSTRY='tstindustry',
                         BETA='tstbeta',
                         DCF='testdcf')

    def test_map_to_year(self):
        sample_line = 'https://www.sec.gov/Archives/edgar/full-index/{}/QTR1/master.idx'
        test_year = '2018'
        expected_result = sample_line.format(test_year)
        res = map_to_year(sample_line, test_year)
        self.assertEqual(expected_result, res)

    def test_map_from_edgar_row(self):
        sample = "|First|13-HF|2018-01-24|edgar/testfile.txt|SomethingElse|"
        expected_result = ('2018-01-24', 'https://www.sec.gov/Archives/edgar/testfile.txt')
        self.assertEqual(expected_result, map_from_edgar_row(sample))

    def test_map_to_bucket_string(self):
        test_dict = self.get_test_dictionary()
        expected_str = '{},{},{},{}'.format(test_dict['COB'],
                                            test_dict['CUSIP'],
                                            test_dict['TICKER'],
                                            test_dict['COUNT'])
        self.assertEqual(expected_str, map_to_bucket_string(test_dict))

    def test_map_to_bq_dict(self):
        test_dict = self.get_test_dictionary()
        test_year = '2018'
        test_dict['EDGAR_YEAR'] = test_year
        res  = map_to_bq_dict(test_dict, test_year)

        for k, v in res.items():
            self.assertEqual(test_dict[k], res[k], 'Test failed for {}: expected:{}'.format(k, v))


import math
import random

import cvzone
import cv2
import numpy as np
from cvzone.HandTrackingModule import HandDetector

import base64
from memory_pic import *

def get_pic(pic_code, pic_name):
    image = open(pic_name, 'wb')
    image.write(base64.b64decode(pic_code))
    image.close()

get_pic(donut_png, 'donut.png')

cap = cv2.VideoCapture(0)  # 0代表自己电脑的摄像头
cap.set(3, 1280)  # 宽
cap.set(4, 720)  # 高

# 手部检测方法，置信度为0.8，最多检测一只手
detector = HandDetector(detectionCon=0.8, maxHands=1)

class SnakeGameClass:
    def __init__(self, pathFood):  # 构造方法
        self.points = []  # 蛇身上所有的点
        self.lengths = []  # 每个点之间的长度
        self.currentLength = 0  # 蛇的总长
        self.allowedLength = 150  # 蛇允许的总长度
        self.previousHead = 0, 0  # 第二个头结点

        self.imgFood = cv2.imread(pathFood, cv2.IMREAD_UNCHANGED)
        self.hFood, self.wFood, _ = self.imgFood.shape
        self.foodPoint = 0, 0
        self.randomFoodLocation()

        self.score = 0
        self.gameOver = False

    def randomFoodLocation(self):
        self.foodPoint = random.randint(100, 1000), random.randint(100, 600)

    def update(self, imgMain, currentHead):  # 实例方法

        if self.gameOver:
            cvzone.putTextRect(imgMain, "Game Over", [300, 400],
                               scale=7, thickness=5, offset=20)
            cvzone.putTextRect(imgMain, f'Your Score:{self.score}', [300, 550],
                               scale=7, thickness=5, offset=20)
        else:
            px, py = self.previousHead
            cx, cy = currentHead

            self.points.append([cx, cy])  # 添加蛇的点列表节点
            distance = math.hypot(cx - px, cy - py)  # 两点之间的距离
            self.lengths.append(distance)  # 添加蛇的距离列表内容
            self.currentLength += distance
            self.previousHead = cx, cy

            # Length Reduction
            if self.currentLength > self.allowedLength:
                for i, length in enumerate(self.lengths):
                    self.currentLength -= length
                    self.lengths.pop(i)
                    self.points.pop(i)
                    if self.currentLength < self.allowedLength:
                        break

            # Check if snake ate the food
            rx, ry = self.foodPoint
            if rx - self.wFood // 2 < cx < rx + self.wFood // 2 and \
                    ry - self.hFood // 2 < cy < ry + self.hFood // 2:
                self.randomFoodLocation()
                self.allowedLength += 50
                self.score += 1
                # print(self.score)

            # Draw Snake
            if self.points:
                for i, point in enumerate(self.points):
                    if i != 0:
                        cv2.line(imgMain, self.points[i - 1], self.points[i], (0, 0, 255), 20)
                # 对列表最后一个点也就是蛇头画为紫色点
                cv2.circle(imgMain, self.points[-1], 20, (200, 0, 200), cv2.FILLED)

            # Draw Food

            imgMain = cvzone.overlayPNG(imgMain, self.imgFood,
                                        (rx - self.wFood // 2, ry - self.hFood // 2))

            cvzone.putTextRect(imgMain, f'Your Score:{self.score}', [50, 80],
                               scale=3, thickness=5, offset=10)

            # Check for Collision
            pts = np.array(self.points[:-2], np.int32)
            pts = pts.reshape((-1, 1, 2))  # 重塑为一个行数未知但只有一列且每个元素有2个子元素的矩阵
            # 第三个参数是False，我们得到的是不闭合的线
            cv2.polylines(imgMain, [pts], False, (0, 200, 0), 3)
            # 参数True表示输出该像素点到轮廓最近距离
            minDist = cv2.pointPolygonTest(pts, (cx, cy), True)
            # print(minDist)

            if -0.05 <= minDist <= 0.05:
                # print("Hit")
                self.gameOver = True
                self.points = []  # 蛇身上所有的点
                self.lengths = []  # 每个点之间的长度
                self.currentLength = 0  # 蛇的总
                self.allowedLength = 150  # 蛇允许的总长度
                self.previousHead = 0, 0  # 第二个头结点
                self.randomFoodLocation()

        return imgMain

game = SnakeGameClass("donut.png")

# 处理每一帧图像
while True:  # 不断迭代更新
    success, img = cap.read()
    # 翻转图像，使自身和摄像头中的自己呈镜像关系
    img = cv2.flip(img, 1)  # 将手水平翻转
    hands, img = detector.findHands(img, flipType=False)  # 左手是左手，右手是右手，映射正确

    if hands:
        lmList = hands[0]['lmList']  # hands是由N个字典组成的列表
        pointIndex = lmList[8][0:2]  # 只要食指指尖的x和y坐标
        img = game.update(img, pointIndex)

    cv2.imshow("Image", img)
    key = cv2.waitKey(1)

    if key == ord('r'):  # 重置游戏
        game.gameOver = False
        game.score = 0

    if key == 27:  # 按下 Esc退出 (27是按键ESC对应的ASCII值)
        break

    if cv2.getWindowProperty("Image", cv2.WND_PROP_AUTOSIZE) < 1:  # 判断是否点击窗口关闭按键
        break

cap.release()
cv2.destroyAllWindows()


import os
import random
import shutil

# Get the list of captured image files
image_files = os.listdir("captured_images")

# Shuffle the image files randomly
random.shuffle(image_files)

# Calculate the split point for train and test images
split_point = int(0.5 * len(image_files))

# Create the parent folder
parent_folder = "dataset"
os.makedirs(parent_folder, exist_ok=True)

# Create the train folder
train_folder = os.path.join(parent_folder, "train")
os.makedirs(train_folder, exist_ok=True)

# Create the test folder
test_folder = os.path.join(parent_folder, "test")
os.makedirs(test_folder, exist_ok=True)

# Move the images to the train folder
for image_file in image_files[:split_point]:
    src = os.path.join("captured_images", image_file)
    dst = os.path.join(train_folder, image_file)
    shutil.move(src, dst)

# Move the images to the test folder
for image_file in image_files[split_point:]:
    src = os.path.join("captured_images", image_file)
    dst = os.path.join(test_folder, image_file)
    shutil.move(src, dst)

print("Images distributed successfully!")


#!"C:\Users\Administrador\AppData\Local\Programs\Python\Python38\python.exe"

"""
5. Definir una función que invierta una cadena dada.
"""
def inversor(frase):
    longitud = len(frase)
    invertida = ""
    indice = -1
    while longitud >= 1:
        invertida += frase[indice]
        indice += -1
        longitud -= 1

    return invertida

frase = "el veloz murcielago, vuela alto en la montaña"
cadena = "Una cadena"

print("frase: " + inversor(frase))
# print("cadena: " + str(contador_vocales(cadena)))


from autobahn.twisted.websocket import WebSocketServerProtocol, \
                                                    WebSocketServerFactory

import armatureio
import json

g_armature = None

class MyServerProtocol(WebSocketServerProtocol):

    def onConnect(self, request):
        print("Client connecting: {0}".format(request.peer))

    def onOpen(self):
        print("WebSocket connection open.")

    def onMessage(self, payload, isBinary):
        global g_armature

        g_armature.read_values()
        jointstate = g_armature.get_named_joint_values()

        # Ignore message and just send back joints
        jointmsg = json.dumps(jointstate)

        ## echo back message verbatim
        self.sendMessage(jointmsg, False)

    def onClose(self, wasClean, code, reason):
        print("WebSocket connection closed: {0}".format(reason))

if __name__ == '__main__':
    global g_armature

    import sys

    from twisted.python import log
    from twisted.internet import reactor

    log.startLogging(sys.stdout)

    g_armature = armatureio.ArmatureReader("COM3")
    joints = armatureio.read_old_json_config("arm_config.json")
    g_armature.add_named_joints(joints)

    factory = WebSocketServerFactory("ws://localhost:9000", debug = False)
    factory.protocol = MyServerProtocol

    reactor.listenTCP(9000, factory)
    reactor.run()


import numpy as np
import pyopencl as cl
import pyopencl.array as cl_array

from PySoap2_gpu.Exceptions import check_for_valid_context

class BroadcastError(Exception):
    pass

class Broadcast:
    device_context = None
    device_queue = None

    device_program = None

    initialized = False

    def __init__(self, context, queue):
        # If this class is initialized, it means that the programs is already on the device
        if Broadcast.initialized:
            return
        Broadcast.device_context = context
        Broadcast.device_queue = queue

        Broadcast.device_program = cl.Program(context, broadcast_across_axis_c_code).build()

        Broadcast.initialized = True

    @staticmethod
    def broadcast_across_0_axis(operation, x_device, y_device):
        """ It is assumed that the operation to be performed is

            x_device 'operation' y_device

            If operation = '-', then this will return x_device - y_device

            Parameters
            ----------
            x_device : (N, ...) cl_array.Array (or (...))
            y_device : (...) cl_array.Array (or (N, ...))
        """
        check_for_valid_context(Broadcast.device_context, x_device, y_device)

        error_msg = f'Broadcasting two arrays must the last dimension equal. ' \
                    f'In particular, x.shape={x_device.shape} not compatible y.shape={y_device.shape}.'
        try:
            return Broadcast._broadcast_across_0_axis(operation, x_device, y_device, error_msg)
        except BroadcastError:
            """ Now assumed that

                x_device : (...) cl_array.Array
                y_device : (N, ...) cl_array.Array
            """
            if operation == "-":  # These are special operation that is sensitive on the position
                return Broadcast._broadcast_across_0_axis('+', -y_device, x_device, error_msg)
            elif operation == '/':
                # Special divide
                return Broadcast._broadcast_across_0_axis('_/', y_device, x_device, error_msg)
            else:
                return Broadcast._broadcast_across_0_axis(operation, y_device, x_device, error_msg)

    @staticmethod
    def _broadcast_across_0_axis(operation, x_device, y_device, error_msg):
        """ Performs broadcasting on the gpu

            Parameters
            ----------
            x_device : (N, ...) cl_array.Array
            y_device : (...) cl_array.Array
            operation : str
                The operation to be performed. Only +,-,*,/ are supported

            y is to be broadcast against x
        """
        N, *input_shape = x_device.shape

        if tuple(input_shape) != y_device.shape:
            raise BroadcastError(error_msg)

        input_shape = tuple(input_shape)
        input_length = np.int32(np.prod(input_shape))
        N = np.int32(N)

        out = cl_array.empty_like(x_device)

        broadcast_program = Broadcast._get_broadcast_program(operation)
        event = broadcast_program(Broadcast.device_queue, (N, input_length), None,
                                  x_device.data, y_device.data, input_length, N, out.data)
        event.wait()
        return out

    @staticmethod
    def _get_broadcast_program(operation):
        if operation == "+":
            return Broadcast.device_program.broadcast_add_across_0_axis
        elif operation == "-":
            return Broadcast.device_program.broadcast_sub_across_0_axis
        elif operation == "*":
            return Broadcast.device_program.broadcast_mul_across_0_axis
        elif operation == "/":
            return Broadcast.device_program.broadcast_div_across_0_axis
        elif operation == "_/":
            return Broadcast.device_program._broadcast_div_across_0_axis
        else:
            raise ValueError(f"{operation} is not a valid operation. Choose from +,-,*,/.")

broadcast_across_axis_c_code = """
__kernel void broadcast_add_across_0_axis(__global const double *x, __global const double *y, const int input_length,
                                          const int N, __global double *out)
{
    int i = get_global_id(0);
    int j = get_global_id(1);

    out[j + i*input_length] = x[j + i*input_length] + y[j];
}

__kernel void broadcast_sub_across_0_axis(__global const double *x, __global const double *y, const int input_length,
                                          const int N, __global double *out)
{
    int i = get_global_id(0);
    int j = get_global_id(1);

    out[j + i*input_length] = x[j + i*input_length] - y[j];
}

__kernel void broadcast_mul_across_0_axis(__global const double *x, __global const double *y, const int input_length,
                                          const int N, __global double *out)
{
    int i = get_global_id(0);
    int j = get_global_id(1);

    out[j + i*input_length] = x[j + i*input_length] * y[j];
}

__kernel void broadcast_div_across_0_axis(__global const double *x, __global const double *y, const int input_length,
                                          const int N, __global double *out)
{
    int i = get_global_id(0);
    int j = get_global_id(1);

    out[j + i*input_length] = x[j + i*input_length] /  y[j];
}

__kernel void _broadcast_div_across_0_axis(__global const double *x, __global const double *y, const int input_length,
                                           const int N, __global double *out)
{
    int i = get_global_id(0);
    int j = get_global_id(1);

    out[j + i*input_length] =  y[j] / x[j + i*input_length];
}
"""


#!/usr/bin/python3
"""web server distribution"""
from fabric.api import *
import os.path

env.user = 'ubuntu'
env.hosts = ["54.146.65.131", "18.208.120.199"]
env.key_filename = "~/.ssh/school"

def do_deploy(archive_path):
    """distributes an archive to your web servers
    """
    if os.path.exists(archive_path) is False:
        return False
    data_path = '/data/web_static/releases/'
    tmp = archive_path.split('.')[0]
    name = tmp.split('/')[1]
    dest = data_path + name

    try:
        put(archive_path, '/tmp')
        run('mkdir -p {}'.format(dest))
        run('tar -xzf /tmp/{}.tgz -C {}'.format(name, dest))
        run('rm -f /tmp/{}.tgz'.format(name))
        run('mv {}/web_static/* {}/'.format(dest, dest))
        run('rm -rf {}/web_static'.format(dest))
        run('rm -rf /data/web_static/current')
        run('ln -s {} /data/web_static/current'.format(dest))
        return True
    except Exception:
        return False


import gc
from os import remove
from unittest import mock

import numpy as np
import pytest

from hyperspy._signals.signal1d import Signal1D
from hyperspy.components1d import Expression, Gaussian, GaussianHF
from hyperspy.io import load

def clean_model_dictionary(d):
    for c in d['components']:
        for p in c['parameters']:
            del p['self']
    return d

class TestModelStoring:

    def setup_method(self, method):
        s = Signal1D(range(100))
        m = s.create_model()
        m.append(Gaussian())
        m.fit()
        self.m = m

    def test_models_getattr(self):
        m = self.m
        s = m.signal
        m.store()
        assert s.models.a is s.models['a']

    def test_models_stub_methods(self):
        m = self.m
        s = m.signal
        m.store()
        s.models.pop = mock.MagicMock()
        s.models.remove = mock.MagicMock()
        s.models.restore = mock.MagicMock()
        s.models.a.restore()
        s.models.a.remove()
        s.models.a.pop()

        assert s.models.pop.call_count == 1
        assert s.models.remove.call_count == 1
        assert s.models.restore.call_count == 1

        assert s.models.pop.call_args[0] == ('a',)
        assert s.models.remove.call_args[0] == ('a',)
        assert s.models.restore.call_args[0] == ('a',)

    def test_models_pop(self):
        m = self.m
        s = m.signal
        m.store()
        s.models.remove = mock.MagicMock()
        s.models.restore = mock.MagicMock()
        s.models.pop('a')
        assert s.models.remove.call_count == 1
        assert s.models.restore.call_count == 1
        assert s.models.remove.call_args[0] == ('a',)
        assert s.models.restore.call_args[0] == ('a',)

    def test_model_store(self):
        m = self.m
        m.store()
        d = m.as_dictionary(True)
        np.testing.assert_equal(
            d,
            m.signal.models._models.a._dict.as_dictionary())

    def test_actually_stored(self):
        m = self.m
        m.store()
        m[0].A.map['values'][0] += 13.33
        m1 = m.signal.models.a.restore()
        assert m[0].A.map['values'] != m1[0].A.map['values']

    def test_models_restore_remove(self):
        m = self.m
        s = m.signal
        m.store('a')
        m1 = s.models.restore('a')
        m2 = s.models.a.restore()
        d_o = clean_model_dictionary(m.as_dictionary())
        d_1 = clean_model_dictionary(m1.as_dictionary())
        d_2 = clean_model_dictionary(m2.as_dictionary())
        np.testing.assert_equal(d_o, d_1)
        np.testing.assert_equal(d_o, d_2)
        assert 1 == len(s.models)
        s.models.a.remove()
        assert 0 == len(s.models)

    def test_store_name_error1(self):
        s = self.m.signal
        with pytest.raises(KeyError):
            s.models.restore('a')

    def test_store_name_error2(self):
        s = self.m.signal
        with pytest.raises(KeyError):
            s.models.restore(3)

    def test_store_name_error3(self):
        s = self.m.signal
        with pytest.raises(KeyError):
            s.models.restore('_a')

    def test_store_name_error4(self):
        s = self.m.signal
        with pytest.raises(KeyError):
            s.models.restore('a._dict')

    def test_store_name_error5(self):
        s = self.m.signal
        self.m.store('b')
        with pytest.raises(KeyError):
            s.models.restore('a')

@pytest.mark.parametrize('save_std', [True, False])
@pytest.mark.parametrize('only_free', [True, False])
@pytest.mark.parametrize('only_active', [True, False])
def test_model_export(tmp_path, save_std, only_free, only_active):
    s = Signal1D(range(100))
    m = s.create_model()
    m.append(Gaussian())
    m.fit()
    m.export_results(tmp_path, save_std=save_std, only_free=only_free,
                     only_active=only_active)

class TestModelSaving:

    def setup_method(self, method):
        s = Signal1D(range(100))
        m = s.create_model()
        m.append(Gaussian(A=13))
        m[-1].name = 'something'
        m.append(GaussianHF(module="numpy"))
        m[-1].height.value = 3
        m.append(Expression(name="Line", expression="a * x + b", a=1, c=0, rename_pars={"b": "c"}))
        self.m = m

    def test_save_and_load_model(self):
        m = self.m
        m.save('tmp.hdf5', overwrite=True)
        s = load('tmp.hdf5')
        assert hasattr(s.models, 'a')
        mr = s.models.restore('a')
        assert mr.components.something.A.value == 13
        assert mr.components.GaussianHF.height.value == 3
        assert mr.components.Line.a.value == 1
        assert mr.components.Line.c.value == 0
        assert mr.components.Line.function(10) == 10

    def teardown_method(self, method):
        gc.collect()        # Make sure any memmaps are closed first!
        remove('tmp.hdf5')

class TestEELSModelSaving:

    def setup_method(self, method):
        s = Signal1D(range(100))
        s.axes_manager[0].offset = 280
        s.set_signal_type("EELS")
        s.add_elements(["C"])
        s.set_microscope_parameters(100, 10, 10)
        m = s.create_model(auto_background=False)
        m.components.C_K.fine_structure_smoothing = 0.5
        m.components.C_K.fine_structure_width = 50
        m.components.C_K.fine_structure_active = True
        self.m = m

    def test_save_and_load_model(self):
        m = self.m
        m.save('tmp.hdf5', overwrite=True)
        l = load('tmp.hdf5')
        assert hasattr(l.models, 'a')
        n = l.models.restore('a')
        assert n[0].fine_structure_width == 50

    def teardown_method(self, method):
        gc.collect()        # Make sure any memmaps are closed first!
        remove('tmp.hdf5')


import multiprocessing

def download_data(q):
    """下载数据"""
    data = [45, 56, 2, 3]

    #向队列中写入数据
    for temp in data:
        q.put(temp)
    print("数据下载完成！")

def recv_data(q):
    """数据处理"""
    recv_list = list()  #创建空列表，用来存数据
    #从队列中获取数据
    while True:
        dataa = q.get()
        recv_list.append(dataa)
        #判断队列中是否还有数据，如果没有则终止循环
        if q.empty():
            break

    print(recv_list)

def main():

    #创建一个队列
    q = multiprocessing.Queue()

    #实例化对象
    t1 = multiprocessing.Process(target=download_data,args=(q,))
    t2 = multiprocessing.Process(target=recv_data,args=(q,))
    #创建两个进程
    t1.start()
    t2.start()

if __name__ == "__main__":
    main()

队列进程配合使用，下载列表.py

队列进程配合使用，下载列表.py


import sys

filename = 'input.txt'

time_lst = []
with open(filename) as file:
    ts = int(file.readline())
    for time in file.readline().strip().split(','):
        if time != 'x':
            time_lst.append(int(time))

min_id = None
min_time = float('inf')

for time in time_lst:
    missed_interval = ts % time
    if missed_interval == 0:
        print(time)
        sys.exit(0)

    catch_time = ts - missed_interval + time

    if catch_time < min_time:
        min_time = catch_time
        min_id = time

wait_time = min_time - ts
print(min_id, min_time)
print(min_id * wait_time)


#!/usr/bin/env python3

import pdb
import sys
import tqdm
import numpy as np
import pandas as pd
import seaborn as sb
import matplotlib.pyplot as plt
import sklearn
from statsmodels.stats.contingency_tables import mcnemar
import os
from pandas import Series, DataFrame

file1 = pd.read_csv(sys.argv[1], delimiter=',')
file2 = pd.read_csv(sys.argv[2], delimiter=',')

corrs1 = file1['correct']
corrs2 = file2['correct']

golds = open(sys.argv[3], 'r').readlines()
training_labels = open('rares.txt', 'r').readlines()
rares = [x.strip().split() for x in training_labels]
rares = [x[1] for x in rares if int(x[0]) < 21]

bothright = 0
YesNo = 0
NoYes = 0
bothwrong = 0

acc1 = 0
acc2 = 0
total = 0

rare_bothright = 0
rare_YesNo = 0
rare_NoYes = 0
rare_bothwrong = 0

rare_acc1 = 0
rare_acc2 = 0
rare_total = 0

statistic = lambda x,y: (x - y)**2 / (x + y)

for c1, c2, true in zip(corrs1, corrs2, golds):
    gold_label = true.split('\t')[0]
    if gold_label not in ['359', '360']:
        if c1 == True and c2 == True:
            bothright += 1
            acc1 += 1
            acc2 += 1
            if gold_label in rares:
                rare_bothright += 1
                rare_acc1 += 1
                rare_acc2 += 1
        elif c1 == True and c2 == False:
            YesNo += 1
            acc1 += 1
            if gold_label in rares:
                rare_YesNo += 1
                rare_acc1 += 1
        elif c1 == False and c2 == True:
            NoYes += 1
            acc2 += 1
            if gold_label in rares:
                rare_NoYes += 1
                rare_acc2 += 1
        elif c1 == False and c2 == False:
            bothwrong += 1
            if gold_label in rares:
                rare_bothwrong += 1
        else:
            print("Error")
            pdb.set_trace()
        total += 1
        if gold_label in rares:
            rare_total += 1

print("sys1 acc:", acc1 / total)
print("sys2 acc:", acc2 / total)

print("manual:", statistic(YesNo, NoYes))
print("statsmodels", mcnemar(np.array([[bothright, YesNo], [NoYes, bothwrong]])))
print("statsmodels exact false", mcnemar(np.array([[bothright, YesNo], [NoYes, bothwrong]]), exact=False))
print("statsmodels correction false", mcnemar(np.array([[bothright, YesNo], [NoYes, bothwrong]]), correction=False))
print("statsmodels exact false correction false", mcnemar(np.array([[bothright, YesNo], [NoYes, bothwrong]]), exact=False, correction=False))

print('\nrare labels!!!')
print('\nthere are {} total labels'.format(total))
print("sys1 acc:", rare_acc1 / rare_total)
print("sys2 acc:", rare_acc2 / rare_total)

print("manual:", statistic(rare_YesNo, rare_NoYes))
print("statsmodels", mcnemar(np.array([[rare_bothright, rare_YesNo], [rare_NoYes, rare_bothwrong]])))
print("statsmodels exact false", mcnemar(np.array([[rare_bothright, rare_YesNo], [rare_NoYes, rare_bothwrong]]), exact=False))
print("statsmodels correction false", mcnemar(np.array([[rare_bothright, rare_YesNo], [rare_NoYes, rare_bothwrong]]), correction=False))
print("statsmodels exact false correction false", mcnemar(np.array([[rare_bothright, rare_YesNo], [rare_NoYes, rare_bothwrong]]), exact=False, correction=False))


import torch
from skimage.color import rgb2hsv

from experts.basic_expert import BasicExpert

class HSVExpert(BasicExpert):
    def __init__(self, full_expert=True):
        self.n_maps = 3
        self.domain_name = "hsv"
        self.str_id = ""
        self.identifier = self.domain_name

    def apply_expert_batch(self, batch_rgb_frames):
        batch_rgb_frames = batch_rgb_frames
        outp_maps = rgb2hsv(batch_rgb_frames)
        hsv_maps = outp_maps.astype('float32').transpose(0, 3, 1, 2) / 255.
        return hsv_maps


from copy import deepcopy
from functools import partial

import jax
import jax.numpy as jnp
import optax
from flax.training.train_state import TrainState
from ml_collections import ConfigDict

from algos.mf_algos.model import update_target_network
from core.core_api import Algo
from utilities.jax_utils import mse_loss, next_rng, value_and_multi_grad

class TD3(Algo):

  @staticmethod
  def get_default_config(updates=None):
    config = ConfigDict()
    config.nstep = 1
    config.discount = 0.99
    config.tau = 0.005
    config.policy_noise = 0.2
    config.noise_clip = 0.5
    config.policy_freq = 2
    config.alpha = 2.5
    config.policy_lr = 3e-4
    config.qf_lr = 3e-4
    config.use_scheduler = False
    config.target_entropy = 0.0

    if updates is not None:
      config.update(ConfigDict(updates).copy_and_resolve_references())
    return config

  def __init__(self, config, policy, qf, max_steps):
    self.config = self.get_default_config(config)
    self.policy = policy
    self.qf = qf
    self.observation_dim = policy.observation_dim
    self.action_dim = policy.action_dim
    self.policy_noise = self.config.policy_noise * policy.max_action
    self.max_action = policy.max_action
    self.max_step = max_steps

    self._total_steps = 0
    self._train_states = {}

    policy_params = self.policy.init(
      {
        'dropout': next_rng(),
        'params': next_rng()
      },
      next_rng(),
      jnp.zeros((10, self.observation_dim)),
    )
    policy_optim = optax.adam(self.config.policy_lr)
    if self.config.use_scheduler:
      schedule_fn = optax.cosine_decay_schedule(
        -self.config.policy_lr, self.max_steps
      )
      policy_optim = optax.chain(
        optax.scale_by_adam(), optax.scale_by_schedule(schedule_fn)
      )
    self._train_states['policy'] = TrainState.create(
      params=policy_params, tx=policy_optim, apply_fn=None
    )

    qf1_params = self.qf.init(
      next_rng(),
      jnp.zeros((10, self.observation_dim)),
      jnp.zeros((10, self.action_dim)),
    )
    self._train_states['qf1'] = TrainState.create(
      params=qf1_params,
      tx=optax.adam(self.config.qf_lr),
      apply_fn=None,
    )
    qf2_params = self.qf.init(
      next_rng(),
      jnp.zeros((10, self.observation_dim)),
      jnp.zeros((10, self.action_dim)),
    )
    self._train_states['qf2'] = TrainState.create(
      params=qf2_params,
      tx=optax.adam(self.config.qf_lr),
      apply_fn=None,
    )

    self._tgt_params = deepcopy(
      {
        'policy': policy_params,
        'qf1': qf1_params,
        'qf2': qf2_params,
      }
    )

    self._model_keys = ('policy', 'qf1', 'qf2')

  @partial(jax.jit, static_argnames=('self', 'policy_update'))
  def _train_step(
    self, train_states, tgt_params, rng, batch, policy_update=False
  ):

    def loss_fn(params, tgt_params, rng):
      observations = batch['observations']
      actions = batch['actions']
      rewards = batch['rewards']
      next_observations = batch['next_observations']
      dones = batch['dones']

      rng, split_rng = jax.random.split(rng)
      # Select next actions according to policy and add noise
      noise = jax.random.truncated_normal(
        split_rng,
        -self.config.noise_clip,
        self.config.noise_clip,
        shape=actions.shape,
        dtype=actions.dtype,
      ) * self.policy_noise

      rng, split_rng = jax.random.split(rng)
      next_action = self.policy.apply(
        params['policy'], rng, next_observations, rngs={'dropout': split_rng}
      )
      next_action = jnp.clip(
        next_action + noise, -self.max_action, self.max_action
      )

      # Compute the target Q values (without gradient)
      tgt_q1 = self.qf.apply(tgt_params['qf1'], next_observations, next_action)
      tgt_q2 = self.qf.apply(tgt_params['qf2'], next_observations, next_action)
      tgt_q = jnp.minimum(tgt_q1, tgt_q2)
      tgt_q = rewards + (1 - dones) * self.config.discount * tgt_q
      tgt_q = jax.lax.stop_gradient(tgt_q)

      # Compute the current Q estimates
      cur_q1 = self.qf.apply(params['qf1'], observations, actions)
      cur_q2 = self.qf.apply(params['qf2'], observations, actions)

      # qf loss
      qf1_loss = mse_loss(cur_q1, tgt_q)
      qf2_loss = mse_loss(cur_q2, tgt_q)
      qf_loss = qf1_loss + qf2_loss

      # Compute the policy loss
      rng, split_rng = jax.random.split(rng)
      new_actions = self.policy.apply(
        params['policy'], rng, observations, rngs={'dropout': split_rng}
      )
      q = self.qf.apply(params['qf1'], observations, new_actions)
      q_abs_mean = jax.lax.stop_gradient(jnp.abs(q).mean())
      lmbda = self.config.alpha / q_abs_mean
      bc_loss = mse_loss(new_actions, actions)
      policy_loss = -lmbda * q.mean() + bc_loss

      loss_collection = {'policy': policy_loss, 'qf1': qf_loss, 'qf2': qf_loss}
      return tuple(loss_collection[key] for key in self.model_keys), locals()

    # Calculate losses and grads
    params = {key: train_states[key].params for key in self.model_keys}
    (_, aux_values), grads = value_and_multi_grad(
      loss_fn, len(self.model_keys), has_aux=True
    )(params, tgt_params, rng)

    # Update qf train states
    train_states['qf1'] = train_states['qf1'].apply_gradients(
      grads=grads[1]['qf1']
    )
    train_states['qf2'] = train_states['qf2'].apply_gradients(
      grads=grads[2]['qf2']
    )

    # Update policy train states if necessary
    if policy_update:
      train_states['policy'] = train_states['policy'].apply_gradients(
        grads=grads[0]['policy']
      )

      # Update target parameters
      tgt_params['policy'] = update_target_network(
        train_states['policy'].params, tgt_params['policy'], self.config.tau
      )
      tgt_params['qf1'] = update_target_network(
        train_states['qf1'].params, tgt_params['qf1'], self.config.tau
      )
      tgt_params['qf2'] = update_target_network(
        train_states['qf2'].params, tgt_params['qf2'], self.config.tau
      )

    metrics = dict(
      policy_loss=aux_values['policy_loss'],
      qf_loss=aux_values['qf_loss'],
      qf1_loss=aux_values['qf1_loss'],
      qf2_loss=aux_values['qf2_loss'],
      bc_loss=aux_values['bc_loss'],
      cur_q1=aux_values['cur_q1'].mean(),
      cur_q2=aux_values['cur_q2'].mean(),
      tgt_q1=aux_values['tgt_q1'].mean(),
      tgt_q2=aux_values['tgt_q2'].mean(),
      tgt_q=aux_values['tgt_q'].mean(),
      lmbda=aux_values['lmbda'],
      q_abs_mean=aux_values['q_abs_mean'],
    )

    return train_states, tgt_params, metrics

  def train(self, batch, update_target_policy):
    self._total_steps += 1
    self._train_states, self._tgt_params, metrics = self._train_step(
      self._train_states, self._tgt_params, next_rng(), batch,
      self._total_steps % self.config.policy_freq == 0
    )
    return metrics

  @property
  def model_keys(self):
    return self._model_keys

  @property
  def train_states(self):
    return self._train_states

  @property
  def train_params(self):
    return {key: self.train_states[key].params for key in self.model_keys}

  @property
  def total_steps(self):
    return self._total_steps


from django.contrib import admin
from django.urls import path

from .views import (
    edit_car, get_all_cars, get_all_cars_by_location, get_all_cars_distances_by_location,
    create_good, get_goods, certain_good, update_good, delete_good, filter_goods
)

urlpatterns = [
    path('car/edit/<int:id>/', edit_car, name='edit_car'),
    path('car/all/', get_all_cars, name='get_all_cars'),
    path('car/all/filter/', filter_goods),
    path('car/location/<int:location_id>/', get_all_cars_by_location, name='get_all_cars_by_location'),
    path('car/distance/<int:location_id>/', get_all_cars_distances_by_location, name='get_all_cars_distances_by_location'),

    path('good/create/', create_good, name='create_good'),
    path('good/all/', get_goods, name='get_goods'),
    path('good/<int:id>/', certain_good, name='certain_good'),
    path('good/update/<int:id>/', update_good, name='update_good'),
    path('good/delete/<int:id>/', delete_good, name='delete_good'),

    path('admin/', admin.site.urls),
]


class Node:
    def __init__(self, val):
        self.val = val
        self.isWord = False
        self.children = {}

class WordDictionary:

    def __init__(self):
        """
        Initialize your data structure here.
        """
        self.root = Node('')

    def addWord(self, word: str) -> None:
        """
        Adds a word into the data structure.
        """
        node = self.root
        for c in word:
            if not c in node.children:
                node.children[c] = Node(c)
            node = node.children[c]
        node.isWord = True

    def _search(self, word, node) -> bool:
        if not word:
            return node.isWord
        childList = []
        if word[0] == '.':
            childList = node.children
        elif word[0] in node.children:
            childList = [word[0]]
        for child in childList:
            if self._search(word[1:], node.children[child]):
                return True
        return False

    def search(self, word: str) -> bool:
        """
        Returns if the word is in the data structure. A word could contain the dot character '.' to represent any one letter.
        """
        return self._search(word, self.root)

# Your WordDictionary object will be instantiated and called as such:
# obj = WordDictionary()
# obj.addWord(word)
# param_2 = obj.search(word)


from mako import runtime, filters, cache
UNDEFINED = runtime.UNDEFINED
__M_dict_builtin = dict
__M_locals_builtin = locals
_magic_number = 5
_modified_time = 1248197032.671
_template_filename='C:\\fox\\rentfox\\rentfox\\templates/base/index.html'
_template_uri='/base/index.html'
_template_cache=cache.Cache(__name__, _modified_time)
_source_encoding='utf-8'
from webhelpers.html import escape
_exports = ['flash', 'header', 'head', 'footer', 'title']

def render_body(context,**pageargs):
    context.caller_stack._push_frame()
    try:
        __M_locals = __M_dict_builtin(pageargs=pageargs)
        self = context.get('self', UNDEFINED)
        next = context.get('next', UNDEFINED)
        __M_writer = context.writer()
        # SOURCE LINE 2
        __M_writer(u'\n<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"\n"http://www.w3.org/TR/html4/strict.dtd">\n\n\n    <title>')
        # SOURCE LINE 7
        __M_writer(escape(self.title()))
        __M_writer(u'</title>\n    ')
        # SOURCE LINE 8
        __M_writer(escape(self.head()))
        __M_writer(u'\n\n\n    ')
        # SOURCE LINE 11
        __M_writer(escape(self.header()))
        __M_writer(u'\n    ')
        # SOURCE LINE 12
        __M_writer(escape(next.heading()))
        __M_writer(u'\n    ')
        # SOURCE LINE 13
        __M_writer(escape(self.flash()))
        __M_writer(u'\n    ')
        # SOURCE LINE 14
        __M_writer(escape(next.body()))
        __M_writer(u'\n    ')
        # SOURCE LINE 15
        __M_writer(escape(self.footer()))
        __M_writer(u'\n\n\n\n')
        # SOURCE LINE 19
        __M_writer(u'\n')
        # SOURCE LINE 22
        __M_writer(u'\n')
        # SOURCE LINE 23
        __M_writer(u'\n')
        # SOURCE LINE 24
        __M_writer(u'\n')
        # SOURCE LINE 33
        __M_writer(u'\n\n')
        return ''
    finally:
        context.caller_stack._pop_frame()

def render_flash(context):
    context.caller_stack._push_frame()
    try:
        session = context.get('session', UNDEFINED)
        __M_writer = context.writer()
        # SOURCE LINE 25
        __M_writer(u'\n')
        # SOURCE LINE 26
        if session.has_key('flash'):
            # SOURCE LINE 27
            __M_writer(u'

')
            __M_writer(escape(session.get('flash')))
            __M_writer(u'

\n    ')
            # SOURCE LINE 28

            del session['flash']
            session.save()

            # SOURCE LINE 31
            __M_writer(u'\n')
        return ''
    finally:
        context.caller_stack._pop_frame()

def render_header(context):
    context.caller_stack._push_frame()
    try:
        h = context.get('h', UNDEFINED)
        __M_writer = context.writer()
        # SOURCE LINE 23
        __M_writer(u'
Home
')
        return ''
    finally:
        context.caller_stack._pop_frame()

def render_head(context):
    context.caller_stack._push_frame()
    try:
        h = context.get('h', UNDEFINED)
        __M_writer = context.writer()
        # SOURCE LINE 20
        __M_writer(u'\n    ')
        # SOURCE LINE 21
        __M_writer(escape(h.stylesheet_link(h.url_for('/css/global.css'))))
        __M_writer(u'\n')
        return ''
    finally:
        context.caller_stack._pop_frame()

def render_footer(context):
    context.caller_stack._push_frame()
    try:
        __M_writer = context.writer()
        # SOURCE LINE 24
        __M_writer(u'

')
        return ''
    finally:
        context.caller_stack._pop_frame()

def render_title(context):
    context.caller_stack._push_frame()
    try:
        __M_writer = context.writer()
        # SOURCE LINE 19
        __M_writer(u'Rentfox')
        return ''
    finally:
        context.caller_stack._pop_frame()


#!/usr/bin/env python
"""
 To use confluent-python:
    python -m pip install confluent-kafka
"""

from confluent_kafka import Consumer, KafkaError

settings = {
        "bootstrap.servers": "localhost:9092",
        "group.id": "tempgroup",
        "client.id": "client-1",
        "enable.auto.commit": True,
        "session.timeout.ms": 6000,
        "default.topic.config": {"auto.offset.reset":"smallest"}
}

c = Consumer(settings)

c.subscribe(["stream-temp"])

try:
    while True:
        msg = c.poll(0.1)
        if msg is None:
            continue
        elif not msg.error():
            print(f"Received: {msg.value().decode('utf-8')}")
        elif msg.error().code() == KafkaError._PARTITION_EOF:
            print(f"End of Partition reached {msg.topic()}/{msg.partition()}")
        else:
            print(f"Error occured: {msg.error().str()}")
except KeyboardInterrupt:
    pass
finally:
    c.close()


import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

data = np.genfromtxt("./timedel_sorted.dat", delimiter='    ').T
#  print(data)

date = data[0]
magA = data[1] + 1.8
sigmaMagA = data[2]
magB = data[3]
sigmaMagB = data[4]

k = 0
plt.figure(num=k, figsize=(10,7))
plt.errorbar(date, magA, yerr=sigmaMagA, label="Image A (mag+1.8)", capsize=2, fmt="o-", color="black")
plt.errorbar(date, magB, yerr=sigmaMagB, label="Image B", capsize=2, fmt="^-", color="black")

ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)

plt.xticks(np.linspace(np.amin(date), np.amax(date), 12))
#  plt.xlim(np.amin(date), np.amax(date))
plt.legend()
plt.xlabel("Julian date")
plt.ylabel("Magnitude")
plt.savefig("timeDelay.pdf", bbox_inches="tight")
plt.close(k)
k += 1

data = np.genfromtxt("./disp.out_MCprob.tab", delimiter="\t").T
timeDel = data[0]
prob = data[1]

def gauss(x, mu, sigma):
    return 1/(sigma * np.sqrt(2*np.pi)) * np.exp(-(x-mu)**2/sigma**2/2)

plt.figure(num=k, figsize=(10,7))
plt.scatter(timeDel, prob, color="black", marker="^", label="from MC")

popt, pcov = curve_fit(gauss, timeDel, prob, p0=[34, 2])
print("Parameters", popt)
print("Covariance matrix", pcov)
xArray = np.linspace(np.amin(timeDel), np.amax(timeDel), 100)
probFit = gauss(xArray, *popt)
plt.plot(xArray, probFit, label="fit", color="black")
plt.legend()

plt.xlabel("Time delay")
plt.ylabel("Probability")
ax = plt.gca()
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
plt.savefig("MCProb.pdf", bbox_inches="tight")
plt.close(k)
k += 1


import ipaddress

from django.contrib.auth import get_user_model
from django.core.management import BaseCommand, call_command, CommandError
from django.db import transaction

from ralph.accounts.tests.factories import RegionFactory
from ralph.assets.models import ConfigurationClass, ConfigurationModule
from ralph.data_importer.management.commands.create_network import \
    Command as NetworkCommand
from ralph.data_importer.management.commands.create_server_model import \
    Command as ServerModelCommand
from ralph.data_importer.management.commands.create_transitions import \
    Command as TransitionsCommand

class Command(BaseCommand):
    """
    Generate data useful to start a fresh and empty Ralph instance.
    """
    @transaction.atomic
    def handle(self, *args, **options):
        try:
            parent_network = ipaddress.ip_network(
                options.get('parent_network')
            )
            dc_name = options.get('dc_name')
            server_room_name = options.get('server_room_name')
            network_address = parent_network.network_address
            dns_1 = ipaddress.ip_address(options.get('dns_1'))
            dns_2 = ipaddress.ip_address(options.get('dns_2'))
            number_of_subnets = int(options.get('number_of_subnets'))
            region = options.get('region')
            configuration_path = options.get('configuration_path')
        except ValueError as e:
            raise CommandError(e)

        self._validate_network(parent_network, number_of_subnets)
        self._validate_configuration_path(configuration_path)

        self.create_users(region)
        self.create_configuration_path(configuration_path)
        TransitionsCommand.create_data_center_asset_transitions()

        NetworkCommand.create_network(
            dc_name=dc_name,
            dns1_address=dns_1,
            dns2_address=dns_2,
            gateway_address=network_address + 1,
            network=parent_network,
            server_room_name=server_room_name
        )
        for _ in range(0, number_of_subnets):
            network = ipaddress.ip_network('{}/{}'.format(network_address, 24))
            NetworkCommand.create_network(
                dc_name=dc_name,
                dns1_address=dns_1,
                dns2_address=dns_2,
                gateway_address=network_address + 1,
                network=network,
                server_room_name=server_room_name,
                create_rack=True
            )
            network_address += 256

        for name in ["A", "B", "C"]:
            ServerModelCommand.create_model(
                    model_name="Model {}".format(name)
            )
            ServerModelCommand.create_model(
                model_name="Blade server model {}".format(name),
                is_blade=True
            )

        call_command('sitetree_resync_apps')

    def add_arguments(self, parser):
        parser.add_argument(
            '-d', '--dc-name',
            default='dc1',
            dest='dc_name',
            help='Data center name.'
        )
        parser.add_argument(
            '-s', '--server-room-name',
            default='server room',
            dest='server_room_name',
            help='Server room name.'
        )
        parser.add_argument(
            '-p', '--parent-network',
            default='10.0.0.0/16',
            dest='parent_network',
            help='Parent network address.'
        )
        parser.add_argument(
            '--dns1',
            default='10.0.0.11',
            dest='dns_1',
            help='Primary DNS server.'
        )
        parser.add_argument(
            '--dns2',
            default='10.0.0.12',
            dest='dns_2',
            help='Secondary DNS server.'
        )
        parser.add_argument(
            '-n', '--number-of-subnets',
            default='3',
            dest='number_of_subnets',
            help='Number of /24 subnets to be generated.'
        )
        parser.add_argument(
            '-r', '--region',
            default='PL',
            dest='region',
            help='Geographical region (eg. your location).'
        )
        parser.add_argument(
            '-c', '--configuration-path',
            default='configuration_module/default',
            dest='configuration_path',
            help='Default configuration path.'
        )

    def _validate_network(self, network, number_of_subnets):
        num_addresses = number_of_subnets * 256
        if network.num_addresses < num_addresses:
            raise CommandError(
                "Network {} will not accommodate {} /24 subnets.".format(
                    network, number_of_subnets
                )
            )
        max_netmask = ipaddress.ip_address('255.255.254.0')
        if network.netmask > max_netmask:
            raise CommandError(
                "Net mask must be /23 or less."
            )

    def _validate_configuration_path(self, configuration_path):
        if len(configuration_path.split('/')) != 2:
            raise CommandError(
                "Configuration path must be a string with no spaces including"
                "one slash."
            )

    def create_users(self, region):
        user_model = get_user_model()
        user, _ = user_model.objects.get_or_create(
            username='admin', is_staff=True, is_superuser=True
        )
        user.regions.add(RegionFactory(name=region))
        user.set_password('admin')
        user.save()

    def create_configuration_path(self, configuration_path):
        configuration_module, configuration_class = configuration_path.split(
            '/'
        )
        module = ConfigurationModule.objects.create(name=configuration_module)
        ConfigurationClass.objects.create(
            class_name=configuration_class, module=module
        )


# ------------------------------------------------
# augment_package for CDeep3M -- NCMIR/NBCR, UCSD
# ------------------------------------------------
import os
import numpy as np
import h5py
from augment_data import augment_img

def augment_package(original, outsubdir, fmnumber, speed):
    allowed_speed = [1, 2, 4, 10]
    if speed not in allowed_speed:
        speed = allowed_speed[abs(np.array(allowed_speed) - speed).argmin()]

    if speed == 10:
        augment_choices = {
            1: [1],
            3: [1],
            5: [1]
        }
    elif speed == 1:
        augment_choices = {
            1: [1, 2, 3, 4, 5, 6, 7, 8],
            3: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
            5: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
        }
    elif speed == 2:
        augment_choices = {
            1: [1, 6, 11, 15],
            3: [1, 2, 3, 4, 13, 14, 15, 16],
            5: [5, 6, 7, 8, 9, 10, 11, 12]
        }
    elif speed == 4:
        augment_choices = {
            1: [2, 3, 6, 7],
            3: [7, 8, 10, 12],
            5: [1, 6, 11, 15]
        }

    do_var = augment_choices.get(fmnumber)

    original_flip = np.flip(original, 0)
    d_details = '/data'

    for i in do_var:
        if i < 9:
            stack_out = augment_img(original, i - 1)
        else:
            stack_out = augment_img(original_flip, i - 9)

        print('Create Hd5 file Variation ', str(i))

        stack_out = stack_out.transpose([1, 2, 0])

        filename = os.path.join(outsubdir, 'image_stacks_v%d.h5' % (i))
        print('Saving: ', filename)
        hdf5_file = h5py.File(filename, mode='w')
        hdf5_file.create_dataset(d_details, data=stack_out)
        hdf5_file.close()


from django.urls import path, include
from . import views

urlpatterns = [
    path('signup/', views.SignUp, name="Registraton"),
    path('login/', views.LogIn, name="LogIn"),
    path('logout/', views.logout_view, name="LogOut"),
    path('ChangePassword/', views.ChangePassword),
    path('profile/<str:pk>', views.get_user, name="Get Profile"),
    path('updateprofile/', views.updateProfile, name="Update Profile"),

    # chat urls
    path('sendd/<str:pk>', views.sendMessages),
    path('rec_msg/<str:uid>/<str:pk>', views.receivedMessages, name = "rec_msg"),
    path('Doctor/<str:user_id>', views.Doctor.as_view(), name = "Doctor"),
    path('detail/<str:uid>/<str:pk>', views.chat_history, name="detail"),
    path('seen/<str:uid>/<str:pk>', views.markMessagesAsSeen),

    # searchURL
    path('search',views.search),

    # get doctors for specialization
    path('doctors/', views.get_allDoctors),

    # track and update the location
    # user side
    path('ambulance/',views.requestAmbulance),
    path('updateUserLoc/', views.update_user_location),
    path('checkConnection/', views.checkConnection),
    path('getConnection/<str:ambid>', views.getConnection),
    # hospital side
    path('hospitalLogin/', views.HosLogin),
    path('HandleRequest/',views.HandleRequest),
    # ambulance
    path('ambLogin/', views.AmbLogin, name="ambulance login"),
    path('updateAmbulanceLoc/', views.update_ambulance_location),

    path('deleteConnection/',views.deleteConnection),

    #First Aid
    path('firstaid/',views.get_firstAid),
    path('firstaid/<str:pk>',views.get_oneFirstAid),
    path('firstaidd/<str:pk>',views.updateFirstAid),

    path('getdata/', views.getdata),
]