import argparse
import math
import os
import numpy as np
import torch
import multiprocessing as mp
from torch.utils.data import Dataset, DataLoader
from tqdm import tqdm
from sklearn.cluster import KMeans
from sklearn_extra.cluster import KMedoids
from medvqa.utils.data_structures import UnionFind
from medvqa.utils.files import (
    get_file_path_with_hashing_if_too_long,
    load_pickle,
    save_pickle,
)
from medvqa.datasets.mimiccxr import MIMICCXR_LARGE_FAST_CACHE_DIR
from medvqa.utils.logging import print_blue, print_bold

class FactEmbeddingsDataset(Dataset):
    def __init__(self, fact_embeddings):
        self.fact_embeddings = torch.tensor(fact_embeddings)

    def __len__(self):
        return len(self.fact_embeddings)

    def __getitem__(self, i):
        return self.fact_embeddings[i]

    def collate_fn(self, batch):
        return torch.stack(batch, dim=0)

_shared_nearest_idxs = None
_shared_similarities = None
def find_most_similar_facts(i, threshold, max_len):
    most_similar = []
    for j in range(len(_shared_nearest_idxs[i])-1, -1, -1):
        nearest_idx = _shared_nearest_idxs[i, j].item()
        if _shared_similarities[i, nearest_idx] < threshold:
            break
        most_similar.append(nearest_idx)
        if len(most_similar) == max_len:
            break
    return most_similar
def find_least_similar_facts(i, threshold, max_len):
    least_similar = []
    for j in range(len(_shared_nearest_idxs[i])):
        nearest_idx = _shared_nearest_idxs[i, j].item()
        if _shared_similarities[i, nearest_idx] > threshold:
            break
        least_similar.append(nearest_idx)
        if len(least_similar) == max_len:
            break
    return least_similar

def deduplicate_facts_with_union_find(facts, fact_embeddings, threshold):
    # run kmeans
    n_clusters = math.ceil(len(facts) / 500) # roughly 500 facts per cluster
    n_clusters = max(n_clusters, 2) # at least 2 clusters
    print(f'Running kmeans with n_clusters={n_clusters}...')
    kmeans = KMeans(n_clusters=n_clusters, n_init='auto', max_iter=300, random_state=0).fit(fact_embeddings)
    labels = kmeans.labels_
    c2idxs = {}
    for i, label in enumerate(labels):
        if label not in c2idxs:
            c2idxs[label] = []
        c2idxs[label].append(i)
    # deduplicate facts
    print('Deduplicating facts...')
    uf = UnionFind(len(facts))
    for idxs in c2idxs.values():
        for i in range(len(idxs)):
            for j in range(i+1, len(idxs)):
                if np.dot(fact_embeddings[idxs[i]], fact_embeddings[idxs[j]]) >= threshold:
                    uf.unionSet(idxs[i], idxs[j])
    # for each set, choose the longest fact
    print('Choosing longest fact for each set...')
    set2idx = {}
    for i in range(len(facts)):
        s = uf.findSet(i)
        if s not in set2idx:
            set2idx[s] = i
        else:
            if len(facts[i]) > len(facts[set2idx[s]]):
                set2idx[s] = i
    # return deduplicated facts and embeddings
    dedup_idxs = list(set2idx.values())
    dedup_idxs.sort()
    print('Number of facts removed:', len(facts) - len(dedup_idxs))
    dedup_facts = [facts[i] for i in dedup_idxs]
    dedup_fact_embeddings = fact_embeddings[dedup_idxs]
    return dedup_facts, dedup_fact_embeddings

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--integrated_embeddings_and_report_annotations_filepath', type=str, required=True)
    parser.add_argument('--finding_only', action='store_true')
    # clustering parameters
    parser.add_argument('--num_kmeans_clusters', type=int, default=500)
    parser.add_argument('--num_kmeans_iterations', type=int, default=300)
    parser.add_argument('--num_kmedoids_clusters', type=int, default=25000)
    parser.add_argument("--num_kmedoids_iterations", type=int, default=300)
    parser.add_argument("--kmedoids_method", type=str, default="alternate", choices=["pam", "alternate"])
    # nearest neighbors parameters
    parser.add_argument("--k", type=int, default=10)
    parser.add_argument("--pos_sim_threshold", type=float, default=0.9)
    parser.add_argument("--neg_sim_threshold", type=float, default=0.3)
    parser.add_argument("--union_find_threshold", type=float, default=0.97)
    parser.add_argument("--max_num_pos", type=int, default=100)
    parser.add_argument("--max_num_neg", type=int, default=100)
    # misc
    parser.add_argument("--batch_size", type=int, default=10000)
    parser.add_argument("--num_workers", type=int, default=4)
    parser.add_argument("--num_processes", type=int, default=8)
    args = parser.parse_args()

    # Load integrated embeddings and report annotations
    print_bold('Loading integrated embeddings and report annotations from:', args.integrated_embeddings_and_report_annotations_filepath)
    integrated_embeddings_and_report_annotations = load_pickle(args.integrated_embeddings_and_report_annotations_filepath)

    facts = integrated_embeddings_and_report_annotations['facts']
    fact_embeddings = integrated_embeddings_and_report_annotations['fact_embeddings']
    print('len(facts):', len(facts))
    print('fact_embeddings.shape:', fact_embeddings.shape)
    if args.finding_only:
        print('Selecting only findings facts...')
        findings_fact_idxs = set()
        for r in integrated_embeddings_and_report_annotations['reports']:
            findings_fact_idxs.update(r['findings_fact_idxs'])
        findings_fact_idxs = list(findings_fact_idxs)
        findings_fact_idxs.sort()
        facts = [facts[i] for i in findings_fact_idxs]
        fact_embeddings = fact_embeddings[findings_fact_idxs]
        print('  len(facts):', len(facts))
        print('  fact_embeddings.shape:', fact_embeddings.shape)
    fact2idx = {f: i for i, f in enumerate(facts)}

    # Find representative facts using kmeans and kmedoids

    representative_facts_filepath = get_file_path_with_hashing_if_too_long(
        folder_path=MIMICCXR_LARGE_FAST_CACHE_DIR,
        prefix='representative_facts',
        strings=[
            args.integrated_embeddings_and_report_annotations_filepath,
            f'num_kmeans_clusters={args.num_kmeans_clusters}',
            f'num_kmeans_iterations={args.num_kmeans_iterations}',
            f'num_kmedoids_clusters={args.num_kmedoids_clusters}',
            f'num_kmedoids_iterations={args.num_kmedoids_iterations}',
            f'kmedoids_method={args.kmedoids_method}',
            f'k={args.k}',
            f'finding_only={args.finding_only}',
            f'union_find_threshold={args.union_find_threshold}',
        ],
        force_hashing=True,
    )
    if os.path.exists(representative_facts_filepath):
        # Load representative facts
        print_bold('Loading representative facts from:', representative_facts_filepath)
        tmp = load_pickle(representative_facts_filepath)
        representative_facts = tmp['dedup_facts'] # use deduplicated facts
        representative_fact_embeddings = tmp['dedup_fact_embeddings'] # use deduplicated fact embeddings
        print('len(representative_facts):', len(representative_facts))
        print('representative_fact_embeddings.shape:', representative_fact_embeddings.shape)
    else:
        # 1. Run kmeans
        print_blue('Running kmeans...')
        kmeans = KMeans(n_clusters=args.num_kmeans_clusters, n_init='auto', max_iter=args.num_kmeans_iterations, random_state=0).fit(fact_embeddings)
        kmeans_labels = kmeans.labels_
        kmeans_c2idxs = {}
        for i, label in enumerate(kmeans_labels):
            if label not in kmeans_c2idxs:
                kmeans_c2idxs[label] = []
            kmeans_c2idxs[label].append(i)

        representative_fact_idxs = set()

        for idxs in kmeans_c2idxs.values():
            cluster_fact_embeddings = fact_embeddings[idxs]
            # 2. Run kmedoids
            print_blue('Running kmedoids...')
            n_clusters = math.ceil(len(idxs) * args.num_kmedoids_clusters / len(fact_embeddings)) # scale number of clusters based on number of facts
            print(f'  n_clusters: {n_clusters}, len(idxs): {len(idxs)}, cluster_fact_embeddings.shape: {cluster_fact_embeddings.shape}')
            kmedoids = KMedoids(n_clusters=n_clusters, metric='cosine', method=args.kmedoids_method,
                                max_iter=args.num_kmedoids_iterations, random_state=0).fit(cluster_fact_embeddings)
            kmedoids_labels = kmedoids.labels_
            kmedoids_centroids = kmedoids.cluster_centers_
            print(f'  kmedoids_labels.shape: {kmedoids_labels.shape}, kmedoids_centroids.shape: {kmedoids_centroids.shape}')
            c2idxs = {}
            for i, label in enumerate(kmedoids_labels):
                if label not in c2idxs:
                    c2idxs[label] = []
                c2idxs[label].append(i)
            for label, sub_idxs in c2idxs.items():
                centroid = kmedoids_centroids[label]
                sub_cluster_fact_embeddings = cluster_fact_embeddings[sub_idxs]
                # Find k nearest facts to centroid
                similarities = np.dot(sub_cluster_fact_embeddings, centroid)
                nearest_sentence_idxs = np.argsort(similarities)[::-1][:args.k]
                # Choose the longest sentence as the representative sentence
                max_len = 0
                max_len_idx = None
                for idx in nearest_sentence_idxs:
                    f = facts[idxs[sub_idxs[idx]]]
                    if len(f) > max_len:
                        max_len = len(f)
                        max_len_idx = idx
                assert max_len_idx is not None
                max_len_idx = idxs[sub_idxs[max_len_idx]]
                assert max_len_idx not in representative_fact_idxs
                representative_fact_idxs.add(max_len_idx)

        print('len(representative_fact_idxs):', len(representative_fact_idxs))
        representative_fact_idxs = list(representative_fact_idxs)
        representative_fact_idxs.sort()
        representative_fact_embeddings = fact_embeddings[representative_fact_idxs]
        representative_facts = [facts[i] for i in representative_fact_idxs]

        # De-duplicate representative facts with union-find
        print_blue('De-duplicating representative facts with union-find...')
        dedup_facts, dedup_fact_embeddings = deduplicate_facts_with_union_find(
            representative_facts, representative_fact_embeddings, args.union_find_threshold)
        print('len(dedup_facts):', len(dedup_facts))
        print('dedup_fact_embeddings.shape:', dedup_fact_embeddings.shape)

        print('Saving representative facts to:', representative_facts_filepath)
        save_pickle({
            'representative_facts': representative_facts,
            'representative_fact_embeddings': representative_fact_embeddings,
            'dedup_facts': dedup_facts,
            'dedup_fact_embeddings': dedup_fact_embeddings,
        }, representative_facts_filepath)

        # Use deduplicated facts and embeddings
        representative_fact_embeddings = dedup_fact_embeddings
        representative_facts = dedup_facts

    # Find positive and negative representative facts for each fact
    pos_and_neg_representative_facts_filepath = get_file_path_with_hashing_if_too_long(
        folder_path=MIMICCXR_LARGE_FAST_CACHE_DIR,
        prefix='pos_and_neg_representative_facts',
        strings=[
            args.integrated_embeddings_and_report_annotations_filepath,
            f'num_kmeans_clusters={args.num_kmeans_clusters}',
            f'num_kmeans_iterations={args.num_kmeans_iterations}',
            f'num_kmedoids_clusters={args.num_kmedoids_clusters}',
            f'num_kmedoids_iterations={args.num_kmedoids_iterations}',
            f'kmedoids_method={args.kmedoids_method}',
            f'k={args.k}',
            f'pos_sim_threshold={args.pos_sim_threshold}',
            f'neg_sim_threshold={args.neg_sim_threshold}',
            f'max_num_pos={args.max_num_pos}',
            f'max_num_neg={args.max_num_neg}',
            f'finding_only={args.finding_only}',
            f'union_find_threshold={args.union_find_threshold}',
        ],
        force_hashing=True,
    )
    if os.path.exists(pos_and_neg_representative_facts_filepath):
        # Load positive and negative representative facts
        print_bold('Loading positive and negative representative facts from:', pos_and_neg_representative_facts_filepath)
        pos_and_neg_representative_facts = load_pickle(pos_and_neg_representative_facts_filepath)
        most_similar = pos_and_neg_representative_facts['most_similar']
        least_similar = pos_and_neg_representative_facts['least_similar']
        print('len(most_similar):', len(most_similar))
        print('len(least_similar):', len(least_similar))
    else:
        print_blue('Finding positive and negative representative facts for each fact...')
        most_similar = [[] for _ in range(len(fact_embeddings))]
        least_similar = [[] for _ in range(len(fact_embeddings))]
        # Create dataset and dataloader
        dataset = FactEmbeddingsDataset(fact_embeddings)
        dataloader = DataLoader(dataset, batch_size=args.batch_size, shuffle=False,
                                num_workers=args.num_workers, collate_fn=dataset.collate_fn)
        r_f_embs = torch.from_numpy(representative_fact_embeddings).cuda()
        offset = 0
        for batch in tqdm(dataloader, mininterval=2):
            batch_embs = batch.cuda()
            similarities = torch.matmul(batch_embs, r_f_embs.T)
            nearest_idxs = torch.argsort(similarities, dim=1)
            similarities = similarities.cpu().numpy()
            nearest_idxs = nearest_idxs.cpu().numpy()
            assert similarities[0, nearest_idxs[0, 0]] < similarities[0, nearest_idxs[0, -1]]
            global _shared_nearest_idxs
            global _shared_similarities
            _shared_nearest_idxs = nearest_idxs
            _shared_similarities = similarities
            # Find negative representative facts
            tasks = [(i, args.neg_sim_threshold, args.max_num_neg) for i in range(len(nearest_idxs))]
            with mp.Pool(processes=args.num_processes) as pool:
                results = pool.starmap(find_least_similar_facts, tasks)
            for i, result in enumerate(results):
                idx = offset + i
                least_similar[idx] = result
            # Find positive representative facts
            tasks = [(i, args.pos_sim_threshold, args.max_num_pos) for i in range(len(nearest_idxs))]
            with mp.Pool(processes=args.num_processes) as pool:
                results = pool.starmap(find_most_similar_facts, tasks)
            for i, result in enumerate(results):
                idx = offset + i
                most_similar[idx] = result

            if offset == 0: # sanity check
                for i in range(len(batch)):
                    if len(most_similar[i]) == 0:
                        continue
                    print(f'facts[{i}]: {facts[i]}')
                    print()
                    print(f'most_similar[{i}]:')
                    for j in range(min(len(most_similar[i]), 5)):
                        print(f'  {representative_facts[most_similar[i][j]]}')
                    print()
                    print(f'least_similar[{i}]:')
                    for j in range(min(len(least_similar[i]), 5)):
                        print(f'  {representative_facts[least_similar[i][j]]}')
                    break # only print one

            offset += len(batch)
        print('len(most_similar):', len(most_similar))
        print('len(least_similar):', len(least_similar))
        print('Saving positive and negative representative facts to:', pos_and_neg_representative_facts_filepath)
        save_pickle({
            'most_similar': most_similar,
            'least_similar': least_similar,
        }, pos_and_neg_representative_facts_filepath)

    # Find positive and negative representative facts for each report
    output = {
        'reports': [],
        'representative_facts': representative_facts,
    }
    orig_facts = integrated_embeddings_and_report_annotations['facts']
    for r in tqdm(integrated_embeddings_and_report_annotations['reports'], mininterval=2):
        if args.finding_only:
            r_fact_idxs = r['findings_fact_idxs']
        else:
            r_fact_idxs = r['findings_fact_idxs'] + r['impression_fact_idxs']
        r_fact_idxs = list(set(r_fact_idxs)) # remove duplicates
        r_fact_idxs = [fact2idx[orig_facts[i]] for i in r_fact_idxs] # use updated indices

        # Find positive representative facts
        pos_representative_fact_idxs = set()
        if len(r_fact_idxs) > 0:
            r_fact_idxs.sort(key=lambda x: len(most_similar[x]), reverse=True)
            for i in range(len(most_similar[r_fact_idxs[0]])):
                for j in range(len(r_fact_idxs)):
                    if len(most_similar[r_fact_idxs[j]]) <= i:
                        break
                    pos_representative_fact_idxs.add(most_similar[r_fact_idxs[j]][i])
                    if len(pos_representative_fact_idxs) == args.max_num_pos:
                        break
                if len(pos_representative_fact_idxs) == args.max_num_pos:
                    break
        pos_representative_fact_idxs = list(pos_representative_fact_idxs)

        # Find negative representative facts
        if len(r_fact_idxs) > 0:
            r_least_similar = set()
            for idx in r_fact_idxs:
                r_least_similar.update(least_similar[idx])
            r_least_similar = list(r_least_similar)
            if len(r_least_similar) > 0:
                r_least_similar_embeddings = representative_fact_embeddings[r_least_similar]
                r_fact_embeddings = fact_embeddings[r_fact_idxs]
                # Compute similarities between least similar facts and report facts
                similarities = np.dot(r_fact_embeddings, r_least_similar_embeddings.T)
                max_similarities = np.max(similarities, axis=0)
                assert len(max_similarities) == len(r_least_similar)
                max_similarities_idxs = np.argsort(max_similarities) # ascending order
                neg_representative_fact_idxs = []
                for idx in max_similarities_idxs:
                    if max_similarities[idx] > args.neg_sim_threshold:
                        break
                    neg_representative_fact_idxs.append(r_least_similar[idx])
                    if len(neg_representative_fact_idxs) == args.max_num_neg:
                        break
            else:
                neg_representative_fact_idxs = []
        else:
            neg_representative_fact_idxs = []

        # Add to output
        output['reports'].append({
            'path': r['path'],
            'part_id': r['part_id'],
            'subject_id': r['subject_id'],
            'study_id': r['study_id'],
            'positive_facts': pos_representative_fact_idxs,
            'negative_facts': neg_representative_fact_idxs,
        })

    # Save output
    output_filepath = get_file_path_with_hashing_if_too_long(
        folder_path=MIMICCXR_LARGE_FAST_CACHE_DIR,
        prefix='pos_and_neg_representative_facts_per_report',
        strings=[
            args.integrated_embeddings_and_report_annotations_filepath,
            f'num_kmeans_clusters={args.num_kmeans_clusters}',
            f'num_kmeans_iterations={args.num_kmeans_iterations}',
            f'num_kmedoids_clusters={args.num_kmedoids_clusters}',
            f'num_kmedoids_iterations={args.num_kmedoids_iterations}',
            f'kmedoids_method={args.kmedoids_method}',
            f'k={args.k}',
            f'pos_sim_threshold={args.pos_sim_threshold}',
            f'neg_sim_threshold={args.neg_sim_threshold}',
            f'max_num_pos={args.max_num_pos}',
            f'max_num_neg={args.max_num_neg}',
            f'finding_only={args.finding_only}'
        ],
        force_hashing=True,
    )
    print_bold('Saving output to:', output_filepath)
    save_pickle(output, output_filepath)
    print('Done!')

if __name__ == '__main__':
    main()


import torch
from torch import Tensor
import torch.nn as nn

class BasicConv(nn.Module) :
    def __init__(self, in_channels, out_channels, kernel_size, stride = 1, padding = 0) :
        super(BasicConv, self).__init__()
        self.bn = nn.BatchNorm2d(in_channels)
        self.relu = nn.ReLU()
        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding)

    def forward(self, x) :
        out = self.bn(x)
        out = self.relu(out)
        out = self.conv(out)

        return out

class Transition(nn.Module) :
    def __init__(self, in_channels) :
        super(Transition, self).__init__()

        self.conv = BasicConv(in_channels, in_channels, 1)
        self.avgpool = nn.AvgPool2d(kernel_size=2, stride= 2)

    def forward(self, x) :

        out = self.conv(x)
        out = self.avgpool(out)

        return out

class Bottleneck(nn.Module) :
    def __init__(self, in_channels, out_channels) :
        super(Bottleneck, self).__init__()

        self.conv1 = BasicConv(in_channels, 4*out_channels, kernel_size = 1)
        self.conv2 = BasicConv(4*out_channels, out_channels, kernel_size=3, stride=1, padding= 1)

    def forward(self, x) :

        out = self.conv1(x)
        out = self.conv2(out)

        return torch.cat((out, x), dim=1)

class DenseBlock(nn.Module) :
    def __init__(self, in_channels, num_layer, growth_rate) :
        super(DenseBlock, self).__init__()

        self.denseblock = self._make_layer(num_layer, in_channels, growth_rate)

    def _make_layer(self, num_layer, init_in_channel, growth_rate) :

        in_channels = init_in_channel
        layers = []
        for i in range(num_layer) :
            layers.append(Bottleneck(in_channels, growth_rate))
            in_channels = in_channels + growth_rate

        return nn.Sequential(*layers)

    def forward(self, x) :

        out = self.denseblock(x)

        return out

class DenseNet(nn.Module) :
    def __init__(self, growth_rate, num_classes=10) :
        super(DenseNet, self).__init__()

        num_channel = 3
        self.conv = BasicConv(num_channel, 16, kernel_size = 7, stride = 2)
        #num_channel = 16

        self.maxpool = nn.MaxPool2d(kernel_size = 3, stride = 2)

        self.denseblock1 = DenseBlock(num_channel, 6, growth_rate)
        num_channel = num_channel + 6 * growth_rate
        self.transition1 = Transition(num_channel)

        self.denseblock2 = DenseBlock(num_channel, 12, growth_rate)
        num_channel = num_channel + 12 * growth_rate
        self.transition2 = Transition(num_channel)

        self.denseblock3 = DenseBlock(num_channel, 24, growth_rate)
        num_channel = num_channel + 24 * growth_rate
        self.transition3 = Transition(num_channel)

        self.denseblock4 = DenseBlock(num_channel, 16, growth_rate)
        num_channel = num_channel + 16 * growth_rate

        self.avgpool = nn.AdaptiveAvgPool2d(1)
        self.fc = nn.Linear(num_channel, num_classes)

        self.feature = nn.Sequential(
            #self.conv,
            #self.maxpool,
            self.denseblock1,
            self.transition1,
            self.denseblock2,
            self.transition2,
            self.denseblock3,
            self.transition3,
            self.denseblock4
        )

    def forward(self, x) :

        out = self.feature(x)
        out = self.avgpool(out)
        out = torch.flatten(out, 1)
        out = self.fc(out)

        return out


from collections import deque
dx = [0, 1, 0, -1]
dy = [1, 0, -1, 0]

def burn():
    for _ in range(len(fire)):
        x, y = fire.popleft()
        for k in range(4):
            nx = x + dx[k]
            ny = y + dy[k]
            if 0 <= nx < N and 0 <= ny < M:
                if arr[nx][ny] != '#' and arr[nx][ny] != '*':
                    arr[nx][ny] = '*'
                    fire.append((nx, ny))

def move():
    isgo = False
    for _ in range(len(start)):
        x, y = start.popleft()
        for k in range(4):
            nx = x + dx[k]
            ny = y + dy[k]
            if 0 <= nx < N and 0 <= ny < M:
                if visited[nx][ny] == 0 and arr[nx][ny] != '#' and arr[nx][ny] != '*':
                    isgo = True
                    visited[nx][ny] = visited[x][y] + 1
                    start.append((nx, ny))
            else:
                return visited[x][y]
    if not isgo:
        return 'IMPOSSIBLE'

T = int(input())
for _ in range(T):
    M, N = map(int, input().split())
    arr = []
    fire = deque()
    start = deque()
    for i in range(N):
        arr.append(list(input().strip()))
        for j in range(M):
            if arr[i][j] == '*':
                fire.append((i, j))
            if arr[i][j] == '@':
                start.append((i, j))
    visited = [[0] * M for _ in range(N)]
    visited[start[0][0]][start[0][1]] = 1

    result = 0
    while True:
        burn()
        result = move()
        if result:
            break

    print(result)


import configparser
import time
from MainClass import EsModules
from datetime import datetime

runtime = datetime.now().strftime('%Y-%m-%d %H-%M-%S')
config = configparser.ConfigParser()
config.read('es_test_config.ini')
ES_environment = config['DEFAULT']['environment']
location_file= config['DEFAULT']['testlib']
# print(ES_environment)

#parsed_info = json.dumps((connection_es.info()), indent=4)
#print(parsed_info)

# search term entered in autosuggest
#search_term = input("Enter Search Term: ")
f=open(location_file,'r',encoding='utf-8')
search_term = f.readlines()
f.close()

# language code used in location alias
alias_lan_code = config['DEFAULT']['languagecode']

# country context value for query
country_ctx = config['DEFAULT']['country_context']

# designates if the index is new
is_New = config['DEFAULT'].getboolean('is_New')

if is_New:
    test_index = "-new"
else:
    test_index = ""
x=0
for line in search_term:
    ssearch_term = search_term[x].strip('\n')
    res = EsModules.es_location_suggest(ES_environment, alias_lan_code, test_index, ssearch_term, country_ctx)
    print(ssearch_term)
    EsModules.query_output(res, ssearch_term,alias_lan_code,ES_environment, runtime)
    x = x + 1
    time.sleep(0.01)


'''

'''

def run_test(test_cases):
    output = []
    for i in range(test_cases):
        n, m = [int(j) for j in input().split()]
        positions = [int(j) for j in input().split()]
        # positions = [1,2,3,4,5]
        result = []
        for i, position in enumerate(positions):
            temp = []
            for j in range(n):
                if j <= position:
                    temp.append(position-j)
                else:
                    temp.append(j-position)
            result.append(temp)
        output.append(result)
    print(output)
    return(output)

run_test(2)


from sqlalchemy import MetaData, String, Table
from sqlalchemy.dialects import postgresql

CIDR_TABLE_COLUMNS = [
    # table name, column name
    ('security_group_rules', 'cidr'),
    ('provider_fw_rules', 'cidr'),
    ('networks', 'cidr'),
    ('networks', 'cidr_v6')]

def upgrade(migrate_engine):
    """Convert String columns holding IP addresses to INET for postgresql."""
    meta = MetaData()
    meta.bind = migrate_engine
    dialect = migrate_engine.url.get_dialect()

    if dialect is postgresql.dialect:
        for table, column in CIDR_TABLE_COLUMNS:
            # can't use migrate's alter() because it does not support
            # explicit casting
            migrate_engine.execute(
                "ALTER TABLE %(table)s "
                "ALTER COLUMN %(column)s TYPE INET USING %(column)s::INET"
                % locals())
    else:
        for table, column in CIDR_TABLE_COLUMNS:
            t = Table(table, meta, autoload=True)
            getattr(t.c, column).alter(type=String(43))

def downgrade(migrate_engine):
    """Convert columns back to the larger String(255)."""
    meta = MetaData()
    meta.bind = migrate_engine
    for table, column in CIDR_TABLE_COLUMNS:
        t = Table(table, meta, autoload=True)
        getattr(t.c, column).alter(type=String(39))


import re

TOKEN_UNSUPPORTED = 0
TOKEN_MC_TEXT = 1
TOKEN_MC_SPACE = 2
TOKEN_MC_WORD = 3
TOKEN_PLAIN_TEXT = 4
TOKEN_PLAIN_SPACE = 5

PATTERNS_TO = [
    (r'[a-zA-Z0-9\.\,?\'\!\/\(\)\&\:\;\=\+\-\_\\\$\@]+', TOKEN_PLAIN_TEXT),
    (r'[\s]+', TOKEN_PLAIN_SPACE)
]

PATTERNS_FROM = [
    (r'[\.-]+', TOKEN_MC_TEXT),
    (r' {7}', TOKEN_MC_WORD), # order matters, look for the longer word, before the shorter space sequence
    (r' {3}', TOKEN_MC_SPACE)
]

class Token:
    def __init__(self, tkn_type: int, value: str, begin: int=None, end: int=None) -> None:
        self.type = tkn_type
        self.value = value
        self.begin = begin
        self.end = end

    def __repr__(self) -> str:
        return "<Token type({}) value({}) begin({}), end({})>".format(self.type, self.value, self.begin, self.end)

    def __eq__(self, __value: object) -> bool:
        if not isinstance(__value, Token):
            return False
        else:
            return self.type == __value.type and self.value == __value.value and self.begin == __value.begin and self.end == __value.end

class Cursor():
    def __init__(self, index: int=0) -> None:
        self.index: int = index

    def advance(self) -> None:
        self.index += 1

    def advance_to(self, location: int) -> None:
        self.index = location

    def __str__(self) -> str:
        return str(self.index)

class UnhandledCharacterException(Exception):
    def __init__(self, character: str, position: int) -> None:
        super().__init__()
        self.character = character
        self.position = position

    def __str__(self) -> str:
         return "Unhandled character {} at index {}.".format(self.character, self.position)

class Lexer:
    def __init__(self, text: str, to_mc: bool=True, strict_mode: bool=True) -> None:
        self.text = text
        self.to_mc = to_mc
        self.strict_mode = strict_mode
        self.cursor = Cursor()

    def tokenize(self):
        tokens = []

        if(self.to_mc):
            patterns = PATTERNS_TO
        else:
            patterns = PATTERNS_FROM

        while self.cursor.index < len(self.text):
            match = None
            for pattern in patterns:
                expression, tkn_type = pattern
                regex = re.compile(expression)
                match = regex.match(self.text, self.cursor.index)
                if match:
                    value = match.group(0)
                    token = Token(tkn_type, value, self.cursor.index, match.end(0))
                    tokens.append(token)
                    break
            if not match:
                if self.strict_mode:
                    raise UnhandledCharacterException(self.text[self.cursor.index], self.cursor)
                else:
                    # If the last token was unsupported, update it
                    last_token = None
                    nmbr_tokens = len(tokens)
                    if nmbr_tokens > 0:
                        last_token = tokens[nmbr_tokens - 1]
                    if last_token is not None and last_token.type == TOKEN_UNSUPPORTED:
                        last_token.value = last_token.value + self.text[self.cursor.index]
                        last_token.end = self.cursor.index + 1
                    else:
                        token = Token(TOKEN_UNSUPPORTED, self.text[self.cursor.index], self.cursor.index, self.cursor.index + 1)
                        tokens.append(token)
                    self.cursor.advance()
            else:
                self.cursor.advance_to(match.end(0))

        return tokens


import heapq
from heapq import *

class SlidingWindowMedian:
    def __init__(self):
        self.maxHeap, self.minHeap = [], []

    def find_sliding_window_median(self, nums, k):
        result = [0.0 for _ in range(len(nums) - k + 1)]
        for i in range(0, len(nums)):
            if not self.maxHeap or nums[i] <= -self.maxHeap[0]:
                heappush(self.maxHeap, -nums[i])
            else:
                heappush(self.minHeap, nums[i])

            self.rebalance_heaps()

            if i - k + 1 >= 0:
                if len(self.maxHeap) == len(self.minHeap):
                    result[i - k + 1] = -self.maxHeap[0] / 2.0 + self.minHeap[0] / 2.0
                else:
                    result[i - k + 1] = -self.maxHeap[0] / 1.0

                element_to_be_removed = nums[i - k + 1]
                if element_to_be_removed <= -self.maxHeap[0]:
                    self.remove(self.maxHeap, -element_to_be_removed)
                else:
                    self.remove(self.minHeap, element_to_be_removed)

                self.rebalance_heaps()

        return result

    def remove(self, heap, element):
        ind = heap.index(element)
        heap[ind] = heap[-1]
        del heap[-1]
        if ind < len(heap):
            heapq._siftup(heap, ind)
            heapq._siftdown(heap, 0, ind)

    def rebalance_heaps(self):
        if len(self.maxHeap) > len(self.minHeap) + 1:
            heappush(self.minHeap, -heappop(self.maxHeap))
        elif len(self.maxHeap) < len(self.minHeap):
            heappush(self.maxHeap, -heappop(self.minHeap))

def main():
    sliding_window_median = SlidingWindowMedian()
    result = sliding_window_median.find_sliding_window_median([1, 2, -1, 3, 5], 2)
    print("Sliding window medians are: " + str(result))

    sliding_window_median = SlidingWindowMedian()
    result = sliding_window_median.find_sliding_window_median([1, 2, -1, 3, 5], 3)
    print("Sliding window medians are: " + str(result))

main()


import numpy as np
from scipy import stats
import pandas as pd
from pathlib import Path
import glob
import ast
import random
import h5py
from tqdm import tqdm

stock = ['CATY']
inputFolder = 'C:\\Users\\I535982\\Documents\\Uni\\Thesis\\Data\\NASDAQ_h5'
iniPriceFolder = 'C:\\Users\\I535982\\Documents\\Uni\\Thesis\\Data\\Output'
outputFolder = 'C:\\Users\\I535982\\Documents\\Uni\\Thesis\\Data\\Simulation\\Data'
timeStep = 30    # seconds
noTick = 15
iteration = 3    # define number of simulated observations

""" Estimate parameters for each trading day and average them """

def transform(df):
    """ Transform each cell of the dataframe to numpy array """
    for col in df.columns.values:
        df[col] = df[col].apply(ast.literal_eval)
        df[col] = df[col].apply(np.array)
    return df

def concat(df, limitOrders, cancelOrders):
    """ Append to limit order and cancel order list """
    for col in df.columns:
        for ind in df.index:
            if col == "limitOrders":
                if len(limitOrders) < ind + 1:
                    limitOrders.append(df.loc[ind, col])
                else:
                    limitOrders[ind] = np.concatenate((limitOrders[ind],df.loc[ind, col]))
            elif col == "cancelOrders":
                if len(cancelOrders) < ind + 1:
                    cancelOrders.append(df.loc[ind, col])
                else:
                    cancelOrders[ind] = np.concatenate((cancelOrders[ind],df.loc[ind, col]))
    return None

def delShares(sign, price, vol):
    """ Delete shares executed or canceled """
    global buyDepth, sellDepth, buyQtyMap, sellQtyMap

    if sign == -1:   #update bid Depth
        buyQtyMap[price] -= min(vol, buyQtyMap[price])
        if buyQtyMap[price] == 0:
            del buyQtyMap[price]
            buyDepth.remove(price)

    else:            #update ask Depth
        sellQtyMap[price] -= min(vol, sellQtyMap[price])
        if sellQtyMap[price] == 0:
            del sellQtyMap[price]
            sellDepth.remove(price)
    return None

def updateOShares():
    """ Update no. of outstanding shares present at i ticks from the opposite side """
    global iniBuy, iniSell, Na, Nb

    for dis in np.arange(0, noTick):
        # update no. of outstanding shares in the bid side

        askPrice = round(buyDepth[0] + (dis + 1)/ 100, 2) if len(buyDepth) > 0 else iniSell
        if askPrice in sellDepth:
            Na[dis] = sellQtyMap[askPrice]
        else:
            Na[dis] = 0

        # update no. of outstanding shares in the ask side
        bidPrice = round(sellDepth[0] - (dis + 1)/100, 2) if len(sellDepth) > 0 else iniBuy
        if bidPrice in buyDepth:
            Nb[dis] = buyQtyMap[bidPrice]
        else:
            Nb[dis] = 0
    return Na, Nb

def updateDepth(price, sign):
    """ Insert bid and ask depth """
    i = 0
    if sign == 1:    # ask order
        for p in sellDepth:
            if price < p:
                sellDepth.insert(i, price)
                return None
            elif price == p:
                return None
            i += 1
        sellDepth.insert(i, price)

    else:
        for p in buyDepth:
            if price > p:
                buyDepth.insert(i, price)
                return None
            elif price == p:
                return None
            i += 1
        buyDepth.insert(i, price)
    return None

def appendDf(t):
    """ Append data to save"""
    global tradeBidSize, tradeAskSize, e, m, ev, bestBL, bestAL
    global timeStamp, bestBidPrice, bestAskPrice, bestBidVol, bestAskVol, bidTradeSize, askTradeSize, OFI, marDep, noEvent

    timeStamp.append(t)
    bestBidPrice.append(bestBL[0]) if t != 0 else bestBidPrice.append(buyDepth[0])
    bestAskPrice.append(bestAL[0]) if t != 0 else bestAskPrice.append(sellDepth[0])
    bestBidVol.append(bestBL[1]) if t != 0 else bestBidVol.append(buyQtyMap[buyDepth[0]])
    bestAskVol.append(bestAL[1]) if t != 0 else bestAskVol.append(sellQtyMap[sellDepth[0]])

    # update trade size
    bidTradeSize.append(tradeBidSize)
    askTradeSize.append(tradeAskSize)
    tradeBidSize = 0
    tradeAskSize = 0

    # update OFI
    OFI.append(e)
    e = 0

    # update market depth
    marDep.append(m)
    m = 0

    # update no. of events of
    noEvent.append(ev)
    ev = 0
    return None

if __name__ == '__main__':
    #  time of each trading day in seconds
    T = int(6.5 * 60 * 60)
    for s in tqdm(stock):
        orderFileFmt = inputFolder + '\\' + s + '\\*Order30s.csv'
        inFileList = glob.glob(orderFileFmt)  # no. of files in the input folder
        trainSize = int(len(inFileList) / 2)

        # initialize price for simulation, elements corresponding with stock list
        iniPriceFmt = iniPriceFolder + '\\' + s + '\\*IniPrice.h5'
        iniPrice = glob.glob(iniPriceFmt)
        iniPrice = h5py.File(iniPrice[0], 'r')
        iniPrice = iniPrice[s][:]

        # initialize parameters for limit and cancel orders
        shape = noTick  # no. of distance from the best opposite side
        alphaTrade = np.zeros((trainSize,))
        alphaLimit, alphaCancel = np.zeros((trainSize, shape)), np.zeros((trainSize, shape))
        locTrade, scaleTrade = np.zeros((trainSize,)), np.zeros((trainSize,))
        locLimit, scaleLimit = np.zeros((trainSize, shape)), np.zeros((trainSize, shape))
        locCancel, scaleCancel = np.zeros((trainSize, shape)), np.zeros((trainSize, shape))

        for i in range(trainSize):
            # initialize list of orders
            limitOrders = []
            cancelOrders = []
            tradeOrders = []

            # read data and save them into lists of orders
            df_temp = pd.read_csv(inFileList[i])  # read csv
            df_temp2 = transform(df_temp.loc[:, ['limitOrders', 'cancelOrders']])  # transform to numpy array
            concat(df_temp2, limitOrders, cancelOrders)  # concatenate files, only for limit orders and cancel orders
            tradeOrders += ast.literal_eval(df_temp.loc[0, 'tradeOrders'])  # concatenate arrays for trade orders
            tradeOrders = np.array(tradeOrders)

            # calculate parameters for trade
            alphaTrade[i] = tradeOrders.size / (2 * T)  # probability of number of trades per second
            tradeData = tradeOrders[(tradeOrders <= np.quantile(tradeOrders, 0.95))]  # remove outliers
            locTrade[i], scaleTrade[i] = stats.norm.fit(np.log(tradeData))  # parameters for volume of trade

            for distance in range(shape):

                # calculate parameters for limit orders at a distance i from the opposite best price
                alphaLimit[i][distance] = limitOrders[distance].size / (2 * T)
                if limitOrders[distance].size > 0:
                    limitData = limitOrders[distance][(limitOrders[distance] <= np.quantile(limitOrders[distance], 0.95))]
                    locLimit[i][distance], scaleLimit[i][distance] = stats.norm.fit(np.log(limitData))
                    # calculate parameters for limit orders at a distance i from the opposite best price
                    alphaCancel[i][distance] = cancelOrders[distance].size / (2 * T * limitOrders[distance].mean())
                else:
                    locLimit[i][distance], scaleLimit[i][distance] = 0, 0
                    alphaCancel[i][distance] = 0

                if cancelOrders[distance].size > 0:
                    cancelData = cancelOrders[distance][(cancelOrders[distance] <= np.quantile(cancelOrders[distance], 0.95))]
                    locCancel[i][distance], scaleCancel[i][distance] = stats.norm.fit(np.log(cancelData))
                else:
                    locCancel[i][distance], scaleCancel[i][distance] = 0, 0

        # average estimated parameters across trading day
        aveAlphaTrade = alphaTrade.mean()
        aveLocTrade = locTrade.mean()
        aveScaleTrade = scaleTrade.mean()
        aveAlphaLimit = alphaLimit.mean(axis=0)
        aveLocLimit = locLimit.mean(axis=0)
        aveScaleLimit = scaleLimit.mean(axis=0)
        aveAlphaCancel = alphaCancel.mean(axis=0)
        aveLocCancel = locCancel.mean(axis=0)
        aveScaleCancel = scaleCancel.mean(axis=0)

        ### SIMULATION ###

        # initialize parameters for simulator
        iniBuy = iniPrice[0][1]
        iniSell = iniPrice[1][1]
        buyDepth, sellDepth = [iniBuy], [iniSell]
        buyQtyMap, sellQtyMap = {iniBuy: 1}, {iniSell: 1}
        Na, Nb = np.zeros((shape,)), np.zeros((shape,))  # number of outstanding shares in the ask side present at i ticks from bid side and ask side
        NaCancel, NbCancel = 0, 0  # weighted sum of shares cancelled at price levels from p - i ticks to p - 1
        A = aveAlphaLimit.sum() * 2  # probability of limit orders
        random.seed(42)

        # initialize parameters for trades, OFI and limit orders
        updateBool = [0]  # list for tracking time to save on dataframe
        step = 1  # iteration indicator
        e = 0  # contribution to OFI
        ev = 0  # event
        m = 0  # market depth and last market depth
        tradeBidSize = 0
        tradeAskSize = 0
        bestBL, bestAL = [0, 0], [0, 0]  # [price, volume] for best limit orders and ask limit orders
        lastBL, lastAL = [0, 0], [0, 0]  # [price, volume] for the last best limit orders and ask limit orders

        # initialize parameters for saving dataframe
        timeStamp = []
        bestBidPrice, bestAskPrice = [], []
        bidTradeSize, askTradeSize = [], []
        bestBidVol, bestAskVol = [], []
        OFI = []
        marDep = []
        noEvent = []  # number of events occur during each interval

        t = 0  # initialize time
        while t <= (T * trainSize * iteration):
            print(t)
            # save data to Dataframe
            updateBool.append(t // timeStep)
            if t == 0 or updateBool[step] != updateBool[step - 1]:
                appendDf(t)

            # update no. of outstanding shares
            Na, Nb = updateOShares()

            # compute cancellation rate
            NaCancel = Na @ aveAlphaCancel
            NbCancel = Nb @ aveAlphaCancel

            # Draw a random event in relative probabilities to choose type of orders
            # event 1 for trade, event 2 for limit orders, event 3 for cancellation at ask side, event 4 for cancellation at bid side
            event = random.choices(population=[1, 2, 3, 4], weights=[2 * aveAlphaTrade, A, NaCancel, NbCancel])[0]

            if event == 1:  # market order
                # draw sign: 1 is ask and -1 is bid
                sign = random.choices(population=[-1, 1])[0]

                # draw volume of market orders
                vol = round(np.exp(stats.norm.rvs(loc=aveLocTrade, scale=aveScaleTrade)))

                # update quantity
                if sign == 1:  # ask order
                    # add ask size
                    tradeAskSize += min(vol, buyQtyMap[buyDepth[0]])
                    delShares(-sign, buyDepth[0], vol)

                else:  # bid order
                    # add bid size
                    tradeBidSize += min(vol, sellQtyMap[sellDepth[0]])
                    delShares(-sign, sellDepth[0], vol)

            elif event == 2:  # limit order
                # draw sign: 1 is ask and -1 is bid
                sign = random.choices(population=[-1, 1])[0]

                # draw distance
                distance = random.choices(population=np.arange(0, noTick), weights=aveAlphaLimit)[0]

                # price of limit order
                if sign == 1:  # ask order
                    price = round(buyDepth[0] + (distance + 1) / 100, 2)
                else:  # bid order
                    price = round(sellDepth[0] - (distance + 1) / 100, 2)

                # draw volume
                vol = round(np.exp(stats.norm.rvs(loc=aveLocLimit[distance], scale=aveScaleLimit[distance])))

                # update quantity
                if sign == 1:  # ask order
                    if price in sellDepth:  # if price exists
                        sellQtyMap[price] += vol
                    else:
                        updateDepth(price, sign)
                        sellQtyMap[price] = vol
                else:  # bid order
                    if price in buyDepth:  # if price exists
                        buyQtyMap[price] += vol
                    else:
                        updateDepth(price, sign)
                        buyQtyMap[price] = vol

            elif event == 3:  # cancelation at ask side
                # draw distance
                distance = random.choices(population=np.arange(0, noTick), weights=Na * aveAlphaCancel)[0]

                price = round(buyDepth[0] + (distance + 1) / 100, 2)

                # draw volume
                vol = round(np.exp(stats.norm.rvs(loc=aveLocCancel[distance], scale=aveScaleCancel[distance])))

                # update quantity
                if price in sellDepth:
                    delShares(1, price, vol)

            elif event == 4:  # cancellation at bid side
                # draw distance
                distance = random.choices(population=np.arange(0, noTick), weights=Nb * aveAlphaCancel)[0]

                price = round(sellDepth[0] - (distance + 1) / 100, 2)

                # draw volume
                vol = round(np.exp(stats.norm.rvs(loc=aveLocCancel[distance], scale=aveScaleCancel[distance])))

                # update quantity
                if price in buyDepth:
                    delShares(-1, price, vol)

            # update best bid and best ask price
            try:
                bestBL = [buyDepth[0], buyQtyMap[buyDepth[0]]]
            except Exception:
                pass

            try:
                bestAL = [sellDepth[0], sellQtyMap[sellDepth[0]]]
            except Exception:
                pass

            # update contribution of the event to the bid and ask queues
            e += (bestBL[0] >= lastBL[0]) * bestBL[1] - (bestBL[0] <= lastBL[0]) * lastBL[1] - (bestAL[0] <= lastAL[0]) * \
                 bestAL[1] + (bestAL[0] >= lastAL[0]) * lastAL[1]

            # update total market depth at the best prices and no. of event
            if (bestBL[1] != lastBL[1]) or (bestAL[1] != lastAL[1]) or (bestBL[0] != lastBL[0]) or (bestAL[0] != lastAL[0]):
                m += bestBL[1] + bestAL[1]
                ev += 1

            # update the last best price and volume and last total market depth
            lastBL = [bestBL[0], bestBL[1]]
            lastAL = [bestAL[0], bestAL[1]]

            step += 1

            # Draw waiting time from exponential distribution with parameter S
            S = 2 * aveAlphaTrade + A + NaCancel + NbCancel
            t += stats.expon.rvs(scale=1 / S)

        # save as a dataframe
        df = pd.DataFrame(list(zip(timeStamp, bestBidPrice, bestAskPrice, OFI, bidTradeSize, askTradeSize, marDep,
                                   noEvent, bestBidVol, bestAskVol)),
                          columns=['timeStamp', 'bestBidPrice', 'bestAskPrice', 'OFI', 'bidTradeSize',
                                   'askTradeSize', 'marDep', 'noEvent', 'bestBidVol', 'bestAskVol'])

        # store main data
        sOutputFolder = outputFolder + "\\" + s
        Path(sOutputFolder).mkdir(parents=True, exist_ok=True)
        outputFName = sOutputFolder + "\\" + s + str(T * trainSize).zfill(2) + ".h5"
        file = open(outputFName, 'w+')
        output = pd.HDFStore(outputFName)
        output['df'] = df
        output.close()


# coding: utf-8

import os
import pathlib

#--------------------------------------------Path Settings---------------------#

PROJECT_PATH = pathlib.Path(__file__).parent.absolute()
JAMENDO_PATH = ''
UNLABEL_PATH = ''

#--------------------------------------------Data Loading Info------------------#
'''
label: sing/nosing: 1/0
FRAME-LEN_HOP-EN_NMEL: '1024_315_80'
'''
PITCH_SHIFTINF_RANGE = 2
TRAIN_STEP = 7
VALID_STEP = 5
TEST_STEP = 3
AUDIO_PROCESSSING_METHOD = '1024_315_80'
AUDIO_PROCESSSING_METHOD_LIST = ['1024_315_80', '2048_512_128']
if AUDIO_PROCESSSING_METHOD == '1024_315_80':
	PROCESSER_DATA_FOLDER_NAME = '1024_315_80'
	SR = 22050	# 1k := 1,000
	FRAME_LEN = 1024 # 0.046s/frame
	HOP_LENGTH = 315 # ~0.014s resolution
	N_MELS = 80
	CUTOFF = 8000  # fmax = 8kHz
	FMIN = 27.5
	FMAX = 8000
	WIN_SIZE = 115  # 1.6 sec
	# CNN_OVERLAP = 7  # Hopsize of 5 for training, 1 for inference
elif AUDIO_PROCESSSING_METHOD == '2048_512_128':
	PROCESSER_DATA_FOLDER_NAME = '2048_512_128'
	SR = 22050	# 1k := 1,000
	FRAME_LEN = 2048 # 0.096s/frame
	HOP_LENGTH = 512 # 0.023s resolution
	N_MELS = 128
	CUTOFF = 8000  # fmax = 8kHz, ... ...
	FMIN = 0
	FMAX = 8192
	WIN_SIZE = 25  # ~0.58s
	# CNN_OVERLAP = 7  # Hopsize of 5 for training, 1 for inference

ENERGY_THRESHOLD = 0.0001

#--------------------------------------------SchCNN Paramaters---------------------------------------------#
DROPOUTRATE = 0.2

#--------------------------------------------Training Settings---------------------------------------------#
THRESHOLD = 0.5  # threshold for binary classification
MODEL_NAME = 'SchCNN'
ZERO_MEAN = True
OPTIM = 'Adam'
LR =  0.00001
MOMENTUM = 0.9
MONITOR = 'acc'
SCHEDULER_FLAG = True	# only be useful when the optimizer is 'SGD'
PS_ARG_FLAG = False

TRAINSET_LIST = ['Jamendo']
VALID_SET_NAME = 'Jamendo'
TEST_SET_NAME = VALID_SET_NAME

EPOCH_NUM = 192
BATCH_SIZE = 64
BATCH_SIZE_UNLABEL = 64
VALID_BATCH_SIZE = 512
GPU_FLAG = True
FINALLY_SAVED_MODEL = ['lastModel.pkl', 'bestModel_acc.pkl', 'bestModel_loss.pkl']
EARLY_STOPPING_EPOCH = EPOCH_NUM + 100

#--------------------------------------------Testing Settigs---------------------------------------------#
# TESTED_MODEL = 'bestModel_acc.pkl'

#--------------------------------------------unlabel data Info---------------------------------------------#
# ['audio_Grammy', 'audio_Oscars_old', audio_GrammyNominees20102020']
# [61, 58, 185]

#--------------------------------------------Experiment Settings-----------------------------------------#
TRAIN_MODE = 'SSL'

TRAIN_MODE_LIST = ['NaiveNN', 'KD', 'SSL', 'SSSL']
if(TRAIN_MODE == 'NaiveNN'):
	pass
elif(TRAIN_MODE == 'KD'): # to validate the similarity hypothesis.
	'''
	KD:= BCE_soft + BCE_hard
	lambda_KD * KD_T * KD_T * BCE_soft + (1 - lambda_KD) * BCE_hard
	'''
	TEACHER_LIST = ['']
	KD_T = 8
	LAMBDA_KD = 0.5
elif(TRAIN_MODE == 'SSL'): # S2019
	'''
	BCE_hard + BCE_hard
	There is no concept of KD, T. only weight is neeeded.
	'''

	HARD_USING = False
	UNLABEL_FOLDER_LIST = [''] # G1
	TEACHER_LIST = ['']
	LAMBDA_SSL = 0.7 # lambda_unlabel
elif(TRAIN_MODE == 'SSSL'):
	'''
	Both modes are organized by unlabeled data loss and well-labeled data loss.
	'soft':
		lambda_unlabel * SSL_T * SSL_T * BCE_soft(SSL_T) + (1 - lambda_unlabel) * KDLoss(KD_T, LAMBDA_KD)
	'multi':
		lambda_unlabel * KDLoss(SSL_T, LAMBDA_SSL_KD) + (1 - lambda_unlabel) * KDLoss(KD_T, LAMBDA_KD)
	'''
	UNLABEL_FOLDER_LIST = ['']
	TEACHER_LIST = ['']
	KD_T = 8
	LAMBDA_KD = 0.5
	KD_SSL_MODE = 'soft' # 'multi'/'soft' # too troublesome & not necessary.
	if(KD_SSL_MODE == 'soft'):
		# BCE_soft + KD
		SSL_T = 8 # T for unlabeled data.
		LAMBDA_UNLABEL = 0.5 # weight for unlabed data loss.
	elif(KD_SSL_MODE == 'multi'):
		# KD + KD
		SSL_T = 8 # T for unlabeled data.
		LAMBDA_UNLABEL = 0.5 # weight for unlabed data loss.
		LAMBDA_SSL_KD = 0.5
else:
	pass

#--------------------------------------------TEACHER Info ---------------------------------------------#
TEACHER_INFO = {}
TEACHER_INFO['g1'] = {'model': 'SchZM',
						'name': 'g1.pkl',
						'description': 'generation: 1'}

#--------------------------------------------UNLABELED_DATA Info ---------------------------------------------#
UNLABELDATA_INFO[''] = {'name': '',
						'audio_track_num': ''}

if __name__ == '__main__':
	pass


import numpy as np
from scipy.signal import butter, lfilter
import os, glob
from keras.models import load_model
import matlab.engine
import hdf5storage

def Readtxt(directory):
    ##Result txt file read in order to get orders
    f = open(directory, 'r', encoding='utf-16')
    j = 0
    Resultline = []
    while True:
        Resultline.append(f.readline())
        j = j + 1
        if Resultline[-1] == "\n":
            Resultline.pop()
            j = j - 1
        if not Resultline[-1]:
            Resultline.pop()
            j = j - 1
            break
#         print(j)
    f.close()
    return [j, Resultline]

### The Methods for Signal Processing ###
def Downsampling(eegData, downsampleRate):
        num = np.floor(eegData.shape[1] / downsampleRate).astype(int)
        Downsampled = np.zeros((eegData.shape[0],num))
        for i in range(num):
            for j in range(eegData.shape[0]):
                Downsampled[j,i] = np.mean(eegData[j,i*downsampleRate:(i+1)*downsampleRate],dtype=np.float64)
        return Downsampled

def Re_referencing(eegData, channelNum, sampleNum):
        after_car = np.zeros((channelNum,sampleNum))
        for i in np.arange(channelNum):
            after_car[i,:] = eegData[i,:] - np.mean(eegData,axis=0)
        return after_car

def butter_bandpass(lowcut, highcut, fs, order=5):
        nyq = 0.5 * fs
        low = lowcut / nyq
        high = highcut / nyq
        b, a = butter(order, [low, high], btype='band')
        return b, a
def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
        b, a = butter_bandpass(lowcut, highcut, fs, order=order)
        y = lfilter(b, a, data)
        return y

#def Epoching(eegData, stims, code, samplingRate, nChannel, epochSampleNum, epochOffset,baseline):
#        Time = stims[np.where(stims[:,1] == code),0][0]
#        Time = np.floor(np.multiply(Time,samplingRate)).astype(int)
#        Time_after = np.add(Time,epochOffset).astype(int)
#        Time_base = np.add(Time,baseline).astype(int)
#        Num = Time.shape
#        Epochs = np.zeros((Num[0], nChannel, epochSampleNum))
#        for j in range(Num[0]):
#            Epochs[j, :, :] = eegData[:,Time_after[j]:Time_after[j] + epochSampleNum]
#            for i in range(nChannel):
#                Epochs[j, i, :] = np.subtract(Epochs[j, i, :], np.mean(eegData[i,Time_after[j]:Time_base[j]]))
#        return [Epochs,Num[0]]

def EpochingNum(eegData, stims, code, samplingRate, nChannel, epochSampleNum, epochOffset, baseline, epochNum): # [-100 100]
        Time = stims[np.where(stims[:,1] == code),0][0]
        Time = np.floor(np.multiply(Time,samplingRate)).astype(int)
        Time_offset = np.add(Time,epochOffset).astype(int)
#        Time_base = np.subtract(Time,baseline).astype(int)
        Time_base = np.add(Time,baseline).astype(int)
        Epochs = np.zeros((epochNum, nChannel, epochSampleNum))
        for j in range(epochNum):
            Epochs[j, :, :] = eegData[:,Time_offset[j]:Time_offset[j] + epochSampleNum]
            for i in range(nChannel):
                Epochs[j, i, :] = np.subtract(Epochs[j, i, :], np.mean(eegData[i,Time_offset[j]:Time_base[j]]))
        return [Epochs,epochNum]

#def EpochingNum(eegData, stims, code, samplingRate, nChannel, epochSampleNum, epochOffset,baseline, epochNum):
#        Time = stims[np.where(stims[:,1] == code),0][0]
#        Time = np.floor(np.multiply(Time,samplingRate)).astype(int)
#        Time_after = np.add(Time,epochOffset).astype(int)
#        Time_base = np.add(Time,baseline).astype(int)
#        Epochs = np.zeros((epochNum, nChannel, epochSampleNum))
#        for j in range(epochNum):
#            Epochs[j, :, :] = eegData[:,Time_after[j]:Time_after[j] + epochSampleNum]
#            for i in range(nChannel):
#                Epochs[j, i, :] = np.subtract(Epochs[j, i, :], np.mean(eegData[i,Time_after[j]:Time_base[j]]))
#        return [Epochs,epochNum]

def DownsamplingOnlineEpoch(Epochs1, Epochs2, Epochs3, Epochs4, Epochs5, Epochs6, downsampleRate):
        num = np.floor(Epochs1.shape[2] / downsampleRate).astype(int)
        Downsampled1 = np.zeros((Epochs1.shape[0],Epochs1.shape[1],num))
        Downsampled2 = np.zeros((Epochs2.shape[0],Epochs2.shape[1],num))
        Downsampled3 = np.zeros((Epochs3.shape[0],Epochs3.shape[1],num))
        Downsampled4 = np.zeros((Epochs4.shape[0],Epochs4.shape[1],num))
        Downsampled5 = np.zeros((Epochs5.shape[0],Epochs5.shape[1],num))
        Downsampled6 = np.zeros((Epochs6.shape[0],Epochs6.shape[1],num))
        for i in range(num):
            for j in range(Epochs1.shape[1]):
                for k in range(Epochs1.shape[0]):
                    Downsampled1[k,j,i] = np.mean(Epochs1[k,j,i*downsampleRate:(i+1)*downsampleRate],dtype=np.float64)
                    Downsampled2[k,j,i] = np.mean(Epochs2[k,j,i*downsampleRate:(i+1)*downsampleRate],dtype=np.float64)
                    Downsampled3[k,j,i] = np.mean(Epochs3[k,j,i*downsampleRate:(i+1)*downsampleRate],dtype=np.float64)
                    Downsampled4[k,j,i] = np.mean(Epochs4[k,j,i*downsampleRate:(i+1)*downsampleRate],dtype=np.float64)
                    Downsampled5[k,j,i] = np.mean(Epochs5[k,j,i*downsampleRate:(i+1)*downsampleRate],dtype=np.float64)
                    Downsampled6[k,j,i] = np.mean(Epochs6[k,j,i*downsampleRate:(i+1)*downsampleRate],dtype=np.float64)
        return [Downsampled1, Downsampled2, Downsampled3, Downsampled4, Downsampled5, Downsampled6, num]

def CNNComputeTarget(eegData, stims, samplingFreq, channelNum, model, Trior):
        sampleNum = eegData.shape[1]
        downsampleRate = 4

#        Downsampling(eegData, downsampleRate)
#        samplingFreq = samplingFreq/4
#        sampleNum = eegData.shape[1]

        #Common Average Reference
        eegData = Re_referencing(eegData, channelNum, sampleNum)

        #Bandpass Filter
        eegData = butter_bandpass_filter(eegData, 0.5, 10, samplingFreq, 4)

        #Epoching
        epochSampleNum = int(np.floor(0.4 * samplingFreq))
        offset = int(np.floor(0.2 * samplingFreq))
        baseline = int(np.floor(0.6 * samplingFreq))
        [Epochs1, Num1] = EpochingNum(eegData, stims, 1, samplingFreq, channelNum, epochSampleNum, offset, baseline, Trior)
        [Epochs2, Num2] = EpochingNum(eegData, stims, 2, samplingFreq, channelNum, epochSampleNum, offset, baseline, Trior)
        [Epochs3, Num3] = EpochingNum(eegData, stims, 3, samplingFreq, channelNum, epochSampleNum, offset, baseline, Trior)
        [Epochs4, Num4] = EpochingNum(eegData, stims, 4, samplingFreq, channelNum, epochSampleNum, offset, baseline, Trior)
        [Epochs5, Num5] = EpochingNum(eegData, stims, 5, samplingFreq, channelNum, epochSampleNum, offset, baseline, Trior)
        [Epochs6, Num6] = EpochingNum(eegData, stims, 6, samplingFreq, channelNum, epochSampleNum, offset, baseline, Trior)

        #Downsampling Epochs
        [Epochs1,Epochs2,Epochs3,Epochs4,Epochs5,Epochs6,epochSampleNum] = DownsamplingOnlineEpoch(Epochs1, Epochs2, Epochs3, Epochs4, Epochs5, Epochs6, downsampleRate)
        samplingFreq = samplingFreq/4

        result = np.zeros((1,6))

        Epochs1 = np.reshape(Epochs1, (Num1,1,channelNum,epochSampleNum))
        Epochs2 = np.reshape(Epochs2, (Num2,1,channelNum,epochSampleNum))
        Epochs3 = np.reshape(Epochs3, (Num3,1,channelNum,epochSampleNum))
        Epochs4 = np.reshape(Epochs4, (Num4,1,channelNum,epochSampleNum))
        Epochs5 = np.reshape(Epochs5, (Num5,1,channelNum,epochSampleNum))
        Epochs6 = np.reshape(Epochs6, (Num6,1,channelNum,epochSampleNum))

        a1 = model.predict(Epochs1)
        a2 = model.predict(Epochs2)
        a3 = model.predict(Epochs3)
        a4 = model.predict(Epochs4)
        a5 = model.predict(Epochs5)
        a6 = model.predict(Epochs6)

        result[0,0] = np.sum(a1[:,1])
        result[0,1] = np.sum(a2[:,1])
        result[0,2] = np.sum(a3[:,1])
        result[0,3] = np.sum(a4[:,1])
        result[0,4] = np.sum(a5[:,1])
        result[0,5] = np.sum(a6[:,1])

        answer = np.argmax(result) + 1
        print(result)
        return answer

def main():
        ###ZeroCNN: Compute accuracy of user data on specific triors
        UserData_Path = "C:/Users/user/WorldSystem/UserData"
        Target_list = []
            #They are stored in the list in the order they are stored.
        Target_list = sorted(glob.glob(UserData_Path + '/*'), key=os.path.getmtime)
        UserNum = np.shape(Target_list)[0]
        Accuracy = np.zeros((UserNum,5))
        samplingFreq = 512
        channelNum = 32
        model = load_model('C:/Users/user/WorldSystem/Zero/ZeroModel/ZeroCNN.h5')
        for i in range(0,UserNum):
            print(str(Target_list[i][35:]))
#            ZeroData_Path = []
#            ZeroData_Path = sorted(glob.glob(Target_list[i] + '/ZeroData/*.mat'), key=os.path.getmtime)
            ZeroTarget_Path = []
            ZeroStims_Path = []
            ZeroTarget_Path = glob.glob(Target_list[i] + '/ZeroTarget/*.txt')
            ZeroData_Path = sorted(glob.glob(Target_list[i] + '/ZeroData/txt_files/eegData/*.out'), key=os.path.getmtime)
            ZeroStims_Path = sorted(glob.glob(Target_list[i] + '/ZeroData/txt_files/stims/*.out'), key=os.path.getmtime)

            ##Result txt file read in order to get orders
            OT = []
            [TryO, OT] = Readtxt(ZeroTarget_Path[0])
            Trior = 1
            ##Find Session Accuracy
            for l in range(0, 5):
                print("Epoch number:",Trior)
                CorrectO = 0
                for k in range(0, TryO):
                    result = 0
#                    mat = hdf5storage.loadmat(ZeroData_Path[k])
#                    eegData = mat['eegData']
#                    stims = mat['stims']
#                    eegData = np.transpose(eegData)
                    eegData = np.loadtxt(ZeroData_Path[k], delimiter = ",")
                    stims = np.loadtxt(ZeroStims_Path[k], delimiter = ",")
                    result = CNNComputeTarget(eegData, stims, samplingFreq, channelNum, model, Trior)
                    if(OT[k][7] == str(result)):
                        CorrectO = CorrectO + 1
                    print("Target:",OT[k][7])
                    print("result:",result)
                Accuracy[i,l] = CorrectO/TryO
                if(l == 0):
                    Trior = Trior * 5
                else:
                    Trior = Trior + 5
            print("Acc:",Accuracy[i,:])
        print("All patient:")
        print(Accuracy)
        print("Acc mean")
        np.mean(Accuracy,axis=0)

if __name__ == "__main__":
    main()


from ariadne import ObjectType
from djangocities.pages.models import Page

site = ObjectType("Site")

@site.field("pages")
def resolve_pages(obj, info):
    return obj.page_set.all()

@site.field("default_page")
def resolve_default_page(obj, info):
    DEFAULT_FILE_NAME = "index.html"
    pages = obj.page_set.filter(site=obj).values_list("file_name", flat=True)

    if len(pages) == 0:
        return ""
    if DEFAULT_FILE_NAME in pages:
        return DEFAULT_FILE_NAME

    return pages[0]


# -*- coding: utf-8 -*
import argparse
import numpy as np
import math

def dim2loc(args):
    shapes = np.array(args.shape.split(',')).astype(np.int32)  # 维度
    offsets = np.array(args.offset.split(',')).astype(np.int32)  # 偏移
    offset = 0
    for i in range(len(shapes)):
        bucket = 1  # 剩余维度容量
        for j in range(i + 1, len(shapes)):
            bucket *= shapes[j]
        offset += offsets[i] * bucket
    print(offset)

def loc2dim(args):
    offset = int(args.offset)  # 偏移
    shapes = np.array(args.shape.split(',')).astype(np.int32)  # 维度
    offsets = []
    print('i', 'offset', 'quotient', 'remainder')
    for i in range(len(shapes)):
        bucket = 1  # 剩余维度容量
        for j in range(i + 1, len(shapes)):
            bucket *= shapes[j]
        quotient = int(math.floor(offset / bucket))  # 商
        remainder = offset % bucket  # 余
        print(i, offset, quotient, remainder)
        offset = remainder
        offsets.append(quotient)
    print(offsets)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("command", help="'dim2loc' or 'loc2dim'")
    parser.add_argument('--shape', required=True, help='matrix shape')
    parser.add_argument('--offset', required=True, help='matrix offset')
    args = parser.parse_args()

    ({'dim2loc': dim2loc, 'loc2dim': loc2dim})[args.command](args)


from flask import Flask, render_template, request, jsonify
import sys
import os
import glob
sys.path.insert(0, '../')
import settings as st
from src.race_manager import RaceManager
from src.race_inferer_wrapper import RaceInfererWrapper
from dotenv import load_dotenv

load_dotenv()

app = Flask(__name__)
app.config['JSON_SORT_KEYS'] = False

RACE_MANAGER = RaceManager().load()
RACE_INFERER = RaceInfererWrapper()

@app.route('/', methods=['GET'])
def get_home():
    return render_template('home.html', races=RACE_MANAGER.races)

@app.route('/race-detail', methods=['POST'])
def post_race_detail():
    race_name = request.form['races']
    race = RACE_MANAGER.races[race_name]
    return render_template('race.html', race=race)

@app.route('/race-detail/visualisation', methods=['POST'])
def post_visualisation():
    race_name = request.form['race_name']
    output_folder = st.files["output_folder"]
    filename = os.path.join(output_folder, f"{race_name}.html")

    if filename not in glob.glob(f'{output_folder}/*.html'):
        RACE_MANAGER.races[race_name].draw()

    json_dict = {'filename': f"/static/output/{race_name}.html"}

    return jsonify(json_dict)

@app.route('/race-detail/statistics', methods=['POST'])
def post_statistic():
    race_name = request.form['race_name']
    json_dict = RACE_MANAGER.races[race_name].get_statistics()

    return jsonify(json_dict)

@app.route('/race-detail/comparison_segments', methods=['POST'])
def get_comparison_segments():
    race_name = request.form['race_name']
    deniv_segment, length_segment, density_segment = RACE_INFERER.get_best_segment(race_name)
    segments = deniv_segment + length_segment + density_segment

    json_dict = {}

    for i, segment_tuple in enumerate(segments):
        second_race = segment_tuple[0]
        segment = segment_tuple[1]

        json_dict[f'seg{i}'] = [second_race, segment.segment_type]

    return jsonify(json_dict)

@app.route('/race-detail/comparison_visualisation', methods=['POST'])
def get_comparison_visualisation():
    race_name = request.form['race_name']
    comparison_race_name = request.form['comparison_race_name']
    segment_type = request.form['segment_type']

    output_folder = st.files["output_folder"]
    filename = f"{race_name}_{comparison_race_name}_{segment_type}.html"
    dir_filename = os.path.join(output_folder, filename)

    if dir_filename not in glob.glob(f'{output_folder}/*.html'):
        RACE_INFERER.draw(race_name, comparison_race_name, segment_type, dir_filename)

    json_dict = {'filename': f"/static/output/{filename}"}

    return jsonify(json_dict)

@app.route('/race-detail/comparison_table', methods=['POST'])
def get_comparison_table():
    race_name = request.form['race_name']
    comparison_race_name = request.form['comparison_race_name']
    segment_type = request.form['segment_type']

    deniv_segment, length_segment, density_segment = RACE_INFERER.get_best_segment(race_name)
    segments = deniv_segment + length_segment + density_segment

    segment = None

    for segment_tuple in segments:
        if segment_tuple[0] == comparison_race_name and segment_tuple[1].segment_type == segment_type:
            segment = segment_tuple[1]
            break

    json_dict = {"seg1": segment.get_statistics(from_race=1), "seg2": segment.get_statistics(from_race=2)}

    return jsonify(json_dict)

@app.route('/race-detail/comparison_graphs', methods=['POST'])
def get_comparison_graphs():
    race_name = request.form['race_name']
    comparison_race_name = request.form['comparison_race_name']
    segment_type = request.form['segment_type']

    deniv_segment, length_segment, density_segment = RACE_INFERER.get_best_segment(race_name)
    segments = deniv_segment + length_segment + density_segment

    segment = None

    for segment_tuple in segments:
        if segment_tuple[0] == comparison_race_name and segment_tuple[1].segment_type == segment_type:
            segment = segment_tuple[1]
            break

    start = segment.points1[0].distance / 1000
    stop = segment.points1[-1].distance / 1000

    json_dict = {'start': start, 'stop': stop}

    return jsonify(json_dict)

if __name__ == '__main__':
    app.run()


from gvsbuild.utils.base_builders import Meson
from gvsbuild.utils.base_expanders import Tarball
from gvsbuild.utils.base_project import Project, project_add

@project_add
class GdkPixbuf(Tarball, Meson):
    def __init__(self):
        Project.__init__(
            self,
            "gdk-pixbuf",
            version="2.42.10",
            repository="https://gitlab.gnome.org/GNOME/gdk-pixbuf",
            archive_url="https://download.gnome.org/sources/gdk-pixbuf/{major}.{minor}/gdk-pixbuf-{version}.tar.xz",
            hash="ee9b6c75d13ba096907a2e3c6b27b61bcd17f5c7ebeab5a5b439d2f2e39fe44b",
            dependencies=[
                "ninja",
                "pkgconf",
                "meson",
                "libtiff-4",
                "libjpeg-turbo",
                "glib",
                "libpng",
            ],
        )
        if self.opts.enable_gi:
            self.add_dependency("gobject-introspection")
            enable_gi = "enabled"
        else:
            enable_gi = "disabled"

        self.add_param("-Dbuiltin_loaders=all")
        self.add_param(f"-Dintrospection={enable_gi}")
        self.add_param("-Dman=false")

    def build(self):
        Meson.build(self)
        self.install(r".\COPYING share\doc\gdk-pixbuf")

    def post_install(self):
        self.exec_cmd(r"%(gtk_dir)s\bin\gdk-pixbuf-query-loaders.exe --update-cache")


#!/home/saeed/Programs/miniconda3/bin/python

from datetime import datetime as dt
from datetime import timedelta as td
from random import gammavariate, choice, seed
from copy import deepcopy as dp
from LatLon import lat_lon as ll
import os, sys

#_________GET INPUT

inp = input("\n+++ Nordic file:\n")

if not os.path.exists("STATION0.HYP"): print("\n+++ No STATION0.HYP found!\n"); sys.exit(0)

#_________CONVERT PHASE

def get_mag():

    seed(1)
    mag = [gammavariate(2,2) for _ in range(500)]
    mag = [_/max(mag) for _ in mag]
    mag = [_*5.5 for _ in mag]

    return mag

random_mag = get_mag()

def get_ot(l):

    if l[16]=="6": l = l[:16]+"59.00"+l[20:]

    for i in [6,8,11,13,16]:

        if not l[i].strip():

            l = l[:i]+"0"+l[i+1:]

    ot = dt.strptime(l[:20], " %Y %m%d %H%M %S.%f")
    lat = float(l[23:30])
    lon = float(l[31:38])
    dep = float(l[39:43])
    erh = 0
    erz = 0
    rms = float(l[52:55])

    if l[56:59].strip() and float(l[56:59])!=0: mag = float(l[56:59])
    else: mag = choice(random_mag)

    return ot, lat, lon, dep, mag, erh, erz, rms

def get_ttim(ot, l, wet, log):

    for i in [18, 19, 20, 22, 23]:

        if not l[i].strip():

            l = l[:i]+"0"+l[i+1:]

    tmp = dp(ot)

    sec = float(l[22:28])
    if sec>=60.00: sec=59.999
    l = l[:22]+'%006.3f'%(sec)+l[28:]
    delayDay = False
    try:
        at = dt.strptime(l[18:28], "%H%M%S.%f")
    except ValueError:
        if float(l[18:20]) == 24:
            l = l[:18]+"00"+l[20:]
            at = dt.strptime(l[18:28], "%H%M%S.%f")
            delayDay = True
    at = at.replace(year=tmp.year, month=tmp.month, day=tmp.day, hour=at.hour, minute=at.minute, second=at.second, microsecond=at.microsecond)
    if delayDay: at = at.replace(day=at.day-1)
    if ot.hour == 23 and at.hour==0: at = at + td(days=1)
    ttim = at-ot
    ttim = ttim.total_seconds()

    if wet>1e-5 and (ttim>100 or ttim<0):
        print("Warning! Travel time exceeded 100s or is negative! Check log.dat")
        log.write(ot.isoformat()+" > "+l)

    return ttim

def get_wet(l):

    if l[14] == " ": return 1
    elif l[14] == "4": return 1e-6
    else: return (0.5)**float(l[14])

def get_pha(l, log):

    if l[10].strip() and l[10].upper()=="P":

        sta = l[:5].strip().upper()
        wet = get_wet(l)
        pha = "P"
        tti = get_ttim(ot, l, wet, log)

        line = "%4s %7.3f %.1e %1s\n"%(sta, tti, wet, pha)
        g.write(line)

    elif l[10].strip() and l[10].upper()=="S":

        sta = l[:5].strip().upper()
        wet = get_wet(l)
        pha = "S"
        tti = get_ttim(ot, l, wet, log)

        line = "%4s %7.3f %.1e %1s\n"%(sta, tti, wet, pha)
        g.write(line)

#________MAIN

with open(inp) as f, open("phase.dat", "w") as g, open("log.dat", "w") as log:

    evn = 0
    flag = False

    for l in f:

        if l.strip() and l[24:43].strip() and l[79]=="1":

            ot, lat, lon, dep, mag, erh, erz, rms = get_ot(l)
            if not ((lat>23)&(lat<42))&((lon>42)&(lon<63)): continue
            evn+=1

            header = "#, YR, MO, DY, HR, MN, SC, LAT, LON, DEP, MAG, EH, EZ, RMS, ID"
            line = "# %22s %7.3f %7.3f %5.1f %4.1f %6.1f %6.1f %5.2f %6d\n"%(ot.strftime("%Y %m %d %H %M %S.%f")[:22],
                                                                             lat, lon, dep, mag, erh, erz, rms, evn)
            g.write(line)
            flag = True

        if flag and not l.strip(): flag = False

        if flag and l.strip() and l[79] in ["4", " "]:

            pha = get_pha(l, log)

#_________CONVERT STATION

with open("STATION0.HYP") as f, open("station.dat", "w") as g:

    sta_list = []

    for l in f:

        if len(l.strip())>20 and [l[13],l[22]] == ["N", "E"]:

            sta = l[:6].strip().upper()
            if sta not in sta_list: sta_list.append(sta)
            else: continue
            lat = ll.Latitude(degree=float(l[6:8]), minute=float(l[8:13])).decimal_degree
            lon = ll.Longitude(degree=float(l[15:17]), minute=float(l[17:21])).decimal_degree
            try:
                elv = float(l[23:27])
            except ValueError:
                elv = 0
            g.write("%4s %7.3f %7.3f %5d\n"%(sta, lat, lon, elv))


import matplotlib
matplotlib.use("TkAgg")
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure
import matplotlib.animation as animation
from matplotlib import style
import tkinter as tk
from tkinter import ttk

style.use("ggplot")

f = Figure(figsize=(5,5), dpi=100)
a = f.add_subplot(111)

def animate(i):
    pullData = open("datadump.csv","r").read()
    dataList = pullData.split('\n')
    xList = []
    yList = []
    mList = []
    nList = []
    for eachLine in dataList:
        if len(eachLine) > 1:
            x, y, m, n = eachLine.split(',')
            xList.append(x)
            yList.append(int(y))
            mList.append(m)
            nList.append(n)

    a.clear()
    a.plot(xList, yList)

class Graph(tk.Tk):

    def __init__(self, *args, **kwargs):
        tk.Tk.__init__(self, *args, **kwargs)

        canvas = FigureCanvasTkAgg(f, self)
        canvas.draw()
        canvas.get_tk_widget().pack()
        canvas._tkcanvas.pack()

app = Graph()
ani = animation.FuncAnimation(f, animate, interval=10)
app.title("Inventory Graph")
app.mainloop()


# coding: utf-8

import traceback
import os
import pandas as pd
from copy import deepcopy
from datetime import timedelta, datetime
from cobra.data.kline import kline_simulator, get_kline
from cobra.data.basic import is_trade_day
from chan import SolidAnalyze, KlineAnalyze
from chan.solid import is_macd_cross

data_path = "./data"
if not os.path.exists(data_path):
    os.mkdir(data_path)

def get_data(sa):
    mark_convert = {"d": 0, "g": 1}
    signals = {"交易标的": sa.symbol, "交易时间": sa.kas['1分钟'].end_dt}
    for freq, ka in sa.kas.items():
        signals[freq + "分型标记"] = mark_convert[ka.fx[-1]['fx_mark']]
        signals[freq + "笔标记"] = mark_convert[ka.bi[-1]['fx_mark']]
        signals[freq + "线段标记"] = mark_convert[ka.xd[-1]['fx_mark']]
        signals[freq + "MACD金叉"] = int(is_macd_cross(ka, direction='up'))
        signals[freq + "MACD死叉"] = int(is_macd_cross(ka, direction='down'))
    print(signals)
    return signals

def trade_simulator(ts_code, end_date, start_date, asset="E", watch_interval=5):
    """单只标的类实盘模拟，研究买卖点变化过程

    :param ts_code: str
        标的代码，如 300033.SZ
    :param end_date: str
        截止日期，如 2020-03-12
    :param start_date: str
        开始日期
    :param asset: str
        tushare 中的资产类型编码
    :param watch_interval: int
        看盘间隔，单位：分钟；默认值为 5分钟看盘一次
    :return: None
    """
    file_signals = os.path.join(data_path, f"{ts_code}_{start_date}_{end_date}_signals.txt")

    end_date = datetime.strptime(end_date.replace("-", ""), "%Y%m%d")
    start_date = datetime.strptime(start_date.replace("-", ""), "%Y%m%d")

    while start_date <= end_date:
        if (asset in ["E", "I"]) and (not is_trade_day(start_date.strftime('%Y%m%d'))):
            start_date += timedelta(days=1)
            continue

        ks = kline_simulator(ts_code, trade_dt=start_date.strftime('%Y-%m-%d'), asset=asset, count=1000)
        for i, klines in enumerate(ks.__iter__(), 1):
            if i % watch_interval != 0:
                continue
            sa = SolidAnalyze(klines)
            print(sa.kas['1分钟'].end_dt)
            try:
                signals = get_data(sa)

                with open(file_signals, 'a', encoding='utf-8') as f:
                    f.write(str(signals) + "\n")
            except:
                traceback.print_exc()

        start_date += timedelta(days=1)

def make_one_day(ts_code, trade_date, asset="E"):
    if "-" in trade_date:
        end_date = datetime.strptime(trade_date, '%Y-%m-%d')
    else:
        end_date = datetime.strptime(trade_date, '%Y-%m-%d')
    start_date = end_date - timedelta(days=1)
    end_dt = end_date + timedelta(days=30)
    start_date = start_date.strftime("%Y-%m-%d")
    end_date = end_date.strftime("%Y-%m-%d")

    if not is_trade_day(start_date):
        return
    print(f"start trade simulator on {start_date}")
    trade_simulator(ts_code=ts_code, start_date=start_date,
                    end_date=end_date, asset=asset, watch_interval=1)

    for freq in ['1min', '5min', '30min']:
        file_signals = os.path.join(data_path, f"{ts_code}_{start_date}_{end_date}_signals.txt")
        signals = [eval(x) for x in open(file_signals, encoding='utf-8').readlines()]
        df = pd.DataFrame(signals)
        kline = get_kline(ts_code, end_dt=end_dt.strftime("%Y-%m-%d %H:%M:%S"), freq=freq, asset=asset)
        ka = KlineAnalyze(kline)
        print(kline.head(), "\n\n")
        xd = deepcopy(ka.xd)
        xd = sorted(xd, key=lambda row: row['dt'], reverse=False)
        print(xd, "\n\n")

        def ___xd_status(dt):
            for x in xd:
                if x['dt'] >= dt:
                    if x['fx_mark'] == 'd':
                        s = "向下段"
                    elif x['fx_mark'] == 'g':
                        s = "向上段"
                    else:
                        raise ValueError
                    return s
            return "o"

        col = f'{freq}线段状态'
        df[col] = df['交易时间'].apply(___xd_status)
        df.to_excel(f"./data/{ts_code}_{start_date}_{end_date}_{freq}.xlsx", index=False)

if __name__ == '__main__':
    ts_code = "000001.SH"
    start_date = "2019-08-01"
    end_date = "2019-10-01"
    asset = 'I'

    start_date = datetime.strptime(start_date, "%Y-%m-%d")
    end_date = datetime.strptime(end_date, "%Y-%m-%d")

    while start_date < end_date:
        start_date += timedelta(days=1)
        make_one_day(ts_code, start_date.strftime("%Y-%m-%d"), asset)


'''
xplsek03 | 2021 | FIT VUT Brno, bakalarska prace Razeni manipulaci pro morici linky
'''

from task.sbh.graph import Graph
from task.bab.edd import Edd
import logging
#from task.output.timer import timer

'''
Funkce pro provadeni Shifting bottleneck heuristiky.
'''
class Sbh:

    '''
    Inicializace harmonogramu nad jeraby a vanami.
    @param vany - seznam van
    @param jeraby - seznam jerabu
    @param clen - jedinec populace
    @return tuple se slovniky harmonogramu
    '''
    def init_harmonogramy(vany, jeraby, clen):

        # harmogram zpracovani na vanach: {id_vany: [operace zpracovavane na teto vane], .. }
        harmonogram_vany = {}

        for vana in vany:
            # filtruj podle konkretniho id vany: seznam operaci, ktere se provadi na jedne vane
            filtered = list(filter(lambda x: (x.id_stroj == vana['_id']), clen['namaceni']))

            # pridej do seznamu harmonogramu, pokud se ve vane bude neco namacet. DEBUG: promenna, self vany je overkill?
            if len(filtered):
                harmonogram_vany[vana['_id']] = filtered

        # harmogram zpracovani na jerabech: {id_stroj: [operace zpracovavane na tomto jerabu], .. }
        harmonogram_jeraby = {}
        # sestav plan na jednotlivych jerabech
        for jerab in jeraby:
            filtered = list(filter(lambda x: (x.id_stroj == jerab), clen['transporty']))
            # pridej do seznamu harmonogramu pokud na jerabu bude neco prepravovaneho. DEBUG: mozna pouzit jinou promennou, overkill
            if len(filtered):
                harmonogram_jeraby[jerab] = filtered

        return (harmonogram_jeraby, harmonogram_vany)

    '''
    SBH - shifting bottleneck heuristika.
    @param graf - graf nad kterym se ma provadet, jednoduchy G nebo uplny G'
    @param slovnik_van
    @param slovnik_produktu
    @param harmonogram - slovnik, klic: stroj hodnota: seznam operaci na stroji
    @param clen - jedinec populace
    @param simple - G = True, G' = False
    @param slovnik_jerabu
    @return cmax nebo None
    '''
    def sbh(graf, slovnik_van, slovnik_produktu, harmonogram, clen, simple=True, slovnik_jerabu=None):

        # pro simple poprve uz mas pridanou jednu vanu v m0, tedy prvni prvek harmonogramu zahod pro simple
        first = True

        # postupne planuj cleny harmonogramu, v jednom harmonogramu je seznam namaceni ruznych jobu v jedne vane nebo seynam transportu
        for m_id, m in harmonogram.items():

            # pouze simple, skipni EDD u 1. vany - pridas jak to je
            if first and simple:
                # proved vypocet fake EDD
                useky = Sbh.fake_edd(m, m_id, slovnik_van, graf)
                # nahrej do grafu disjunktivni spojnice prvni vany, id vem z harmonogram (harmonogram_vany)
                Graph.add_disjunctive(graf, useky, slovnik_van, slovnik_jerabu, simple)
                first = False
                continue

            # v kazde iteraci pocitas vlastni cmax
            cmax, pozitivni_cyklus = Graph.bellman(graf, node_from="start", node_to="end")
            if pozitivni_cyklus:
                return None

            # proved branch and bound pro konkretni stroj. best = Node() | None pokud to nenajde reseni
            node = Edd.branch_and_bound(slovnik_van, slovnik_jerabu, graf, m, cmax, is_namaceni=simple)

            # BaB nenasel vysledek - chyba aproximace
            if not node:
                logging.debug('CHYBA BAB')
                return None

            # vloz do grafu ziskane spojnice - o hodnote exp_max te vany nebo LT + ET u uplneho grafu
            Graph.add_disjunctive(graf, node.casove_useky, slovnik_van, slovnik_jerabu, simple)

            # over na pozitivni cyklus
            cmax, pozitivni_cyklus = Graph.bellman(graf, node_from="start", node_to="end")
            if pozitivni_cyklus:
                # odted vime ze obsahuje pozitivni cyklus
                # odeber pridane disjunktivni hrany
                Graph.remove_disjunctive(graf, node.casove_useky)
                # vytvor LB a UB pro kazdy prvek, rozmezi v jakem muze swapovat s dalsim prvkem
                Sbh.init_lb_ub(node.casove_useky)
                # postupne prehazuj dva prvky dokud nenajdes odpovidajici harmonogram na stroji, co netvori pozitivni cyklus
                cmax = Graph.sbh_reparation(graf, node.casove_useky, slovnik_van, slovnik_jerabu, simple)
                if not cmax:
                    return None

            # updatuj - serad podle noveho - zaznam harmonogramu, kvuli velkemu grafu - aby vedel ktere disjunktivni aktualni spojnice ma vytvaret
            mapa = [i['stroj'].graphid for i in node.casove_useky]
            m.sort(key=lambda x: mapa.index(x.graphid))

        # vrat vypocitany cmax
        return cmax

    '''
    Pomocna funkce pro generovani poradi upper a lower bounds, ktere budou respektovany
    pri prehazovani kombinace prvku v pripade opravy sekvence v SBH v kroku 4.
    @param useky - casove useky vracene z Node()
    '''
    def init_lb_ub(useky):

       for j in range(len(useky)):
            # jdi doleva, defaultne nastav 0
            useky[j]['lb'] = 0
            it = 0
            for l in reversed(useky[:j]):
                if l['stroj'].id_produkt == useky[j]['stroj'].id_produkt:
                    useky[j]['lb'] = j - it
                    break
                it += 1
            # jdi doprava
            useky[j]['ub'] = len(useky) - 1
            for i, r in enumerate(useky[j:]):
                if r['stroj'].id_produkt == useky[j]['stroj'].id_produkt:
                    useky[j]['ub'] = j + i

    '''
    Pridani operaci prvniho stroje bez provadeni EDD. Pouze u jednoducheho grafu.
    @param m - seznam operaci provadenych na stroji
    @param id - id stroje
    @param slovnik_van
    @param graf - jednoduchy graf G'
    @return vygenerovany seznam casovych useku
    '''
    def fake_edd(m, id, slovnik_van, graf):

        # finalni useky
        useky = []
        # projdi seznam polozek
        for item in m:
            rj, pozitivni_cyklus = Graph.bellman(graf, node_from="start", node_to=item.graphid)
            if pozitivni_cyklus:
                return None
            pj = slovnik_van[id]['exp_max']

            useky.append({'start': rj, 'end': rj + pj, 'stroj': item})

            # je potreba useky projit a nastavit precedence v ramci produktu, aby nektere - pokud jsou - nebezely zaraz na stejne vane
            m_index = 0  # index v ramci m, pokud te zajima prava polozka tak je to m_index+1
            for left,right in zip(useky[:-1], useky[1:]):

                # pokud ten co ma nasledovat za nim prekryva cas, posun ho
                if right['start'] < left['end']:
                    delta = left['end'] - right['start']
                    right['start'] += delta
                    right['end'] += delta

                try:
                    # predchazejici uzel, muze byt v jine vane
                    prev = m[m_index+1].prev.prev
                    # pokud by se ta operace nevesla do mezery mezi temi co jsi posunul, musis zvetsit mezeru
                    if slovnik_van[prev.id_stroj]['exp_max'] > right['start'] - left['end']:
                        delta = left['end'] + slovnik_van[prev.id_stroj]['exp_max'] - right['start']
                        right['start'] += delta
                        right['end'] += delta

                except AttributeError:
                    pass

                m_index += 1

        return useky


from queue import Queue
from models import OptionTrainingLabels, OptionHistorical
from app import db
from thread_workers import LongCallWorker
from time import sleep
from datetime import datetime

def calculate_long_call_training_labels():
    all_dates = []
    scrapped_dates = []
    for value in db.session.query(OptionHistorical.trade_date).distinct():
        all_dates.append(value.trade_date)
    print("[{timestamp}]: Finished all dates".format(timestamp=datetime.now()))
    for value in db.session.query(OptionTrainingLabels.trade_date).filter(OptionTrainingLabels.trade_type == "Long Call").distinct():
        scrapped_dates.append(value.trade_date)
    print("[{timestamp}]: Finished scrapped dates".format(timestamp=datetime.now()))
    dates_to_do = set(all_dates) - set(scrapped_dates)
    print("[{timestamp}]: Creating queue".format(timestamp=datetime.now()))
    queue = Queue()
    for i in dates_to_do:
        queue.put(i)
    print("[{timestamp}]: Spawning workers".format(timestamp=datetime.now()))
    for x in range(2):
        print("[{timestamp}]: Create Worker {num}".format(timestamp=datetime.now(), num = x+1))
        worker = LongCallWorker(queue)
        worker.daemon = True
        worker.start()
        print("[{timestamp}]: Start Worker {num}".format(timestamp=datetime.now(), num=x+1))
    queue.join()

    #for day in dates_to_do:
    #    print("[{timestamp}]: Working on {trade_day}".format(timestamp=datetime.now(), trade_day=day))
    #    query_string = """WITH max_price_data AS (SELECT ticker, experiation_date, strike, max(call_bid_price) as max_sales_price FROM option_historical WHERE trade_date >= to_date('{day_working_on}', 'YYYY-MM-DD') AND (ticker, experiation_date, strike) = ANY(SELECT ticker, experiation_date, strike FROM option_historical WHERE trade_date = to_date('{day_working_on}', 'YYYY-MM-DD')) GROUP BY ticker, experiation_date, strike)
    #                      SELECT option_historical.ticker, option_historical.experiation_date, option_historical.strike, option_historical.trade_date, option_historical.call_ask_price as buy_price, max_price_data.max_sales_price
    #                      FROM option_historical JOIN max_price_data
    #                      ON option_historical.ticker = max_price_data.ticker
    #                      AND option_historical.experiation_date = max_price_data.experiation_date
    #                      AND option_historical.strike = max_price_data.strike
    #                      AND option_historical.trade_date = to_date('{day_working_on}', 'YYYY-MM-DD')
    #                      """.format(day_working_on = day)
    #
    #    data = pd.read_sql(query_string, db.engine)
    #    data['trade_type'] = "Long Call"
    #    data['label'] = (data['max_sales_price'] - data['buy_price']) / data['buy_price']
    #    data.drop(['buy_price', 'max_sales_price'], inplace=True, axis=1)
    #    data.dropna(inplace=True)
    #    copy_dataframe_to_database(data, OptionTrainingLabels, with_index=False)