#!/opt/anaconda3/bin/python
"""
# Author: Yuzhe Li
# Created Time : Wed Sep 16 09:46:22 CST 2020

# File Name: SCALE.py
# Description: Single-Cell ATAC-seq Analysis via Latent feature Extraction.
    Input:
        scATAC-seq data
    Output:
        1. latent feature
        2. cluster assignment
        3. imputation data
"""

import time
import torch
import numpy as np
import pandas as pd
import scanpy as sc
import anndata as ad
import scipy
import os
import argparse

from scale import SCALE
from scale.dataset import SingleCellDataset, load_data, filter_features, filter_cells

from scale.utils import read_labels, cluster_report, estimate_k, binarization
from scale.plot import plot_embedding

from sklearn.preprocessing import MaxAbsScaler
from torch.utils.data import DataLoader

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='SCALE: Single-Cell ATAC-seq Analysis via Latent feature Extraction')
    parser.add_argument('--dataset', '-d', type=str, help='input dataset path')
    parser.add_argument('--n_centroids', '-k', type=int, help='cluster number')
    parser.add_argument('--outdir', '-o', type=str, default='output/', help='Output path')
    parser.add_argument('--verbose', action='store_true', help='Print loss of training process')
    parser.add_argument('--pretrain', type=str, default=None, help='Load the trained model')
    parser.add_argument('--lr', type=float, default=0.002, help='Learning rate')
    parser.add_argument('--batch_size', '-b', type=int, default=32, help='Batch size')
    parser.add_argument('--gpu', '-g', default=0, type=int, help='Select gpu device number when training')
    parser.add_argument('--seed', type=int, default=18, help='Random seed for repeat results')
    parser.add_argument('--encode_dim', type=int, nargs='*', default=[1024, 128], help='encoder structure')
    parser.add_argument('--decode_dim', type=int, nargs='*', default=[], help='encoder structure')
    parser.add_argument('--latent', '-l', type=int, default=10, help='latent layer dim')
    parser.add_argument('--low', '-x', type=float, default=0.01, help='Remove low ratio peaks')
    parser.add_argument('--high', type=float, default=0.9, help='Remove high ratio peaks')
    parser.add_argument('--min_peaks', type=float, default=100, help='Remove low quality cells with few peaks')
    parser.add_argument('--log_transform', action='store_true', help='Perform log2(x+1) transform')
    parser.add_argument('--max_iter', '-i', type=int, default=30000, help='Max iteration')
    parser.add_argument('--weight_decay', type=float, default=5e-4)
    parser.add_argument('--impute', action='store_true', help='Save the imputed data')
    parser.add_argument('--binary', action='store_true', help='Save binary imputed data')
    #     parser.add_argument('--no_tsne', action='store_true', help='Not save the tsne embedding')
    parser.add_argument('--emb', type=str, default='UMAP')
    parser.add_argument('--reference', '-r', default=None, type=str, help='Reference celltypes')
    parser.add_argument('--transpose', '-t', action='store_true', help='Transpose the input matrix')

    args = parser.parse_args()

    # Set random seed
    seed = args.seed
    np.random.seed(seed)
    torch.manual_seed(seed)

    if torch.cuda.is_available():  # cuda device
        device = 'cuda'
        torch.cuda.set_device(args.gpu)
    else:
        device = 'cpu'
    batch_size = args.batch_size

    normalizer = MaxAbsScaler()

    adata = load_data(args.dataset,transpose=args.transpose)
    args.min_peaks = int(args.min_peaks) if args.min_peaks >= 1 else args.min_peaks
    total_cells = adata.shape[0]
    min_cells = int(args.low * total_cells)
    max_cells = int(args.high * total_cells)
    filter_features(adata, min_cells=min_cells)
    filter_features(adata, max_cells=max_cells)
    filter_cells(adata,min_peaks=args.min_peaks)

    dataset = SingleCellDataset(adata,transforms=[normalizer.fit_transform])
    trainloader = DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=True)
    testloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, drop_last=False)

    cell_num = dataset.shape[0]
    input_dim = dataset.shape[1]

    if args.n_centroids is None:
        k = min(estimate_k(dataset.data.T), 30)
        print('Estimate k {}'.format(k))
    else:
        k = args.n_centroids
    lr = args.lr
    name = args.dataset.strip('/').split('/')[-1]

    outdir = args.outdir
    if not os.path.exists(outdir):
        os.makedirs(outdir)

    print("\n**********************************************************************")
    print("  SCALE: Single-Cell ATAC-seq Analysis via Latent feature Extraction")
    print("**********************************************************************\n")
    print("======== Parameters ========")
    print(
        'Cell number: {}\nPeak number: {}\nn_centroids: {}\nmax_iter: {}\nbatch_size: {}\ncell filter by peaks: {}\nrare peak filter: {}\ncommon peak filter: {}'.format(
            cell_num, input_dim, k, args.max_iter, batch_size, args.min_peaks, args.low, args.high))
    print("============================")

    dims = [input_dim, args.latent, args.encode_dim, args.decode_dim]
    model = SCALE(dims, n_centroids=k)
    #     print(model)

    if not args.pretrain:
        print('\n## Training Model ##')
        model.init_gmm_params(testloader)
        model.fit(trainloader,
                  lr=lr,
                  weight_decay=args.weight_decay,
                  verbose=args.verbose,
                  device=device,
                  max_iter=args.max_iter,
                  name=name,
                  outdir=outdir
                  )
    #         torch.save(model.to('cpu').state_dict(), os.path.join(outdir, 'model.pt')) # save model
    else:
        print('\n## Loading Model: {}\n'.format(args.pretrain))
        model.load_model(args.pretrain)
        model.to(device)

    ### output ###
    print('outdir: {}'.format(outdir))
    # 1. latent feature
    feature = model.encodeBatch(testloader, device=device, out='z')
    # pd.DataFrame(feature).to_csv(os.path.join(outdir, 'feature.txt'), sep='\t', header=False)
    dataset.obsm['latent'] = feature

    # 2. cluster assignments
    pred = model.predict(testloader, device)
    cluster=pd.Series(pred, index=dataset.barcode,dtype="category")
    dataset.obs['cluster'] = cluster

    # 3. imputed data
    if args.impute or args.binary:
        recon_x = model.encodeBatch(testloader, device, out='x', transforms=[normalizer.inverse_transform])
        if args.binary:
            print("Saving binary imputed data")
            recon_x = binarization(recon_x, dataset.data).T
            #imputed_dir = outdir + '/binary_imputed/'
            #os.makedirs(imputed_dir, exist_ok=True)
            #scipy.io.mmwrite(imputed_dir + 'count.mtx', recon_x)
            #pd.Series(dataset.peaks).to_csv(imputed_dir + 'peak.txt', sep='\t', index=False, header=None)
            #pd.Series(dataset.barcode).to_csv(imputed_dir + 'barcode.txt', sep='\t', index=False, header=None)
            dataset.layers['binary'] = recon_x.T
        elif args.impute:
            print("Saving imputed data")
            #recon_x = pd.DataFrame(recon_x.T, index=dataset.peaks, columns=dataset.barcode)
            #recon_x.to_csv(os.path.join(outdir, 'imputed_data.txt'), sep='\t')
            dataset.layers['impute'] = recon_x

    dataset.write(outdir + 'adata.h5ad', compression='gzip')

        #     torch.save(model.to('cpu').state_dict(), os.path.join(outdir, 'model.pt')) # save model

    #     if not args.no_tsne:
    print("Plotting embedding")
    if args.reference:
        ref = pd.read_csv(args.reference, sep='\t', header=None, index_col=0)[1]
        labels = ref.reindex(dataset.barcode, fill_value='unknown')
    else:
        labels = pred
    plot_embedding(feature, labels, method=args.emb,
                   save=os.path.join(outdir, 'emb_{}.pdf'.format(args.emb)),
                   save_emb=os.path.join(outdir, 'emb_{}.txt'.format(args.emb)))
#         plot_embedding(feature, labels,
#                        save=os.path.join(outdir, 'tsne.pdf'), save_emb=os.path.join(outdir, 'tsne.txt'))


import uuid
from readingdb.format import route_sort_key
from readingdb.utils import timestamp
import time
from typing import Any, Callable, Dict, List, Set, Tuple

import boto3
from boto3.dynamodb.conditions import Key

from readingdb.clean import *
from readingdb.reading import Reading, geohash_partition_key
from readingdb.route import Route
from readingdb.constants import *

class DB():
    ITEM_KEY = 'Items'
    LAST_EVAL_KEY = 'LastEvaluatedKey'

    def __init__(
        self,
        url,
        resource_name='dynamodb',
        region_name='ap-southeast-2',
        config=None,
        max_page_readings=1800,
    ):
        self.db = boto3.resource(
            resource_name,
            region_name=region_name,
            endpoint_url=url,
            config=config,
        )

        # DynamoDB only returns 1MB of data in a single query
        # use paginator on queries that stand a chance of returning
        # > 1MB of data.
        self.client = self.db.meta.client
        self.paginator = self.client.get_paginator('query')
        self.scan_paginator = self.client.get_paginator('scan')
        self.max_page_readings = max_page_readings

        self.reading_table = self.db.Table(Constants.READING_TABLE_NAME)
        self.org_table = self.db.Table(Constants.ORG_TABLE_NAME)

    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -------------------------- GENERAL ------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------

    def __paginate_table(
        self,
        table_name: str,
        decode_fn: Callable[[Dict[str, Any]], Any],
        query_key: str = None,
        query_value: str = None,
    )-> Any:
        if query_key is not None:
            pg = self.paginator.paginate(
                TableName=table_name,
                KeyConditionExpression=Key(query_key).eq(query_value)
            )
        else:
            pg = self.scan_paginator.paginate(
                TableName=table_name,
            )

        items = []
        for page in pg:
            for item in page[self.ITEM_KEY]:
                decode_fn(item)
                items.append(item)

        return items

    def all_tables(self) -> List[Any]:
        return list(self.db.tables.all())

    def create_reading_db(
        self,
    ) -> Tuple[Any, Any]:
        self.reading_table = self.__make_reading_table()
        self.org_table = self.__make_org_table()
        return (self.reading_table, self.org_table)

    def teardown_reading_db(self) -> None:
        self.__delete_table(Constants.READING_TABLE_NAME)
        self.__delete_table(Constants.ORG_TABLE_NAME)

    def __delete_table(self, table_name) -> None:
        self.db.Table(table_name).delete()

    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -------------------------- ROUTE --------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------

    def remove_route(self, route_id: str):
        self.org_table.delete_item(
            Key={
                Constants.PARTITION_KEY: Constants.ROUTE_PK,
                Constants.SORT_KEY: route_sort_key(route_id)
            }
        )

    def put_route(self, route: Route):
        return self.org_table.put_item(Item=route.item_data())

    def get_route(self, route_id: str) -> Dict[str, Any]:
        response = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(Constants.ROUTE_PK) &
                Key(Constants.SORT_KEY).eq(route_sort_key(route_id))
        )

        if len(response[self.ITEM_KEY]) < 1:
            return None

        item = response[self.ITEM_KEY][0]
        Route.decode_item(item)

        return item

    def route_geohashes(self, route_id: str) -> Set[str]:
        route = self.get_route(route_id)

        if route is None:
            return set()

        return set(route[Constants.ROUTE_HASHES])

    def routes_for_user(self, user_id: str) -> List[Dict[str, Any]]:
        groups = self.groups_for_user(user_id)
        routes = self.all_routes()

        return [r for r in routes if r[Constants.GROUP_ID] in groups]

    def all_routes(self) -> List[Dict[str, Any]]:
        response = self.org_table.query(
            KeyConditionExpression=Key(Constants.PARTITION_KEY).eq(Constants.ROUTE_PK)
        )

        return [Route.decode_item(item) for item in response[self.ITEM_KEY]]

    def update_route_name(self, route_id: str, name: str) -> None:
        r: Dict[str, Any] = self.get_route(route_id)

        if r[Constants.NAME] != name:
            self.org_table.update_item(
                Key={
                    Constants.PARTITION_KEY: Constants.ROUTE_PK,
                    Constants.SORT_KEY: route_sort_key(route_id)
                },
                UpdateExpression=f'set {Constants.NAME} = :r, {Constants.LAST_UPDATED} = :l',
                ExpressionAttributeValues={
                    ':r': name,
                    ':l': int(time.time())
                },
            )

    def set_route_status(self, route_id: str, status: int) -> None:
        r: Dict[str, Any] = self.get_route(route_id)

        if r[Constants.STATUS] != status:
            self.org_table.update_item(
                Key={
                    Constants.PARTITION_KEY: Constants.ROUTE_PK,
                    Constants.SORT_KEY: route_sort_key(route_id)
                },
                UpdateExpression=f'set {Constants.STATUS} = :r, {Constants.LAST_UPDATED} = :l',
                ExpressionAttributeValues={
                    ':r': status,
                    ':l': int(time.time())
                },
            )

    def __make_org_table(self):
        return self.db.create_table(
            TableName=Constants.ORG_TABLE_NAME,
            KeySchema=[
                {
                    'AttributeName':  Constants.PARTITION_KEY,
                    'KeyType': 'HASH'
                },
                {
                    'AttributeName': Constants.SORT_KEY,
                    'KeyType': 'RANGE'
                }
            ],
            AttributeDefinitions=[
                {
                    'AttributeName': Constants.PARTITION_KEY,
                    'AttributeType': 'S'
                },
                {
                    'AttributeName': Constants.SORT_KEY,
                    'AttributeType': 'S'
                },
            ],
            ProvisionedThroughput={
                'ReadCapacityUnits': 100,
                'WriteCapacityUnits': 100
            }
        )

    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -------------------------- READING ------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------

    def put_readings(self, readings: List[Reading]):
        with self.reading_table.batch_writer() as batch:
            for r in readings:
                batch.put_item(Item=r.item_data())

    def get_route_readings(self, route_id: str) -> List[Dict[str, Any]]:
        geohashes = self.route_geohashes(route_id)
        all_readings = []

        for geohash in geohashes:
            hash_readings = self.__paginate_table(
                Constants.READING_TABLE_NAME,
                Reading.decode,
                Constants.PARTITION_KEY,
                geohash_partition_key(geohash, Constants.PREDICTION),
            )
            all_readings.extend([r for r in hash_readings if r[Constants.ROUTE_ID] == route_id])

        return all_readings

    def geohash_readings(self, geohash: str, reading_type: str) -> List[Dict[str, Any]]:
        response = self.reading_table.query(
            KeyConditionExpression=Key(Constants.PARTITION_KEY).eq(geohash_partition_key(geohash, reading_type))
        )
        return response[self.ITEM_KEY]

    def put_reading(self, reading: Reading):
        return self.reading_table.put_item(Item=reading.item_data())

    def delete_reading_items(self, readings: List[Dict[str, str]])-> None:
        for reading in readings:
            self.reading_table.delete_item(
                Key=Reading.get_key(reading)
            )

    def __make_reading_table(self):
        return self.db.create_table(
            TableName=Constants.READING_TABLE_NAME,
            KeySchema=[
                {
                    'AttributeName': Constants.PARTITION_KEY,
                    'KeyType': 'HASH'
                },
                {
                    'AttributeName': Constants.SORT_KEY,
                    'KeyType': 'RANGE'
                }
            ],
            AttributeDefinitions=[
                {
                    'AttributeName': Constants.PARTITION_KEY,
                    'AttributeType': 'S'
                },
                {
                    'AttributeName': Constants.SORT_KEY,
                    'AttributeType': 'S'
                },

            ],
            ProvisionedThroughput={
                'ReadCapacityUnits': 500,
                'WriteCapacityUnits': 500
            }
        )

    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # --------------------------- LAYERS ------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------

    def put_layer(
        self,
        layer_id: str,
        readings: List[Dict[str, Any]] = None,
        name:str=None
    ) -> str:
        if readings is not None:
            existing_readings = self.layer_readings([layer_id])
            existing_reading_ids = set()
            current_reading_ids = set()

            for reading in readings:
                current_reading_ids.add(reading[Constants.READING_ID])

            for reading in existing_readings:
                reading_id = reading[Constants.READING_ID]
                existing_reading_ids.add(reading_id)
                if reading_id not in current_reading_ids:
                    self.remove_reading_from_layer(layer_id, reading)

            for reading in readings:
                if reading[Constants.READING_ID] not in existing_reading_ids:
                    self.add_reading_to_layer(layer_id, reading)

        item = {
            Constants.PARTITION_KEY: Constants.LAYER_PK,
            Constants.SORT_KEY: self.__layer_sort_key(layer_id),
        }

        if name is not None:
            item[Constants.LAYER_NAME] = name

        self.org_table.put_item(Item=item)

        return layer_id

    def remove_readings_from_layer(self, layer_id: str, readings: List[Dict[str, Any]]):
        for reading in readings:
            self.remove_reading_from_layer(layer_id, reading)

    def remove_reading_from_layer(self, layer_id: str, reading: Dict[str, Any]):
        self.org_table.delete_item(Key={
            Constants.PARTITION_KEY: self.__layer_reading_primary_key(layer_id),
            Constants.SORT_KEY: reading[Constants.READING_ID]
        })

    def add_readings_to_layer(self, layer_id:str, readings: List[Dict[str, Any]]) -> None:
        for reading in readings:
            self.add_reading_to_layer(layer_id, reading)

    def add_reading_to_layer(self, layer_id: str, reading: Dict[str, Any]):
        self.org_table.put_item(Item={
            Constants.PARTITION_KEY: self.__layer_reading_primary_key(layer_id),
            Constants.SORT_KEY: reading[Constants.READING_ID],
            Constants.READING_ID: reading[Constants.READING_ID],
            Constants.GEOHASH: reading[Constants.GEOHASH],
            Constants.READING_TYPE: reading[Constants.READING_TYPE],
        })

    def layer_ids_for_user(self, user_id: str):
        layer_ids = []

        group_ids = self.groups_for_user(user_id)
        for id in group_ids:
            layer_ids.extend(self.layer_ids_for_group(id))

        return layer_ids

    def layers_for_user(self, user_id:str) -> List[Dict[str, Any]]:
        layers = []

        group_ids = self.groups_for_user(user_id)
        for id in group_ids:
            layer_ids = self.layer_ids_for_group(id)

            for layer_id in layer_ids:
                layers.append(self.get_layer(layer_id))

        return layers

    def readings_for_layer_id(self, layer_id:str) -> List[Dict[str, Any]]:
        return self.layer_readings([layer_id])

    def layer_readings(self, layer_ids: str) -> List[Dict[str, Any]]:
        readings = []

        geohashes = set()
        reading_ids = set()

        for layer_id in layer_ids:
            identifiers = self.__layer_reading_identifiers(layer_id)

            for identifier in identifiers:
                geohashes.add(identifier[Constants.GEOHASH])
                reading_ids.add(identifier[Constants.SORT_KEY])

        for geohash in geohashes:
            geohash_readings = self.geohash_readings(geohash, Constants.PREDICTION)
            for geohash_reading in geohash_readings:
                if geohash_reading[Constants.READING_ID] in reading_ids:
                    readings.append(geohash_reading)

        return readings

    def identifiers_for_layers(self, layer_ids: List[str]):
        all_identifiers = []

        for layer_id in layer_ids:
            all_identifiers.extend(self.__layer_reading_identifiers(layer_id))

        return all_identifiers

    def __layer_reading_identifiers(self, layer_id: str) -> Dict[str, Any]:
        resp = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(self.__layer_reading_primary_key(layer_id))
        )

        return resp[self.ITEM_KEY]

    def get_layer(self, layer_id: str) -> Dict[str, Any]:
        response = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(Constants.LAYER_PK) &
                Key(Constants.SORT_KEY).eq(self.__layer_sort_key(layer_id))
        )

        if len(response[self.ITEM_KEY]) == 0:
            return None

        return response[self.ITEM_KEY][0]

    def __layer_id_from_layer_data(self, layer_data: List[Dict[str, Any]]) -> str:
        return layer_data[Constants.SORT_KEY].split('#')[1]

    def __layer_sort_key(self, layer_id: str) -> str:
        return f'Layer#{layer_id}'

    def __layer_reading_primary_key(self, layer_id: str):
        return f'LayerReading#{layer_id}'
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # ---------------------------- GROUPS -----------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------

    def put_group(self, group_id: str, name=None) -> None:
        item = {
            Constants.PARTITION_KEY: Constants.GROUP_PK,
            Constants.SORT_KEY: self.__group_key(group_id),
            Constants.GROUP_ID: group_id
        }

        if name is not None:
            item[Constants.GROUP_NAME] = name

        self.org_table.put_item(Item=item)

    def set_group_name(self, group_id:str, name:str) -> None:
        self.put_group(group_id, name)

    def get_group(self, group_id: str) -> Dict[str, Any]:
        resp = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(Constants.GROUP_PK) &
                Key(Constants.SORT_KEY).eq(self.__group_key(group_id))
        )

        return resp[self.ITEM_KEY][0]

    def user_add_group(self, user_id: str, group_id: str) -> None:
        self.org_table.put_item(Item={
            Constants.PARTITION_KEY: self.__user_group_pk(user_id),
            Constants.SORT_KEY: self.__group_key(group_id),
        })

    def user_remove_group(self, user_id: str, group_id: str) -> None:
        self.org_table.delete_item(Key={
            Constants.PARTITION_KEY: self.__user_group_pk(user_id),
            Constants.SORT_KEY: self.__group_key(group_id)
        })

    def group_add_layer(self, group_id: str, layer_id: str) -> None:
        self.org_table.put_item(Item={
            Constants.PARTITION_KEY: self.__group_key(group_id),
            Constants.SORT_KEY: self.__layer_group_pk(layer_id),
        })

    def group_remove_layer(self, group_id: str, layer_id: str) -> None:
        self.org_table.delete_item(Key={
            Constants.PARTITION_KEY: self.__group_key(group_id),
            Constants.SORT_KEY: self.__layer_group_pk(layer_id),
        })

    def groups_for_user(self, user_id: str) -> List[str]:
        response = self.org_table.query(KeyConditionExpression=
            Key(Constants.PARTITION_KEY).eq(self.__user_group_pk(user_id)))

        group_ids = []

        for item in response[self.ITEM_KEY]:
            group_ids.append(self.__id_from_key(item[Constants.SORT_KEY]))

        return group_ids

    def user_has_group(self, user_id: str, group_id: str) -> bool:
        response = self.org_table.query(KeyConditionExpression=
            Key(Constants.PARTITION_KEY).eq(self.__user_group_pk(user_id))
        )

        for item in response[self.ITEM_KEY]:
            if self.__id_from_key(item[Constants.SORT_KEY]) == group_id:
                return True

        return False

    def __user_group_pk(self, user_id: str) -> str:
        return f'UserGroup#{user_id}'

    def __id_from_key(self, agid: str) -> str:
        return agid.split("#")[-1]

    def layer_ids_for_group(self, group_id: str) -> str:
        response = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(self.__group_key(group_id))
        )

        layer_ids = []

        for item in response[self.ITEM_KEY]:
            layer_ids.append(self.__id_from_key(item[Constants.SORT_KEY]))

        return layer_ids

    def __layer_group_pk(self, layer_id: str) -> str:
        return f'LayerGroup#{layer_id}'

    def __group_key(self, acess_group_id: str) -> str:
        return f'Group#{acess_group_id}'

    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # --------------------------- ORG ---------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------

    def put_org(self, org_name:str, default_group_id: str = None) -> Dict[str, Any]:
        if default_group_id is None:
            default_group_id = str(uuid.uuid1())

        item = {
            Constants.PARTITION_KEY: Constants.ORG_PK,
            Constants.SORT_KEY: self.__org_sk(org_name),
            Constants.ORG_NAME: org_name,
            Constants.ORG_GROUP: default_group_id
        }

        self.org_table.put_item(Item=item)

        return item

    def get_orgs(self) -> List[Dict[str, Any]]:
        response = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(Constants.ORG_PK)
        )

        return response[self.ITEM_KEY]

    def get_org(self, org_name: str) -> Dict[str, Any]:
        response = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(Constants.ORG_PK) &
                Key(Constants.SORT_KEY).eq(self.__org_sk(org_name)),
        )

        if len(response[self.ITEM_KEY]) > 0:
            return response[self.ITEM_KEY][0]
        return None

    def __org_sk(self, org_id:str) -> str:
        return f'Org#{org_id}'

    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -------------------------- USER ---------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------
    # -----------------------------------------------------------------

    def user_data(self, org_name:str, user_id: str)-> Dict[str, Any]:
        response = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(self.__user_org_key(org_name)) &
                Key(Constants.SORT_KEY).eq(self.__user_sort_key(user_id))
        )

        return response[self.ITEM_KEY][0]

    def all_users(self, org_name:str) -> List[Dict[str, Any]]:
        resp = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(self.__user_org_key(org_name))
        )

        return resp[self.ITEM_KEY]

    def all_user_ids(self, org_name:str) -> List[str]:
        user_ids = []
        response = self.org_table.query(
            KeyConditionExpression=
                Key(Constants.PARTITION_KEY).eq(self.__user_org_key(org_name))
        )

        for item in response[self.ITEM_KEY]:
            user_ids.append(item[Constants.USER_ID])

        return user_ids

    def put_user(self, org_name, user_id: str) -> None:
        self.org_table.put_item(Item={
            Constants.PARTITION_KEY: self.__user_org_key(org_name),
            Constants.SORT_KEY: self.__user_sort_key(user_id),
            Constants.TIMESTAMP: timestamp(),
            Constants.USER_ID: user_id,
            Constants.ORG_NAME: org_name
        })

        org = self.get_org(org_name)

        self.user_add_group(user_id, org[Constants.ORG_GROUP])

    def __user_org_key(self, org_name: str) -> str:
        return f'Org#{org_name}'

    def __user_sort_key(self, user_id: str) -> str:
        return f'User#{user_id}'


"""
monobit.raw - raw binary font files

(c) 2019--2021 Rob Hagemans
licence: https://opensource.org/licenses/MIT
"""

import logging

from ..base.binary import ceildiv, bytes_to_bits
from ..formats import loaders, savers
from ..font import Font
from ..glyph import Glyph
from ..streams import FileFormatError
from ..base import boolean, pair

@loaders.register('dos', 'bin', 'rom', 'raw', name='raw binary')
def load(instream, where=None, cell:pair=(8, 8), n_chars:int=None, offset:int=0, strike:boolean=False):
    """Load font from raw binary."""
    # get through the offset
    # we don't assume instream is seekable - it may be sys.stdin
    instream.read(offset)
    if strike:
        cells = load_strike(instream, cell, n_chars)
    else:
        cells = load_aligned(instream, cell, n_chars)
    return Font(cells)

@savers.register(loader=load)
def save(fonts, outstream, where=None):
    """Save font to raw byte-aligned binary (DOS font)."""
    if len(fonts) > 1:
        raise FileFormatError('Can only save one font to BDF file.')
    save_aligned(outstream, fonts[0])

def save_aligned(outstream, font, encoding=None):
    """Save fixed-width font to byte-aligned bitmap."""
    # check if font is fixed-width and fixed-height
    if font.spacing != 'character-cell':
        raise FileFormatError(
            'This format only supports character-cell fonts.'
        )
    for glyph in font.glyphs:
        outstream.write(glyph.as_bytes())

def load_strike(instream, cell, n_chars):
    """Load fixed-width font from bitmap strike."""
    width, height = cell
    # n_chars must be given for strikes
    # assume byte-aligned at end of strike only
    strike_bytes = ceildiv(n_chars * width, 8)
    rombytes = instream.read(strike_bytes * height)
    # flatten strikes
    rows = [
        rombytes[_strike*strike_bytes : (_strike+1)*strike_bytes]
        for _strike in range(height)
    ]
    # convert to bits
    drawn = [bytes_to_bits(_row) for _row in rows]
    # clip out glyphs
    cells = [
        [_strike[_n*width:(_n+1)*width] for _strike in drawn]
        for _n in range(n_chars)
    ]
    return cells

def load_aligned(instream, cell, n_chars):
    """Load fixed-width font from byte-aligned bitmap."""
    width, height = cell
    width_bytes = ceildiv(width, 8)
    if n_chars is None:
        rombytes = instream.read()
        # get number of chars in extract
        n_chars = ceildiv(len(rombytes), width_bytes * height)
    else:
        rombytes = instream.read(n_chars * width_bytes * height)
    return parse_aligned(rombytes, width, height, n_chars)

def parse_aligned(rombytes, width, height, n_chars, offset=0):
    """Load fixed-width font from byte-aligned bitmap."""
    bytesize = ceildiv(width, 8) * height
    # get chunks
    glyphbytes = [
        rombytes[offset+_ord*bytesize : offset+(_ord+1)*bytesize]
        for _ord in range(n_chars)
    ]
    # concatenate rows
    cells = [
        Glyph.from_bytes(_bytes, width)
        for _bytes in glyphbytes
    ]
    return cells


import pandas as pd
from flask import Flask, request, jsonify
from flask_cors import CORS
import json

app = Flask(__name__)
CORS(app)

@app.route("/",methods=['GET'])

def default():
	return "Famous Quotes", 200

@app.route("/allquotes",methods=["GET"])

def allquotes():
	content = {"data": []}
	table = pd.read_csv("selectquotes.csv")
	for i in range(len(table)):
		d = {
		"Author Name": table.iloc[i]["author"],
		"Author Quote": table.iloc[i]["quote"],
		"Category": table.iloc[i]["category"]
		}
		content["data"].append(d)

	with open("allquotes.json","w") as file:
		json.dump(content, file, ensure_ascii=False)

	return jsonify(content), 200

if __name__ == "__main__":
	app.run(port=4500)

# READING THE DETAILS

# table = pd.read_csv("Student Details.csv")

# print(table.iloc[0]) # Returns first record data only with their column headers
# print(table.iloc[0]["Students"]) # Returns first's records data from the Students column
# print(table.iloc[0]["Ages"]) # Returns first's records data from the Ages column

# print("-------------------------------------------")

# mylist = []

# for i in range(len(table)):
# 	d = {"Serial No":table.iloc[i]["Serials"],
# 	"Name":table.iloc[i]["Students"],
# 	"Age": table.iloc[i]["Ages"]}
# 	mylist.append(d)
# print(mylist)


from pymavlink import mavutil
import time
# Create the connection
master = mavutil.mavlink_connection('udpin:127.0.0.1:14550')
# Wait a heartbeat before sending commands
master.wait_heartbeat()

boot_time = time.time()

def set_rc_channel_pwm(channel_id, pwm=1500):
    """ Set RC channel pwm value
    Args:
        channel_id (TYPE): Channel ID
        pwm (int, optional): Channel pwm value 1100-1900
    """
    if channel_id < 1 or channel_id > 18:
    	print("Channel does not exist.")
    	return

    # Mavlink 2 supports up to 18 channels:
    # https://mavlink.io/en/messages/common.html#RC_CHANNELS_OVERRIDE
    rc_channel_values = [65535 for _ in range(18)]
    rc_channel_values[channel_id - 1] = pwm
    master.mav.rc_channels_override_send(
        master.target_system,                # target_system
        master.target_component,             # target_component
        *rc_channel_values)                  # RC channel list, in microseconds.
	#master.mav.rc_channels_raw_send(int(1e3 * (time.time() - boot_time)),0,1500,1500,pwm,1500,1500,1500,1500,1500,254)
def select_mode(mode):
    if mode not in master.mode_mapping():
        print('Unknown mode : {}'.format(mode))
        print('Try:', list(master.mode_mapping().keys()))
        sys.exit(1)
    mode_id = master.mode_mapping()[mode]

    master.mav.set_mode_send(
    master.target_system,
    mavutil.mavlink.MAV_MODE_FLAG_CUSTOM_MODE_ENABLED,
    mode_id)

def mannual_control(pitch,roll,thrust,yaw):
    master.mav.manual_control_send(
        master.target_system,
        pitch,
        roll,
        thrust,
        yaw,
        0)

select_mode('ALT_HOLD')
master.arducopter_arm()
# Set some roll
#set_rc_channel_pwm(2, 1600)

# Set some yaw
#set_rc_channel_pwm(4, 1600)

#set mode to alt_hold

t = time.time()

alt = 0
while True:
    try:
        alt = master.recv_match(type='AHRS2',blocking=False).to_dict()['altitude']
        print(alt)
    except:
        pass

    #set_rc_channel_pwm(3, 1600)
    '''
    if(time.time()-t > 50000):
        set_rc_channel_pwm(3, 1500)
    else:
        set_rc_channel_pwm(3, 1600)
    '''
    if alt < 587:
        master.mav.manual_control_send(
        master.target_system,
        0,
        0,
        800,
        0,
        0)
    else:
        master.mav.manual_control_send(
        master.target_system,
        0,
        0,
        500,
        0,
        0)

    time.sleep(0.1)


# for testing
import time

# required
import numpy as np
from scipy import signal
import RPi.GPIO as GPIO
from smbus2 import SMBus, i2c_msg
from PCA_Constants import *
import gphoto2 as gp
import os

# consts
OE_PIN = 7

def capture_seq(camera,buffer,settings):
    t1 = time.time()
    camera.capture_sequence(buffer, format='yuv', use_video_port=settings.video_port, burst=settings.burst)
    t2 = time.time()
    print("Time for capturing image sequence: {}'s".format(t2-t1))
    return buffer

# normalize to 8-bit greyscale
def gs(im):
    im = im-im.min()
    im = 255*im/im.max()
    return np.uint8(im)

# function that does a major component of the image processing
def improcess(logY,f,fr):
    Edges = np.array([signal.convolve(im,f,mode='same') for im in logY])
    m = np.median(Edges,0)
    im = signal.convolve(m,fr,'same')
    return im

# computes g - the integration kernal
def calculate_g(h,w,p,dataType):
    x,y = h+2*p,w+2*p
    a = np.zeros((2*x , 2*y), dtype=dataType)
    a[x+1,y+1] = 4
    a[x+2,y+1] = -1
    a[x-0,y+1] = -1
    a[x+1,y+2] = -1
    a[x+1,y-0] = -1
    A = np.fft.fft2(a)
    index = A==0
    A[index]=1
    G = 1/A
    G[index]=0
    g = np.real(np.fft.ifft2(G))
    return g

# Enabling the PCA_Driver
def enable_PCA():
    GPIO.output(OE_PIN,0)

# Disabling the PCA_Driver
def disable_PCA():
    GPIO.output(OE_PIN,1)

def initPCADriver(address,bus):
    # Initializing GPIO Stuff
    try:
        writePCARegister(address,MODE1,MODE1_DATA,bus)
    except OSError:
        print("Test Array connections")
    try:
        writePCARegister(address,PWMALL,0x00,bus)
    except OSError:
        print("Test Array connections")

    try:
        writePCARegister(address,IREFALL,IREFALL_4MA,bus)

    except OSError:
        print("Test Array connections")

    else: return True

def init_LED_array(bus):
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(OE_PIN,GPIO.OUT)
    disable_PCA()
    for chip in [ADDR1,ADDR2,ADDR3,ADDR4]:
        if (not initPCADriver(chip,bus)):
            print("driver: {} init failed\n".format(chip))

    #if here then all settings wrote successfully
    # write them all to 0
    for chip in [ADDR1,ADDR2,ADDR3,ADDR4]:
        for i in range(24):
            writePCARegister(chip,0x0A+i,0,bus)

    enable_PCA()

def writePCARegister(chipAddr, regAddr, data,bus):
    cmd = [regAddr|CTRL_NO_AIF, data]
    msg = i2c_msg.write(chipAddr|ADDR_WRITE,cmd)
    bus.i2c_rdwr(msg)
    return True

# sweeps led array light to get light variation
def sweep_LED(intensity,bus):
    time.sleep(0.05)
    t1 = time.time()
    light_column(0,intensity,bus)
    for i in range(1,12):
        light_column(i,intensity,bus)
        time.sleep(0.01)
    light_column(0,0,bus)
    for i in range(1,12):
        light_column(i,0,bus)
        time.sleep(0.01)
    t2 = time.time()
    return True

def light_column(x1,intensity,bus):
        x2 = x1+12
        for chip in [ADDR1,ADDR2,ADDR3,ADDR4]:
            writePCARegister(chip,0x0A+x1,intensity,bus)
            writePCARegister(chip,0x0A+x2,intensity,bus)

# drives the shadow map to LED for shadow removal
def drive_LED(smap,bus):
    for i,row in enumerate(smap):
        if i < 2:
            chip = ADDR4
        elif i < 4:
            chip = ADDR3
        elif i < 6:
            chip = ADDR2
        elif i < 8:
            chip = ADDR1

        if i % 2:
            o = 12
        else:
            o = 0

        for j,val in enumerate(row):
            writePCARegister(chip,0x09+23-j-o,val,bus)

    return True

# waits for camera to be pressed, at the moment is input from keyboard
def wait():
#     while(1):
#         input("Hit enter to run...")
#         break

    # 5 second counter
    t1 = time.time()
    count = 0
    while time.time() < t1+5:
        count+=1
        time.sleep(1)
        print(count)

    return True

# captures the final image
def capture_image(Cannon):
    path = Cannon.capture(gp.GP_CAPTURE_IMAGE)
    return path


def printMatrix(m):
    for row in m:
        print(row)

def matrixMult(A,B):
    m = len(A) # Rows of A
    n = len(A[0]) # Columns of A
    p = len(B[0]) # Columns of B
    r = len(B) # Rows of B
    if n != r:
        print("""Unfortunately, the number of columns of matrix A and \n
              the number of rows of matrix B are not equal, so the matrix multiplication cannot be done.""")
        return None
    C = [[None for i in range(p)] for j in range(m)]
    for i in range(m):
        for j in range(p):
            C[i][j] = 0
            for k in range(r):
                x = A[i][k]
                y = B[k][j]
                C[i][j] += x*y
    return C

# Testing code
# Test1
A = [[ 2, -3, 3],
[-2, 6, 5],
[ 4, 7, 8]]
B = [[-1, 9, 1],
[ 0, 6, 5],
[ 3, 4, 7]]
C = matrixMult(A, B)
if not C == None:
    printMatrix(C)
# Test2
A = [[ 2, -3, 3, 0],
[-2, 6, 5, 1],
[ 4, 7, 8, 2]]
B = [[-1, 9, 1],
[ 0, 6, 5],
[ 3, 4, 7]]
C = matrixMult(A, B)
if not C == None:
    printMatrix(C)
# Test3
A = [[ 2, -3, 3, 5],
[-2, 6, 5, -2]]
B = [[-1, 9, 1],
[ 0, 6, 5],
[ 3, 4, 7],
[ 1, 2, 3]]
C = matrixMult(A, B)
if not C == None:
    printMatrix(C)


import json
from knlp.seq_labeling.NER.trie_seg.inference import TrieInference
from knlp.common.constant import KNLP_PATH

class clue_submit:
    """
    这个类主要用作将trie预测结果转换为clue的提交格式。
    """
    def __init__(self):
        self.in_json_file = KNLP_PATH + r'\knlp\data\cluener_public\test.json'
        self.out_json_file = KNLP_PATH + r'\knlp\data\cluener_public\predict.json'
        self.in_json = open(self.in_json_file, encoding='utf-8')

    def eval_ner(self):
        out = open(self.out_json_file, 'w')
        for line in self.in_json.readlines():
            line = eval(line)
            tmp_dict = {}
            id = line["id"]
            text = line["text"]
            tmp_dict["id"] = int(id)
            tmp_dict["label"] = {}
            trieTest = TrieInference()
            list_word, list_label = trieTest.knlp_seg(text)
            for idx, word in enumerate(list_word):
                if list_label[idx] != 'O':
                    if word in tmp_dict["label"].keys():
                        tmp_dict["label"][list_label[idx]][word] = [[text.find(word), text.find(word) + len(word) - 1]]
                    else:
                        tmp_dict["label"][list_label[idx]] = {}
                        tmp_dict["label"][list_label[idx]][word] = [[text.find(word), text.find(word) + len(word) - 1]]
            j = json.dumps(tmp_dict, ensure_ascii=False)
            out.write(j)
            out.write('\n')
            # print(tmp_dict)
        out.close()

if __name__ == '__main__':
    # trieTest = TrieInference()
    # print(trieTest.get_DAG("你会和星级厨师一道先从巴塞罗那市中心兰布拉大道的laboqueria市场的开始挑选食材，", trieTest._trie))
    # print(trieTest.knlp_seg("销售冠军：辐射3-Bethesda"))
    evalner = clue_submit()

    # test_file = open(evalner.in_json_file, encoding='utf-8')
    # for line in test_file.readlines():
    #     words = line["text"]
    #     word_out, label_out = trieTest.knlp_seg(words)
    #     evalner.eval_ner()
    evalner.eval_ner()


#!/usr/bin/env python
"""
Update fits headers in the command line.

Depends on the pyfits package.

Author: Nicolas Barbey
"""
import pyfits
import sys, getopt

LINE_WIDTH = 75
KEY_WIDTH = 8

def update_header(filename, ext, rec, val):
    """
    Update value in an fits header
    """
    fits = pyfits.fitsopen(filename, "update")
    for e in ext:
        for r, v in zip(rec, val):
            fits[e].header.update(r, v)
    fits.flush()

def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hv:r:e:", ["help", "val=", "rec=", "ext="])
    except(getopt.GetoptError, err):
        print(err)
        usage()
        sys.exit(2)
    filenames = args
    ext = list()
    rec = list()
    val = list()
    if len(args) == 0:
        usage()
        return
    for o, a in opts:
        if o in ("-h", "--help"):
            usage()
            sys.exit()
        elif o in ("-e", "--ext"):
            ext.append(int(a))
        elif o in ("-r", "--rec"):
            rec.append(a)
        elif o in ("-v", "--val"):
            val.append(a)
        else:
            assert False, "unhandled option"
    # update file
    for filename in filenames:
        update_header(filename, ext, rec, val)

def usage():
    print(__usage__)

__usage__ = """Usage: fitsupdate [options] [filename(s)]

Update a fits file header keyword.

Options:
  -h, --help        Show this help message and exit
  -e, --ext         Extension number to update.
  -r, --rec         Record string to update.
  -v, --val         Update record value with val option.
"""

# to call from command line
if __name__ == "__main__":
    main()


# Студент-физик получил неожиданный результат при использовании кода force = G • mass1 • mass2 / radius * radius
# для вычисления значения по формуле F = Gm1m2/r2• Объясните проблему и исправьте код.

G = 10
mass1 = mass2 = 5
radius = 10
forceFstCase = G * mass1 * mass2 / radius**2 #1 вариант решения
forceSndCase = (G * mass1 * mass2) / (radius * radius) #2 вариант решения

print(forceFstCase)
print(forceSndCase)


import argparse
import pickle
from sklearn.utils import class_weight
from nltk.corpus import stopwords
import nltk
from nltk.wsd import lesk
from nltk.corpus import wordnet as wn
from utils import clean_str, clean_str_simple_version, show_statisctic, clean_document
import sys
from nltk import tokenize
import collections
from collections import Counter
import random
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

def display_topics(model, feature_names, no_top_words):
	keywords_dic = {}
	for topic_idx, topic in enumerate(model.components_):
		print("Topic %d:" % (topic_idx))
		klist = [feature_names[i] for i in topic.argsort()[:-no_top_words - 1:-1]]
		print(" ".join(klist))
		for k in klist:
			if not k in keywords_dic:
				keywords_dic[k] = []
			keywords_dic[k].append(topic_idx)
	return keywords_dic

def Generate_LDA(dataset):
	doc_content_list = []
	doc_sentence_list = []
	f = open('data/' + dataset + '_corpus.txt', 'rb')
	for line in f.readlines():
	    doc_content_list.append(line.strip().decode('latin1'))
	    doc_sentence_list.append(tokenize.sent_tokenize(clean_str_simple_version(doc_content_list[-1], dataset)))
	f.close()

	# Remove the rare words
	doc_content_list = clean_document(doc_sentence_list, dataset)

	doc_list = []
	for doc in doc_content_list:
		temp = ''
		for sen in doc:
			temp += (' '.join(sen) + ' ')
		doc_list.append(temp)

	vectorizer = CountVectorizer(stop_words='english',max_df=0.98)
	vector = vectorizer.fit_transform(doc_list)
	feature_names = vectorizer.get_feature_names()
	lda = LatentDirichletAllocation(n_components=args.topics,learning_method='online',learning_offset=50.,random_state=0).fit(vector)

	keywords_dic = display_topics(lda, feature_names, args.topn)
	print(len(keywords_dic))
	print(keywords_dic)

	pickle.dump(keywords_dic, open('data/' + dataset + '_LDA.p', "wb" ) )

if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('--dataset', default='20ng', help='dataset name: 20ng/R8/R52/ohsumed/mr')
	parser.add_argument('--topn', type=int, default=10, help='top n keywords')
	parser.add_argument('--topics', type=int, help='number of topics')
	args = parser.parse_args()
	print(args)

	Generate_LDA(args.dataset)


import numpy as np
from numba import jit
import os
import sys
import sklearn # for shuffle

def split(X, y, train_ratio):
    X = np.array(X)
    y = np.array(y)
    train_x, train_y = X[:int(X.shape[0]*train_ratio)], y[:int(y.shape[0]*train_ratio)]
    valid_x, valid_y = X[int(X.shape[0]*train_ratio):], y[int(y.shape[0]*train_ratio):]
    return train_x, train_y, valid_x, valid_y

@jit(nogil=True, parallel=True)
def normalized(a, axis=-1, order=2):
    l2 = np.atleast_1d(np.linalg.norm(a, order, axis))
    l2[l2==0] = 1
    return a / np.expand_dims(l2, axis)

def eprint(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

class Activation:
    def __init__(self):
        pass

    @staticmethod
    def linear(x):
        return x

    @staticmethod
    def linear_dx(x):
        return np.vectorize(lambda _: 1.)(x)

    @staticmethod
    def relu(x):
        return np.vectorize(lambda xi: max(xi, 0.))(x)

    @staticmethod
    def relu_dx(x):
        return np.vectorize(lambda xi: 1. if xi > 0. else 0.)(x)

    @staticmethod
    def selu(x):
        l = 1.0507009873554804934193349852946
        a = 1.6732632423543772848170429916717
        return np.vectorize(lambda xi: l*xi if xi > 0. else l*a*(np.exp(xi)-1.))(x)

    @staticmethod
    def selu_dx(x):
        l = 1.0507009873554804934193349852946
        a = 1.6732632423543772848170429916717
        return np.vectorize(lambda xi: l if xi > 0. else l*a*np.exp(xi))(x)

    @staticmethod
    def leaky_relu(x):
        return np.vectorize(lambda xi: xi if xi > 0. else 0.01*xi)(x)

    @staticmethod
    def leaky_relu_dx(x):
        return np.vectorize(lambda xi: 1. if xi > 0. else 0.01)(x)

    @staticmethod
    def softmax(x, epslion=1e-10):
        """Compute softmax values for each sets of scores in x."""
        x = np.array(x)
        e_x = np.exp(x)
        return e_x / (e_x.sum() + epslion)

    @staticmethod
    def softmax_dx(x, epslion=1e-10):
        ''' e^xi * (e^x0 + e^x1 + ... ) / (e^x0 + e^x1 + ...)^2 '''
        x = np.array(x)
        sum_e_x = np.exp(x).sum()
        return np.vectorize(lambda xi: np.exp(xi)*(sum_e_x - np.exp(xi)) / (sum_e_x*sum_e_x + epslion) )(x)

class Initializer:
    def __init__(self):
        pass

    @staticmethod
    def orthogonal(shape, gain=1.):
        num_rows = 1
        for dim in shape[:-1]:
            num_rows *= dim
        num_cols = shape[-1]
        flat_shape = (num_rows, num_cols)

        a = np.random.normal(0.0, 1.0, flat_shape)
        u, _, v = np.linalg.svd(a, full_matrices=False)
        # Pick the one with the correct shape.
        q = u if u.shape == flat_shape else v
        q = q.reshape(shape)
        return gain * q[:shape[0], :shape[1]]

    @staticmethod
    def zeros(shape):
        return np.zeros(shape)

    @staticmethod
    def constant(shape, value=0.):
        return np.full(shape, value)

class Loss:
    def __init__(self):
        pass

    @staticmethod
    def cross_entropy(x, t, epslion=1e-10):
        ''' return cross entropy loss of input-x and target-t'''
        x = np.array(x)
        t = np.array(t)
        return np.vectorize(lambda xi, ti: -( ti * np.log(xi+epslion) + (1.-ti) * np.log(1.-xi-epslion) ))(x, t).sum()

    # cross_entropy_dx simply doesn't work alone

    @staticmethod
    def softmax_cross_entropy(x, t):
        return Loss.cross_entropy(Activation.softmax(x), t)

    @staticmethod
    def softmax_cross_entropy_dx(x, t):
        return Activation.softmax(x) - t

    @staticmethod
    def mse(x, t):
        x = np.array(x)
        t = np.array(t)
        return 0.5 * np.sqrt(np.square(x-t).sum())

    @staticmethod
    def mse_dx(x, t):
        x = np.array(x)
        t = np.array(t)
        return x-t

class Optimizer:
    def __init__(self): pass

    class sgd:
        def __init__(self, momentum=0., nesterov=False):
            self.last_updates_w = 0
            self.last_updates_b = 0
            self.last_grad_w = 0
            self.last_grad_b = 0
            self.momentum = momentum
            self.nesterov = nesterov

        @jit(nogil=True, parallel=True)
        def cal_updates(self, grad_w, grad_b, lr):
            if self.nesterov:
                updates_w = ( grad_w + self.momentum*(self.last_updates_w + grad_w - self.last_grad_w) ) * lr
                updates_b = ( grad_b + self.momentum*(self.last_updates_b + grad_b - self.last_grad_b) ) * lr
            else:
                updates_w = (grad_w + self.momentum * self.last_updates_w) * lr
                updates_b = (grad_b + self.momentum * self.last_updates_b) * lr

            # update last weight and bias
            self.last_grad_w = grad_w
            self.last_grad_b = grad_b
            self.last_updates_w = updates_w
            self.last_updates_b = updates_b

            return updates_w, updates_b

class LRScheduler:
    def __init__(self): pass

    class reduce_on_loss_plateau:
        def __init__(self, lr_decay=0.5, patience=20):
            self.best_loss = np.inf
            self.wait = 0
            self.patience = patience
            self.lr_decay = lr_decay

        def new_lr(self, loss, old_lr, verbose=True):
            self.wait += 1
            if self.wait < self.patience:
                if self.best_loss > loss:
                    if verbose: eprint('best_loss: %.4f -> %.4f' % (self.best_loss, loss))
                    self.wait = 0 # reset
                    self.best_loss = loss
                return old_lr
            else:
                self.wait = 0 # reset
                if verbose: eprint('reduce_lr: %.6f -> %.6f' % (old_lr, old_lr*self.lr_decay))
                return old_lr * self.lr_decay

class EarlyStop:
    def __init__(self, patience=40):
        self.best_loss = np.inf
        self.wait = 0
        self.patience = patience

    def need_stop(self, loss, verbose=True):
        self.wait += 1
        if self.wait < self.patience:
            if self.best_loss > loss:
                self.wait = 0 # reset
                self.best_loss = loss
            return False
        if verbose: eprint('early stop!')
        return True

class MLPClassifier:
    class __fc_layer:
        def __init__(
                self,
                input_layer,
                n_unit=1024,
                activation='relu',
                weight_initializer='constant',
                bias_initializer='zeros'
            ):
            self.input_dim = input_layer if type(input_layer) is int else input_layer.n_unit
            self.n_unit = n_unit
            self.activation = getattr(Activation, activation.lower())
            self.activation_dx = getattr(Activation, activation.lower()+'_dx')
            self.weight_initializer = getattr(Initializer, weight_initializer.lower())
            self.bias_initializer = getattr(Initializer, bias_initializer.lower())
            self.w = self.weight_initializer(shape=(n_unit, self.input_dim))
            self.b = self.bias_initializer(shape=(n_unit))

            self.__cache = dict()

            self.next_layer = None
            if type(input_layer) is type(self):
                self.parent_layer = input_layer
                input_layer.next_layer = self
            else:
                self.parent_layer = None

        @jit(nogil=True, parallel=True)
        def __call__(self, x):
            self.__cache['x'] = x
            self.__cache['wx'] = np.matmul(self.w, x) + self.b
            return self.activation(self.__cache['wx'])

        @jit(nogil=True, parallel=True)
        def __grad(self):
            delta = self.__cache['delta'].reshape(self.__cache['delta'].shape[0], 1)
            x = self.__cache['x'].reshape(1, self.__cache['x'].shape[0])
            grad_w = np.matmul(delta, x)
            grad_b = self.__cache['delta']
            return grad_w, grad_b

        @jit(nogil=True, parallel=True)
        def backpropagation(self, loss_dx_delta):
            if self.next_layer is None: # output_layer
                self.__cache['delta'] = loss_dx_delta * self.activation_dx( self.__cache['wx'] )
            else:
                self.__cache['delta'] = np.matmul(np.transpose(self.next_layer.w), loss_dx_delta) * self.activation_dx( self.__cache['wx'] )

            grad_w, grad_b = self.__grad()

            if self.parent_layer is not None: # not the first layer
                rtn_grad_w, rtn_grad_b = self.parent_layer.backpropagation(self.__cache['delta'])
                rtn_grad_w.append(grad_w)
                rtn_grad_b.append(grad_b)
            else:
                rtn_grad_w, rtn_grad_b = [grad_w], [grad_b]

            if self.next_layer is None: # output_layer
                rtn_grad_w = np.array(rtn_grad_w)
                rtn_grad_b = np.array(rtn_grad_b)
            return rtn_grad_w, rtn_grad_b

        @jit(nogil=True, parallel=True)
        def update(self, updates_w, updates_b):
            self.w = self.w - updates_w[-1]
            self.b = self.b - updates_b[-1]

            if not(self.parent_layer is None): self.parent_layer.update(updates_w[:-1], updates_b[:-1])

    def __init__(
            self,
            hidden_layer_sizes=128,
            n_hidden_layers=10,
            activation='selu',
            solver='sgd',
            momentum=0.,
            nesterov=False,
            loss='cross_entropy',
            weight_initializer='orthogonal',
            bias_initializer='zeros',
            batch_size=32,
            learning_rate=0.01,
            lr_decay_on_plateau=0.5,
            lr_decay_patience=15,
            early_stop_patience=40
        ):
        loss_need_convert = ['cross_entropy']

        self.hidden_layer_sizes = hidden_layer_sizes
        self.batch_size = batch_size
        self.hidden_layers = None
        self.output_layer = None
        self.activation = activation
        self.weight_initializer = weight_initializer
        self.bias_initializer = bias_initializer
        self.learning_rate = learning_rate
        self.n_hidden_layers = n_hidden_layers
        self.lr_scheduler = LRScheduler.reduce_on_loss_plateau(lr_decay=lr_decay_on_plateau, patience=lr_decay_patience)
        self.early_stop = EarlyStop(patience=early_stop_patience)
        self.optimizer = getattr(Optimizer, solver.lower())(momentum=momentum, nesterov=nesterov)

        # softmax + cross_entropy -> softmax_cross_entropy
        self.output_layer_activation = 'linear' if loss.lower() in loss_need_convert else 'softmax'
        loss = 'softmax_' + loss.lower() if loss.lower() in loss_need_convert else loss
        self.loss = getattr(Loss, loss.lower())
        self.loss_dx = getattr(Loss, loss.lower()+'_dx')

        self.input_dim = -1
        self.output_dim = -1

    def __forward_pass(self, x):
        rtn = x
        for h in self.hidden_layers:
            rtn = h(rtn)
        return self.output_layer(rtn)

    def __backpropagation(self, predict_y, true_y):
        loss_dx = self.loss_dx(predict_y, true_y)
        return self.output_layer.backpropagation(loss_dx) # all gradient

    def __update(self, updates_w, updates_b):
        self.output_layer.update(updates_w, updates_b)

    def __batchlize(self, X, y):
        for index_batch in range(y.shape[0] // self.batch_size):
            index_start = self.batch_size * index_batch
            index_end = index_start + self.batch_size
            xi = X[index_start: index_end]
            yi = y[index_start: index_end]
            yield xi, yi

    def __add_hidden_layer(self, input_layer):
        self.hidden_layers.append(self.__fc_layer(
                                    input_layer=input_layer,
                                    n_unit=self.hidden_layer_sizes,
                                    activation=self.activation,
                                    weight_initializer=self.weight_initializer,
                                    bias_initializer=self.bias_initializer
                                ))
        return self.hidden_layers[-1]

    def __accuracy_loss(self, X, y):
        ''' return total accuracy and average loss '''
        loss = 0.
        n_currect = 0
        X = np.array(X)
        predict_y = self.predict(X)
        for py, yi in zip(predict_y, y):
            if np.argmax(py) == np.argmax(yi): n_currect += 1
            loss += self.loss(py, yi)
        return n_currect / X.shape[0], loss / X.shape[0]

    def __init_network(self):
        self.hidden_layers = list()

        last_layer = self.input_dim
        for i in range(self.n_hidden_layers):
            last_layer = self.__add_hidden_layer(last_layer)

        self.output_layer = self.__fc_layer(
                                    input_layer=last_layer,
                                    n_unit=self.output_dim,
                                    activation=self.output_layer_activation,
                                    weight_initializer=self.weight_initializer,
                                    bias_initializer=self.bias_initializer
                                )
        return None

    def fit(self, X, y, valid_set=None, epochs=100, shuffle=True, verbose=True):
        X = np.array(X) # (?, input_dim)
        y = np.array(y) # (?, output_dim)

        history = {
            'loss': list(),
            'acc': list(),
            'valid_loss': list(),
            'valid_acc': list()
        }

        self.input_dim = X.shape[1]
        self.output_dim = y.shape[1]

        self.__init_network()

        for index_epoch in range(epochs):
            if shuffle: X, y = sklearn.utils.shuffle(X, y)
            for batch_x, batch_y in self.__batchlize(X, y):
                # calculate average gradient
                grad_w_sum = None
                grad_b_sum = None
                for xi, yi in zip(batch_x, batch_y):
                    predict_y = self.__forward_pass(xi)
                    grads_w, grads_b = self.__backpropagation(predict_y, yi)
                    grad_w_sum = grads_w if grad_w_sum is None else grad_w_sum + grads_w
                    grad_b_sum = grads_b if grad_b_sum is None else grad_b_sum + grads_b
                # calculate updates
                grad_w_avg = grad_w_sum / self.batch_size
                grad_b_avg = grad_b_sum / self.batch_size

                updates_w, updates_b = self.optimizer.cal_updates(grad_w_avg, grad_b_avg, self.learning_rate)
                self.__update(updates_w, updates_b)

            # result
            train_acc, train_loss = self.__accuracy_loss(X[:self.batch_size], y[:self.batch_size])
            if valid_set is not None:
                valid_acc, valid_loss = self.__accuracy_loss(valid_set[0], valid_set[1])

            history['loss'].append(train_loss)
            history['acc'].append(train_acc)
            history['valid_loss'].append(valid_loss)
            history['valid_acc'].append(valid_acc)

            # update learning rate
            self.learning_rate = self.lr_scheduler.new_lr(valid_loss if valid_set is not None else train_loss, self.learning_rate, verbose=verbose)

            if verbose:
                eprint('epochs: %d/%d lr: %.6f' % (index_epoch, epochs, self.learning_rate), end=' ')
                eprint('loss: %.4f acc: %.4f ' % (train_loss, train_acc), end='\n' if valid_set is None else '')
                if valid_set is not None:
                    eprint('valid_loss: %.4f valid_acc: %.4f' % (valid_loss, valid_acc))

            if self.early_stop.need_stop(valid_loss if valid_set is not None else train_loss, verbose=verbose):
                return history

        return history

    def predict(self, X):
        rtn = list()
        for x in X:
            if self.output_layer_activation == 'linear':
                rtn.append(Activation.softmax(self.__forward_pass(x)))
        return np.array(rtn)

if __name__ == '__main__':
    from sklearn import datasets
    from sklearn.preprocessing import OneHotEncoder

    iris = datasets.load_iris()
    X = normalized(iris.data)
    y = OneHotEncoder().fit_transform(iris.target.reshape(iris.target.shape[0], 1)).toarray()

    X, y = sklearn.utils.shuffle(X, y)

    # split to training and validation set
    train_x, train_y, valid_x, valid_y = split(X, y, 0.7)

    mlp_clf = MLPClassifier(
                    hidden_layer_sizes=128,
                    n_hidden_layers=9,
                    activation='selu',
                    solver='sgd',
                    momentum=0.4,
                    nesterov=False,
                    loss='cross_entropy',
                    weight_initializer='orthogonal',
                    bias_initializer='zeros',
                    batch_size=32,
                    learning_rate=0.01,
                    lr_decay_on_plateau=0.5,
                    lr_decay_patience=15,
                    early_stop_patience=40)
    history = mlp_clf.fit(train_x, train_y, valid_set=(valid_x, valid_y), epochs=2000, verbose=True)


import unittest
from problem5 import findSmallestFactor

class MyTestCase(unittest.TestCase):
    def test_findSmallestFactor(self):
        testdata = {5:60, 7:420, 10:2520, 13:360360, 20:232792560}
        for key in testdata:
            self.assertEqual(findSmallestFactor(key),testdata[key])

if __name__ == '__main__':
    unittest.main()


"""
* By M.K
* Licensed under the MIT License.
* https://github.com/MI-K253
* https://github.com/MI-K253/Python-Plays-Game
"""

import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.layers import Dense, Conv2D, Dropout, MaxPooling2D, Input, Flatten
from tensorflow.keras.callbacks import TensorBoard
from datetime import datetime
import os

Image_Height, Image_Width = 40, 45
Num_Classes = 10
Epochs, Batch_Size = 15, 32
# TB_LogDir = f"logs/Speed-Controller-{Epochs}epochs-{datetime.now()}"
# TB_CallBack = TensorBoard(log_dir=TB_LogDir, histogram_freq=1)

X = np.array(np.load(r"Data/X.npy", allow_pickle=True))
Y = np.array(np.load(r"Data/Y.npy", allow_pickle=True))
X = np.reshape(X, (X.shape[0], Image_Height, Image_Width, 1))             # Reshaping data  --> (nmber, height, width, channels)
print(f"{X}\n\n{Y}\n\nLengths:\tX: {len(X)}  Y:{len(Y)}\nShapes:\tX: {np.shape(X)}  Y: {np.shape(Y)}\n")

def main():
	model = keras.Sequential()
	model.add(Input(shape=(Image_Height, Image_Width, 1)))

	model.add(Conv2D(32, (3, 3), strides=(1, 1), padding="same", activation="relu"))
	model.add(MaxPooling2D())

	model.add(Flatten())

	model.add(Dense(1024, activation="relu"))
	model.add(Dense(512, activation="relu"))
	model.add(Dropout(0.3))
	model.add(Dense(128, activation="relu"))
	model.add(Dense(Num_Classes, activation="softmax"))

	model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
	model.fit(X, Y, epochs=Epochs, batch_size=Batch_Size, validation_split=0.1, verbose=1)
	model.save(r"Speed_Controller.h5")

	return

if __name__ == "__main__":
	main()

# END


import scipy.interpolate as interpolate
import xarray as xr

def get_interpfunc(da=None, dim='time',
                   bounds_error=False, fill_value='extrapolate'):
    '''
    Linearly interpolates `da` along dimension `dim`.  This function
    returns a function that evaluates `da` at a set of values along
    dimension `dim`.

    Parameters
    ----------
    da : xarray.DataArray
    dim : string
          name of dimension to interpolate along
    '''

    x = da.coords[dim]
    y = da.values
    axis = da.dims.index(dim)
    func = interpolate.interp1d(x=x, y=y, axis=axis,
                                bounds_error=bounds_error,
                                fill_value=fill_value)

    def callf(coords=None, name_dim=None):
        data = func(coords)

        dims_interp = list(da.dims)
        coords_interp = [coords if d == dim else da.coords[d]
                         for d in dims_interp]

        if name_dim:
            dims_interp[dims_interp.index(dim)] = name_dim

        da_interp = xr.DataArray(data,
                                 dims=dims_interp, coords=coords_interp,
                                 attrs=da.attrs)
        return da_interp

    return callf

def interp_layers2levels(ds, vars=None):
    '''
    Interpolate layer values to levels.

    Args:
        ds: xarray.Dataset
        vars: list of variables in `ds` for which layers values are to be
              interpolated onto levels. Defaults to an empty list.
    Returns:
        ds: xarray.Dataset, with new variable for the level values added
            for each variable in `vars`
    '''
    if not vars:
        vars = []

    for var in vars:
        try:
            da = ds[var]
        except KeyError:
            continue
        else:
            interpfunc = get_interpfunc(da, dim='lev')
            da_interp = interpfunc(coords=ds.coords['ilev'],
                                   name_dim='ilev')
            ds['i' + var] = (da_interp.dims, da_interp, da_interp.attrs)
    return ds


from datetime import datetime, timedelta
from unittest import TestCase

from algotrader.entities.timespan import TimeSpan
from algotrader.pipeline.sources.ib_history import IBHistorySource
from algotrader.providers.ib.interactive_brokers_connector import InteractiveBrokersConnector

class TestIBSource(TestCase):
    def setUp(self) -> None:
        super().setUp()
        self.ib_connector = InteractiveBrokersConnector()

    def tearDown(self) -> None:
        super().tearDown()
        self.ib_connector.kill()

    def test(self):
        symbol = "AAPL"
        from_time = datetime.now() - timedelta(days=30)
        source = IBHistorySource(self.ib_connector, [symbol], TimeSpan.Day, from_time)

        candles = list(source.read())
        self.assertTrue(len(candles) > 10)

        for candle in candles:
            self.assertEqual(symbol, candle.symbol)
            self.assertEqual(TimeSpan.Day, candle.time_span)
            self.assertTrue(candle.timestamp < datetime.now())


import os
import cv2
import numpy as np

FolderPathString = "./MAG COVID EVAL/mag-05312021/demo-images/A-3-G/"
filename = "37"
img_file_path_green = FolderPathString + filename + "-demo-only green.jpg"
img_file_path_red = FolderPathString + filename + "-demo-only red.jpg"
img_file_path_mag = FolderPathString + filename + "-demo-only mag.jpg"
img_green = cv2.imread(img_file_path_green)
img_red = cv2.imread(img_file_path_red)
img_mag = cv2.imread(img_file_path_mag)

img_gray_green = cv2.cvtColor(img_green, cv2.COLOR_BGR2GRAY)
num_green_beads = np.count_nonzero(img_gray_green)
sum_green_beads = np.sum(img_gray_green)
img_gray_red = cv2.cvtColor(img_red, cv2.COLOR_BGR2GRAY)
num_red_beads = np.count_nonzero(img_gray_red)
sum_red_beads = np.sum(img_gray_red)
img_gray_mag = cv2.cvtColor(img_mag, cv2.COLOR_BGR2GRAY)
num_mag_beads = np.count_nonzero(img_gray_mag)

print("green intensity ratio", sum_green_beads/num_mag_beads/2 )
print("green pixel ratio", num_green_beads/num_mag_beads/2 )
print("red intensity ratio", sum_red_beads/num_mag_beads/2 )
print("red pixel ratio", num_red_beads/num_mag_beads/2 )


from typing import List, Callable

import json
from time import sleep
from datetime import date
from os import path

from api import BilibiliApi
from writer import write_raw_data

BASE_PATH = './archive'
NAP_TIME = .5

def save_tags(api: BilibiliApi, loc: str, name: str, aids: List[str]) -> BilibiliApi.RAW_DATA_T:
    all_tags = {}
    for aid in aids:
        sleep(NAP_TIME)
        tag_list = api.get_tag(aid)
        for tag in tag_list:
            if tag['tag_id'] in all_tags:
                all_tags[tag['tag_id']]['day_count'] += 1
            else:
                all_tags[tag['tag_id']] = {'data': tag, 'day_count': 1}
    write_raw_data(all_tags, path.join(loc, 'Tags', name))

def save_raw(api: BilibiliApi, func: Callable[[], BilibiliApi.RAW_DATA_T], loc: str, name: str):
    raw_data = func()
    write_raw_data(raw_data, path.join(loc, 'Raw', name))

    aids = (video['aid'] for video in raw_data)
    save_tags(api, loc, name, aids)

def save_raw_today():
    date_str = date.today().isoformat()
    loc = path.join(BASE_PATH, 'Bilibili', date_str)

    api = BilibiliApi()
    save_raw(api, BilibiliApi.get_most_popular, loc, 'most_popular.json')
    save_raw(api, BilibiliApi.get_highest_ranked, loc, 'highest_ranked.json')

if __name__ == '__main__':
    save_raw_today()


import torch
import torch.nn as nn
import torch.nn.functional as F
import sys
#sys.path.append('../../')
import argparse
import os
from functools import partial
import time
import statistics

from robustbench.utils import clean_accuracy, load_model
import autoattack
try:
    import other_utils
except ImportError:
    from autoattack import other_utils
from utils_tta import load_dataset, eval_fast, get_wc_acc, parse_args
from apgd_tta import apgd_restarts

#

def load_models(modelname, device, modeldir, modeldir_rb):
    if modelname == 'rebuffi_orig':
        model = load_model('Rebuffi2021Fixing_70_16_cutmix_extra',
            model_dir=modeldir_rb)
        model.to(device)
        model.eval()
        return model

    elif modelname == 'rebuffi_sodef':
        from models import create_model
        new_model = create_model(device, modeldir_rb)
        ckpt = torch.load(f'{modeldir}/Rebuffi2021Fixing_70_16_cutmix_extra/full.pth')
        new_model.load_state_dict(ckpt['state_dict'])
        new_model.eval()
        return new_model

    elif modelname == 'trades':
        from wideresnet import WideResNet
        model = WideResNet()
        ckpt = torch.load(f'{modeldir}/EXP/pre_train_model/wide10_trades_eps8_tricks.pt')
        model.load_state_dict(ckpt)
        model.to(device)
        model.eval()
        return model

    elif modelname == 'trades_sodef':
        from models_small import create_model
        model = create_model(device)
        ckpt = torch.load(f'{modeldir}/EXP/nips_model/full.pth')
        model.load_state_dict(ckpt['state_dict'])
        model.eval()
        return model

class ModelWithFeatures(nn.Module):
    """ to use both logits and intermediate features
    """
    def __init__(self, composite_model):
        super().__init__()
        self.composite_model = composite_model

    def forward(self, x, return_fts=False):
        if not return_fts:
            return self.composite_model(x)

        else:
            fts = self.composite_model[0](x)
            out = self.composite_model[1](fts)
            out = self.composite_model[2](out)
            return out, fts

if __name__ == '__main__':
    args = parse_args()
    device = 'cuda'

    if args.eps > 1. and args.norm == 'Linf':
        args.eps /= 255.
    if not args.data_path is None:
        assert os.path.exists(args.data_path)

    x_test, y_test = load_dataset(args.dataset, args.n_ex, device=device,
        data_dir=args.data_dir)
    print('data loaded')

    model = load_models(args.model, device, args.model_dir, args.model_dir_rb)
    print('model loaded')

    logger = other_utils.Logger(None)
    savedir = f'./results/{args.model}'
    other_utils.makedir(savedir)
    logsdir = f'./logs/{args.model}'
    other_utils.makedir(logsdir)
    logger.log_path = f'{logsdir}/stats.txt'

    if args.loss in ['l2', 'l1', 'linf']:
        assert 'sodef' in args.model
        test_model = ModelWithFeatures(model)
        test_model.eval()
    else:
        test_model = model

    runname = f'{args.attack}_1_{args.n_ex}_{args.norm}_eps_{args.eps:.5f}'

    if not args.indices is None:
        notes_run = f'_{args.indices}'
        ind = args.indices.split('-')
        ind_restr = list(range(int(ind[0]), int(ind[1])))
        print(ind_restr)
        sys.stdout.flush()
        x_test, y_test = x_test[ind_restr], y_test[ind_restr]
        runname += notes_run

    if args.attack == 'eval_fast':
        short_version = False #True
        #runname = f'{args.attack}_1_{args.n_ex}'
        runname += f'{"_short" if short_version else ""}'
        #runname += f'_{args.norm}_eps_{args.eps:.5f}'
        x_adv = eval_fast(test_model, x_test, y_test, eps=args.eps,
            savedir=savedir, bs=args.batch_size, short_version=short_version,
            eot_iter=args.eot_test, norm=args.norm,
            log_path=f'{savedir}/{runname}.txt')
        torch.save(x_adv, f'{savedir}/{runname}.pth')

    elif args.attack == 'maxloss':
        runname += f'_niter_{args.n_iter}_loss_{args.loss}'
        x_adv = eval_maxloss(test_model, x_test, y_test, eps=args.eps,
            savedir=savedir, #eot_iter=args.eot_test,
            verbose=True, seed=args.seed, norm=args.norm,
            bs=args.batch_size, loss=args.loss, n_iter=args.n_iter,
            log_path=f'{savedir}/{runname}.txt')
        torch.save(x_adv, f'{savedir}/{runname}.pth')

    elif args.attack == 'eval_with_square':
        runname += f'_niter_{args.n_iter}_loss_{args.loss}'
        runname += f'_nrestarts_{args.n_restarts}'
        x_adv = eval_with_square(test_model, x_test, y_test, eps=args.eps,
            savedir=savedir, #eot_iter=args.eot_test, #verbose=Truer
            seed=args.seed, opt_loss=True, n_restarts=args.n_restarts,
            bs=args.batch_size, loss=args.loss, n_iter=args.n_iter, norm=args.norm,
            log_path=f'{logsdir}/{runname}.txt')
        torch.save(x_adv, f'{savedir}/{runname}.pth')

    elif args.attack in ['interm', 'interm_ge', 'apgd', 'apgd_ge', 'apgd_on_features']:
        runname += f'_niter_{args.n_iter}_loss_{args.loss}' + \
            f'_nrestarts_{args.n_restarts}_init_{args.init}'
        if not args.step_size is None:
            runname += f'_stepsize_{args.step_size:.3f}'
        x_init = None
        n_batches = math.ceil(args.n_ex / args.batch_size)
        x_adv, x_best = x_test.clone(), x_test.clone()
        for c in range(n_batches):
            x_test_curr = x_test[c * args.batch_size:(c + 1) * args.batch_size]
            y_test_curr = y_test[c * args.batch_size:(c + 1) * args.batch_size]
            x_adv_curr, x_best_curr = apgd_restarts(test_model,
                x_test_curr, y_test_curr, n_restarts=args.n_restarts,
                verbose=True, n_iter=args.n_iter, loss=args.loss, eps=args.eps,
                norm=args.norm, log_path=f'{logsdir}/{runname}.txt')
            x_adv[c * args.batch_size:(c + 1) * args.batch_size] = x_adv_curr.clone()
            x_best[c * args.batch_size:(c + 1) * args.batch_size] = x_best_curr.clone()
        torch.save(x_adv, f'{savedir}/{runname}.pth')
        torch.save(x_best, f'{savedir}/{runname}_best.pth')

    elif args.attack == 'test_points':
        if not args.data_path is None:
            runname += f' {args.data_path.split("/")[-1]}'
            x_adv = torch.load(args.data_path)
        else:
            runname += ' clean points'
            x_adv = x_test.clone()
        print(x_adv.shape)

    elif args.attack in 'find_wc':
        xs = [
            # feature attacks
            'apgd_1_1000_Linf_eps_0.03137_niter_100_loss_l1_nrestarts_1_init_None.pth',
            'apgd_1_1000_Linf_eps_0.03137_niter_25_loss_l1_nrestarts_4_init_None.pth',
            #
            # AA on the original model
            'eval_fast_1_1000_Linf_eps_0.03137.pth',
            #
            #'eval_with_square_1_1000_Linf_eps_0.03137_niter_5000_loss_ce_nrestarts_1.pth'
            ]
        xs = [f'./results/rebuffi_sodef/{c}' for c in xs]
        for x in xs:
            assert os.path.exists(x), f'missing {x}'
        xtr = [
            # transfer apgd (points with best loss for unsuccessful cases are used)
            'apgd_1_1000_Linf_eps_0.03137_niter_100_loss_dlr-targeted_nrestarts_9_init_None_best.pth',
            'apgd_1_1000_Linf_eps_0.03137_niter_99_loss_cw_nrestarts_1_init_None_best.pth',
            'apgd_1_1000_Linf_eps_0.03137_niter_99_loss_dlr-targeted_nrestarts_3_init_None_best.pth',
            'apgd_1_1000_Linf_eps_0.03137_niter_99_loss_ce_nrestarts_1_init_None_best.pth',
            ]
        xtr = [c for c in xtr if args.norm in c]
        xtr = [f'./results/rebuffi_orig/{c}' for c in xtr]
        for x in xtr:
            assert os.path.exists(x), f'missing {x}'
        logger.log('\n'.join(xs))
        xs = [torch.load(x).cpu() for x in xs]
        logger.log('\n'.join(xtr))
        xs += [torch.load(x).cpu() for x in xtr]
        xs = xs[::-1]
        for i, x in enumerate(xs):
            other_utils.check_imgs(xs[i].cpu(), x_test.cpu(), args.norm)
        with torch.no_grad():
            acc, x_adv = get_wc_acc(model, xs, y_test,
                bs=args.batch_size, device=device, eot_test=1, logger=logger,
                loss=args.loss)
        #torch.save(x_adv, f'{savedir}/{runname}.pth')

    l_acc = []
    logger.log(runname)
    for c in range(args.eot_test):
        if c == 1:
            startt = time.time() # skip first since it's usually slower
        acc = clean_accuracy(model, x_adv, y_test, device=device,
            batch_size=args.batch_size)

        if c == 0:
            str_imgs = other_utils.check_imgs(x_adv.to(x_test.device), x_test, norm=args.norm)
            logger.log(str_imgs)
        logger.log(f'[{args.model}] robust acc={acc:.1%}')
    try:
        evalt = time.time() - startt
    except:
        evalt = 0.
    logger.log(f'runs={args.eot_test - 1} single run time={evalt / (args.eot_test - 1):.3f} s')
    logger.log('')


from flask import Flask, render_template, request, send_file
from werkzeug import secure_filename
import numpy as np
import pandas as pd
import time
import json

app = Flask(__name__, static_url_path='')

myDictObj = { "type":"Feature", "properties":{"title": "Medical Emergency","marker-color": "#ff0000" \
,"marker-symbol": "danger","line": "red"} \
, "geometry":{"type":"Point","coordinates":[-77.04404229438322,38.85341638599062]} }

myDictObj2 = { "type":"Feature", "properties":{"title": "Medical Center","iconSize":[60,60],"description": "recommended location for a medical center", "marker-color": "#0000ff" \
,"marker-symbol": "hospital","line": "blue"} \
, "geometry":{"type":"Point","coordinates":[-77.04404229438322,38.85341638599062]} }

@app.route('/')
def root():
	return app.send_static_file('index.html')

@app.route('/submit', methods = ['GET', 'POST'])
def upload_file():
	if request.method == 'POST':
		k = int(request.form['text'])
		f = request.files['file']
		f.save(secure_filename(f.filename))
		df = pd.read_csv(f.filename)

		rows = df.shape[0]
		groupsize = int(rows/k)
		currentAvgLat=0
		currentAvgLon=0
		current=0
		centersLeft=k

		g=0
		update2 = open("static/medcenters.txt","w")
		update = open("static/stations2.geojson","w")
		update.write('{\n  "type": "FeatureCollection",\n  "crs": {\n    "type": "name",\n    "properties": {\n     "name": "urn:ogc:def:crs:OGC:1.3:CRS84"')
		update.write('\n    }\n   },\n   "features": [\n    ')
		a1=0
		a2=0
		for a, b, c in zip(df.lat,df.lng,df.addr):
			myDictObj["geometry"]["coordinates"]=[b,a]
			myDictObj["properties"]["description"]=c

			currentAvgLon+=b
			currentAvgLat+=a
			current+=1
			if current == groupsize or (g==df.shape[0]-1 and centersLeft>0):
				myDictObj2["geometry"]["coordinates"]=[float(float(currentAvgLon)/float(groupsize)),float(float(currentAvgLat)/float(groupsize))]
				update2.write(str(float(float(currentAvgLon)/float(groupsize))) + " " + str(float(float(currentAvgLat)/float(groupsize))) +"\n")
				current=0
				currentAvgLat=0
				centersLeft-=1
				currentAvgLon=0
				update.write(str(json.dumps(myDictObj2, sort_keys=False, indent=3)) + (",\n" if g<df.shape[0] else "\n  ]\n}"))

			a1+=a
			a2+=b
			g+=1
			update.write(str(json.dumps(myDictObj, sort_keys=False, indent=3)) + (",\n" if g<df.shape[0] else "\n  ]\n}"))
		update.close()
		update2.close()

		return render_template('map.html',avglat=(a1/g),avglon=(a2/g))

if __name__ == '__main__':
	app.run(debug=True)


import random
i = 0
z = 0
inty = 0
floaty = 0
n = random.randint(1, 10)
print('Podaj ', n,' liczb')
while i < n:
    x = float(input("Podaj liczbę całkowitą, nieujemną"))
    if x.is_integer():
        inty += 1
    else:
        floaty += 1
    z = z + x
    i += 1
    if x < 0:
            print('Liczba ujemna! Średnia nie zmienia się')
            continue
print('Średnia podanych liczb wynosi: ', z/i)
print('Podałeś ', floaty,' floatów i ', inty,' intów')


from scipy import stats
from src.significance_testing.load_and_inspect import get_results

def summary(file1, file2):
    df1, metrics = get_results(file1)
    df2, metrics = get_results(file2)
    for metric in metrics:
        sig_ttest(df1[metric], df2[metric], metric, file1, file2)

def sig_ttest(list_results1, list_results2, metric, file1, file2):
    t2, p2 = stats.ttest_ind(list_results1, list_results2)
    #print("p = " + str(p2))
    stars = ''
    better = ''
    if p2 < 0.05:
        stars +='*'
        if p2 < 0.001:
            stars += '*'
        if list_results1.mean() > list_results2.mean():
            better = file1[:-4]
        else:
            better = file2[:-4]
    print('{}: {} {} is higher'.format(metric, stars, better))
    return

if __name__=='__main__':
    file1 = 'lstm_los_no_diag.csv'
    file2 = 'lstm_los.csv'
    summary(file1, file2)


class Main:
    @staticmethod
    def menu():
        user_input = input('''Hello. How Would you like to proceed?
                        1. IF YOU ARE A LIBRARIAN ENTER 1
                        2. IF YOU ARE A USER ENTER 2
                        3. Enter 3 to Exit .  ''')
        if user_input == "1":
            from Lib import Library
        elif user_input == "2":
            from Users import User
        else:
            print("Enter the Correct Output")

L = Main
L.menu()


"""
Routes to interact with the database. (API endpoints.)

This should be called from the frontend to get data from the database.
Filtering and sorting should be done on the frontend, not the backend.
"""
import os
import shutil
import sqlite3
from datetime import datetime

import flask
import markupsafe
import requests

import audits
import inventory
import teams
import user
from __init__ import DATABASE, LABEL_SERVER, MIN_DAYS, MAX_DAYS, MIN_LABELS, MAX_LABELS
from db import Settings
from inventory import Item
from sanitizer import VALIDATORS, MINMAX, sanitize, handle_api_exception, APIException
from utils import login_required, next_july

api = flask.Blueprint('api', __name__)

@api.route('/items', methods=['GET'])
@login_required(admin_only=True, api=True)
def get_items() -> flask.Response:
    """Get all items in the database for frontend display."""
    items = inventory.get_all()
    return flask.jsonify([item.api_repr() for item in items])

@api.route('/items/available', methods=['GET'])
@login_required(admin_only=True, api=True)
def get_items_available() -> flask.Response:
    """Get all items in the database for frontend display."""
    items = inventory.get_all_available()
    return flask.jsonify([item.api_repr() for item in items])

@api.route('/items/unavailable', methods=['GET'])
@login_required(admin_only=True, api=True)
def get_items_unavailable() -> flask.Response:
    """Get all items in the database for frontend display."""
    items = inventory.get_all_unavailable()
    return flask.jsonify([item.api_repr() for item in items])

@api.route('/items/overdue', methods=['GET'])
@login_required(admin_only=True, api=True)
def get_items_overdue() -> flask.Response:
    """Get all items in the database for frontend display."""
    items = inventory.get_all_overdue()
    return flask.jsonify([item.api_repr() for item in items])

@api.route('/items/user/<userid>', methods=['GET'])
@login_required(admin_only=True, api=True)
def get_items_by_userid(userid: str) -> flask.Response:
    items = inventory.get_all_unavailable()
    return flask.jsonify([item.api_repr() for item in items if item.borrowed_to == userid])

@api.route('/items', methods=['POST'])
@login_required(admin_only=True)
@handle_api_exception
def add_item() -> flask.Response:
    """Add an item to the database."""
    # START: Validation
    validation_map = {
        'id': VALIDATORS.UNIQUE_ID,
        'name': VALIDATORS.NAME,
        'category': VALIDATORS.CATEGORY,
    }
    item_dict = sanitize(validation_map, flask.request.form)
    # END: Validation

    item = {key: val for key, val in item_dict.items()}
    item = Item(**item)
    inventory.add(item)
    return flask.Response(f'La til {item.id} i databasen.', status=201)

@api.route('/items/<item_id>', methods=['PUT'])
@login_required(admin_only=True)
@handle_api_exception
def edit_item(item_id: str) -> flask.Response:
    """Edit an item in the database."""
    # START: Validation
    validation_map = {
        'id': VALIDATORS.UNIQUE_OR_SAME_ID,
        'name': VALIDATORS.NAME,
        'category': VALIDATORS.CATEGORY,
    }
    item_dict = sanitize(validation_map, flask.request.form, {'id': item_id})
    # END: Validation

    item = {key: val for key, val in item_dict.items()}
    item = Item(**item)
    inventory.edit(item_id, item)
    return flask.Response(f'Redigerte {item_id} i databasen.', status=200)

@api.route('/items/<item_id>', methods=['DELETE'])
@login_required(admin_only=True)
@handle_api_exception
def delete_item(item_id: str) -> flask.Response:
    """Delete an item from the database."""
    inventory.delete(item_id)
    return flask.Response(f'Slettet {item_id} fra databasen.', status=200)

@api.route('/items/<item_id>/label/<variant>/preview', methods=['GET'])
@login_required(admin_only=True)
@handle_api_exception
def get_label_preview(item_id: str, variant: str = 'qr') -> flask.Response:
    """Get a label preview for an item."""
    item = inventory.get(item_id)
    url = f'{LABEL_SERVER}/preview?id={item.id}&name={item.name}&variant={variant}'
    return flask.redirect(url)

@api.route('/items/<item_id>/label/print', methods=['POST'])
@login_required(admin_only=True)
@handle_api_exception
def print_label(item_id: str) -> flask.Response:
    # START: Validation
    validation_map = {
        'print_label_count': VALIDATORS.INT,
        'print_label_count_minmax': MINMAX(MIN_LABELS, MAX_LABELS),
        'print_label_type': VALIDATORS.LABEL_TYPE,
    }
    form = sanitize(validation_map, flask.request.form)
    # END: Validation

    variant = form.get('print_label_type', 'qr')
    count = int(form.get('print_label_count', '1'))
    item = inventory.get(item_id)
    url = f'{LABEL_SERVER}/print?id={item.id}&name={item.name}&variant={variant}&count={count}'
    try:
        response = requests.post(url)
    except requests.exceptions.RequestException as e:
        return flask.Response(str(e), status=500)
    return flask.Response(response.text, status=response.status_code)

@api.route('/book/out', methods=['POST'])
@login_required(admin_only=True)
@handle_api_exception
def book_equipment() -> flask.Response:
    """Book out equipment for a user."""
    # START: Validation
    validation_map = {
        'user': VALIDATORS.UNIQUE_ID,
        'days': VALIDATORS.INT,
        'days_minmax': MINMAX(MIN_DAYS, MAX_DAYS),
        'equipment': VALIDATORS.ITEM_LIST_EXISTS,
    }
    form = sanitize(validation_map, flask.request.form)
    # END: Validation
    userid = form.get('user')
    days = form.get('days')

    for item in form.get('equipment'):
        inventory.register_out(item_id=item, userid=userid, days=days)
    return flask.Response(f'Utstyret ble utlevert til {user.get(userid).get("name")}.', status=200)

@api.route('/return/<item_id>', methods=['POST'])
@login_required(admin_only=True)
@handle_api_exception
def return_equipment(item_id: str) -> flask.Response:
    """Return equipment from a user."""
    inventory.register_in(item_id=item_id)
    return flask.Response('Utstyr ble innlevert.', status=200)

@api.route('/user/<userid>', methods=['GET'])
@login_required(admin_only=True, api=True)
def get_user(userid: str) -> flask.Response:
    """Get user as JSON."""
    u = user.get(userid)
    return flask.jsonify(u)

@api.route('/users', methods=['POST'])
@login_required()
def register_student() -> flask.Response:
    """Add/update a class in the database."""
    con = sqlite3.connect(DATABASE)
    cur = con.cursor()

    selected_classroom = flask.request.form.get('classroom')
    try:
        sanitize({'classroom': VALIDATORS.GROUP}, {'classroom': selected_classroom})
    except APIException as e:
        return flask.abort(e.status_code, e.message)

    data = {
        'name': flask.session.get('user').name,
        'email': flask.session.get('user').email,
        'userid': flask.session.get('user').userid,
        'classroom': selected_classroom,
        'updated_at': datetime.now().isoformat(),
        'expires_at': next_july().isoformat()
    }
    cur.execute(
        'REPLACE INTO users (name, email, userid, classroom, updated_at, expires_at) '
        'VALUES (:name, :email, :userid, :classroom, :updated_at, :expires_at)',
        data)
    con.commit()
    con.close()
    return flask.redirect(flask.request.referrer)

@api.route('/groups', methods=['PUT'])
@login_required(admin_only=True)
@handle_api_exception
def update_groups() -> flask.Response:
    """Update a class in the database."""
    new_groups = list()
    for group in flask.request.form.get('groups').split('\n'):
        if not group.strip():
            continue
        sanitize({'group': VALIDATORS.GROUP_NAME}, {'group': group.strip()})
        new_groups.append(group.strip())

    con = sqlite3.connect(DATABASE)
    cur = con.cursor()
    # noinspection SqlWithoutWhere
    cur.execute('DELETE FROM groups')
    con.commit()
    for group in new_groups:
        cur.execute('INSERT INTO groups (classroom) VALUES (?)', (group,))

    con.commit()
    con.close()
    return flask.Response('Gruppene ble oppdatert.', status=200)

@api.route('/categories', methods=['PUT'])
@login_required(admin_only=True)
@handle_api_exception
def update_categories() -> flask.Response:
    """Update every category in the database."""
    new_categories = list()
    for category in flask.request.form.get('categories').split('\n'):
        if not category.strip():
            continue
        sanitize({'category': VALIDATORS.CATEGORY_NAME}, {'category': category.strip()})
        new_categories.append(category.strip())

    con = sqlite3.connect(DATABASE)
    cur = con.cursor()
    # noinspection SqlWithoutWhere
    cur.execute('DELETE FROM categories')
    con.commit()
    for category in new_categories:
        cur.execute('INSERT INTO categories (name) VALUES (?)', (category,))

    con.commit()
    con.close()
    return flask.Response('Kategoriene ble oppdatert.', status=200)

@api.route('/registrer_avvik', methods=['POST'])
@login_required(admin_only=True)
@handle_api_exception
def registrer_avvik() -> flask.Response:
    """Log a defect."""
    item_id = flask.request.form.get('id') or ''
    log_text = flask.request.form.get('text') or ''

    if item_id:
        txt = f'Avvik på utstyr {item_id}: {log_text}'
    else:
        txt = f'Generelt avvik: {log_text}'

    audits.audit('AVVIK', markupsafe.escape(txt))
    return flask.Response(f'Avvik registrert: {markupsafe.escape(txt)}', status=200)

@api.route('/send_report', methods=['POST'])
@login_required(admin_only=True, api=True)
@handle_api_exception
def send_report() -> flask.Response:
    """Send a report to the specified teams webhooks.

    Can be used with a cron job to send reports automatically, e.g.:
    15 8 * 1-6,8-12 MON curl -X POST "http://localhost:5000/send_report?interval=7&token=<token>"

    If the interval parameter is set, the report will only be sent
    if the last report was sent more than interval days ago.
    """
    interval = flask.request.args.get('interval')

    if interval:
        sanitize({'interval': VALIDATORS.INT}, flask.request.args)
        current_date = datetime.now().date()
        last_sent = datetime.fromtimestamp(float(Settings.get('report_last_sent') or 0)).date()
        if last_sent and (current_date - last_sent).days < int(interval):
            raise APIException(f'Ikke sendt - mindre enn {interval} dager siden forrige rapport.', 200)

    return teams.send_report()

@api.route('/users/prune_inactive', methods=['POST'], endpoint='prune_inactive_users')
@login_required(admin_only=True, api=True)
def prune_inactive_users() -> flask.Response:
    """Prune users that have not been updated in a while.

    Example cronjob:
    0 1 1 7 * curl -X POST "http://localhost:5000/users/prune_inactive?token=<token>"
    """
    # TODO: run this every day as a background task here instead of an external cronjob?
    user.prune_inactive()
    return flask.Response('Inaktive brukere ble fjernet.', status=200)

@api.route('/users/me', methods=['DELETE'])
@login_required()
def delete_me() -> flask.Response:
    """Delete the currently logged in user."""
    u = flask.session.get("user")
    user.delete(u.userid)
    flask.session.clear()
    return flask.Response(f'Bruker {u.name} ble slettet.', status=200)

@api.route('/backup/<filename>', methods=['POST'])
@login_required(admin_only=True, api=True)
@handle_api_exception
def backup(filename: str) -> flask.Response:
    """Backup the database with the given filename to the data/backups folder.

    Recommended to use a cronjob to run this every week, e.g.:
    0 1 * * 1 curl -X POST "http://localhost:5000/backup/backup-$(date +\%Y-w\%V).sqlite?token=<token>"
    """
    if not filename.endswith('.sqlite'):
        raise APIException('Ugyldig filnavn (Må avsluttes med .sqlite).', 400)

    if not os.path.exists('data/backups'):
        os.mkdir('data/backups')

    shutil.copyfile(DATABASE, f'data/backups/{filename}')
    return flask.Response(f'Databasen ble sikkerhetskopiert til {filename}.', status=200)

@api.route('/bulletin', methods=['PUT'])
@login_required(admin_only=True)
@handle_api_exception
def update_bulletin() -> flask.Response:
    """Update the bulletin."""
    bulletin_title = flask.request.form.get('bulletin_title') or ''
    bulletin = flask.request.form.get('bulletin') or ''

    Settings.set('bulletin_title', markupsafe.escape(bulletin_title))
    Settings.set('bulletin', markupsafe.escape(bulletin))
    return flask.Response('Bulletin ble oppdatert.', status=200)