""" TensorMONK :: architectures """

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from ..layers import Convolution, ResidualComplex, PrimaryCapsule, \
    RoutingCapsule

class CapsuleNet(nn.Module):
    r"""Dynamic routing between capsules - https://arxiv.org/pdf/1710.09829.pdf
    for MNIST (tensor_size = (1, 1, 28, 28)) -- works for FashionMNIST

    Args:
        tensor_size: shape of tensor in BCHW
            (None/any integer >0, channels, height, width)
        n_labels: 10, MNIST/FashionMNIST
        primary_n_capsules: number of capsules in primary capsule layer
        primary_capsule_length: length of each capsule in primary capsule layer
        routing_capsule_length: number of capsules in routing capsule
        routing_iterations: length of each capsule in routing capsule
        replicate_paper: replicates https://arxiv.org/pdf/1710.09829.pdf
        activation: None/relu/relu6/lklu/elu/prelu/tanh/sigm/maxo/rmxo/swish
        dropout: 0. - 1., default = 0.1 with dropblock=True
        normalization: None/batch/group/instance/layer/pixelwise
        pre_nm: if True, normalization -> activation -> convolution else
            convolution -> normalization -> activation

    Return:
        embedding (a torch.Tensor), rec_tensor (a torch.Tensor),
            rec_loss (a torch.Tensor)
    """
    def __init__(self,
                 tensor_size: tuple = (6, 1, 28, 28),
                 n_labels: int = 10,
                 primary_n_capsules: int = 8,
                 primary_capsule_length: int = 32,
                 routing_capsule_length: int = 16,
                 routing_iterations: int = 3,
                 replicate_paper: bool = True,
                 activation: str = "relu",
                 dropout: float = 0.,
                 normalization: str = None,
                 pre_nm: bool = False,
                 *args, **kwargs):
        super(CapsuleNet, self).__init__()

        if replicate_paper:
            primary_n_capsules = 8
            primary_capsule_length = 32
            routing_capsule_length = 16
            routing_iterations = 3
            block = Convolution
            self.InitialConvolutions = \
                Convolution(tensor_size, filter_size=9, out_channels=256,
                            strides=1, pad=False, activation="relu")
            _tensor_size = self.InitialConvolutions.tensor_size
        else:  # You can be creative!
            block = Convolution
            kwargs = {"activation": activation, "dropout": dropout,
                      "normalization": normalization, "pre_nm": pre_nm}
            tp = [Convolution(tensor_size, 5, 64, 1, False, **kwargs),
                  ResidualComplex((6,  64, 24, 24), 3, 256, 1, True, **kwargs),
                  ResidualComplex((6, 256, 24, 24), 3, 256, 1, True, **kwargs),
                  ResidualComplex((6, 256, 24, 24), 3, 256, 1, True, **kwargs),
                  ResidualComplex((6, 256, 24, 24), 3, 256, 1, True, **kwargs),
                  Convolution((6, 256, 24, 24), 5, 64, 1, False, "", **kwargs)]
            self.InitialConvolutions = nn.Sequential(tp)
            _tensor_size = self.InitialConvolutions[-1].tensor_size
        print("InitialConvolutions output size :: ", _tensor_size)

        # block can be replaced with any module available in NeuralLayers
        self.Primary = PrimaryCapsule(_tensor_size, filter_size=9,
                                      out_channels=256, strides=2,
                                      pad=False, activation="",
                                      dropout=dropout, batch_nm=False,
                                      pre_nm=False, block=block,
                                      n_capsules=primary_n_capsules,
                                      capsule_length=primary_capsule_length)
        print("Primary capsule output size :: ",
              self.Primary.tensor_size)
        self.Routing = RoutingCapsule(self.Primary.tensor_size,
                                      n_capsules=n_labels,
                                      capsule_length=routing_capsule_length,
                                      iterations=routing_iterations)

        print("Routing capsule output size :: ", self.Routing.tensor_size)
        self.Reconstruction = \
            nn.Sequential(nn.Linear(n_labels*routing_capsule_length, 512),
                          nn.ReLU(),
                          nn.Linear(512, 1024),
                          nn.ReLU(),
                          nn.Linear(1024, int(np.prod(tensor_size[1:]))))

        self.tensor_size = (6, n_labels)
        self.input_tensor_size = tensor_size

    def forward(self, tensor, targets):
        tensor_size = tensor.size()

        # CapsuleNet
        tensor_deep = self.InitialConvolutions(tensor)
        tensor_primary = self.Primary(tensor_deep)
        embedding = self.Routing(tensor_primary)

        # Reconstruction (only during training)
        #   remove 'if' loop if you like to view the rec_tensor on test data
        rec_tensor = None
        rec_loss = 0.
        if self.training:
            identity = torch.eye(self.tensor_size[1])
            if targets.is_cuda:
                identity = identity.cuda()
            onehot_targets = identity.index_select(dim=0,
                                                   index=targets.view(-1))

            rec_tensor = (embedding *
                          onehot_targets[:, :, None]).view(tensor_size[0], -1)
            rec_tensor = torch.tanh(self.Reconstruction(rec_tensor))
            rec_loss = F.mse_loss(rec_tensor.view(tensor_size[0], -1),
                                  tensor.view(tensor_size[0], -1))
            rec_tensor = rec_tensor.view(*tensor_size)

        return embedding, rec_tensor, rec_loss

# from tensormonk.layers import *
# tensor_size = (2, 1, 28, 28)
# targets = torch.LongTensor([1, 2])
# tensor = torch.rand(*tensor_size)
# test = CapsuleNet(tensor_size)
# test(tensor, targets)[2]


import tensorflow as tf
import numpy as np

def main():

    x_train = arrayFromFile("TrainDigitX.idx")
    y_train = arrayFromFile("TrainDigitY.idx")
    x_test = arrayFromFile("TestDigitX.idx")
    y_test = arrayFromFile("TestDigitY.idx")

    model = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(input_shape=(784,)),
    tf.keras.layers.Dense(100, activation=tf.nn.relu),
    tf.keras.layers.Dense(10, activation=tf.nn.softmax)
    ])
    model.compile(optimizer='adam',
                loss='sparse_categorical_crossentropy',
                metrics=['accuracy'])

    model.fit(x_train, y_train, epochs=2)
    print(model.evaluate(x_test, y_test))

# read file into a np array
def arrayFromFile(filename):

    print("reading", filename)

    data = bytes()
    dimensions = []
    size = 0
    data = data
    dataTypeSize = 1
    startingIndex = 0

    with open(filename,'rb') as file:
        string = file.read()
        data = bytes(string)

    dataType = data[2]

    if dataType == 0x8 or dataType == 0x9:
        dataTypeSize = 1
    elif dataType == 0xB:
        dataTypeSize = 2
    elif dataType == 0xC or dataType == 0xD:
        dataTypeSize = 4
    elif dataType == 0xE:
        dataTypeSize = 8

    dimensionCount = data[3]
    index = 4

    for d in range(dimensionCount):

        size = 0

        for i in range(4):
            size = size<<8 | data[index+i]

        dimensions.append(size)
        index += 4

    startingIndex = index
    dataList = list(data[startingIndex:])
    array = np.array(dataList)
    array = array.astype(float)

    if dimensionCount == 3:
        array = array.reshape(dimensions[0], len(dataList)//dimensions[0])
        array /= 255

    return array

# after all functions are defined, run main code
if __name__ == "__main__": main()


"""
StreamRecorder: resting-state recording
=======================================

A resting-state recording is a simple offline recording during which the brain
activity of a subject is measured in the absence of any stimulus or task. A
resting-state recording can be designed with a `~bsl.StreamRecorder`.
"""

# %%

# Authors: Mathieu Scheltienne <mathieu.scheltienne@fcbg.ch>
#
# License: LGPL-2.1

# sphinx_gallery_thumbnail_path = '_static/stream_recorder/stream_recorder_cli.gif'

# %%
# .. warning::
#
#     Both `~bsl.StreamPlayer` and `~bsl.StreamRecorder` create a new process
#     to stream or record data. On Windows, mutliprocessing suffers a couple of
#     restrictions. The entry-point of a multiprocessing program should be
#     protected with ``if __name__ == '__main__':`` to ensure it can safely
#     import and run the module. More information on the
#     `documentation for multiprocessing on Windows
#     <https://docs.python.org/2/library/multiprocessing.html#windows>`_.
#
# This example will use a sample EEG resting-state dataset that can be retrieve
# with :ref:`bsl.datasets <api/utils:Datasets>`. The dataset is stored in the user home
# directory in the folder ``bsl_data`` (e.g. ``C:\Users\User\bsl_data``).

# %%

import os
import time
from pathlib import Path

import mne

from bsl import StreamPlayer, StreamRecorder, datasets
from bsl.triggers import MockTrigger

# %%
#
# To simulate an actual signal coming from an LSL stream, a `~bsl.StreamPlayer`
# is used with a 40 seconds resting-state recording.

stream_name = "StreamPlayer"
fif_file = datasets.eeg_resting_state.data_path()
player = StreamPlayer(stream_name, fif_file)
player.start()
print(player)

# %%
#
# For this example, the folder ``bsl_data/examples`` located in the user home
# directory will be used to stored recorded files. To ensure its existence,
# `os.makedirs` is used.

record_dir = Path("~/bsl_data/examples").expanduser()
os.makedirs(record_dir, exist_ok=True)
print(record_dir)

# %%
#
# For this simple offline recording, the goal is to start a
# `~bsl.StreamRecorder`, send an event on a trigger to mark the beginning of
# the resting-state recording, wait for a defined duration, and stop the
# recording.
#
# By default, a `~bsl.StreamRecorder` does not require any argument. The
# current working directory is used to record data from all available streams
# in files named based on the date/time timestamp at which the recorder is
# started.
#
# To record only a subset of the available streams with a specific file name
# and in a specific directory, the arguments ``record_dir``, ``fname`` and
# ``stream_name`` must be provided.
#
# For this example, the directory used to store recordings is
# ``bsl_data/examples`` and the file name will start with
# ``example-resting-state``.
#
# .. note::
#
#     By default, the `~bsl.StreamRecorder.start` method is blocking and will
#     wait for the recording to start. This behavior can be changed with the
#     ``blocking`` argument.

recorder = StreamRecorder(record_dir, fname="example-resting-state")
recorder.start()
print(recorder)

# %%
#
# Now that a `~bsl.StreamRecorder` is started and is acquiring data, a trigger
# to mark the beginning of the segment of interest is created. For this
# example, a `~bsl.triggers.MockTrigger` is used, but this example
# would be equally valid with a different type of trigger.

trigger = MockTrigger()

# %%
#
# To mark the beginning of the segment of interest in the recording, a signal
# is sent on the trigger. For this example, the event value (1) is used.

trigger.signal(1)

# %%
#
# Finally, after the appropriate duration, the recording is interrupted.

time.sleep(2)  # 2 seconds duration
del trigger
recorder.stop()
print(recorder)

# %%
#
# A `~bsl.StreamRecorder` records data in ``.pcl`` format. This file can be
# open with `pickle.load`, and is automatically converted to a `~mne.io.Raw`
# FIF file in a subdirectory ``fif``. The recorded files name syntax is:
#
# - If ``fname`` is not provided: ``[date/time timestamp]-[stream]-raw.fif``
# - If ``fname`` is provided: ``[fname]-[stream]-raw.fif``
#
# Where ``stream`` is the name of the recorded LSL stream. Thus, one file is
# created for each stream being recorded.

fname = record_dir / "fif" / "example-resting-state-StreamPlayer-raw.fif"
raw = mne.io.read_raw_fif(fname, preload=True)
print(raw)
events = mne.find_events(raw, stim_channel="TRIGGER")
print(events)

# %%
#
# As for the `~bsl.StreamPlayer`, the `~bsl.StreamRecorder` can be used as a
# context manager. The context manager takes care of starting and stopping the
# recording.

with StreamRecorder(record_dir):
    time.sleep(1)

# %%
#
# As for the `~bsl.StreamPlayer`, the `~bsl.StreamRecorder` can be started via
# command-line when a LSL stream is accessible on the network.
#
# Example assuming:
#
# - the current working directory is ``bsl_data`` in the user home directory
# - the stream to connect to is named ``MyStream``
# - the recorded file naming scheme is ``test-[stream]-raw.fif``, i.e.
#   ``test-MyStream-raw.fif``
#
# .. code-block:: console
#
#     $ bsl_stream_recorder -d examples -f test -s MyStream
#
# .. image:: ../../_static/stream_recorder/stream_recorder_cli.gif
#    :alt: StreamRecorder
#    :align: center

# %%
# | Stop the mock LSL stream used in this example.

player.stop()


#Tic tac toe game easy way

xo_list = ["_"] * 9

def winner(pattern):
    if xo_list[0] + xo_list[1] + xo_list[2] == pattern \
        or xo_list[3] + xo_list[4] + xo_list[5] == pattern \
        or xo_list[6] + xo_list[7] + xo_list[8] == pattern \
        or xo_list[0] + xo_list[3] + xo_list[6] == pattern \
        or xo_list[1] + xo_list[4] + xo_list[7] == pattern \
        or xo_list[2] + xo_list[5] + xo_list[8] == pattern \
        or xo_list[0] + xo_list[4] + xo_list[8] == pattern \
        or xo_list[2] + xo_list[4] + xo_list[6] == pattern:
        return True

# To convert the coordinates to position in list
def converter(x,y):
    i = int(y) + 2
    j = int(x) - 1
    return (j * 3 + i) - 3

# Display the matrix
def display_matrix(xo_list):
    print("---------")
    for _i in range(0, 3):
        print(f"| {xo_list[_i * 3]} {xo_list[_i * 3 + 1]} {xo_list[_i * 3 + 2]} |")
    print("---------")

# To check if the cell contains a "X" or "O"
def check_position(xo_list, position):
    if xo_list[position] == "_":
        return False
    else:
        return True

# To check that coordinate values are a number
def check_format(x, y):
    try:
        x = int(x)
        y = int(y)
        return True
    except:
        return False

# To check range of the coordinates
def check_range(x, y):
    if 0 < int(x) < 4 and 0 < int(y) < 4:
        return True
    else:
        return False

display_matrix(xo_list)
x = None
y = None
turn = 1

while xo_list.count("_") > 0 or not(winner("XXX")) or not(winner("OOO")):
    x, y = input("Enter the coordinates: ").split()

    while check_format(x, y) == False:
        print("You should enter numbers!")
        x, y = input("Enter the coordinates: ").split()
        #check_format(x, y)

    while check_range(x, y) == False:
        print("Coordinates should be from 1 to 3!")
        x, y = input("Enter the coordinates: ").split()
        #check_range(x, y)

    position = converter(x, y)

    while check_position(xo_list, position) == True:
        print("This cell is occupied! Choose another one!")
        x, y = input("Enter the coordinates: ").split()
        position = converter(x, y)
        check_position(xo_list, position)

    if turn % 2 != 0:
        xo_list[position] = "X"
    else:
        xo_list[position] = "O"

    display_matrix(xo_list)
    turn += 1

    if winner("XXX"):
        print("X wins")
        break

    if winner("OOO"):
        print("O wins")
        break

    if turn == 10:
        print("Draw")
        break


"""
Tres de Tres - Partidos

- alimentar: Alimentar los partidos a partir de un archivo CSV
"""
from pathlib import Path
import csv
import click

from lib.safe_string import safe_string

from citas_admin.app import create_app
from citas_admin.extensions import db

from citas_admin.blueprints.tdt_partidos.models import TdtPartido

app = create_app()
db.app = app

@click.group()
def cli():
    """Tres de Tres - Partidos"""

@click.command()
@click.argument("entrada_csv")
def alimentar(entrada_csv):
    """Alimentar a partir de un archivo CSV"""
    ruta = Path(entrada_csv)
    if not ruta.exists():
        click.echo(f"AVISO: {ruta.name} no se encontró.")
        return
    if not ruta.is_file():
        click.echo(f"AVISO: {ruta.name} no es un archivo.")
        return
    click.echo("Alimentando partidos...")
    contador = 0
    with open(ruta, encoding="utf8") as puntero:
        rows = csv.DictReader(puntero)
        for row in rows:
            TdtPartido(
                nombre=safe_string(row["nombre"], to_uppercase=True, save_enie=True),
                siglas=safe_string(row["siglas"], to_uppercase=True, save_enie=True),
                estatus=row["estatus"],
            ).save()
            contador += 1
            if contador % 100 == 0:
                click.echo(f"  Van {contador}...")
    click.echo(f"{contador} partidos alimentados.")

cli.add_command(alimentar)


import numpy as np

# create a sample dataset of vectors
dataset = [
    np.array([1.0, 2.0]),
    np.array([2.0, 3.0]),
    np.array([4.0, 5.0]),
    np.array([6.0, 7.0]),
]

# Query vector
Query_vector = np.array([3.0, 4.0])

# calculate cosine similarity between vectors
def cosine_similarity(vec1, vec2):
    dot_product = np.dot(vec1, vec2)
    norm1 = np.linalg.norm(vec1)
    norm2 = np.linalg.norm(vec2)
    similarity = dot_product / (norm2 * norm1)
    return similarity

# Perform similarity search
similarities = [cosine_similarity(Query_vector, vec) for vec in dataset]

# Find the index of the most similar vector
most_similar_index = np.argmax(similarities)
most_similar_vector = dataset[most_similar_index]

print("Query vector :", Query_vector)
print()
print("Sample dataset for the Given set: ", dataset)
print()
print("Most Similar Vector:", most_similar_vector)


import sys
import os
import re
import json
import time
import socket
import codecs
import multiprocessing
from urlparse import urlparse

from setproctitle import setproctitle
import distance
import arrow

from xbake import __version__, __date__
from xbake.common.logthis import *
from xbake.mscan import util, out
from xbake.common import fsutil
from xbake.mscan import mdb
from xbake.mscan.mdb import MCMP

class DSTS:
    """mkey string enum"""
    NEW = 'new'
    UNCHANGED = 'unchanged'
    RENAMED = 'renamed'

# File match regexes
fregex = [
            r"^(\[(?P<fansub>[^\]]+)\])[\s._]*(?P<series>.+?)(?:[\s._]-[\s._]|[\._])(?:(?P<special>(NCOP|NCED|OP|ED|PV|OVA|ONA|Special|Insert|Preview|Lite|Short)\s*-?\s*[0-9]{0,2})|(?:[eEpP]{2}[\s._]*)?(?P<epnum>[0-9]{1,3}))(?P<version>[vep]{1,2}[0-9]{1,2}([-,][0-9]{1,2})?)?",
            r"^(\[(?P<fansub>[^\]]+)\])?[\s._]*(?P<series>.+?)(?P<season>[0-9]{1,2})x(?P<epnum>[0-9]{1,2})(.*)$",
            r"^(\[(?P<fansub>[^\]]+)\])?[\s._]*(?P<series>.+)[\.\-_ ]SE?(?P<season>[0-9]{1,2})EP?(?P<epnum>[0-9]{1,2})(?:[\.\-_ ](?P<eptitle>.+?))?[\.\-_\[\( ]+(?:([0-9]{3,4}p|web|aac|bd|tv|hd|x?264)+)",
            r"^(\[(?P<fansub>[^\]]+)\])?[\s._]*(?P<series>.+)[\._](?P<epnum>[0-9]{1,4})[\._](.*)$",
            r"^(?P<series>.+?)[\-_ ](?P<epnum>[0-9]{2})[\-_ ](.*)$",
            r"^(\[(?P<fansub>[^\]]+)\])?[\s._]*(?P<series>.+)[\._ ]-[\._ ][sS](?P<season>[0-9]{1,2}) ?[eE](?P<epnum>[0-9]{1,2})(.*)$",
            r"^(?P<series>.+)[sS](?P<season>[0-9]{1,2}) ?[eE](?P<epnum>[0-9]{1,2})(.*)$",
            r"^(?P<series>.+?) (?P<season>[0-9]{1,2}) (?P<epnum>[0-9]{1,2}) (.*)$",
            r"^(?P<series>.+) - (?P<epnum>[0-9]{1,2})(.*)$",
            r"^(?P<series>.+?)(?P<epnum>[0-9]{1,4})\.(.+)$",
            r"^(?P<series>.+?)(?P<epnum>[0-9]{2,3})(.+)$",
            r"^(?P<epnum>[0-9]{2,4})(.+)$"
         ]

# File extension filter
fext = re.compile(r'\.(avi|mkv|mpg|mpeg|wmv|vp8|ogm|mp4|mpv)', re.I)

config = None

def run(xconfig):
    """
    Implements --scan mode
    """
    global config, monjer
    config = xconfig

    # Check input filename
    if not config.run['infile']:
        failwith(ER.OPT_MISSING, "option infile required (-i/--infile)")
    else:
        if not os.path.exists(config.run['infile']):
            failwith(ER.OPT_BAD, "path/file [%s] does not exist" % (config.run['infile']))
        if config.run['single'] and not os.path.isfile(config.run['infile']):
            failwith(ER.OPT_BAD, "file [%s] is not a regular file; --single mode is used when scanning only one file" % (config.run['infile']))
        elif not config.run['single'] and not os.path.isdir(config.run['infile']):
            failwith(ER.OPT_BAD, "file [%s] is not a directory; use --single mode if scanning only one file" % (config.run['infile']))

    # Set proctitle
    try:
        setproctitle("xbake: scanning %s" % (config.run['infile']))
    except:
        pass

    # Examine and enumerate files
    if config.run['single']:
        new_files, flist = scan_single(config.run['infile'], config.scan['mforce'], config.scan['nochecksum'], config.scan['savechecksum'])
    else:
        if config.run['tsukimi'] is True:
            tstatus('scanlist', scanlist=get_scanlist(config.run['infile'], config.scan['follow_symlinks']))
        new_files, flist = scan_dir(config.run['infile'], config.scan['follow_symlinks'], config.scan['mforce'], config.scan['nochecksum'], config.scan['savechecksum'], int(config.scan['procs']))

    # Scrape for series information
    if new_files > 0:
        mdb.series_scrape(config)

    # Build host data
    hdata = {
                'hostname': socket.getfqdn(),
                'tstamp': time.time(),
                'duration': 0, # FIXME
                'topmost': os.path.realpath(config.run['infile']),
                'command': ' '.join(sys.argv),
                'version': __version__
            }

    # Build main output structure
    odata = {
                'scan': hdata,
                'files': flist,
                'series': mdb.get_tdex()
            }

    # Parse outfile
    if not config.run['outfile']:
        config.run['outfile'] = config.scan['output']

    # If no file defined, or '-', write to stdout
    if not config.run['outfile'] or config.run['outfile'] == '-':
        config.run['outfile'] = '/dev/stdout'

    # Parse URLs
    ofp = urlparse(config.run['outfile'])

    tstatus('output', event='start', output=config.run['outfile'])
    if ofp.scheme == 'mongodb':
        # Write to Mongo
        logthis(">> Output driver: Mongo", loglevel=LL.VERBOSE)
        if ofp.hostname is None:
            cmon = config.mongo
            logthis("Using existing MongoDB configuration; URI:", suffix=cmon['uri'], loglevel=LL.DEBUG)
        else:
            cmon = {'uri': config.run['outfile']}
            logthis("Using new MongoDB URI:", suffix=cmon['uri'], loglevel=LL.DEBUG)
        ostatus = out.to_mongo(odata, cmon)
    elif ofp.scheme == 'http' or ofp.scheme == 'https':
        # Send via HTTP(S) to a listening XBake daemon, or other web service
        logthis(">> Output driver: HTTP/HTTPS", loglevel=LL.VERBOSE)
        ostatus = out.to_server(odata, config.run['outfile'], config)
    else:
        # Write to file or stdout
        logthis(">> Output driver: File", loglevel=LL.VERBOSE)
        ostatus = out.to_file(odata, ofp.path)

    if ostatus['status'] == "ok":
        logthis("*** Scanning task completed successfully.", loglevel=LL.INFO)
        tstatus('complete', status='ok', files=len(flist), series=len(mdb.get_tdex()))
        return 0
    elif ostatus['status'] == "warning":
        logthis("*** Scanning task completed, with warnings.", loglevel=LL.WARNING)
        tstatus('complete', status='warning')
        return 49
    else:
        logthis("*** Scanning task failed.", loglevel=LL.ERROR)
        tstatus('complete', status='fail')
        return 50

# run config setting map for xattribs
setmap = {
                'ignore': "xbake.ignore",
                'series': "media.seriesname",
                'season': "media.season",
                'episode': "media.episode",
                'tvdb_id': "media.xref.tvdb",
                'mal_id': "media.xref.mal",
                'tdex_id': "xbake.tdex",
                'fansub': "media.fansub"
            }

def setter(xconfig):
    """
    Set overrides for a file or directory
    """
    global setmap
    infile = xconfig.run['infile']

    setout = {}
    for k, v in setmap.iteritems():
        if k in xconfig.run:
            if xconfig.run[k]:
                setout[v] = xconfig.run[k]

    logthis("Setting overrides:\n", suffix=print_r(setout), loglevel=LL.VERBOSE)
    fsutil.xattr_set(infile, setout)
    logthis("Overrides set OK.", ccode=C.GRN, loglevel=LL.INFO)
    return 0

def unsetter(xconfig):
    """
    Remove overrides for a file or directory
    """
    infile = xconfig.run['infile']
    dlist = list(fsutil.xattr_get(infile))
    logthis("Removing overrides:\n", suffix=print_r(dlist), loglevel=LL.VERBOSE)
    fsutil.xattr_del(infile, dlist)
    logthis("Overrides cleared.", ccode=C.GRN, loglevel=LL.INFO)
    return 0

def get_scanlist(dpath, dreflinks=True):
    """
    Return a list of files to be scanned by scan_dir()
    """
    dryout = scan_dir(dpath, dreflinks, dryrun=True)[1]
    return dryout.values()

def scan_dir(dpath, dreflinks=True, mforce=False, nochecksum=False, savechecksum=True, procs=0, dryrun=False):
    """
    Scan a directory recursively; follows symlinks by default
    """
    ddex = {}
    new_files = 0

    if dryrun is False:
        ## Set up workers and IPC
        if procs == 0:
            procs = multiprocessing.cpu_count()
        mp_inq = multiprocessing.Queue()
        mp_outq = multiprocessing.Queue()
        mp_tdexq = multiprocessing.Queue()

        ## Start queue runners
        wlist = []
        for wid in range(procs): # pylint: disable=unused-variable
            cworker = multiprocessing.Process(name="xbake: scanrunner", target=scanrunner, args=(mp_inq, mp_outq, mp_tdexq))
            wlist.append(cworker)
            cworker.start()

    ## Enumerate files
    for tdir, dlist, flist in os.walk(unicode(dpath), followlinks=dreflinks):  # pylint: disable=unused-variable
        # get base & parent dir names
        tdir_base = os.path.split(tdir)[1]  # pylint: disable=unused-variable
        tdir_parent = os.path.split(os.path.split(tdir)[0])[1]  # pylint: disable=unused-variable
        ovrx = {}

        # Get xattribs
        ovrx = parse_xattr_overrides(tdir)

        # Check if ignore flag is set for this directory (xattribs only)
        if ovrx.has_key('ignore'):
            logthis("Skipping directory, has 'ignore' flag set in xattribs:", suffix=tdir, loglevel=LL.INFO)
            continue

        # Parse overrides for this directory
        ovrx.update(parse_overrides(tdir))

        if dryrun is False:
            logthis("*** Scanning files in directory:", suffix=tdir, loglevel=LL.INFO)

        # enum files in this directory
        for xv in flist:
            xvreal = os.path.realpath(unicode(tdir + '/' + xv))
            xvbase, xvext = os.path.splitext(xv)  # pylint: disable=unused-variable

            # Skip .xbake file
            if unicode(xv) == unicode('.xbake'): continue

            # Skip unsupported filetypes, non-regular files, and broken symlinks
            if not os.path.exists(xvreal):
                logthis("Skipping broken symlink:", suffix=xvreal, loglevel=LL.WARNING)
                continue
            if not os.path.isfile(xvreal):
                logthis("Skipping non-regular file:", suffix=xvreal, loglevel=LL.VERBOSE)
                continue
            if not fext.match(xvext):
                logthis("Skipping file with unsupported extension:", suffix=xvreal, loglevel=LL.DEBUG)
                continue

            # Skip file if on the overrides 'ignore' list
            if check_overrides(ovrx, xv):
                logthis("Skipping file. Matched rule in override ignore list:", suffix=xvreal, loglevel=LL.INFO)
                continue

            # Create copy of override object and strip-out unneeded values and flags
            ovrx_sub = clean_overrides(ovrx)

            # Get file properties
            if dryrun is True:
                ddex[new_files] = os.path.realpath(tdir + '/' + xv)
                new_files += 1
            else:
                mp_inq.put({'rfile': xvreal, 'ovrx': ovrx_sub, 'mforce': mforce, 'nochecksum': nochecksum, 'savechecksum': savechecksum})

    ## Tend the workers
    if dryrun is False:
        # Pump terminators at the end of the queue
        for wid in range(procs):
            mp_inq.put({'EOF': True})

        # Monitor scanrunner progress
        logthis("File enumeration complete. Waiting for scanrunner to complete...", loglevel=LL.DEBUG)
        while len(wlist) > 0:
            # pull scan data off the outbound queue
            for tk in range(mp_outq.qsize()):  # pylint: disable=unused-variable
                try:
                    xfile, xdata = mp_outq.get(block=False)
                    logthis("got file from queue:", suffix=xfile, loglevel=LL.DEBUG)
                    ddex[xfile] = xdata
                    new_files += 1
                except:
                    pass

            # pull series data from tdex queue
            for tk in range(mp_tdexq.qsize()):  # pylint: disable=unused-variable
                try:
                    xkey, xdata = mp_tdexq.get(block=False)
                    logthis("got series from tdex queue:", suffix=xkey, loglevel=LL.DEBUG)
                    if xkey in mdb.tdex:
                        mdb.tdex[xkey]['count'] += xdata['count']
                    else:
                        mdb.tdex[xkey] = xdata
                except:
                    pass

            # check to see if the kids have died yet
            for wk, wid in enumerate(wlist):
                if not wid.is_alive():
                    logthis("Scanrunner is complete; pid =", suffix=wid.pid, loglevel=LL.DEBUG)
                    del(wlist[wk])
                    break

    return (new_files, ddex)

def scan_single(dfile, mforce=False, nochecksum=False, savechecksum=True):
    """
    Scan a single media file
    """
    ddex = {}
    new_files = 0

    # Parse overrides for directory the file is in
    tdir = os.path.dirname(os.path.realpath(dfile))
    ovrx = parse_xattr_overrides(tdir)
    ovrx.update(parse_overrides(tdir))
    ovrx = clean_overrides(ovrx)

    dasc = scanfile(dfile, ovrx=ovrx, mforce=mforce, nochecksum=nochecksum, savechecksum=savechecksum)
    if dasc:
        ddex[dfile] = dasc
        new_files += 1

    return (new_files, ddex)

def scanrunner(in_q, out_q, tdex_q):
    """
    Process queue runner
    """
    hproc = multiprocessing.current_process()
    setproctitle("xbake: scanrunner")
    while True:
        # pop next job off the queue; will block until a job is available
        thisjob = in_q.get()
        if thisjob.get('EOF') is not None:
            logthis("Got end-of-queue marker; terminating; pid =", suffix=hproc.pid, loglevel=LL.DEBUG)
            for tkey, tshow in mdb.tdex.iteritems():
                tdex_q.put((tkey, tshow))
            # we need to wait until the master process pulls our items from the queue
            while tdex_q.qsize() > 0 or out_q.qsize() > 0:
                time.sleep(0.1)
            os._exit(0)
        out_q.put((thisjob['rfile'], scanfile(**thisjob)))

def scanfile(rfile, ovrx={}, mforce=False, nochecksum=False, savechecksum=True):
    """
    Examine file: obtain filesystem stats, checksum, ownership; file/path are parsed
    and episode number, season, and series title extracted; file examined with
    mediainfo and container, video, audio, subtitle track info, and chapter data extracted
    """
    dasc = {}

    # get file parts
    xvreal = rfile
    tdir, xv = os.path.split(xvreal)
    xvbase, xvext = os.path.splitext(xv)

    # get base & parent dir names
    tdir_base = os.path.split(tdir)[1]
    tdir_parent = os.path.split(os.path.split(tdir)[0])[1]

    logthis("Examining file:", suffix=xv, loglevel=LL.INFO)
    tstatus('scanfile', event='start', filename=xv)

    # Get xattribs
    fovr = {}
    fovr.update(ovrx)
    fovr.update(parse_xattr_overrides(xvreal))
    if fovr.has_key('ignore'):
        logthis("File has 'ignore' flag set via override; skipping", loglevel=LL.INFO)
        return False

    # Get file path information
    dasc['dpath'] = {'base': tdir_base, 'parent': tdir_parent, 'full': tdir}
    dasc['fpath'] = {'real': xvreal, 'base': xvbase, 'file': xv, 'ext': xvext.replace('.', '')}

    # Stat, Extended Attribs, Ownership
    dasc['stat'] = util.dstat(xvreal)
    dasc['owner'] = {'user': util.getuser(dasc['stat']['uid']), 'group': util.getgroup(dasc['stat']['gid'])}

    # Modification key (MD5 of inode number + mtime + filesize)
    mkey_id = util.getmkey(dasc['stat'])
    dasc['mkey_id'] = mkey_id

    # Determine file status (new, unchanged, or file unchanged but moved/renamed)
    xzist = mdb.mkey_match(mkey_id, xvreal)
    if xzist == MCMP.RENAMED:
        xstatus = DSTS.RENAMED
    elif xzist == MCMP.NOCHG:
        xstatus = DSTS.UNCHANGED
    else:
        xstatus = DSTS.NEW

    dasc['status'] = xstatus

    # Check status and carry on as needed
    if xstatus == DSTS.UNCHANGED:
        logthis("File unchanged:", suffix=xv, loglevel=LL.INFO)
        if mforce:
            logthis("File unchanged, but scan forced. Flag --mforce in effect.", loglevel=LL.WARNING)
        else:
            return False

    # Retrieve or caclulate checksums
    if fovr.has_key('md5') and fovr.has_key('ed2k') and fovr.has_key('crc32'):
        dasc['checksum'] = {'md5': fovr['md5'], 'ed2k': fovr['ed2k'], 'crc32': fovr['crc32']}
        logthis("Using checksum information from extended file attributes", loglevel=LL.VERBOSE)
    else:
        if not nochecksum:
            logthis("Calculating checksum...", loglevel=LL.INFO)
            dasc['checksum'] = util.checksum(xvreal)
            if savechecksum:
                save_checksums(xvreal, dasc['checksum'])

    # Get mediainfo
    dasc['mediainfo'] = util.mediainfo(xvreal, config)

    # Determine series information from path and filename
    dasc['fparse'], dasc['tdex_id'] = parse_episode_filename(dasc, fovr)

    # Record last time this entry was updated (UTC)
    last_up = arrow.utcnow().timestamp
    logthis("last_updated =", suffix=last_up, loglevel=LL.DEBUG)
    dasc['last_updated'] = last_up

    return dasc

def parse_episode_filename(dasc, ovrx={}, single=False, longep=False):
    """
    Determines series name, season, episode, and special release data from
    episode filenames and directory path. Outputs the data as the 'fparse' array.
    """
    fparse = {'series': None, 'season': None, 'episode': None, 'special': None}
    tdex_id = None
    dval = dasc['fpath']['base']

    # Regex matching rounds
    for rgx in fregex:
        logthis("Trying regex:", suffix=rgx, loglevel=LL.DEBUG2)
        mm = re.search(rgx, dval, re.I)
        if mm:
            mm = mm.groupdict()
            # determine series name
            if mm.has_key('series'):
                if not single:
                    ldist = distance.nlevenshtein(dasc['dpath']['base'].lower(), mm['series'].lower())
                    if ldist < 0.26:
                        sser = filter_fname(dasc['dpath']['base'])
                        logthis("Using directory name for series name (ldist = %0.3f)" % (ldist), loglevel=LL.DEBUG)
                    else:
                        sser = filter_fname(mm['series'])
                        logthis("Using series name extracted from filename (ldist = %0.3f)" % (ldist), loglevel=LL.DEBUG)
                else:
                    sser = filter_fname(mm['series'])
                    logthis("Using series name extracted from filename", loglevel=LL.DEBUG)
            else:
                # Check base directory name; if it has the season number or season name
                sspc = re.match(r'(season|s)\s*(?P<season>[0-9]{1,2})', dasc['dpath']['base'], re.I)
                if sspc:
                    # Grab series name from the parent directory; tuck the season number away for later
                    sspc = sspc.groupdict()
                    mm['season'] = sspc['season']
                    sser = dasc['dpath']['parent']
                else:
                    # Directory name should be series name (if you name your directories properly!)
                    sser = dasc['dpath']['base']

            # Parse out the fansub group name
            if mm.get('fansub', None) is not None:
                fansub = mm['fansub'].strip()
            else:
                fansub = None

            # Grab season name from parsed filename; if it doesn't exist, assume Season 1
            snum = mm.get('season', '1')
            if snum is None: snum = '1'

            # Get episode number
            epnum = mm.get('epnum', '0')
            if epnum is None: epnum = '0'

            # Fix episode number, if necessary
            if not longep:
                if int(epnum) > 100:
                    # For numbers over 100, assume SSEE encoding
                    # (ex: 103 = Season 1, Episode 3)
                    epnum = int(mm['epnum'][-2:])
                    snum = int(mm['epnum'][:(len(mm['epnum']) -2)])

            # Get special episode type
            special = mm.get('special', "")
            if special: special = special.strip()

            # Set overrides
            if ovrx:
                if ovrx.has_key('season'):
                    snum = int(ovrx['season'])
                    logthis("Season set by override. Season:", suffix=snum, loglevel=LL.VERBOSE)

                if ovrx.has_key('series_name'):
                    sser = ovrx['series_name']
                    logthis("Series name set by override. Series:", suffix=sser, loglevel=LL.VERBOSE)

                if ovrx.has_key('fansub'):
                    fansub = ovrx['fansub']
                    logthis("Fansub group set by override. Fansub:", suffix=fansub, loglevel=LL.VERBOSE)

            logthis("Matched [%s] with regex:" % (dval), suffix=rgx, loglevel=LL.DEBUG)
            logthis("> Ser[%s] Se#[%s] Ep#[%s] Special[%s] Fansub[%s]" % (sser, snum, epnum, special, fansub), loglevel=LL.DEBUG)

            # Build output fparse array
            fparse = {'series': sser, 'season': int(snum), 'episode': int(epnum), 'special': special, 'fansub': fansub}

            # Add series to tdex
            tdex_id = mdb.series_add(sser, ovrx)

            break

    return (fparse, tdex_id)

def filter_fname(fname):
    """
    Take an input title string that was extracted from a filename, and
    if there are more periods or underscores than spaces, then replace them
    (underscores or periods) with spaces.
    """
    # Count number of spaces, periods, and underscores
    spc = fname.count(' ')
    spd = fname.count('.')
    spu = fname.count('_')

    # No spaces? Something's up
    if spc == 0:
        if spd > spu:
            nout = fname.replace('.', ' ')
        else:
            nout = fname.replace('_', ' ')
        # fix double-spaces
        nout = nout.replace('  ', ' ')
    else:
        # return without changes
        nout = fname

    return fname

def save_checksums(fname, chksums):
    """
    Save checksums to xattribs
    """
    ox = {
            'checksum.md5': chksums.get('md5', ''),
            'checksum.ed2k': chksums.get('ed2k', ''),
            'checksum.crc32': chksums.get('crc32', '')
         }
    logthis("Setting checksum xattribs:", suffix=ox, loglevel=LL.DEBUG)
    fsutil.xattr_set(fname, ox)

xaov_map = {
                'media.seriesname': "series_name",
                'media.season': "season",
                'media.episode': "episode",
                'media.xref.tvdb': "tvdb_id",
                'media.xref.mal': "mal_id",
                'media.fansub': "fansub",
                'checksum.md5': "md5",
                'checksum.ed2k': "ed2k",
                'checksum.crc32': "crc32",
                'checksum.sha1': "sha1",
                'xbake.ignore': "ignore",
                'xbake.tdex': "tdex"
            }

def parse_xattr_overrides(xpath):
    """
    Parse overrides from extended file attributes
    """
    xrides = {}
    xatr = fsutil.xattr_get(xpath)

    for xk, xv in xatr.iteritems():
        if xaov_map.has_key(xk):
            logthis("Got override from xattrib [user.%s]: %s ->" % (xk, xaov_map[xk]), suffix=xv, loglevel=LL.VERBOSE)
            xrides[xaov_map[xk]] = xv

    return xrides

def parse_overrides(xpath):
    """
    Parse overrides file (./.xbake)
    This is a JSON file with special settings for a particular directory.
    Settings include: File ignore list, Series name, tvdb series ID,
    and season number
    """
    xfile = os.path.realpath(xpath + "/" + '.xbake')
    xrides = {}
    if os.path.exists(xfile):
        logthis("Overrides persent for this directory. Parsing file:", suffix=xfile, loglevel=LL.VERBOSE)
        try:
            with codecs.open(xfile, 'r', 'utf-8') as f:
                xrides = json.load(f)
            logthis("Parsed overrides successfully:\n", suffix=print_r(xrides), loglevel=LL.DEBUG)
        except IOError as e:
            logexc(e, "Failed to read overrides file")
        except UnicodeDecodeError as e:
            logexc(e, "Failed to decode overrides file (%s)" % (xfile))
        except ValueError as e:
            logexc(e, "Failed to parse JSON from overrides file")
    else:
        logthis("No overrides for this directory. File does not exist:", suffix=xfile, loglevel=LL.DEBUG)
    return xrides

def check_overrides(ovx, cfile):
    """
    Check override ignore list for matches.
    Return: True if match (file should be skipped/ignored)
            False if no match (file should be processed as usual)
    """
    if isinstance(ovx, list):
        if 'ignore' in ovx:
            for ii in ovx['ignore']:
                if unicode(ii) == unicode(cfile):
                    return True
    return False

def clean_overrides(ovrx):
    """
    Remove unnecessary keys from overrides structure; return new cleaned copy
    """
    ovrx_sub = util.deepcopy(ovrx)
    if 'ignore' in ovrx_sub: del(ovrx_sub['ignore'])
    if 'md5' in ovrx_sub: del(ovrx_sub['md5'])
    if 'crc32' in ovrx_sub: del(ovrx_sub['crc32'])
    if 'ed2k' in ovrx_sub: del(ovrx_sub['ed2k'])
    return ovrx_sub


import pygame
from pygame.locals import *

class GameObject(object):

    animation_swap_time = 75

    def __init__(self):
        self.clips = []
        self.rect  = None
        self.speed = 0
        self.image = None

    def __init__(self, imgPath):
        self.speed = 0
        self.clips = []
        self.image = pygame.image.load(imgPath)
        self.rect  = self.image.get_rect()
        self.current_clip = 0

    def setPos(self, x, y):
        self.rect.left = x
        self.rect.top  = y

    def move(self, xspeed, yspeed):
        self.rect.left += xspeed
        self.rect.top  += yspeed

    def setImage(self, image):
        self.image = image
        self.rect = image.get_rect()
        self.current_clip = -1

    def draw(self):
        if len(self.clips) == 0:
            pygame.display.get_surface().blit(self.image, self.rect)
        else:
            if self.current_clip == 0:
                self.rect.width = self.clips[self.current_clip].width
            self.rect.height = self.clips[self.current_clip].height
            # print self.clips
            pygame.display.get_surface().blit(self.image.subsurface(self.clips[self.current_clip]), self.rect)


import os, sys

import CMGTools.RootTools.fwlite.Config as cfg

def load():
    #load the libaries needed
    from ROOT import gROOT,gSystem
    gSystem.Load("libCintex")
    gROOT.ProcessLine('ROOT::Cintex::Cintex::Enable();')

    #now the RootTools stuff
    gSystem.Load("libCMGToolsExternal")

load()
### sys.path.append( '/'.join( [ os.environ['CMSSW_BASE'],
###                                  'python/CMG/JetIDAnalysis/analyzers'] ))

jetAna = cfg.Analyzer(
    'JetIDAnalyzer',
    ptCut = 20,
    ## use pat::Jets
    ## jetCollection = ('selectedPatJetsAK5','std::vector<pat::Jet>'),
    jetCollection = ('selectedPatJets','std::vector<pat::Jet>'),
    ## jetCollection = ('selectedPatJetsPFlow','std::vector<pat::Jet>'),
    ## or cmg::Jets
    ## jetCollection = ('cmgPFJetSel','std::vector<cmg::PFJet>'),
    doJetIdHisto    = True,
    dumpTree        = False,
    applyPFLooseId  = True,
    jetIdMva = ( 0, "%s/src/CMGTools/External/data/mva_JetID.weights.xml" % os.getenv("CMSSW_BASE"), "JetID" ),
    ## genJetsCollection =  ("ak5GenJets","vector<reco::GenJet>"),
    genJetsCollection =  (("selectedPatJets","genJets"),"vector<reco::GenJet>"),
    ## genJetsCollection =  (("selectedPatJetsPFlow","genJets"),"vector<reco::GenJet>"),
    useGenLeptons = False,
)


##############################################################################
# 6.1.7 어느 갱신 방법을 이용할 것인가?
# SGD, 모멘텀, AdaGrad, Adam의 학습 패턴을 비교합니다.
##############################################################################

import sys,os
sys.path.append(os.pardir)  # 부모 디렉터리의 파일을 가져올 수 있도록 설정
import numpy as np
import matplotlib.pyplot as plt
from collections import OrderedDict
from common.optimizer import *

# 함수 f
def f(x, y):
    return x ** 2 / 20.0 + y ** 2

# f의 미분 함수
def df(x, y):
    return x / 10.0, 2.0 * y

init_pos = (-7.0, 2.0)

# 초기화
params = {}
params['x'], params['y'] = 0, 0
grads = {}
grads['x'], grads['y'] = 0, 0

"""Optimezer 별로 학습 패턴 비교"""
optimizers = OrderedDict()
optimizers["SGD"] = SGD(lr=0.95)
optimizers["Momentum"] = Momentum(lr=0.1)
optimizers["AdaGrad"] = AdaGrad(lr=1.5)
optimizers["RMSprop"] = RMSprop(lr=0.2)
# optimizers["Adam"] = Adam(lr=0.3)

# 크게 10번만 가보자
# optimizers["SGD"] = SGD(lr=1.05)
# optimizers["Momentum"] = Momentum(lr=0.8)

# 작게 99번만 가보자
# optimizers["SGD"] = SGD(lr=0.35)
# optimizers["Momentum"] = Momentum(lr=0.033)

# 더 작게...
# optimizers["Momentum 0.01"] = Momentum(lr=0.01)   # 380
# optimizers["Momentum 0.001"] = Momentum(lr=0.001) # 3500

idx = 1
# optimizer 별로 실행해서 비교해보자
for key in optimizers:
    optimizer = optimizers[key]
    x_history = []
    y_history = []
    params['x'], params['y'] = init_pos[0], init_pos[1]  # x와 y 초기값 설정

    # 30회 반복
    for i in range(30):
        x_history.append(params['x'])
        y_history.append(params['y'])

        # 기울기 구하기
        grads['x'], grads['y'] = df(params['x'], params['y'])   # 미분

        # 기울기 최적화 업데이트
        #   - 예)     초기값 세팅된 1회 : params: {'x': -7.0, 'y': 2.0} & grads : {'x': 0, 'y': 0}
        #   - 예) 최적화 업데이트 후 2회 : params: {'x': -5.5, 'y': 0.5} & grads : {'x': -0.7, 'y': 4.0}
        #         ...
        optimizer.update(params, grads)

    # x축과 y축의 값
    x = np.arange(-10, 10, 0.01)  # -10 <= x < 10 (0.01 만큼씩) : x: [-10. - 9.99 - 9.98...   9.97   9.98   9.99]
    y = np.arange(-5, 5, 0.01)    #  -5 <= y <  5 (0.01 만큼씩) : y: [ -5. - 4.99 - 4.98...   4.97   4.98   4.99]

    # x축을 나타내는 점들, y축을 나타내는 점들 => 하나의 그리드
    X, Y = np.meshgrid(x, y)
    # 최적화된 기울기를 함수 f 결과 값
    Z = f(X, Y)

    # 외곽선 단순화
    mask = Z > 7
    Z[mask] = 0  # 결과값 중 7보다 큰 것을(True) 0으로 masking.

    # 그래프 그리기 (2 X 2, )
    plt.subplot(2, 2, idx)
    idx += 1
    plt.plot(x_history, y_history, 'o-', color="red")
    plt.contour(X, Y, Z)    # X, Y, Z 윤곽 표시 (등고선)
    plt.ylim(-10, 10)
    plt.xlim(-10, 10)
    plt.plot(0, 0, '+')  # (0,0) 위치 표시
    # colorbar()
    # spring()
    plt.title(key)
    plt.xlabel("x")
    plt.ylabel("y")

plt.show()


from scipy.io import loadmat
import h5py  # tables  # or
import numpy as np
import os
import torch
import pickle

class RenderWithTransportMat():

    def __init__(self, transportMatFname, lightHeight, doHalfMat=True):
        self.transportMatFname = transportMatFname
        self.mat_ImgHeight = lightHeight
        # self.renderHeight = renderHeight

        self.doHalfMat = doHalfMat

        self.matT, self.maskT = self.load()
        # self.renderVectors = None

    def load(self):
        typeMat = 'half' if self.doHalfMat else 'full'
        # print('[Render]loading: %s' % (os.path.split(self.transportMatFname)[1]))
        assert os.path.exists(self.transportMatFname), 'FileNotFound %s' % self.transportMatFname
        ext = os.path.split(self.transportMatFname)[1].split('.')[-1]
        if ext == 'mat':
            try:
                mat = loadmat(self.transportMatFname)
                matT = mat['T'].astype('float32')
                maskT = mat['mask'].astype('bool')
            except NotImplementedError:
                mat = h5py.File(self.transportMatFname, 'r')
                matT = mat.get('T')[:].astype('float32').transpose()
                maskT = mat.get('mask')[:].astype('bool').transpose()
                mat.close()
        elif ext == 'pkl':
            with open(self.transportMatFname, 'rb') as infile:
                mat = pickle.load(infile)
                matT = mat['T'].astype('float32')
                maskT = mat['mask'].astype('bool')
        elif ext == 'msgpack':
            import lz4.frame, msgpack, msgpack_numpy
            from functools import partial
            lz4open = partial(lz4.frame.open, block_size=lz4.frame.BLOCKSIZE_MAX1MB,
                            compression_level=lz4.frame.COMPRESSIONLEVEL_MIN)
            with lz4open(self.transportMatFname, "rb") as infile:
                raw = infile.read()
                mat = msgpack.unpackb(raw, object_hook=msgpack_numpy.decode, max_str_len=2**32-1)
                matT = mat[b'T'].astype('float32')
                maskT = mat[b'mask'].astype('bool')
        # matT.shape == [r*r, eH*eW]
        return matT, maskT

    def rendering(self, ims, dst='vector'):
        if type(ims).__module__.find('numpy') >= 0:
            if self.doHalfMat:
                ims = ims.astype('float32')
                H = ims.shape[1]
                ims = ims[:, 0: H // 2, :, :]
                render_vector = self.renderingNP(ims)
                if dst == 'vector':
                    out = render_vector
                elif dst == 'image':
                    out = self.reshapeNP(render_vector)
            return out
        elif type(ims).__module__.find('torch') >= 0:
            if self.doHalfMat:
                H = self.mat_ImgHeight
                ims = ims[:, 0: H // 2, :, :]
                # print("Ims",ims.shape)
                render_vector = self.renderingTorch(ims)
                if dst == 'vector':
                    out = render_vector
                elif dst == 'image':
                    # print("Render", render_vector.shape)
                    out = self.reshapeTorch(render_vector)
                    out = out.permute(0,3,1,2)
                    # print("Out ", out.shape)
            return out

    def renderingTorch(self, ims):
        matT = torch.Tensor(self.matT).to(ims.device)
        nPixels = matT.shape[0]
        N, H, W, C = ims.shape
        assert H * W == matT.shape[1], '[ERROR] transportMat %d doesn\'t match image size %d' % (H * W, matT.shape[1])

        # operation from matlab, reshape in column order
        ims = ims.permute([0, 2, 1, 3])
        lights = ims.contiguous().view((-1, H * W, C))  # [N, #pixel, C]
        # transpose and reshape
        lightsT = lights.permute([1, 0, 2])  # [W*H, N, C]
        lightsTR = lightsT.contiguous().view([W * H, -1])  # [W*H, N*C]
        vectors = torch.matmul(matT, lightsTR)  # [#pixel, W*H] x [W*H, N*C)] -> [#pixel, N*C]
        vectors = vectors.view([nPixels, -1, C])  # [#pixel, N, C]
        vectors = vectors.permute([1, 0, 2])  # [N, #pixel, C]
        renderVectors = vectors
        return renderVectors

    def _reshapeTorch(self, tl_WHxC):
        # todo batch reshape
        C = tl_WHxC.shape[1]
        img = torch.zeros((self.maskT.shape[0], self.maskT.shape[1], C)).to(tl_WHxC.device)
        # img[torch.Tensor(self.maskT.astype(int)).nonzero()] = tl_WHxC. # non differentiable
        img[torch.Tensor(self.maskT.astype(int)).to(tl_WHxC.device) == 1] = tl_WHxC

        img = img.permute([1, 0, 2])
        return img

    def reshapeTorch(self, renderVectors):
        N = renderVectors.shape[0]
        ims = [self._reshapeTorch(renderVectors[i]) for i in range(N)]
        return torch.stack(ims)

    def renderingNP(self, ims):
        matT = self.matT
        nPixels = matT.shape[0]
        N, H, W, C = ims.shape
        assert H * W == matT.shape[1], '[ERROR] transportMat %d doesn\'t match image size %d' % (H * W, matT.shape[1])

        # operation from matlab, reshape in column order
        ims = np.transpose(ims, axes=[0, 2, 1, 3])
        lights = np.reshape(ims, (-1, H * W, C))  # [N, #pixel, C]
        # transpose and reshape
        lightsT = np.transpose(lights, axes=[1, 0, 2])  # [W*H, N, C]
        lightsTR = np.reshape(lightsT, [W * H, -1])  # [W*H, N*C]
        vectors = np.matmul(matT, lightsTR)  # [#pixel, W*H] x [W*H, N*C)] -> [#pixel, N*C]
        vectors = np.reshape(vectors, [nPixels, -1, C])  # [#pixel, N, C]
        vectors = np.transpose(vectors, axes=[1, 0, 2])  # [N, #pixel, C]
        renderVectors = vectors
        return renderVectors

    def _reshapeNP(self, tl_WHxC):
        # todo batch reshape
        C = tl_WHxC.shape[1]
        img = np.zeros((self.maskT.shape[0], self.maskT.shape[1], C))
        img[self.maskT] = tl_WHxC
        img = np.ndarray.transpose(img, [1, 0, 2])
        return img

    def reshapeNP(self, renderVectors):
        N = renderVectors.shape[0]
        ims = [self._reshapeNP(renderVectors[i]) for i in range(N)]
        return np.asarray(ims)

    def render_top_down(self, img,chw=True):
        if chw:
            img = img.permute(0,2,3,1)
        if type(img).__module__.find('numpy') >= 0:
            reverse_img = np.flip(img, [1])

        elif type(img).__module__.find('torch') >= 0:
            reverse_img = torch.flip(img, [1])
        else :
            print('[ERROR] Input img is neither torch nor numpy')

        return {"top" : self.rendering(img, dst='image'), "bottom": self.rendering(reverse_img, dst='image')}

def example():
    import time
    im_h = 256
    img_dim = 64
    rtm = RenderWithTransportMat(transportMatFname='transportMat.BumpySphereMiddle.top.e64.r64.half.mat', lightHeight=img_dim, doHalfMat=True)
    ims = np.ones([256, img_dim, img_dim*2, 3])
    print(ims.shape)
    # ims shape = [b,h,w,c]
    # pred shape = [b,c,h,w]

    # pred = [256,512]
    # pred = [64,128] = > [32,128]*2
    # what it wants = [128,64]=> [64,64]*2
    p = rtm.render_top_down(torch.from_numpy(ims).type('torch.FloatTensor'))
    print(p['top'].shape)
    print(p['bottom'].shape)

if __name__ == '__main__':
    import sys
    print(sys.version)
    example()


# -*- mode: python ; coding: utf-8 -*-

block_cipher = None

added_files = [('.env', '.'),('*.py','.')]
a = Analysis(['esriOverwrite.py'],
             pathex=[],
             binaries=[],
             datas=added_files,
             hiddenimports=["requests_ntlm","arcgis", "pyodbc", "sqlalchemy", 'dotenv', 'tqdm'],
             hookspath=[],
             hooksconfig={},
            runtime_hooks=[],
            excludes=['arcpy'],
             win_no_prefer_redirects=False,
             win_private_assemblies=False,
             cipher=block_cipher,
             noarchive=False)
a.datas += Tree('./arcgis', prefix='arcgis')
pyz = PYZ(a.pure, a.zipped_data,
             cipher=block_cipher)

exe = EXE(pyz,
          a.scripts,
          [],
          exclude_binaries=True,
          name='esriOverwrite',
          debug=False,
          bootloader_ignore_signals=False,
          strip=False,
          upx=True,
          console=True,
          disable_windowed_traceback=False,
          target_arch=None,
          codesign_identity=None,
          entitlements_file=None )
coll = COLLECT(exe,
               a.binaries,
               a.zipfiles,
               a.datas,
               strip=False,
               upx=True,
               upx_exclude=[],
               name='esriOverwrite')


import graphviz
from ShExJSG.ShExJ import IRIREF, TripleConstraint, NodeConstraint, ObjectLiteral
from pyshex.utils.schema_loader import SchemaLoader

symbol = {
    "class": "oval",
    "datatype": "octagon",
    "literal": "rectangle",
    "iri": "diamond",
    "bnode": 'point',
    "oneof": 'record'
}

def shex2dot(shex, graphviz_name="default.png", rankdir="LR"):
    def process_tc(tc):
        dotschema.node(startshape.replace(":", ""), startshape, shape=symbol["iri"])
        if isinstance(tc, TripleConstraint):
            if tc.max == None and tc.min == None:
                arrowhead = "normal"
            elif tc.max == 1 and tc.min == 0:
                arrowhead = "teeodot"
            elif tc.max == -1 and tc.min == 1:
                arrowhead = "crowtee"
            elif tc.max == -1 and tc.min == 0:
                arrowhead = "crowdot"
            if isinstance(tc.valueExpr, IRIREF):
                node = tc.valueExpr
                predicate = tc.predicate
                for key in prefixmap.keys():
                    node = node.replace(key, prefixmap[key])
                    predicate = predicate.replace(key, prefixmap[key] + ":")
                dotschema.node(node, node, shape=symbol["iri"])
                dotschema.edge(shape.id.split("/")[-1], node, label=predicate, arrowhead=arrowhead)
            elif isinstance(tc.valueExpr, NodeConstraint):
                if tc.valueExpr.datatype:
                    datatype = tc.valueExpr.datatype
                    predicate = tc.predicate
                    for key in prefixmap.keys():
                        datatype = datatype.replace(key, prefixmap[key] + ":")
                        predicate = predicate.replace(key, prefixmap[key] + ":")
                    dotschema.node(
                        shape.id.split("/")[-1] + tc.valueExpr.datatype.split("/")[-1] + tc.predicate.split("/")[-1],
                        datatype, shape=symbol["datatype"])
                    dotschema.edge(shape.id.split("/")[-1],
                                   shape.id.split("/")[-1] + tc.valueExpr.datatype.split("/")[-1] +
                                   tc.predicate.split("/")[
                                       -1], label=predicate, arrowhead=arrowhead)
                elif tc.valueExpr.values:
                    oneofs = []
                    predicate = tc.predicate
                    for value in tc.valueExpr.values:
                        if isinstance(value, ObjectLiteral):
                            oneofs.append(value.value)
                        else:
                            for key in prefixmap.keys():
                                try:
                                    value = value.replace(key, prefixmap[key] + ":")
                                except:
                                    value = "a"
                            oneofs.append(value)
                    for key in prefixmap.keys():
                        predicate = predicate.replace(key, prefixmap[key] + ":")
                    dotschema.node(
                        shape.id.split("/")[-1] + "|".join(oneofs).replace(":", "") + tc.predicate.split("/")[-1],
                        "{" + "|".join(oneofs) + "}", shape=symbol["oneof"])
                    dotschema.edge(shape.id.split("/")[-1],
                                   shape.id.split("/")[-1] + "|".join(oneofs).replace(":", "") +
                                   tc.predicate.split("/")[
                                       -1], label=predicate)

                elif tc.valueExpr.nodeKind:
                    dotschema.node(tc.valueExpr.nodeKind, tc.valueExpr.nodeKind.split("/")[-1],
                                   shape=symbol[tc.valueExpr.nodeKind])
                    dotschema.edge(shape.id.split("/")[-1], tc.valueExpr.nodeKind.split("/")[-1],
                                   label=tc.predicate.split("/")[-1], arrowhead="teedot")
                elif tc.valueExpr.xone:

                    dotschema.node(tc.valueExpr.xone[0].identifier, tc.valueExpr.xone[0].identifier.split("/")[-1],
                                   shape=symbol["oneof"])
                    dotschema.edge(shape.id.split("/")[-1], tc.valueExpr.xone[0].identifier.split("/")[-1],
                                   label=tc.predicate.split("/")[-1], arrowhead="teedot")
                else:
                    pass
                    # print("No valueExpr")
            else:
                pass
                # print("No valueExpr")

    dotschema = graphviz.Digraph(graphviz_name, format="png")
    dotschema.attr(rankdir=rankdir)
    prefixmap = dict()
    for line in shex.splitlines():
        if line.startswith("PREFIX"):
            line = line.replace("PREFIX", "")
            prefix, uri = line.split(": ")
            prefix = prefix.strip()
            uri = uri.strip()
            prefixmap[uri.replace("<", "").replace(">", "")] = prefix
            dotschema.node(prefix, uri, shape="none", style="invis")

    loader = SchemaLoader()
    schema = loader.loads(shex)

    for shape in schema.shapes:
        if id in (dir(shape)):
            continue
        startshape = shape.id
        for key in prefixmap.keys():
            startshape = startshape.replace(key, prefixmap[key] + ":")
        if "expressions" in dir(shape.expression):
            for tc in shape.expression.expressions:
                process_tc(tc)
        else:
            tc = shape.expression
            process_tc(tc)

    return dotschema

def view_graphviz(dotschema):
    return dotschema.view()

def save_graphviz(dotschema, filename):
    return dotschema.render(filename)


from django.views import generic
from django.urls import reverse_lazy
from .models import NewsStory
from .forms import StoryForm

class IndexView(generic.ListView):
    template_name = 'news/index.html'

    def get_queryset(self):
        '''Return all news stories.'''
        return NewsStory.objects.order_by('pub_date')

    def get_context_data(self, **kwargs):
        print(self.request.GET.get('search'))
        search_var = self.request.GET.get('search',"")
        context = super().get_context_data(**kwargs)
        # context['latest_stories'] = NewsStory.objects.order_by('-pub_date')[:3]
        # context['all_stories'] = NewsStory.objects.order_by('-pub_date')[3:100]
        # context['latest_stories'] = NewsStory.objects.order_by('-pub_date').filter(title__contains=search_var)[:3]
        # context['all_stories'] = NewsStory.objects.order_by('-pub_date').filter(title__contains=search_var)[3:]
        context['latest_stories'] = NewsStory.objects.order_by('-pub_date').filter(author__username__contains=search_var)[:3]
        context['all_stories'] = NewsStory.objects.order_by('-pub_date').filter(author__username__contains=search_var)[3:]
        return context

class StoryView(generic.DetailView):
    model = NewsStory
    template_name = 'news/story.html'
    context_object_name = 'story'

class AddStoryView(generic.CreateView):
    form_class = StoryForm #from \news\forms.py
    context_object_name = 'storyForm' #just a name for the object
    template_name = 'news/createStory.html'
    success_url = reverse_lazy('news:index') #the path we can post things to when they are succesful

    #overiding form_valid on generic.CreateView
    def form_valid(self, form):
        #set author to logged in user
        form.instance.author = self.request.user
        return super().form_valid(form)


from playwright.sync_api import sync_playwright
from pathlib import Path
from urllib.parse import urlparse, urlunparse
import time

"""
https://playwright.dev/python/docs/api/class-route
"""
def custom_headers(url) -> dict:
    res = urlparse(url)
    loc = res.netloc
    host = urlunparse([res.scheme, res.netloc, '', '', '', ''])
    page_url = urlunparse([res.scheme, res.netloc, res.path, '', '', ''])
    return {
        'authority': loc,
        'accept': 'application/json, text/javascript, */*; q=0.01',
        'accept-language': 'zh-CN,zh;q=0.9',
        'origin': host,
        'referer': host,
        'sec-ch-ua': '"Google Chrome";v="113", "Chromium";v="113", "Not-A.Brand";v="24"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"macOS"',
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'same-site',
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',
        'x-referer-page': page_url,
        'x-rp-client': 'h5_1.0.0',
    }

with sync_playwright() as p:
    url = 'https://item.jd.com/7836786.html#crumb-wrap'
    extra_http_headers = custom_headers(url)
    browser = p.chromium.launch(headless=False)
    context = browser.new_context(extra_http_headers=extra_http_headers)
    page = context.new_page()
    page.goto(url)
    print(context.storage_state())
    time.sleep(5)
    page.screenshot(path='example1.png')
    browser.close()


import numpy as np
from scipy.stats import multivariate_normal
from scipy.special import gamma
from environment import Environment
from sklearn.neighbors import KernelDensity

import torch
import time

class Trainer():

    def __init__(self, agent,
                 OBS_LEAK=1e-3,
                 EPSILON=1e-3,
                 ref_prob='unif',
                 final=False,
                 monte_carlo=False,
                 Q_learning=False,
                 KL_reward=False,
                 augmentation=True,
                 KL_correction=False,
                 Q_ref_correction=False,
                 BATCH_SIZE=20,
                 KL_centering=True,
                 rtg_centering=True):
        self.agent = agent
        self.nb_trials = 0
        self.init_trial(update=False)
        self.nb_visits = np.zeros(self.agent.env.N_obs)
        self.obs_score = np.zeros(self.agent.env.N_obs)
        self.nb_visits_final = np.zeros(self.agent.env.N_obs)
        self.obs_score_final = np.zeros(self.agent.env.N_obs)
        self.mem_obs_final = []
        self.mem_total_reward = []
        self.mem_total_reward_test = []
        self.mem_mean_rtg = []
        self.mem_KL_final = []
        self.mem_t_final = []
        self.mem_t_final_test = []
        self.OBS_LEAK = OBS_LEAK
        self.EPSILON = EPSILON
        self.HIST_HORIZON = agent.HIST_HORIZON
        self.mem_V = {}
        self.ref_prob = ref_prob
        self.final = final
        self.monte_carlo = monte_carlo
        self.Q_learning = Q_learning
        self.KL_reward = KL_reward
        self.augmentation = augmentation
        self.KL_correction = KL_correction
        self.Q_ref_correction = Q_ref_correction
        self.BATCH_SIZE=BATCH_SIZE
        self.KL_centering = KL_centering
        self.rtg_centering = rtg_centering

    def init_trial(self, update=True):
        if update:
            self.nb_trials += 1
        self.total_reward = 0
        self.trajectory = []
        self.action_history = []
        self.actions_set_history = []
        self.reward_history = []
        self.rtg_history = []
        self.ktg_history = []

    def calc_state_probs(self):
        self.obs_score / np.sum(self.obs_score)

    def calc_final_state_probs(self):
        return self.obs_score_final / np.sum(self.obs_score_final)

    def calc_ref_probs(self, obs, EPSILON=1e-3):
        if self.ref_prob == 'unif':
            # EXPLORATION DRIVE
            if self.augmentation:
                p = np.zeros(self.agent.N_obs)
                if self.final:
                    p[np.where(self.nb_visits_final > 0)] = 1 / np.sum(self.nb_visits_final > 0)
                else:
                    p[np.where(self.nb_visits > 0)] = 1 / np.sum(self.nb_visits > 0)
            else:
                p = np.ones(self.agent.N_obs) / self.agent.N_obs
            return p
        else:
            # SET POINT
            ref_probs = np.ones(self.agent.env.N_obs) * EPSILON / self.agent.env.N_obs
            ref_probs[int(self.ref_prob)] += (1 - EPSILON)
            return ref_probs

    def KL(self, obs, done=False, verbose=False):
        if self.final: # Only final state for probability calculation
            if done:
                final_state_probs = self.calc_final_state_probs()
                ref_probs = self.calc_ref_probs(obs, EPSILON=self.EPSILON)
                return np.log(final_state_probs[obs]) - np.log(ref_probs[obs])
            else:
                return 0
        else:
            state_probs = self.calc_state_probs()
            ref_probs = self.calc_ref_probs(obs, EPSILON=self.EPSILON)
            return np.log(state_probs[obs]) - np.log(ref_probs[obs])  # +1

    def calc_sum_future_KL(self, obs, obs_or_time, done, actions_set=None):
        sum_future_KL = self.KL(obs, done=done)
        if not done:
            next_values = self.agent.set_Q_obs(obs_or_time,
                                               Q=self.agent.Q_KL,
                                               actions_set=actions_set)
            next_sum = self.agent.softmax_expectation(obs_or_time,
                                                      next_values,
                                                      actions_set=actions_set)
            sum_future_KL += self.agent.GAMMA * next_sum
        return sum_future_KL

    def online_KL_err(self, past_obs_or_time, past_action, obs, obs_or_time, done=False):
        sum_future_KL = self.calc_sum_future_KL(obs, obs_or_time, done)
        return self.agent.Q_KL(past_obs_or_time, past_action) - sum_future_KL

    def calc_sum_future_rewards(self, reward, obs_or_time, done, actions_set=None):
        sum_future_rewards = reward
        if not done:
            next_values = self.agent.set_Q_obs(obs_or_time,
                                               Q=self.agent.Q_ref,
                                               actions_set=actions_set)
            next_sum = self.agent.softmax_expectation(obs_or_time,
                                                        next_values,
                                                        actions_set=actions_set)
            sum_future_rewards += self.agent.GAMMA * next_sum

        return sum_future_rewards

    def calc_TD_err_ref(self, sum_future_rewards, past_obs_or_time, past_action):
        return self.agent.PREC * (self.agent.Q_ref(past_obs_or_time, past_action) - sum_future_rewards)

    def online_TD_err_ref(self, past_obs_or_time, past_action, obs_or_time, reward, done=False):
        sum_future_rewards = self.calc_sum_future_rewards(reward, obs_or_time, done)
        return self.calc_TD_err_ref(sum_future_rewards, past_obs_or_time, past_action)

    def calc_TD_err_var(self, sum_future_rewards, sum_future_KL, past_obs, past_obs_or_time, past_action):
        if self.Q_learning:
            mult_Q = 0
        else:
            mult_Q = 1
        return self.agent.PREC * (self.agent.Q_var(past_obs_or_time, past_action) - sum_future_rewards) \
                                  + 1 / self.agent.BETA * mult_Q * sum_future_KL

    def online_TD_err_var(self, past_obs, past_obs_or_time, past_action, obs, obs_or_time, reward, done=False):
        sum_future_rewards = self.calc_sum_future_rewards(reward, obs_or_time, done)
        if self.rtg_centering:
            sum_future_rewards -= np.mean(self.agent.Q_ref_tab)
        sum_future_KL = self.agent.Q_KL(past_obs_or_time, past_action)
        if self.KL_centering:
            sum_future_KL -= np.mean(self.agent.Q_KL_tab)
        return self.calc_TD_err_var(sum_future_rewards, sum_future_KL, past_obs, past_obs_or_time, past_action)

    def online_update(self, past_obs, past_action, obs, reward, done, past_time, current_time, actions_set=None):
        if self.agent.isTime:
            past_obs_or_time = past_time
            obs_or_time = current_time
        else:
            past_obs_or_time = past_obs
            obs_or_time = obs
        if not self.Q_learning:
            self.agent.Q_KL_tab[past_obs_or_time, past_action] -= self.agent.ALPHA * self.online_KL_err(past_obs_or_time,
                                                                                past_action,
                                                                                obs,
                                                                                obs_or_time,
                                                                                done=done)
        if not self.agent.do_reward:
            self.agent.Q_ref_tab[past_obs_or_time, past_action] -= self.agent.ALPHA * self.online_TD_err_ref(past_obs_or_time,
                                                                                past_action,
                                                                                obs_or_time,
                                                                                reward,
                                                                                done=done)
        self.agent.Q_var_tab[past_obs_or_time, past_action] -= self.agent.ALPHA * self.agent.Q_VAR_MULT * self.online_TD_err_var(past_obs,
                                                                                past_obs_or_time,
                                                                                past_action,
                                                                                obs,
                                                                                obs_or_time,
                                                                                reward,
                                                                                done=done)
    def monte_carlo_update(self, done):
        if done:
            final_time = self.agent.get_time()
            liste_KL = np.zeros(final_time+1)
            liste_sum_KL = np.zeros(final_time)
            liste_rtg = np.zeros(final_time+1)

            if not self.Q_learning:
                # FIRST LOOP
                for time in range(final_time):
                    new_obs = self.trajectory[time + 1]
                    test_done = final_time == time + 1
                    liste_KL[time] = self.KL(new_obs, done=test_done)
                if self.KL_correction:
                    liste_KL[final_time] = self.calc_sum_future_KL(new_obs, new_obs, False)

                # SECOND LOOP
                for time in range(final_time):
                    liste_sum_KL[time]  = np.sum(np.array(liste_KL[time:]) * \
                                           self.agent.GAMMA **(np.arange(time, final_time+1) - time))
                    self.ktg_history.append(liste_sum_KL[time])

            if self.agent.do_reward:
                for time in range(final_time):
                    liste_rtg[time] = np.sum(np.array(self.reward_history[time:]) * \
                                          self.agent.GAMMA **(np.arange(time, final_time) - time))
                    self.rtg_history.append(liste_rtg[time])

            # THIRD LOOP
            if self.KL_centering:
                mean_KL_final = np.mean(self.mem_KL_final[-100:])
            if self.agent.do_reward and self.rtg_centering:
                mean_mean_rtg = np.mean(self.mem_mean_rtg[-100:])
            for time in range(final_time):                ## !!!! faux dans le cas "full KL" et "full reward" !!!! TODO ##
                past_obs = self.trajectory[time]
                if self.agent.isTime:
                    past_obs_or_time = time
                else:
                    past_obs_or_time = self.trajectory[time]
                past_action = self.action_history[time]
                sum_future_KL = liste_sum_KL[time]  #
                if self.KL_centering:
                    sum_future_KL -= mean_KL_final
                sum_future_rewards = liste_rtg[time]
                if self.agent.do_reward and self.rtg_centering:
                    sum_future_rewards -= mean_mean_rtg

                if self.nb_trials > 10:
                    TD_err_ref = self.calc_TD_err_ref(sum_future_rewards, past_obs_or_time, past_action)
                    self.agent.Q_ref_tab[past_obs_or_time, past_action] -= self.agent.ALPHA * TD_err_ref

                    TD_err_var = self.calc_TD_err_var(sum_future_rewards,
                                                      sum_future_KL ,
                                                      past_obs,
                                                      past_obs_or_time,
                                                      past_action)
                    self.agent.Q_var_tab[past_obs_or_time, past_action] -= self.agent.ALPHA * TD_err_var

    def run_episode(self, train=True, render=False, verbose=False):
        self.agent.init_env()
        self.init_trial(update=train)
        obs = self.agent.get_observation()
        self.trajectory.append(obs)
        tic = time.clock()

        while True:

            past_time = self.agent.get_time()
            past_obs = obs #self.agent.get_observation()
            actions_set = None

            ########### STEP #############
            if train:
                past_obs_or_time, past_action, obs_or_time, reward, done = self.agent.step(actions_set = actions_set)
            else:
                past_obs_or_time, past_action, obs_or_time, reward, done = self.agent.step(actions_set = actions_set, test=True)
            ##############################

            current_time = self.agent.get_time()
            obs = self.agent.get_observation()

            self.action_history.append(past_action)
            self.actions_set_history.append(actions_set)
            self.trajectory.append(obs)

            self.nb_visits[obs] += 1
            self.obs_score *= 1 - self.OBS_LEAK
            self.obs_score[obs] += 1
            self.agent.num_episode += 1

            if done:
                self.nb_visits_final[obs] += 1
                self.obs_score_final *= 1 - self.OBS_LEAK
                self.obs_score_final[obs] += 1
                self.mem_obs_final += [obs]

            if past_time == 0 and train:
                self.state_probs = self.calc_state_probs()
                self.ref_probs = self.calc_ref_probs(obs)

            mem_reward = reward
            if not self.agent.do_reward:
                reward = 0
            if self.KL_reward:
                reward -= self.KL(obs, done=done)
            self.reward_history.append(reward)
            self.total_reward += mem_reward

            ########### LEARNING STEP #############
            if train:
                if self.monte_carlo:
                    self.monte_carlo_update(done)
                else:
                    self.online_update(past_obs, past_action, obs, reward, done, past_time, current_time, actions_set=actions_set)
            #######################################

            if render and type(self.agent.env) is not Environment:
                self.agent.env.render()

            if done:
                if train:
                    KL_final = self.KL(obs, done=done)
                    self.mem_KL_final.append(KL_final)
                    self.mem_t_final.append(current_time)
                    self.mem_total_reward.append(self.total_reward)
                    self.mem_mean_rtg.append(np.mean(self.rtg_history))
                    if verbose:
                        print('obs:', obs, 'final KL loss:', KL_final) #, 'final time:', current_time, 'total reward:', self.total_reward)
                    if self.nb_trials % 100 == 0 and not self.agent.isTime:
                        V = np.zeros(self.agent.N_obs)
                        for obs in range(self.agent.N_obs):
                            V[obs] = self.agent.softmax_expectation(obs, self.agent.set_Q_obs(obs), actions_set=actions_set)
                        self.mem_V[self.nb_trials] = V
                else:
                    self.mem_t_final_test.append(current_time)
                    self.mem_total_reward_test.append(self.total_reward)
                break
        toc = time.clock()
        if verbose:
            print('Time elapsed :', toc-tic)

class Q_learning_trainer(Trainer):

    def __init__(self, agent,
                 EPSILON=1e-3,
                 OBS_LEAK=1e-3,
                 ref_prob='unif',
                 final=False,
                 monte_carlo=False,
                 KL_reward=False,
                 augmentation=False,
                 KL_correction=False,
                 Q_ref_correction=False,
                 BATCH_SIZE=20,
                 KL_centering=True,
                 rtg_centering=True):
        super().__init__(agent,
                         EPSILON=EPSILON,
                         OBS_LEAK=OBS_LEAK,
                         ref_prob=ref_prob,
                         final=final,
                         monte_carlo=monte_carlo,
                         Q_learning=True,
                         KL_reward=KL_reward,
                         augmentation=augmentation,
                         KL_correction=KL_correction,
                         Q_ref_correction=Q_ref_correction,
                         BATCH_SIZE=BATCH_SIZE,
                         KL_centering=KL_centering,
                         rtg_centering=rtg_centering)

class KL_Q_learning_trainer(Trainer):

    def __init__(self, agent,
                 EPSILON=1e-3,
                 OBS_LEAK=1e-3,
                 ref_prob='unif',
                 final=False,
                 monte_carlo=False,
                 KL_reward=True,
                 augmentation=True,
                 KL_correction=False,
                 Q_ref_correction=False,
                 BATCH_SIZE=20,
                 KL_centering=True,
                 rtg_centering=True):
        super().__init__(agent,
                         EPSILON=EPSILON,
                         OBS_LEAK=OBS_LEAK,
                         ref_prob=ref_prob,
                         final=final,
                         monte_carlo=monte_carlo,
                         Q_learning=True,
                         KL_reward=KL_reward,
                         augmentation=augmentation,
                         KL_correction=KL_correction,
                         Q_ref_correction=Q_ref_correction,
                         BATCH_SIZE=BATCH_SIZE,
                         KL_centering=KL_centering,
                         rtg_centering=rtg_centering)

class One_step_variational_trainer(Trainer):

    def __init__(self, agent,
                 EPSILON=1e-3,
                 OBS_LEAK=1e-3,
                 ref_prob='unif',
                 final=False,
                 monte_carlo=False,
                 Q_learning=False,
                 KL_reward=False,
                 augmentation=True,
                 KL_correction=False,
                 Q_ref_correction=False,
                 BATCH_SIZE=20,
                 KL_centering=True,
                 rtg_centering=True):
        super().__init__(agent,
                         EPSILON=EPSILON,
                         OBS_LEAK=OBS_LEAK,
                         ref_prob=ref_prob,
                         final=final,
                         monte_carlo=monte_carlo,
                         Q_learning=False,
                         KL_reward=KL_reward,
                         augmentation=augmentation,
                         KL_correction=KL_correction,
                         Q_ref_correction=Q_ref_correction,
                         BATCH_SIZE=BATCH_SIZE,
                         KL_centering=KL_centering,
                         rtg_centering=rtg_centering)

class Final_variational_trainer(Trainer):

    def __init__(self, agent,
                 EPSILON=1e-3,
                 OBS_LEAK=1e-3,
                 ref_prob='unif',
                 final=False,
                 monte_carlo=False,
                 Q_learning=False,
                 KL_reward=False,
                 augmentation=True,
                 KL_correction=False,
                 Q_ref_correction=False,
                 BATCH_SIZE=20,
                 KL_centering=True,
                 rtg_centering=True):
        super().__init__(agent,
                         EPSILON=EPSILON,
                         OBS_LEAK=OBS_LEAK,
                         ref_prob=ref_prob,
                         final=final,
                         monte_carlo=monte_carlo,
                         Q_learning=Q_learning,
                         KL_reward=KL_reward,
                         augmentation=augmentation,
                         KL_correction=KL_correction,
                         Q_ref_correction=Q_ref_correction,
                         BATCH_SIZE=BATCH_SIZE,
                         KL_centering=KL_centering,
                         rtg_centering=rtg_centering)


# tensorflow_version 2.0
from flask import Flask, jsonify, request
import json
import requests
from keras.models import Sequential
from keras.layers import Activation, Dense, Dropout, LSTM
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.metrics import mean_absolute_error
# %matplotlib inline
def predict_data(coin,limit):
    endpoint = 'https://min-api.cryptocompare.com/data/histoday'
    res = requests.get(endpoint + '?fsym={}&tsym=CAD&limit=1750'.format(coin))
    hist = pd.DataFrame(json.loads(res.content)['Data'])
    hist = hist.set_index('time')
    hist.index = pd.to_datetime(hist.index, unit='s')
    target_col = 'close'

    hist.drop(["conversionType", "conversionSymbol"], axis = 'columns', inplace = True)

    hist.tail(5)

    hist['high_low_avg'] = (hist['high'] + hist['low']) / 2
    hist = hist.drop(['high'], axis=1)
    hist = hist.drop(['low'], axis=1)
    hist.tail(5)

    def train_test_split(df, test_size=0.2):
        split_row = len(df) - int(test_size * len(df))
        train_data = df.iloc[:split_row]
        test_data = df.iloc[split_row:]
        return train_data, test_data

    train, test = train_test_split(hist, test_size=0.2)

    def normalise_zero_base(df):
        return df / df.iloc[0] - 1

    def extract_window_data(df, window_len=5, zero_base=True):
        window_data = []
        for idx in range(len(df) - window_len):
            tmp = df[idx: (idx + window_len)].copy()
            if zero_base:
                tmp = normalise_zero_base(tmp)
            window_data.append(tmp.values)
        return np.array(window_data)

    def prepare_data(df, target_col, window_len=10, zero_base=True, test_size=0.2):
        train_data, test_data = train_test_split(df, test_size=test_size)
        X_train = extract_window_data(train_data, window_len, zero_base)
        X_test = extract_window_data(test_data, window_len, zero_base)
        y_train = train_data[target_col][window_len:].values
        y_test = test_data[target_col][window_len:].values
        if zero_base:
            y_train = y_train / train_data[target_col][:-window_len].values - 1
            y_test = y_test / test_data[target_col][:-window_len].values - 1

        return train_data, test_data, X_train, X_test, y_train, y_test

    def build_lstm_model(input_data, output_size, neurons=100, activ_func='linear',
                        dropout=0.2, loss='mse', optimizer='adam'):
        model = Sequential()
        model.add(LSTM(neurons, input_shape=(input_data.shape[1], input_data.shape[2])))
        model.add(Dropout(dropout))
        model.add(Dense(units=output_size))
        model.add(Activation(activ_func))

        model.compile(loss=loss, optimizer=optimizer)
        return model

    np.random.seed(42)
    window_len = 5

    test_size = 0.2
    zero_base = True
    lstm_neurons = 100
    epochs = 20
    batch_size = 32
    loss = 'mse'
    dropout = 0.2
    optimizer = 'adam'

    train, test, X_train, X_test, y_train, y_test = prepare_data(
        hist, target_col, window_len=window_len, zero_base=zero_base, test_size=test_size)

    model = build_lstm_model(
        X_train, output_size=1, neurons=lstm_neurons, dropout=dropout, loss=loss,
        optimizer=optimizer)
    history = model.fit(
        X_train, y_train, validation_data=(X_test, y_test), epochs=epochs, batch_size=batch_size, verbose=1, shuffle=True)

    targets = test[target_col][window_len:]
    preds = model.predict(X_test).squeeze()

    preds = test[target_col].values[:-window_len] * (preds + 1)
    preds = pd.Series(index=targets.index, data=preds)
    return preds.values.tolist()
import flask
app = Flask(__name__)
@app.route('/predict', methods=['GET','POST'])
def predict():
    try:
        if flask.request.method == 'GET':
            coin = flask.request.args.get('coin')
            limit = flask.request.args.get('limit')
            data={}
            data['status']="true"
            data['message']="Data Fetched"
            data['data']=predict_data(coin,limit)

        return str(data)
    except:
        data={}
        data['status']="false"
        data['message']="Error"
        data['data']="null"
        return str(data)

if __name__ == "__main__":
   app.run(host='0.0.0.0')


import numpy as np
import pandas as pd

def nonzeroloadings(B):
    """
    This function calculates the number of nonzero loadings. Requires the loadings matrix B, and
    returns the number of nonzero loadings per column.

    :param B: loading matrix
    :return: number of nonzero loadings (int)
    """
    nonzero_count = (B != 0).astype(int).sum(axis=0)
    return nonzero_count

def variance(X, V):
    """
    This function returns the variance (not the adjusted variance) based on the formulas provided
    by Zou, Hastie, and Tibshirani (2006). The function requires the matrices X and V as input,
    and gives the variance as output.
    :param X: input data
    :param V: weight data
    :return: variance, diagonal of the covariance matrix
    """
    k = V.shape[0]
    X = X.iloc[:, :k]
    sigma = X.T @ X
    Z = V.T @ sigma @ V
    print(Z.shape)
    variance = np.trace(Z.T @ Z)
    Z_array = Z.to_numpy()
    diagonal = Z_array.T @ Z_array
    diagonal = diagonal.diagonal()
    return variance, diagonal

def tex_output(tables):
    """
    This function can be used to generate LaTeX output from multiple tables. Requires one or more
    tables as input, and yields the LaTeX source code as output.
    """
    width = tables[0].shape[0]
    for table in tables:
        if(table.shape[0] != width):
            return "The provided tables do not have the same dimensions."
        else:
            pass