"""
==================================================================
Principal Component Regression vs Partial Least Squares Regression
==================================================================

This example compares `Principal Component Regression
<https://en.wikipedia.org/wiki/Principal_component_regression>`_ (PCR) and
`Partial Least Squares Regression
<https://en.wikipedia.org/wiki/Partial_least_squares_regression>`_ (PLS) on a
toy dataset. Our goal is to illustrate how PLS can outperform PCR when the
target is strongly correlated with some directions in the data that have a
low variance.

PCR is a regressor composed of two steps: first,
:class:`~sklearn.decomposition.PCA` is applied to the training data, possibly
performing dimensionality reduction; then, a regressor (e.g. a linear
regressor) is trained on the transformed samples. In
:class:`~sklearn.decomposition.PCA`, the transformation is purely
unsupervised, meaning that no information about the targets is used. As a
result, PCR may perform poorly in some datasets where the target is strongly
correlated with *directions* that have low variance. Indeed, the
dimensionality reduction of PCA projects the data into a lower dimensional
space where the variance of the projected data is greedily maximized along
each axis. Despite them having the most predictive power on the target, the
directions with a lower variance will be dropped, and the final regressor
will not be able to leverage them.

PLS is both a transformer and a regressor, and it is quite similar to PCR: it
also applies a dimensionality reduction to the samples before applying a
linear regressor to the transformed data. The main difference with PCR is
that the PLS transformation is supervised. Therefore, as we will see in this
example, it does not suffer from the issue we just mentioned.

"""

# %%
# The data
# --------
#
# We start by creating a simple dataset with two features. Before we even dive
# into PCR and PLS, we fit a PCA estimator to display the two principal
# components of this dataset, i.e. the two directions that explain the most
# variance in the data.
import matplotlib.pyplot as plt
import numpy as np

from sklearn.decomposition import PCA

rng = np.random.RandomState(0)
n_samples = 500
cov = [[3, 3], [3, 4]]
X = rng.multivariate_normal(mean=[0, 0], cov=cov, size=n_samples)
pca = PCA(n_components=2).fit(X)

plt.scatter(X[:, 0], X[:, 1], alpha=0.3, label="samples")
for i, (comp, var) in enumerate(zip(pca.components_, pca.explained_variance_)):
    comp = comp * var  # scale component by its variance explanation power
    plt.plot(
        [0, comp[0]],
        [0, comp[1]],
        label=f"Component {i}",
        linewidth=5,
        color=f"C{i + 2}",
    )
plt.gca().set(
    aspect="equal",
    title="2-dimensional dataset with principal components",
    xlabel="first feature",
    ylabel="second feature",
)
plt.legend()
plt.show()

# %%
# For the purpose of this example, we now define the target `y` such that it is
# strongly correlated with a direction that has a small variance. To this end,
# we will project `X` onto the second component, and add some noise to it.

y = X.dot(pca.components_[1]) + rng.normal(size=n_samples) / 2

fig, axes = plt.subplots(1, 2, figsize=(10, 3))

axes[0].scatter(X.dot(pca.components_[0]), y, alpha=0.3)
axes[0].set(xlabel="Projected data onto first PCA component", ylabel="y")
axes[1].scatter(X.dot(pca.components_[1]), y, alpha=0.3)
axes[1].set(xlabel="Projected data onto second PCA component", ylabel="y")
plt.tight_layout()
plt.show()

# %%
# Projection on one component and predictive power
# ------------------------------------------------
#
# We now create two regressors: PCR and PLS, and for our illustration purposes
# we set the number of components to 1. Before feeding the data to the PCA step
# of PCR, we first standardize it, as recommended by good practice. The PLS
# estimator has built-in scaling capabilities.
#
# For both models, we plot the projected data onto the first component against
# the target. In both cases, this projected data is what the regressors will
# use as training data.
from sklearn.cross_decomposition import PLSRegression
from sklearn.decomposition import PCA
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=rng)

pcr = make_pipeline(StandardScaler(), PCA(n_components=1), LinearRegression())
pcr.fit(X_train, y_train)
pca = pcr.named_steps["pca"]  # retrieve the PCA step of the pipeline

pls = PLSRegression(n_components=1)
pls.fit(X_train, y_train)

fig, axes = plt.subplots(1, 2, figsize=(10, 3))
axes[0].scatter(pca.transform(X_test), y_test, alpha=0.3, label="ground truth")
axes[0].scatter(
    pca.transform(X_test), pcr.predict(X_test), alpha=0.3, label="predictions"
)
axes[0].set(
    xlabel="Projected data onto first PCA component", ylabel="y", title="PCR / PCA"
)
axes[0].legend()
axes[1].scatter(pls.transform(X_test), y_test, alpha=0.3, label="ground truth")
axes[1].scatter(
    pls.transform(X_test), pls.predict(X_test), alpha=0.3, label="predictions"
)
axes[1].set(xlabel="Projected data onto first PLS component", ylabel="y", title="PLS")
axes[1].legend()
plt.tight_layout()
plt.show()

# %%
# As expected, the unsupervised PCA transformation of PCR has dropped the
# second component, i.e. the direction with the lowest variance, despite
# it being the most predictive direction. This is because PCA is a completely
# unsupervised transformation, and results in the projected data having a low
# predictive power on the target.
#
# On the other hand, the PLS regressor manages to capture the effect of the
# direction with the lowest variance, thanks to its use of target information
# during the transformation: it can recognize that this direction is actually
# the most predictive. We note that the first PLS component is negatively
# correlated with the target, which comes from the fact that the signs of
# eigenvectors are arbitrary.
#
# We also print the R-squared scores of both estimators, which further confirms
# that PLS is a better alternative than PCR in this case. A negative R-squared
# indicates that PCR performs worse than a regressor that would simply predict
# the mean of the target.

print(f"PCR r-squared {pcr.score(X_test, y_test):.3f}")
print(f"PLS r-squared {pls.score(X_test, y_test):.3f}")

# %%
# As a final remark, we note that PCR with 2 components performs as well as
# PLS: this is because in this case, PCR was able to leverage the second
# component which has the most preditive power on the target.

pca_2 = make_pipeline(PCA(n_components=2), LinearRegression())
pca_2.fit(X_train, y_train)
print(f"PCR r-squared with 2 components {pca_2.score(X_test, y_test):.3f}")


# coding: utf-8

# In[17]:

import csv
import numpy as np
from collections import Counter
import time
from datetime import datetime, timedelta
from itertools import groupby
import operator

File4 = []
a = []
item = 0
File = []
File2 = []
File3 = []
ip = []
end = []
Final_sess1 = []
Final_sess2 = []
Final_sess3 = []
Final = []

# open input file and count requests with id, grouping id regardless of session
with open('/Users/christianbeynis/Desktop/Github/Edgar-Analytics/input/log.csv', 'rt') as f:

    reader = csv.reader(f)
    next(reader, None)
    for key, group in groupby(reader, lambda x: (x[2],x[0])):
        for thing in group:
            File.append(key)
    File2 = Counter(File)
    for k,v in Counter(File).items():
        ip.append([k,v])

f.close()

#Retrieve the date
with open('/Users/christianbeynis/Desktop/Github/Edgar-Analytics/input/log.csv', 'rt') as f:
    next(f)
    date = f.readline().split(',')[1]
    date = datetime.strptime(date, '%m/%d/%y')
    date.strftime('%Y-%m-%d')
    f.close()

# Retrieve the final time and convert into date format
t_final = File[-1:][0][0]
t_final_s = 360*int(t_final[:1])+60*int(t_final[2:4])+int(t_final[5:])
T_final = date + timedelta(seconds= t_final_s)

#find inactivity time
t_inac1 = open('/Users/christianbeynis/Desktop/Github/Edgar-Analytics/input/inactivity_period.txt','r')
t_inac = int(t_inac1.read())
t_inac1.close()

# id and session time from the begining in seconds, requests
values = set(map(lambda x:x[0][1], ip))
File3 = [[[360*int(y[0][0][:1])+60*int(y[0][0][2:4])+int(y[0][0][5:]),y[1], y[0][1]] for y in ip if y[0][1]==x] for x in values]

# separate sessions according to activity
for item in range(len(File3)):
    a.append(np.diff([pair[0] for pair in File3[item]]))

    if len(a[item]) != 0:
        for value in range(len(a[item])):
            File3[item][value+1][0] = a[item][value]

for item in range(len(File3)):
    for element in range(len(File3[item])):
        if element >0 and File3[item][element][0] > t_inac :
                File4.append([File3[item][element-1]])
                File4.append([File3[item][element]])
                File3[item][element][0] = File3[item][element][0] + 1
        else:
            if element == len(File3[item]) - 1:
                 File4.append(File3[item])

# end is the file with id t initial, t final, duration, request with 1 second inclusive
for item2 in range(len(File4)):
    end.append([File4[item2][0][2],
                date + timedelta(seconds=int(File4[item2][0][0])),
                (sum(t for t, Req, id_ in File4[item2])+1-File4[item2][0][0]),
                (sum(Req for t, Req, id_ in File4[item2]))])

# ordering for session finishing before final time,
# the one that would eventually or not finish at the final time.
# They are active 2s before the end of session
# session active that are stopped at the final time, they pull a request
for item in range(len(end)):
    t_i = end[item][1]
    t_f = end[item][1] + timedelta(seconds= int(end[item][2])-1)
    if  t_f + timedelta(seconds= t_inac) < T_final:
        Final_sess1.append([end[item][0],t_i, t_f,end[item][2], end[item][3]])
    else:
        if t_f < T_final:
            Final_sess2.append([end[item][0],t_i, t_f,end[item][2], end[item][3]])
        else:
            Final_sess3.append([end[item][0],t_i, t_f,end[item][2],end[item][3]])

#sorting and regrouping of all cases lists
Final_sess1 = sorted(Final_sess1, key=operator.itemgetter(2))
Final_sess2 = sorted(Final_sess2, key=operator.itemgetter(1))
Final_sess3 = sorted(Final_sess3, key=operator.itemgetter(1))
Final = Final_sess1 + Final_sess2 + Final_sess3

#write file
sessionization  = open('/Users/christianbeynis/Desktop/Github/Edgar-Analytics/insight_testsuite/temp/output/sessionization.txt', 'w')
for lines in range(len(Final)):
    for itemized in range(len(Final[lines])):
        sessionization.write(str(Final[lines][itemized]))
        if itemized+1 != len(Final[lines]) :
            sessionization.write(',')
    if lines-1 != len(Final):
        sessionization.write('\n')

sessionization.close()


import pytest
from geom2d.Vector import Vector

@pytest.fixture
def create_vector():
    """
    Crea instancias de vector con los parámetros introducidos
    """
    def _create_vector(x, y):
        return Vector(x, y)
    return _create_vector

@pytest.mark.parametrize("x1, y1, x2, y2, expected", [
    (1.0, 2.0, 1.0, 2.0, True),
    (3.0, 4.0, -3.0, -4.0, False),
])
def test_equality_of_vectors(create_vector, x1, y1, x2, y2, expected):
    v1 = create_vector(x1, y1)
    v2 = create_vector(x2, y2)
    assert (v1 == v2) == expected

@pytest.mark.parametrize("x1, y1, x2, y2, result_x, result_y", [
    (-1.0, -2.0, 3.0, 4.0, 2.0, 2.0),
    (0.0, 0.0, -1.0, -1.0, -1.0, -1.0),
    (5.0, -3.0, -2.0, 3.0, 3.0, 0.0),
])
def test_addition_of_vectors(create_vector, x1, y1, x2, y2, result_x, result_y):
    v1 = create_vector(x1, y1)
    v2 = create_vector(x2, y2)
    result = v1 + v2
    assert result.x == result_x and result.y == result_y

01. Programación avanzada en Python/Ejercicio_4_clases_paquetes/geom2d/tests/test_vector.py


from flask import render_template
from flask_babel import gettext as _
from pgadmin.utils.ajax import internal_server_error
from pgadmin.utils.exception import ExecuteError
from pgadmin.browser.server_groups.servers.databases.schemas.utils \
    import DataTypeReader
from pgadmin.browser.server_groups.servers.utils import parse_priv_from_db, \
    parse_priv_to_db
from pgadmin.browser.server_groups.servers.databases.utils \
    import make_object_name
from functools import wraps

def get_template_path(f):
    """
    This function will behave as a decorator which will prepare
    the template path based on database server version.
    """

    @wraps(f)
    def wrap(*args, **kwargs):
        # Here args[0] will hold the connection object
        conn_obj = args[0]
        if 'template_path' not in kwargs:
            kwargs['template_path'] = 'columns/sql/#{0}#'.format(
                conn_obj.manager.version)

        return f(*args, **kwargs)
    return wrap

@get_template_path
def get_parent(conn, tid, template_path=None):
    """
    This function will return the parent of the given table.
    :param conn: Connection Object
    :param tid: Table oid
    :param template_path: Optional template path
    :return:
    """

    SQL = render_template("/".join([template_path,
                                    'get_parent.sql']), tid=tid)
    status, rset = conn.execute_2darray(SQL)
    if not status:
        raise ExecuteError(rset)

    schema = ''
    table = ''
    if 'rows' in rset and len(rset['rows']) > 0:
        schema = rset['rows'][0]['schema']
        table = rset['rows'][0]['table']

    return schema, table

def _check_primary_column(data):
    """
    To check if column is primary key
    :param data: Data.
    """
    if 'attnum' in data and 'indkey' in data:
        # Current column
        attnum = str(data['attnum'])

        # Single/List of primary key column(s)
        indkey = str(data['indkey'])

        # We will check if column is in primary column(s)
        if attnum in indkey.split(" "):
            data['is_pk'] = True
            data['is_primary_key'] = True
        else:
            data['is_pk'] = False
            data['is_primary_key'] = False

def _fetch_inherited_tables(tid, data, fetch_inherited_tables, template_path,
                            conn):
    """
    This function will check for fetch inherited tables, and return inherited
    tables.
    :param tid: Table Id.
    :param data: Data.
    :param fetch_inherited_tables: flag to fetch inherited tables.
    :param template_path: Template path.
    :param conn: Connection.
    """
    if fetch_inherited_tables:
        SQL = render_template("/".join(
            [template_path, 'get_inherited_tables.sql']), tid=tid)
        status, inh_res = conn.execute_dict(SQL)
        if not status:
            return True, internal_server_error(errormsg=inh_res)
        for row in inh_res['rows']:
            if row['attrname'] == data['name']:
                data['is_inherited'] = True
                data['tbls_inherited'] = row['inhrelname']
    return False, ''

@get_template_path
def column_formatter(conn, tid, clid, data, edit_types_list=None,
                     fetch_inherited_tables=True, template_path=None):
    """
    This function will return formatted output of query result
    as per client model format for column node
    :param conn: Connection Object
    :param tid: Table ID
    :param clid: Column ID
    :param data: Data
    :param edit_types_list:
    :param fetch_inherited_tables:
    :param template_path: Optional template path
    :return:
    """

    # To check if column is primary key
    _check_primary_column(data)

    # Fetch length and precision
    data = fetch_length_precision(data)

    # We need to fetch inherited tables for each table
    is_error, errmsg = _fetch_inherited_tables(
        tid, data, fetch_inherited_tables, template_path, conn)

    if is_error:
        return errmsg

    # We need to format variables according to client js collection
    if 'attoptions' in data and data['attoptions'] is not None:
        data['attoptions'] = parse_column_variables(data['attoptions'])

    # Need to format security labels according to client js collection
    if 'seclabels' in data and data['seclabels'] is not None:
        seclabels = []
        for seclbls in data['seclabels']:
            k, v = seclbls.split('=')
            seclabels.append({'provider': k, 'label': v})

        data['seclabels'] = seclabels

    # Get formatted Column Options
    if 'attfdwoptions' in data and data['attfdwoptions'] != '':
        data['coloptions'] = parse_options_for_column(data['attfdwoptions'])

    # We need to parse & convert ACL coming from database to json format
    SQL = render_template("/".join([template_path, 'acl.sql']),
                          tid=tid, clid=clid)
    status, acl = conn.execute_dict(SQL)

    if not status:
        return internal_server_error(errormsg=acl)

    # We will set get privileges from acl sql so we don't need
    # it from properties sql
    data['attacl'] = []

    for row in acl['rows']:
        priv = parse_priv_from_db(row)
        data.setdefault(row['deftype'], []).append(priv)

    # we are receiving request when in edit mode
    # we will send filtered types related to current type
    type_id = data['atttypid']

    if edit_types_list is None:
        edit_types_list = []
        SQL = render_template("/".join([template_path,
                                        'edit_mode_types.sql']),
                              type_id=type_id)
        status, rset = conn.execute_2darray(SQL)
        edit_types_list = [row['typname'] for row in rset['rows']]

    # We will need present type in edit mode
    edit_types_list.append(data['typname'])
    data['edit_types'] = sorted(edit_types_list)
    data['cltype'] = DataTypeReader.parse_type_name(data['cltype'])
    return data

def parse_options_for_column(db_variables):
    """
        Function to format the output for variables.

        Args:
            db_variables: Variable object

                Expected Object Format:
                    ['option1=value1', ..]
                where:
                    user_name and database are optional
        Returns:
            Variable Object in below format:
                {
                'variables': [
                    {'name': 'var_name', 'value': 'var_value',
                    'user_name': 'user_name', 'database': 'database_name'},
                    ...]
                }
                where:
                    user_name and database are optional
        """
    variables_lst = []

    if db_variables is not None:
        for row in db_variables:
            # The value may contain equals in string, split on
            # first equals only
            var_name, var_value = row.split("=", 1)
            var_dict = {'option': var_name, 'value': var_value}
            variables_lst.append(var_dict)
    return variables_lst

@get_template_path
def get_formatted_columns(conn, tid, data, other_columns,
                          table_or_type, template_path=None,
                          with_serial=False):
    """
    This function will iterate and return formatted data for all
    the columns.
    :param conn: Connection Object
    :param tid: Table ID
    :param data: Data
    :param other_columns:
    :param table_or_type:
    :param template_path: Optional template path
    :return:
    """
    SQL = render_template("/".join([template_path, 'properties.sql']),
                          tid=tid, show_sys_objects=False)

    status, res = conn.execute_dict(SQL)
    if not status:
        raise ExecuteError(res)

    all_columns = res['rows']
    edit_types = {}
    # Add inherited from details from other columns - type, table
    for col in all_columns:
        edit_types[col['atttypid']] = []
        for other_col in other_columns:
            if col['name'] == other_col['name']:
                col['inheritedfrom' + table_or_type] = \
                    other_col['inheritedfrom']

        if with_serial:
            # Here we assume if a column is serial
            serial_seq_name = make_object_name(
                data['name'], col['name'], 'seq')
            # replace the escaped quotes for comparison
            defval = (col.get('defval', '') or '').replace("''", "'").\
                replace('""', '"')

            if serial_seq_name in defval and defval.startswith("nextval('")\
                    and col['typname'] in ('integer', 'smallint', 'bigint'):

                serial_type = {
                    'integer': 'serial',
                    'smallint': 'smallserial',
                    'bigint': 'bigserial'
                }[col['typname']]

                col['displaytypname'] = serial_type
                col['cltype'] = serial_type
                col['typname'] = serial_type
                col['defval'] = ''

    data['columns'] = all_columns

    if 'columns' in data and len(data['columns']) > 0:
        SQL = render_template("/".join([template_path,
                                        'edit_mode_types_multi.sql']),
                              type_ids=",".join(map(lambda x: str(x),
                                                    edit_types.keys())))
        status, res = conn.execute_2darray(SQL)
        for row in res['rows']:
            edit_types[row['main_oid']] = sorted(row['edit_types'])

        for column in data['columns']:
            column_formatter(conn, tid, column['attnum'], column,
                             edit_types[column['atttypid']], False)

    return data

def _parse_column_actions(final_columns, column_acl):
    """
    Check action and access for it.
    :param final_columns: final column list
    :param column_acl: Column access.
    """
    for c in final_columns:
        if 'attacl' in c:
            if 'added' in c['attacl']:
                c['attacl']['added'] = parse_priv_to_db(
                    c['attacl']['added'], column_acl
                )
            elif 'changed' in c['attacl']:
                c['attacl']['changed'] = parse_priv_to_db(
                    c['attacl']['changed'], column_acl
                )
            elif 'deleted' in c['attacl']:
                c['attacl']['deleted'] = parse_priv_to_db(
                    c['attacl']['deleted'], column_acl
                )
        if 'cltype' in c:
            # check type for '[]' in it
            c['cltype'], c['hasSqrBracket'] = \
                type_formatter(c['cltype'])

        c = convert_length_precision_to_string(c)

def _parse_format_col_for_edit(data, columns, column_acl):
    """
    This function parser columns for edit mode.
    :param data: Data from req.
    :param columns: Columns list from data
    :param column_acl: Column access.
    """
    for action in ['added', 'changed']:
        if action in columns:
            final_columns = []
            for c in columns[action]:
                if 'inheritedfrom' not in c:
                    final_columns.append(c)

            _parse_column_actions(final_columns, column_acl)

            data['columns'][action] = final_columns

def parse_format_columns(data, mode=None):
    """
    This function will parse and return formatted list of columns
    added by user.

    :param data:
    :param mode:
    :return:
    """
    column_acl = ['a', 'r', 'w', 'x']
    columns = data['columns']
    # 'EDIT' mode
    if mode is not None:
        _parse_format_col_for_edit(data, columns, column_acl)
    else:
        # We need to exclude all the columns which are inherited from other
        # tables 'CREATE' mode
        final_columns = []

        for c in columns:
            if c.get('inheritedfrom', None) is None:
                final_columns.append(c)

        # Now we have all lis of columns which we need
        # to include in our create definition, Let's format them
        for c in final_columns:
            if 'attacl' in c:
                c['attacl'] = parse_priv_to_db(
                    c['attacl'], column_acl
                )

            if 'cltype' in c:
                # check type for '[]' in it
                c['cltype'], c['hasSqrBracket'] = type_formatter(c['cltype'])

            c = convert_length_precision_to_string(c)

        data['columns'] = final_columns

    return data

def convert_length_precision_to_string(data):
    """
    This function is used to convert length & precision to string
    to handle case like when user gives 0 as length.

    :param data:
    :return:
    """

    if 'attlen' in data and data['attlen'] == '':
        data['attlen'] = None
    elif 'attlen' in data and data['attlen'] is not None:
        data['attlen'] = str(data['attlen'])

    if 'attprecision' in data and data['attprecision'] == '':
        data['attprecision'] = None
    elif 'attprecision' in data and data['attprecision'] is not None:
        data['attprecision'] = str(data['attprecision'])

    return data

def type_formatter(data_type):
    """
    We need to remove [] from type and append it
    after length/precision so we will set flag for
    sql template.

    :param data_type:
    :param template_path: Optional template path
    :return:
    """

    if '[]' in data_type:
        return data_type[:-2], True
    else:
        return data_type, False

def fetch_length_precision(data):
    """
    This function is used to fetch the length and precision.

    :param data:
    :return:
    """
    # Find length & precision of column data type
    fulltype = DataTypeReader.get_full_type(
        data['typnspname'], data['typname'],
        data['isdup'], data['attndims'], data['atttypmod'])

    length = False
    precision = False
    if 'elemoid' in data:
        length, precision, typeval = \
            DataTypeReader.get_length_precision(data['elemoid'])

    # Set length and precision to None
    data['attlen'] = None
    data['attprecision'] = None

    import re

    # If we have length & precision both
    if length and precision:
        match_obj = re.search(r'(\d+),(\d+)', fulltype)
        if match_obj:
            data['attlen'] = match_obj.group(1)
            data['attprecision'] = match_obj.group(2)
    elif length:
        # If we have length only
        match_obj = re.search(r'(\d+)', fulltype)
        if match_obj:
            data['attlen'] = match_obj.group(1)
            data['attprecision'] = None

    return data

def parse_column_variables(col_variables):
    # We need to format variables according to client js collection
    spcoptions = []
    if col_variables is not None:
        for spcoption in col_variables:
            k, v = spcoption.split('=')
            spcoptions.append({'name': k, 'value': v})
    return spcoptions


from prettytable import PrettyTable

table = PrettyTable()

table.field_names = ["Pokemon", "Type"]
table.add_row(["Pickachu", "Electric"])
table.add_row(["Squirtle", "Water"])
table.add_row(["Charmander", "Fire"])

table.align["Pokemon"] = "l"
table.align["Type"] = "r"
print(table)


import pygame
import random

# Common settings
HEIGHT = 400
WIDTH = 600
FPS = 30

# COLOURS
BLACK = (72, 0, 72)
BLUE = (0, 0, 255)
RED = (255, 0, 0)
GREEN = (0, 255, 0)
WHITE = (255, 255, 255)
YELLOW = (255, 255, 0)

# initialize pygame window and sound
pygame.init()
pygame.mixer.init()
screen = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("Classic Snake")
clock = pygame.time.Clock()

# Global variables

def Score(score):
    font = pygame.font.Font(None, 50)
    value = font.render("Score: " + str(score), True, GREEN)
    screen.blit(value, (0, 0))

def snake(snake_block, snake_list):
    for h in snake_list:
        pygame.draw.rect(screen, YELLOW, (h[0], h[1], snake_block, snake_block))

def message(msg):
    font = pygame.font.Font(None, 30)
    message = font.render(msg, True, RED)
    screen.blit(message, (WIDTH//6, HEIGHT//3))

# Game loop
def game_loop():
    x = WIDTH//2
    y = HEIGHT//2
    x_new = 0
    y_new = 0
    snake_block = 10
    snake_speed = 15
    snake_list = []
    length_snake = 1
    game_close = False
    foodx = random.randrange(0, (WIDTH-snake_block)//10)*10
    foody = random.randrange(0, (HEIGHT-snake_block)//10)*10

    # Game loop
    running = True
    while running:

        while game_close == True:
            screen.fill(BLACK)
            message("Game Over, Press Q-to quit or R-Play again")
            pygame.display.update()
            for event in pygame.event.get():
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_q:
                        running = False
                        game_close = False
                    if event.key == pygame.K_r:
                        game_loop()

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                running = False
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT:
                    x_new = -snake_block
                    y_new = 0
                elif event.key == pygame.K_RIGHT:
                    x_new = snake_block
                    y_new = 0
                elif event.key == pygame.K_UP:
                    y_new = -snake_block
                    x_new = 0
                elif event.key == pygame.K_DOWN:
                    y_new = snake_block
                    x_new = 0

        if x >= WIDTH or x < 0 or y >= HEIGHT or y < 0:
            game_close = True

        x += x_new
        y += y_new
        snake_Head = []
        snake_Head.append(x)
        snake_Head.append(y)
        snake_list.append(snake_Head)
        if len(snake_list) > length_snake:
            del snake_list[0]

        for h in snake_list[:-1]:
            if h == snake_Head:
                game_close = True

        screen.fill(BLACK)
        pygame.draw.rect(screen, RED, [foodx, foody, snake_block, snake_block])
        snake(snake_block, snake_list)
        Score(length_snake - 1)

        if x == foodx and y == foody:
            foodx = random.randrange(0, (WIDTH-snake_block)//10)*10
            foody = random.randrange(0, (HEIGHT-snake_block)//10)*10
            length_snake += 1

        clock.tick(snake_speed)
        pygame.display.update()

game_loop()
pygame.quit()


from typing import List
from pathlib import Path

def insert_line_breaks(
    text: str,
    max_chars_per_line: int,
    prefix: str = "",
    suffix: str = "",
    newline="\n",
    is_japanese=False,
) -> str:
    """
    Insert line breaks into the text to ensure that each line has a maximum number of characters.

    Args:
        text (str): The input text.
        max_chars_per_line (int): The maximum number of characters allowed per line.
        prefix (str, optional): The prefix to be added at the beginning of each line. Defaults to "".
        suffix (str, optional): The suffix to be added at the end of each line. Defaults to "".
        newline (str, optional): The newline character to be used. Defaults to "\n".

    Returns:
        str: The text with line breaks inserted.
    """
    if is_japanese:
        words = text
        max_chars_per_line = max_chars_per_line // 2
    else:
        words = text.split()

    lines = []
    current_line = ""
    for word in words:
        if len(current_line) + len(word) <= max_chars_per_line:
            current_line += word
            if not is_japanese:
                current_line += " "
        else:
            lines.append(current_line.strip())
            current_line = word
            if not is_japanese:
                current_line += " "
    lines.append(current_line.strip())
    return prefix + newline.join(lines) + suffix

def overwrite_image(
    img_url: str | None, image_path: str, img_ignore_paths: List[str]
) -> str | None:
    import requests

    if img_url is None:
        return None

    if img_url in img_ignore_paths:
        return image_path

    response = requests.get(img_url)
    if response.status_code == 200:
        Path(image_path).parent.mkdir(parents=True, exist_ok=True)
        with open(image_path, "wb") as f:
            f.write(response.content)
    return image_path


import os
from datetime import datetime
import pandas as pd     # -> pip install pandas

output_file_name = 'ip_summary.csv'

BASE_DIR = os.path.dirname(os.path.realpath(__file__))
ip_log_folder_path = os.path.join(BASE_DIR,'IP_Log')

date_max = datetime.strptime('3000-01-01 00:00:00', "%Y-%m-%d %H:%M:%S" )
date_min = datetime.strptime('1900-01-01 00:00:00', "%Y-%m-%d %H:%M:%S" )

#df = pd.DataFrame(columns=['ip','xray','nginx','ngpr','total','probe_percent','first_seen','last_seen','nginx_req','ngpr_req'])
Dict_list = []

def read_ip_log(file_name):
    print('opening '+file_name)
    with open(file_name, "r") as f:
        count = 0
        while True:
            count += 1
            line = f.readline().replace('\n','').replace('\r','')
            if line:
                X = parse_line(line)
                add_line_to_dataframe(X)
            else:
                break

    print('-> total line read : '+str(count))

    return True

def parse_line(mystr):
    x = mystr.split('\t, ')
    if( len(x)==4):
        return {'ip':x[0] , 'access':x[1] , 'time':x[2] , 'req':x[3][:11]}
    elif( len(x)==3):
        return {'ip':x[0] , 'access':x[1] , 'time':x[2] , 'req':''}
    else:
        raise Exception('line parse err. invalid log file?')

def add_line_to_dataframe(X):
    id_bool = [ (s['ip']==X['ip']) for s in Dict_list]
    n = sum(id_bool)
    if ( n==1 ):
        D = convert_line_to_dict(X , Dict_list[ id_bool.index(True) ] )
    elif( n==0 ):
        D = convert_line_to_dict(X , None)
        Dict_list.append(D)
    else:
        raise Exception('err in algorithm!')

def convert_line_to_dict(X , old_dict=None):

    if(old_dict == None):
        D = {'ip':X['ip'],'xray':0,'nginx':0,'ngpr':0,'total':0,'probe_percent':0,'first_seen':date_max,'last_seen':date_min,'nginx_req':'','ngpr_req':''}
    else :
        D = old_dict

    if( X['access'] == 'XRAY'):
        D['xray'] = D['xray'] + 1
    elif( X['access'] == 'NGINX' ):
        D['nginx'] = D['nginx'] + 1
        D['nginx_req'] = X['req']
    elif( X['access'] == 'NG-PR' ):
        D['ngpr'] = D['ngpr'] + 1
        D['ngpr_req'] = X['req']
    else:
        raise Exception('err in line')

    D['total'] = D['xray']+D['nginx']+D['ngpr']

    D['probe_percent'] = (D['nginx']+D['ngpr'])/(D['total']+1e-8)

    this_time = datetime.strptime(X['time'], "%Y-%m-%d %H:%M:%S" )
    if( this_time < D['first_seen'] ):
        D['first_seen'] = this_time

    if( this_time > D['last_seen'] ):
        D['last_seen'] = this_time

    return D

file_list = [f for f in os.listdir(ip_log_folder_path) if os.path.isfile(os.path.join(ip_log_folder_path, f))]

for f in file_list :
    # open only txt file or log file generated by pyprox
    if( f[-4:]=='.txt') or (f[-3]=='_'):
        myfile = os.path.join(ip_log_folder_path,f)
        read_ip_log(myfile)

df = pd.DataFrame(Dict_list)
new_df = df.sort_values('probe_percent',ascending=False)
new_df.to_csv(os.path.join(BASE_DIR,output_file_name))

print('summary writed to '+ os.path.join(BASE_DIR,output_file_name) )


from __future__ import unicode_literals

import posixpath

from indico.core.config import config
from indico.core.db import db
from indico.core.storage import StoredFileMixin
from indico.util.string import return_ascii, strict_unicode
from indico.web.flask.util import url_for

class DesignerImageFile(StoredFileMixin, db.Model):
    __tablename__ = 'designer_image_files'
    __table_args__ = {'schema': 'indico'}

    # Image files are not version-controlled
    version_of = None

    #: The ID of the file
    id = db.Column(
        db.Integer,
        primary_key=True
    )
    #: The designer template the image belongs to
    template_id = db.Column(
        db.Integer,
        db.ForeignKey('indico.designer_templates.id'),
        nullable=False,
        index=True
    )

    template = db.relationship(
        'DesignerTemplate',
        lazy=False,
        foreign_keys=template_id,
        backref=db.backref(
            'images',
            cascade='all, delete-orphan',
            lazy=True
        )
    )

    @property
    def download_url(self):
        return url_for('designer.download_image', self)

    @property
    def locator(self):
        return dict(self.template.locator, image_id=self.id, filename=self.filename)

    def _build_storage_path(self):
        path_segments = ['designer_templates', strict_unicode(self.template.id), 'images']
        self.assign_id()
        filename = '{}-{}'.format(self.id, self.filename)
        path = posixpath.join(*(path_segments + [filename]))
        return config.ATTACHMENT_STORAGE, path

    @return_ascii
    def __repr__(self):
        return '<DesignerImageFile({}, {}, {}, {})>'.format(
            self.id,
            self.template_id,
            self.filename,
            self.content_type
        )


import argparse

import torch
import torch.nn as nn
import torchvision
import torch as th
import os
import torch.nn.functional as F
from functools import partial
import torch
import random

from guided_diffusion.script_util import (
    model_and_diffusion_defaults,
    classifier_defaults,
    create_model_and_diffusion,
    create_classifier,
    add_dict_to_argparser,
    args_to_dict,
)
import torch
import numpy as np

class Jitter(nn.Module):
    def __init__(self, lim: int = 32):
        super().__init__()
        self.lim = lim

    def forward(self, x: torch.tensor) -> torch.tensor:
        off1 = random.randint(-self.lim, self.lim)
        off2 = random.randint(-self.lim, self.lim)
        return torch.roll(x, shifts=(off1, off2), dims=(2, 3))

class ColorJitter(nn.Module):
    def __init__(self, batch_size: int, shuffle_every: bool = False, mean: float = 1., std: float = 1.):
        super().__init__()
        self.batch_size, self.mean_p, self.std_p = batch_size, mean, std
        self.mean = self.std = None
        self.shuffle()
        self.shuffle_every = shuffle_every

    def shuffle(self):
        self.mean = (torch.rand((self.batch_size, 3, 1, 1,)).cuda() - 0.5) * 2 * self.mean_p
        self.std = ((torch.rand((self.batch_size, 3, 1, 1,)).cuda() - 0.5) * 2 * self.std_p).exp()

    def forward(self, img: torch.tensor) -> torch.tensor:
        if self.shuffle_every:
            self.shuffle()
        return (img - self.mean) / self.std

class ColorJitterR(ColorJitter):
    def forward(self, img: torch.tensor) -> torch.tensor:
        if self.shuffle_every:
            self.shuffle()
        return (img * self.std) + self.mean

def get_model_diffusion_classifier(args):
    model, diffusion = create_model_and_diffusion(
        **args_to_dict(args, model_and_diffusion_defaults().keys())
    )
    print('loading diffusion model...')
    print(args.model_path)
    checkpoint = torch.load(args.model_path)
    model.load_state_dict(checkpoint)
    print('done')
    model.eval()
    model.requires_grad_(False)
    model = torch.nn.DataParallel(model).cuda()
    if len(args.classifier_path) > 2:
        print("loading classifier...")
        classifier = create_classifier(**args_to_dict(args, classifier_defaults().keys()))
        classifier_checkpoint = torch.load(args.classifier_path)
        classifier.load_state_dict(classifier_checkpoint)
        print('done')
        classifier.eval()
        classifier.requires_grad_(False)
        classifier = torch.nn.DataParallel(classifier).cuda()
    else:
        classifier = None

    return model, diffusion, classifier

def get_parser():
    defaults = dict(
        clip_denoised=True,
        num_samples=1,
        batch_size=16,
        use_ddim=False,
        model_path="",
        classifier_path="",
        classifier_scale=1.0,
    )
    defaults.update(model_and_diffusion_defaults())
    defaults.update(classifier_defaults())
    parser = argparse.ArgumentParser()
    parser.add_argument('--max_time', type=int, default=10)
    parser.add_argument('--data_dir', default='/fs/cml-datasets/ImageNet/ILSVRC2012/train')
    parser.add_argument('--val_data_dir', default='/fs/cml-datasets/ImageNet/ILSVRC2012/val')
    parser.add_argument('--workers', default=5, type=int)
    add_dict_to_argparser(parser, defaults)
    return parser

class IdentityBN(torch.nn.Module):
    def __init__(self, *args, **kwargs):
        super(IdentityBN, self).__init__()

    def forward(self, x):
        return x

# We may want to use a pretrained model
def get_model(in_channels=6, remove_bn=False):
    norm_layer = IdentityBN if remove_bn else nn.BatchNorm2d
    resnet18 = torchvision.models.resnet18(num_classes=2, norm_layer=norm_layer)
    resnet18.conv1 = nn.Conv2d(in_channels, resnet18.conv1.out_channels, kernel_size=7, stride=2, padding=3, bias=False)
    return resnet18

def cond_fn(x, t, y=None, args=None, classifier=None):
    assert y is not None
    with th.enable_grad():
        x_in = x.detach().requires_grad_(True)
        logits = classifier(x_in, t)
        log_probs = F.log_softmax(logits, dim=-1)
        selected = log_probs[range(len(logits)), y.view(-1)]
        return th.autograd.grad(selected.sum(), x_in)[0] * args.classifier_scale

def model_fn(x, t, y=None, args=None, model=None):
    # assert y is not None
    return model(x, t, y if args.class_cond else None)

def get_noisy_denoised_incremental(args, model, diffusion, classifier, image, label):
    image, label = image.cuda(), label.cuda()
    times = torch.ones(image.shape[0], dtype=torch.long).cuda() * (args.max_time - 1)
    perturbed = diffusion.q_sample(image, times)
    model_kwargs = {}
    model_kwargs["y"] = label
    sample_fn = (
        diffusion.p_sample_loop if not args.use_ddim else diffusion.ddim_sample_loop
    )
    sample = sample_fn(
        partial(model_fn, model=model, args=args),
        (args.batch_size, 3, args.image_size, args.image_size),
        clip_denoised=args.clip_denoised,
        model_kwargs=model_kwargs,
        cond_fn=partial(cond_fn, classifier=classifier, args=args),
        device=torch.device('cuda'),
        progress=True,
        max_time=args.max_time,
        noise=perturbed,
    )

    return sample

class Evaluator(object):
    def __init__(self, num_class):
        self.num_class = num_class
        self.confusion_matrix = np.zeros((self.num_class,)*2)

    def Pixel_Accuracy(self):
        Acc = np.diag(self.confusion_matrix).sum() / self.confusion_matrix.sum()
        return Acc

    def Pixel_Accuracy_Class(self):
        Acc = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(axis=1)
        Acc = np.nanmean(Acc)
        return Acc

    def Mean_Intersection_over_Union(self):
        MIoU = np.diag(self.confusion_matrix) / (
                    np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) -
                    np.diag(self.confusion_matrix))
        MIoU = np.nanmean(MIoU)
        return MIoU

    def Frequency_Weighted_Intersection_over_Union(self):
        freq = np.sum(self.confusion_matrix, axis=1) / np.sum(self.confusion_matrix)
        iu = np.diag(self.confusion_matrix) / (
                    np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) -
                    np.diag(self.confusion_matrix))

        FWIoU = (freq[freq > 0] * iu[freq > 0]).sum()
        return FWIoU

    def _generate_matrix(self, gt_image, pre_image):
        mask = (gt_image >= 0) & (gt_image < self.num_class)
        label = self.num_class * gt_image[mask].astype('int') + pre_image[mask]
        count = np.bincount(label, minlength=self.num_class**2)
        confusion_matrix = count.reshape(self.num_class, self.num_class)
        return confusion_matrix

    def add_batch(self, gt_image, pre_image):
        assert gt_image.shape == pre_image.shape
        self.confusion_matrix += self._generate_matrix(gt_image, pre_image)

    def reset(self):
        self.confusion_matrix = np.zeros((self.num_class,) * 2)

class OptimizerDetails:
    def __init__(self):
        self.num_steps = None
        self.operation_func = None
        self.optimizer = None # handle it on string level
        self.lr = None
        self.loss_func = None
        self.max_iters = 0
        self.loss_cutoff = None
        self.lr_scheduler = None
        self.warm_start = None
        self.old_img = None
        self.fact = 0.5
        self.print = False
        self.print_every = None
        self.folder = None
        self.tv_loss = None
        self.guidance_3 = False
        self.optim_guidance_3_wt = 0
        self.other_guidance_func = None
        self.other_criterion = None
        self.original_guidance = False
        self.sampling_type = None
        self.loss_save = None

class Operation:
    def __init__(self, args, operation, shape, progressive=False):
        self.args = args
        self.model, self.diffusion, self.classifier = get_model_diffusion_classifier(self.args)
        self.progressive = progressive
        self.operation = operation
        self.shape = shape

    def operator(self, label, operated_image, max_time=None):
        max_time = max_time or self.args.max_time
        model_kwargs = {}
        model_kwargs["y"] = label

        sample_fn = self.diffusion.ddim_sample_loop_operation
        sample = sample_fn(
            partial(model_fn, model=self.model, args=self.args),
            self.shape,
            operated_image=operated_image,
            operation=self.operation,
            clip_denoised=self.args.clip_denoised,
            model_kwargs=model_kwargs,
            cond_fn=partial(cond_fn, classifier=self.classifier, args=self.args),
            device=torch.device('cuda'),
            progress=self.progressive
        )

        return sample

class Normalize(object):
    def __call__(self, sample):
        return (sample - 0.5) * 2

def un_normalize(image):
    return (image + 1) / 2


## EVALUATION ON VQA-E DATASET

import os
import sys

import argparse
import json
import progressbar
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torch.autograd import Variable
import numpy as np
import pickle
from build_vocab import Vocabulary
from model_explain import Sen_Sim
from model_explain import Relev_Check, CaptionVocabCandidate, CaptionVocabCandidateBox

from address_server_XAI import *

sys.path.append(addr_BAN)

from codes.base_model import BiSU

from codes.dataset import Dictionary, VQAFeatureDataset
from data_loader import BottomUp_get_loader, VQAE_FineTunning_loader
import codes.utils as utils
from model import Encoder_HieStackedCorr, DecoderRNN, BAN_HSC, DecoderTopDown
from address_server_XAI import *
import codes.base_model as base_model
from model_VQAE import EnsembleVQAE
import pdb

import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--num_hid', type=int, default=1280)
    parser.add_argument('--model', type=str, default='ban')
    parser.add_argument('--op', type=str, default='c')
    parser.add_argument('--label', type=str, default='XAI')
    parser.add_argument('--gamma', type=int, default=2)
    parser.add_argument('--split', type=str, default='validation')
    parser.add_argument('--input', type=str, default='saved_models/ban')
    parser.add_argument('--output', type=str, default='results')
    parser.add_argument('--batch_size', type=int, default=256)
    parser.add_argument('--debug', type=bool, default=True)
    parser.add_argument('--logits', type=bool, default=False)
    parser.add_argument('--index', type=int, default=0)
    parser.add_argument('--epoch', type=int, default=12)
    parser.add_argument('--hsc_epoch', type=int, default=16)
    parser.add_argument('--hsc_path', type=str, default='models/', help='path for resuming hsc pre-trained models')
    parser.add_argument('--BUTD_path', type=str, default='models_BUTD/',
                        help='path for resuming hsc pre-trained models')
    parser.add_argument('--embed_size', type=int, default=256, help='dimension of word embedding vectors')
    parser.add_argument('--hidden_size', type=int, default=512, help='dimension of lstm hidden states')
    parser.add_argument('--vocab_path', type=str, default='data/vocab_threshold_0.pkl', help='path for vocabulary wrapper')
    parser.add_argument('--vocab_vqae_path', type=str, default='data/VQA_E_vocab.pkl',
                        help='path for VQA_E vocabulary wrapper')
    parser.add_argument('--num_layers', type=int, default=1, help='number of layers in lstm')
    parser.add_argument('--save_fig_loc', type=str, default='saved_figs_with_caption/')
    parser.add_argument('--x_method', type=str, default='weight_only') # mean, NoAtt, sum, weight_only
    parser.add_argument('--t_method', type=str, default='uncorr') # mean, uncorr
    parser.add_argument('--s_method', type=str, default='BestOne') # BestOne, BeamSearch
    parser.add_argument('--LRdim', type=int, default=64)
    parser.add_argument('--model_num', type=int, default=1)
    parser.add_argument('--isBUTD', type=bool, default=False)
    parser.add_argument('--isUnion', type=bool, default=False)
    parser.add_argument('--isFeeding', type=bool, default=False)
    parser.add_argument('--paramH', type=int, default=256, help='dimension of lstm hidden states')
    parser.add_argument('--result_json_path', type=str, default='test_json/', help='saving path for captioning json')
    parser.add_argument('--test_target', type=str, default='COCO', help='COCO or KARPATHY')
    parser.add_argument('--method', type=str, default='VQAE&LRCN', help='applied method')
    parser.add_argument('--model_vqaE', type=str, default='model_XAI/vqaE')
    parser.add_argument('--model_vqaE_BUTD', type=str, default='model_XAI/vqaE_BUTD')
    parser.add_argument('--selfAtt', type=bool, default=False, help='self Attention?')
    parser.add_argument('--input_selfAtt', type=str, default='selfAtt-30-Final.pth')
    args = parser.parse_args()
    return args

def get_question(q, dataloader):
    str = []
    dictionary = dataloader.dataset.dictionary
    for i in range(q.size(0)):
        str.append(dictionary.idx2word[q[i]] if q[i] < len(dictionary.idx2word) else '_')
    return ' '.join(str)

def get_answer(p, dataloader):
    _m, idx = torch.max(p,1)
    label_from_vqa=torch.cuda.LongTensor(p.size()).fill_(0)
    for i in range(p.size(0)):
        label_from_vqa[i,idx[i]]=1
    return idx

def get_answer2(p, dataloader):
    _m, idx = p.max(0)
    return dataloader.dataset.label2ans[idx.item()]

def get_image(q_id, Dict_qid2vid, data_root=addr_test_imgs): #q_id => single scalar
    img_id=Dict_qid2vid[q_id]
    img_name='COCO_test2015_%012d.jpg' % img_id
    img_path=os.path.join(data_root,img_name)
    img=plt.imread(img_path)

    return img

def showPlot(points):
    plt.figure()
    fig, ax = plt.subplots()
    # this locator puts ticks at regular intervals
    loc = ticker.MultipleLocator(base=0.2)
    ax.yaxis.set_major_locator(loc)
    plt.plot(points)

def check_captions(caption_generator, dataloader,Dict_qid2vid, vocab,save_fig_loc,x_method_, t_method_, s_method_, args_, method, selfAtt=None):
    N = len(dataloader.dataset)
    idx = 0
    bar = progressbar.ProgressBar(maxval=N).start()
    print('t_method : {}, x_method : {}, isBUTD : {}, isUnion : {}'.format(t_method_,x_method_,args.isBUTD,args.isUnion))
    captions_list_all = []
    img_id_list=[]

    print('processing-{}-'.format(method))

    for i,(v, b, q, ans, img_ids, _, num_objs) in enumerate(dataloader):
        bar.update(idx)
        batch_size = v.size(0)
        with torch.no_grad():
            num_objs=num_objs.cuda()
            v = Variable(v).cuda()
            q=q.squeeze(1)

            if (method == 'MEAN_LRCN'):
                generated_captions, logits, att, encoded_feats, _ = caption_generator.generate_caption_n_context(v, b, q, t_method='mean', x_method='weight_only',
                                                      s_method=s_method_, useVQA=True, obj_nums=num_objs)
            elif (method == 'VQAE_LRCN'):
                if(selfAtt):
                    v=selfAtt(v)
                generated_captions, logits, att = caption_generator.generate_caption(v, b, q,
                                                                                t_method='mean',
                                                                                x_method='weight_only',
                                                                                s_method='BestOne',
                                                                                model_num=1, obj_nums=num_objs)
            elif (method == 'MEAN_BUTD'):
                generated_captions, logits, att, encoded_feats, Vmat = caption_generator.generate_caption_n_context(v, b, q, t_method='mean', x_method='weight_only',
                                                      s_method=s_method_,
                                                      isBUTD=True, useVQA=True, obj_nums=num_objs)
            elif (method == 'VQAE_BUTD'):
                if (selfAtt):
                    v = selfAtt(v)
                generated_captions, logits, att = caption_generator.generate_caption(v, b, q,
                                                                                t_method='mean',
                                                                                x_method='weight_only',
                                                                                s_method='BestOne',
                                                                                model_num=1,
                                                                                isBUTD=True, obj_nums=num_objs)
            elif (method == 'OURS_LRCN' or method == 'OURS_LRCN_Feeding'):

                generated_captions, logits, att, encoded_feats, _ = caption_generator.generate_caption_n_context(v, b, q, t_method='uncorr', x_method='weight_only',
                                                      s_method=s_method_, useVQA=True, obj_nums=num_objs)
            else:
                generated_captions, logits, att, encoded_feats, Vmat = caption_generator.generate_caption_n_context(
                    v, b, q, t_method='uncorr', x_method='weight_only', s_method=s_method_,
                    isBUTD=True, isUnion=True, useVQA=True, obj_nums=num_objs)

            idx += batch_size
            bar.update(idx)
            question_list = []
            answer_list = []
            captions_list = []
            for idx2 in range(len(img_ids)):
                question_list.append(get_question(q.data[idx2], dataloader))
                if (s_method_ == 'BestOne'):
                    caption = [vocab.idx2word[generated_captions[idx2][w_idx].item()] for w_idx in
                               range(generated_captions.size(1))]
                    captions_list.append(caption)
                elif (s_method_ == 'BeamSearch'):
                    for tmp_idx in range(generated_captions.size(0)):
                        # pdb.set_trace()
                        caption = [vocab.idx2word[generated_captions[tmp_idx][w_idx].item()] for w_idx in
                                   range(generated_captions.size(1))]
                        captions_list.append(caption)
                answer_list.append(get_answer2(logits.data[idx2], dataloader))

            if (method == 'OURS_LRCN_Feeding' or method == 'OURS_BUTD_Feeding'):
                ##################################################################### EXPLAIN ##################################################################################

                word_candidate_idx_in_coco_vocab = CaptionVocabCandidateBox(question_list, answer_list, vocab)
                if (method == 'OURS_BUTD_Feeding'):
                    generated_captions = caption_generator.generate_explain(Vmat, encoded_feats,
                                                                                word_candidate_idx_in_coco_vocab,
                                                                                t_method='uncorr',
                                                                                x_method='weight_only',
                                                                                s_method=s_method_,
                                                                                isBUTD=True,
                                                                                isUnion=True,
                                                                                model_num=args_.model_num)
                else:
                    generated_captions = caption_generator.generate_explain(None, encoded_feats,
                                                                                word_candidate_idx_in_coco_vocab,
                                                                                t_method='uncorr',
                                                                                x_method='weight_only',
                                                                                s_method=s_method_,
                                                                                model_num=args_.model_num)
                captions_list = []
                for idx2 in range(len(img_ids)):
                    if (s_method_ == 'BestOne'):
                        caption = [vocab.idx2word[generated_captions[idx2][w_idx].item()] for w_idx in
                                   range(generated_captions.size(1))]
                        captions_list.append(caption)

                # print(captions_list[0])

                ##################################################################### EXPLAIN ##################################################################################
            for caption in captions_list:
                caption = caption_refine(caption, model_num=args_.model_num)
                #pdb.set_trace()
                captions_list_all.append(caption)

        img_id_list.extend(img_ids)

    bar.update(idx)
    result_json=make_json(captions_list_all, img_id_list)

    with open(args.result_json_path + '/VQAEexplain_%s_results.json' \
              % (method), 'w') as f:

        json.dump(result_json, f)

def make_json(captions, ImgIds):
    utils.assert_eq(len(captions), len(ImgIds))
    results = []
    for i in range(len(captions)):
        result = {}
        result['image_id'] = ImgIds[i]
        result['ann_id'] = ImgIds[i]
        result['caption'] = captions[i]
        results.append(result)
    return results

#######################################################################################

def caption_refine(explains, NumBeams=1, model_num=1):

    x_caption = ''
    for num_sen in range(NumBeams):
        if(NumBeams > 1):
            explain=explains[num_sen]
        else:
            explain=explains
        for word_ in explain:
            if word_ == '<start>':
                if model_num <7:
                    x_caption = ''
            elif word_ == '<end>':
                break
            else:
                x_caption = x_caption + ' ' + word_

        if (NumBeams > 1):
            x_caption=x_caption+'\n'

    return x_caption

#######################################################################################`

if __name__ == '__main__':
    args = parse_args()
    if (args.t_method == 'uncorr'):
        args.isUnion = True

    torch.backends.cudnn.benchmark = True

    name = args.split
    Dict_qid2vid=[]

    if not os.path.exists(
            args.result_json_path):
        os.makedirs(args.result_json_path)

    dictionary_vqa = Dictionary.load_from_file('codes/tools/data/dictionary.pkl')

    # Visualization through eval_dataset

    n_device = torch.cuda.device_count()
    batch_size = args.batch_size * n_device

    # Load vocabulary wrapper
    with open(args.vocab_path, 'rb') as f:
        vocab = pickle.load(f)

    with open(args.vocab_vqae_path, 'rb') as f:
        vocab_VQAE = pickle.load(f)

    vqaE_loader = VQAE_FineTunning_loader('eval', dictionary_vqa, vocab, args.batch_size, shuffle=False, num_workers=0)

    # Build the models
    encoder_LRCN_mean = Encoder_HieStackedCorr(args.embed_size, 2048, model_num=args.model_num, LRdim=args.LRdim).to(
        device)
    encoder_LRCN_uni = Encoder_HieStackedCorr(args.embed_size, 2048, model_num=args.model_num, LRdim=args.LRdim).to(
        device)
    decoder_LRCN_mean = DecoderRNN(args.embed_size, args.hidden_size, len(vocab), args.num_layers).to(device)
    decoder_LRCN_uni = DecoderRNN(args.embed_size, args.hidden_size, len(vocab), args.num_layers).to(device)
    decoder_vqae_LRCN = DecoderRNN(args.embed_size, args.hidden_size, len(vocab_VQAE), args.num_layers).to(device)

    encoder_BUTD_mean = Encoder_HieStackedCorr(args.embed_size, 2048, model_num=args.model_num, LRdim=args.LRdim).to(
        device)
    encoder_BUTD_uni = Encoder_HieStackedCorr(args.embed_size, 2048, model_num=args.model_num, LRdim=args.LRdim).to(
        device)
    decoder_BUTD_mean = DecoderTopDown(args.embed_size, 2048, args.hidden_size, args.hidden_size, len(vocab),
                                       args.num_layers, paramH=args.paramH)
    decoder_BUTD_uni = DecoderTopDown(args.embed_size, 2048, args.hidden_size, args.hidden_size, len(vocab),
                                      args.num_layers, paramH=args.paramH)
    decoder_vqae_BUTD = DecoderTopDown(args.embed_size, 2048, args.hidden_size, args.hidden_size, len(vocab_VQAE),
                                       args.num_layers, paramH=args.paramH).to(device)

    if (args.selfAtt):
        selfAtt = BiSU(256, 2048, 1)
        selfAtt.to(device)
        print('selfAttention applied')
    else:
        selfAtt=None

    # Build the models
    encoder = Encoder_HieStackedCorr(args.embed_size, 2048, model_num=args.model_num, LRdim=args.LRdim).to(device)
    if(args.isBUTD):

        decoder = DecoderTopDown(args.embed_size, 2048, args.hidden_size, args.hidden_size, len(vocab), args.num_layers,
                                 paramH=args.paramH).to(device)
        args.save_fig_loc=args.save_fig_loc+args.hsc_path
        #args.hsc_path='models_BUTD/'
        model_file='model-{}.pth'.format(20)
    else:

        decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab), args.num_layers).to(device)
        model_file = 'model-{}.pth'.format(16)

    constructor = 'build_%s' % args.model
    model = getattr(base_model, constructor)(vqaE_loader.dataset, args.num_hid, args.op,
                                             args.gamma).cuda() #<= this is the BAN-VQAE
    model_BUTD = getattr(base_model, constructor)(vqaE_loader.dataset, args.num_hid, args.op, args.gamma).cuda()

    model_BAN = getattr(base_model, constructor)(vqaE_loader.dataset, args.num_hid, args.op,
                                             args.gamma).cuda()  # <= this is the BAN

    def process(args, model, model_BUTD, model_BAN, eval_loader,Dict_qid2vid,vocab):

        model_hsc_path_LRCN_mean = os.path.join(
            args.hsc_path, 'mean', 'model{}_LR{}'.format(args.model_num, args.LRdim), 'model-{}.pth'.format(16))
        model_hsc_path_BUTD_mean = os.path.join(
            args.BUTD_path, 'mean', 'model{}_LR{}'.format(args.model_num, args.LRdim),
            'model-{}.pth'.format(20))
        model_hsc_path_LRCN_uni = os.path.join(
            args.hsc_path, 'uncorr', 'model{}_LR{}'.format(args.model_num, args.LRdim), 'model-{}.pth'.format(16))
        model_hsc_path_BUTD_uni = os.path.join(
            args.BUTD_path, 'uncorr', 'model{}_LR{}'.format(args.model_num, args.LRdim),
            'model-{}.pth'.format(20))

        print('loading hsc_LRCN mean datas %s' % model_hsc_path_LRCN_mean)
        model_hsc_LRCN_data_mean = torch.load(model_hsc_path_LRCN_mean)
        print('loading hsc_LRCN uni datas %s' % model_hsc_path_LRCN_uni)
        model_hsc_LRCN_data_uni = torch.load(model_hsc_path_LRCN_uni)
        print('loading hsc_BUTD mean datas %s' % model_hsc_path_BUTD_mean)
        model_hsc_BUTD_data_mean = torch.load(model_hsc_path_BUTD_mean)
        print('loading hsc_BUTD uni datas %s' % model_hsc_path_BUTD_uni)
        model_hsc_BUTD_data_uni = torch.load(model_hsc_path_BUTD_uni)

        encoder_LRCN_mean.load_state_dict(model_hsc_LRCN_data_mean['encoder_state'])
        encoder_LRCN_uni.load_state_dict(model_hsc_LRCN_data_uni['encoder_state'])
        decoder_LRCN_mean.load_state_dict(model_hsc_LRCN_data_mean['decoder_state'])
        decoder_LRCN_uni.load_state_dict(model_hsc_LRCN_data_uni['decoder_state'])

        encoder_BUTD_mean.load_state_dict(model_hsc_BUTD_data_mean['encoder_state'])
        encoder_BUTD_uni.load_state_dict(model_hsc_BUTD_data_uni['encoder_state'])
        decoder_BUTD_mean.load_state_dict(model_hsc_BUTD_data_mean['decoder_state'])
        decoder_BUTD_uni.load_state_dict(model_hsc_BUTD_data_uni['decoder_state'])

        model_vqae_path = args.model_vqaE + '/vqaE-19-Final.pth'

        model_vqae_BUTD_path = args.model_vqaE_BUTD + '/vqaE-20-Final.pth'

        model_BAN_path = args.input + '/model%s.pth' % \
                 ('' if 0 > 12 else '_epoch%d' % 12)

        print('loading %s' % model_vqae_path)
        model_vqae_LRCN_data = torch.load(model_vqae_path)

        print('loading %s' % model_vqae_BUTD_path)
        model_vqae_BUTD_data = torch.load(model_vqae_BUTD_path)

        print('loading %s' % model_BAN_path)
        model_BAN_data = torch.load(model_BAN_path)

        if (args.selfAtt):
            print('loading %s' % args.input_selfAtt)
            selfAtt_data = torch.load(os.path.join('model_XAI', 'vqaE_selfAtt', args.input_selfAtt))
            selfAtt.load_state_dict(selfAtt_data.get('model_state', selfAtt_data))

            model_vqae_path = args.model_vqaE + '/vqaE-19-Final.pth'
            model_vqae_LRCN_data = torch.load(model_vqae_path)
            print('reloading %s' % model_vqae_path)
            selfAtt.train(False)

        model = nn.DataParallel(model).cuda()
        model_BUTD = nn.DataParallel(model_BUTD).cuda()
        model_BAN= nn.DataParallel(model_BAN).cuda()

        ensemble_LRCN = EnsembleVQAE(model, decoder_vqae_LRCN).to(device)
        ensemble_LRCN.load_state_dict(model_vqae_LRCN_data['model_state'])

        ensemble_BUTD = EnsembleVQAE(model_BUTD, decoder_vqae_BUTD).to(device)
        ensemble_BUTD.load_state_dict(model_vqae_BUTD_data['model_state'])

        model_BAN.load_state_dict(model_BAN_data['model_state'])

        # pdb.set_trace()

        model.train(False)
        model_BUTD.train(False)
        ensemble_LRCN.train(False)
        encoder_LRCN_mean.train(False)
        encoder_LRCN_uni.train(False)
        decoder_LRCN_mean.train(False)
        decoder_LRCN_uni.train(False)

        ensemble_BUTD.train(False)
        encoder_BUTD_mean.train(False)
        encoder_BUTD_uni.train(False)
        decoder_BUTD_mean.train(False)
        decoder_BUTD_uni.train(False)

        caption_generator_LRCN_mean = BAN_HSC(model_BAN, encoder_LRCN_mean, decoder_LRCN_mean, vocab).to(device)
        caption_generator_LRCN_uni = BAN_HSC(model_BAN, encoder_LRCN_uni, decoder_LRCN_uni, vocab).to(device)
        caption_generator_BUTD_mean = BAN_HSC(model_BUTD, encoder_BUTD_mean, decoder_BUTD_mean, vocab).to(device)
        caption_generator_BUTD_uni = BAN_HSC(model_BUTD, encoder_BUTD_uni, decoder_BUTD_uni, vocab).to(device)

        ################################################################################################################

        #Concatenate Encoder-Decoder to model and check whether the model generates correct captions based on visual cues
        methods = ['VQAE_LRCN', 'OURS_LRCN', 'MEAN_LRCN', 'OURS_LRCN_Feeding', 'VQAE_BUTD', 'OURS_BUTD', 'MEAN_BUTD', 'OURS_BUTD_Feeding']
        vocabs=[vocab_VQAE, vocab, vocab, vocab, vocab_VQAE, vocab, vocab, vocab]
        caption_generators=[ensemble_LRCN, caption_generator_LRCN_uni, caption_generator_LRCN_mean, caption_generator_LRCN_uni, ensemble_BUTD,
         caption_generator_BUTD_uni, caption_generator_BUTD_mean, caption_generator_BUTD_uni]

        for idx in range(1,len(methods)):
            check_captions(caption_generators[idx], eval_loader, Dict_qid2vid,vocabs[idx],args.save_fig_loc,args.x_method, args.t_method, args.s_method, args, methods[idx],selfAtt=selfAtt)

        ################################################################################################################

    process(args, model, model_BUTD, model_BAN, vqaE_loader,Dict_qid2vid,vocab)


import os
import sys
from utils import *
from datetime import datetime

import telegram
from telegram.ext import (Updater, CommandHandler, MessageHandler, Filters,
                          ConversationHandler)

def start(update, context):
    update.message.reply_text("ברוכ/ה הבא/ה לרכבוט!")
    update.message.reply_text("הכנס תחנת מוצא:")
    return CHOOSE_ORIGIN

def get_depart_station(update, context):
    user = update.message.from_user
    input_text = update.message.text
    logger.info("User {}, departure input: {}".format(user, input_text))
    # TODO - Validate input and fuzzy lookup
    context.user_data['depart_station'], score = get_fuzzy_station_name(update.message.text)
    update.message.reply_text("תחנת מוצא: {}".format(context.user_data['depart_station']))
    update.message.reply_text("הכנס/י תחנת יעד:")
    return CHOOSE_DEST

def get_dest_station(update, context):
    user = update.message.from_user
    logger.info("User {}, destination input: {}".format(user, update.message.text))
    # TODO - Validate input and fuzzy lookup
    context.user_data['dest_station'], score = get_fuzzy_station_name(update.message.text)
    update.message.reply_text("תחנת יעד: {}".format(context.user_data['dest_station']))

    if 'depart_station' in context.user_data and 'dest_station' in context.user_data:
        depart_station = context.user_data['depart_station']
        dest_station = context.user_data['dest_station']
        if depart_station == dest_station:
            update.message.reply_text("אנא הכנס/י תחנת מוצא ויעד שונות.")
            update.message.reply_text("הכנס תחנת מוצא:")
            return CHOOSE_ORIGIN
    else:
        update.message.reply_text("נתונים חסרים. אנא התחל/י מחדש")
        return ConversationHandler.END

    update.message.reply_text("עכשיו?", reply_markup=telegram.ReplyKeyboardMarkup([['לא', 'כן']]))

    return CHOOSE_TIME

def get_choose_time(update, context):
    user = update.message.from_user
    context.user_data['time_keyboard_input'] = update.message.text
    logger.info("User {}, keyboard 'now' input: {}".format(user, context.user_data['time_keyboard_input']))
    if context.user_data['time_keyboard_input'] == "כן":
        update.message.reply_text("מחפש רכבות קרובות...", reply_markup=telegram.ReplyKeyboardRemove())
        timestamp = datetime.now()
        get_route(update, context, timestamp)
        update.message.reply_text("איך עוד אפשר לעזור?",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['סיימתי', 'חיפוש חדש', 'צור תזכורת']]))
        return PAST_ROUTE
    elif context.user_data['time_keyboard_input'] == "לא":
        update.message.reply_text("מתי?",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['בתאריך...', 'מחר', 'היום']]))
    else:
        update.message.reply_text("לא הצלחתי להבין.", reply_markup=telegram.ReplyKeyboardRemove())
        update.message.reply_text("לחפש רכבות לעכשיו?", reply_markup=telegram.ReplyKeyboardMarkup([['לא', 'כן']]))

        return CHOOSE_TIME
    return PARSE_DAY

def get_parsed_day(update, context):
    user = update.message.from_user
    context.user_data['day_keyboard_input'] = update.message.text
    logger.info("User {}, parsed day input: {}".format(user, context.user_data['day_keyboard_input']))
    if context.user_data['day_keyboard_input'] == "היום":
        context.user_data['date_input'] = datetime.now()
    elif context.user_data['day_keyboard_input'] == "מחר":
        context.user_data['date_input'] = datetime.now() + timedelta(hours=24)
    elif context.user_data['day_keyboard_input'] == "בתאריך...":
        update.message.reply_text("הכנס תאריך:", reply_markup=telegram.ReplyKeyboardRemove())
        return CUSTOM_DAY
    else:
        update.message.reply_text("לא הצלחתי להבין.", reply_markup=telegram.ReplyKeyboardRemove())
        update.message.reply_text("בחר חלון תאריכים לחיפוש רכבות:",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['בתאריך...', 'מחר', 'היום']]))
        return PARSE_DAY

    update.message.reply_text("באיזו שעה?", reply_markup=telegram.ReplyKeyboardRemove())

    return PARSE_HOUR

def get_custom_day(update, context):
    user = update.message.from_user
    manual_day_input = update.message.text
    logger.info("User {}, manual day input: {}".format(user, manual_day_input))
    found_time = parse(manual_day_input)
    if found_time is None or found_time.year < CUR_YEAR:
        update.message.reply_text("לא הצלחתי להבין.", reply_markup=telegram.ReplyKeyboardRemove())
        update.message.reply_text("בחר חלון תאריכים לחיפוש רכבות:",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['בתאריך...', 'מחר', 'היום']]))
        return PARSE_DAY
    context.user_data['date_input'] = found_time
    update.message.reply_text("אוקיי, ה-{}".format(context.user_data['date_input'].strftime("%d/%m/%Y")),
                              reply_markup=telegram.ReplyKeyboardRemove())
    update.message.reply_text("באיזו שעה?", reply_markup=telegram.ReplyKeyboardRemove())

    return PARSE_HOUR

def get_parsed_hour(update, context):
    user = update.message.from_user
    logger.info("User {}, parsed time input: {}".format(user, update.message.text))
    context.user_data['time_input'] = update.message.text
    context.user_data['time_input'] = get_fuzzy_time(context.user_data['time_input'])
    if context.user_data['time_input'] is None:
        update.message.reply_text("לא הצלחתי להבין מתי. נסה/י שוב.")
        return PARSE_HOUR
    context.user_data['timestamp_input'] = datetime.combine(context.user_data['date_input'].date(),
                                                            context.user_data['time_input'].time())
    update.message.reply_text("מחפש רכבות עבור: {}"
                              .format(context.user_data['timestamp_input'].strftime('%d/%m/%y %H:%M')))
    if (datetime.now() - context.user_data['timestamp_input']).total_seconds() > 30:
        update.message.reply_text("לא ניתן לקבל לוחות זמנים עבור רכבות עבר. הכנס/י תאריך ושעה אחרים.")
        update.message.reply_text("מתי?", reply_markup=telegram.ReplyKeyboardMarkup([['בתאריך...', 'מחר', 'היום']]))
        return PARSE_DAY
    get_route(update, context, context.user_data['timestamp_input'])
    update.message.reply_text("איך עוד אפשר לעזור?",
                              reply_markup=telegram.ReplyKeyboardMarkup([['סיימתי', 'חיפוש חדש', 'צור תזכורת']]))
    return PAST_ROUTE

def past_route(update, context):
    user = update.message.from_user
    context.user_data['conv_end_message'] = update.message.text
    logger.info("User {}, past route input: {}".format(user, context.user_data['conv_end_message']))
    if context.user_data['conv_end_message'] == 'סיימתי':
        return happy_end(update, context)
    elif context.user_data['conv_end_message'] == 'חיפוש חדש':
        update.message.reply_text("הכנס תחנת מוצא:", reply_markup=telegram.ReplyKeyboardRemove())
        return CHOOSE_ORIGIN
    elif context.user_data['conv_end_message'] == 'צור תזכורת':
        update.message.reply_text("באיזו שעה להתריע?", reply_markup=telegram.ReplyKeyboardRemove())
        return TIME_SCHEDULE
    else:
        update.message.reply_text("לא הצלחתי להבין.")
        update.message.reply_text("איך עוד אפשר לעזור?",
                                  reply_markup=telegram.ReplyKeyboardMarkup([['סיימתי', 'חיפוש חדש', 'צור תזכורת']]))
        return PAST_ROUTE

def get_time_schedule(update, context):
    user = update.message.from_user
    sched_time_message = update.message.text
    logger.info("User {}, sched time input: {}".format(user, sched_time_message))
    found_time = get_fuzzy_time(sched_time_message)
    if found_time is None:
        update.message.reply_text("לא הצלחתי להבין מתי. נסה/י להזין שעה בפורמט כמו '08:30':")
        return TIME_SCHEDULE
    context.user_data['time_schedule'] = found_time.time()
    update.message.reply_text("אוקיי, ב{}".format(context.user_data['time_schedule'].strftime("%H:%M")))
    update.message.reply_text("באיזה ימים? הכנס/י רשימה מהצורה 'א,ב,ג...'.")
    return DAY_SCHEDULE

def get_day_schedule(update, context):
    user = update.message.from_user
    sched_day_message = update.message.text
    try:
        sanitized_days = [DAYS_DICT[x] for x in list(sched_day_message.replace(" ", "").replace(",", ""))]
        if len(sanitized_days) == 0:
            raise ValueError("לא נמצאו ימים")
    except:
        update.message.reply_text("לא הצלחתי להבין את טווח הימים. נסה/י להזין שוב ללא גרשיים או פיסוק מיותר.")
        return DAY_SCHEDULE

    # Pack needed job data in context.user_data
    update.message.reply_text("אתריע בימים: {}".format([REV_DAYS_DICT[x] for x in sanitized_days]))
    context.user_data['chat_id'] = update.effective_chat.id
    context.user_data['days'] = sanitized_days
    found_time = context.user_data['time_schedule']
    logger.info("User {}, adding schedule at time {}, days: {}".format(user, found_time, sanitized_days))
    context.job_queue.run_daily(notify, found_time, days=tuple(sanitized_days), context=context)
    update.message.reply_text("התראה נוספה בהצלחה.")
    return happy_end(update, context)

def timeout(update, context):
    user = context.effective_user
    chat_id = context.message.chat_id
    logger.info("User {} timed out".format(user))
    update.send_message(chat_id=chat_id, text="עקב חוסר פעילות נותקת מהמערכת.", reply_markup=telegram.ReplyKeyboardRemove())
    update.send_message(chat_id=chat_id, text="להתחלה לחצ/י על /start", reply_markup=telegram.ReplyKeyboardRemove())
    return ConversationHandler.END

def cancel(update, context):
    user = update.message.from_user
    logger.info("User {} canceled the conversation.".format(user))
    return happy_end(update, context)

def happy_end(update, context):
    chat_id = update.effective_chat.id
    context.bot.send_message(chat_id=chat_id, text="רכבוט שמח לעזור!", reply_markup=telegram.ReplyKeyboardRemove())
    context.bot.send_message(chat_id=chat_id, text="להתחלה לחצ/י על /start", reply_markup=telegram.ReplyKeyboardRemove())
    return ConversationHandler.END

def error(update, context):
    """Log Errors caused by Updates."""
    logger.warning('Update {} caused error {}'.format(update, context.error))

def main():

    # Get the dispatcher to register handlers
    dp = updater.dispatcher

    # Add conversation handler with states
    conv_handler = ConversationHandler(
        entry_points=[CommandHandler('start', start)],

        states={
            CHOOSE_ORIGIN: [MessageHandler(Filters.text, get_depart_station)],
            CHOOSE_DEST: [MessageHandler(Filters.text, get_dest_station)],
            CHOOSE_TIME: [MessageHandler(Filters.text, get_choose_time)],
            PARSE_DAY: [MessageHandler(Filters.text, get_parsed_day)],
            CUSTOM_DAY: [MessageHandler(Filters.text, get_custom_day)],
            PARSE_HOUR: [MessageHandler(Filters.text, get_parsed_hour)],
            PAST_ROUTE: [MessageHandler(Filters.text, past_route)],
            TIME_SCHEDULE: [MessageHandler(Filters.text, get_time_schedule)],
            DAY_SCHEDULE: [MessageHandler(Filters.text, get_day_schedule, pass_job_queue=True)],
            ConversationHandler.TIMEOUT: [MessageHandler(None, timeout)]
        },

        fallbacks=[CommandHandler('cancel', cancel)],

        conversation_timeout=CONVERSATION_TIMEOUT
    )

    dp.add_handler(conv_handler)

    # log all errors
    dp.add_error_handler(error)

    # Start the Bot
    updater.start_polling()

    # Run the bot until you press Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT. This should be used most of the time, since
    # start_polling() is non-blocking and will stop the bot gracefully.
    updater.idle()

if __name__ == '__main__':

    mode = os.getenv("MODE")
    TOKEN = os.getenv("TOKEN")
    logger.info("Starting bot")

    updater = Updater(TOKEN, use_context=True)

    if mode == "dev":
        logger.info("Dev mode")
        updater.start_polling()

        # app_url = "https://localhost/"
        # wakeup_wrapper(app_url)

    elif mode == "prod":
        logger.info("Production mode")

        PORT = int(os.environ.get("PORT", "8443"))
        HEROKU_APP_NAME = os.environ.get("HEROKU_APP_NAME")
        # Code from https://github.com/python-telegram-bot/python-telegram-bot/wiki/Webhooks#heroku
        updater.start_webhook(listen="0.0.0.0",
                              port=PORT,
                              url_path=TOKEN)
        updater.bot.set_webhook("https://{}.herokuapp.com/{}".format(HEROKU_APP_NAME, TOKEN))

        # app_url = "https://{}.herokuapp.com/".format(HEROKU_APP_NAME)
        # wakeup_wrapper("127.0.0.1")

    else:
        logger.error("No MODE specified!")
        sys.exit(1)

    main()


from django.shortcuts import redirect,render
from ngoma.models import Song,Album
from django.http import HttpResponse

def home_base(request):
    # if request.user.is_authenticated:
    #     return redirect('ngoma:home')

    albums = Album.objects.filter(shared=True)
    song_set = ''
    if albums:
        for album in albums:
            song_set = Song.objects.filter(album=album.id)
            for song in song_set:
                # path = song.song_file.path.split('.')
                # print(path)
                # if len(path==3):
                #     song_name = path[1]
                #     song_file_type = path[2]
                #     song.song_title = song_name[60:]
                #     song.file_type = file_type
                #     song.save()
                # else:
                song_name = song.song_file.path[:-4]
                song.song_title = song_name[60:]
                song.file_type = song.song_file.path[-4:]
                song.save()

                song_set = Song.objects.filter(album = album.id)

    context = {
        'songs': song_set,
        'albums': albums
    }

    return render(request, 'main/home_base.html', context)

def login_redirect(request):
    return redirect('ngoma:home')


# encoding: utf-8

import pytest

from web.contentment.taxonomy import Taxonomy

@pytest.fixture(scope='module')
def connection(request):
	from mongoengine import connect
	from .fixture import apply

	connection = connect('testing')
	apply()

	request.addfinalizer(lambda: connection.drop_database('testing'))

	return connection

@pytest.fixture
def root(connection):
	from web.component.asset.model import Asset
	return Asset.objects.get(path='/careers.illicohodes.com')

@pytest.fixture
def models(connection):
	from collections import namedtuple

	Models = namedtuple('Models', 'parent child1 child2')
	TModel.objects.delete()
	return Models(
		TModel.objects.create(name='parent', path='/'),
		TModel.objects.create(name='child1', path='1'),
		TModel.objects.create(name='child2', path='2'),
	)

class TModel(Taxonomy):
	pass

@pytest.mark.usefixtures('connection')
class TestTaxonomy:
	def test_siblings(self, root):
		assert root.children[1].siblings.count() == 5

	def test_next(self, root):
		subroot = root.children[0].children[1]
		nexts = list(root.children[0].children[2:4])
		assert subroot.nextAll.count() == 2
		assert list(subroot.nextAll) == nexts
		c = subroot
		count = 0
		while True:
			c = c.next
			if c is None:
				break
			count += 1
		assert count == 2

	def test_prev(self, root):
		subroot = root.children[0].children[2]
		prevs = list(root.children[0].children[0:2])
		assert subroot.prevAll.count() == 2
		assert list(subroot.prevAll) == prevs
		c = subroot
		count = 0
		while True:
			c = c.prev
			if c is None:
				break
			count += 1
		assert count == 2

	def test_insert_detach(self, models):
		assert models.parent.children.count() == 0
		assert models.child1.parents == []

		models.parent.insert(0, models.child1)
		models.parent.insert(1, models.child2)
		assert list(models.parent.children) == [models.child1, models.child2]

		models.child2.detach()
		assert list(models.parent.children) == [models.child1]

		models.parent.insert(0, models.child2)
		assert list(models.parent.children) == [models.child2, models.child1]

		models.child1.append(models.child2)
		assert list(models.parent.children) == [models.child1] and list(models.child1.children) == [models.child2]
		assert list(models.parent.contents) == [models.child1, models.child2]

		models.child1.empty()
		assert list(models.parent.contents) == [models.child1]
		assert TModel.objects.count() == 2

	def test_append_prepend(self, models):
		models.parent.append(models.child1)
		assert list(models.parent.children) == [models.child1]
		models.parent.append(models.child2)
		assert list(models.parent.children) == [models.child1, models.child2]

		models.parent.prepend(models.child2)
		assert list(models.parent.children) == [models.child2, models.child1]

	def test_after_before(self, models):
		models.parent.append(models.child1)
		models.child1.after(models.child2)
		assert list(models.parent.children) == [models.child1, models.child2]

		models.child1.before(models.child2)
		assert list(models.parent.children) == [models.child2, models.child1]

	def test_pivoted(self, models):
		models.child1.appendTo(models.parent)
		assert list(models.parent.children) == [models.child1]
		models.child2.prependTo(models.parent)
		assert list(models.parent.children) == [models.child2, models.child1]

		models.child1.insertBefore(models.child2)
		assert list(models.parent.children) == [models.child1, models.child2]
		models.child1.insertAfter(models.child2)
		assert list(models.parent.children) == [models.child2, models.child1]

	def test_extend(self, models):
		parent2 = TModel.objects.create(name='par2', path='/P2')
		models.child1.appendTo(models.parent)
		models.child2.appendTo(parent2)

		models.parent.extend(parent2)
		assert list(models.parent.children) == [models.child1, models.child2]

	def test_replace(self, models):
		replacer = TModel.objects.create(name='replacer', path='r')
		models.child1.appendTo(models.parent)
		models.child1.reload(-1)
		assert TModel.objects.count() == 4
		replacer.replace(models.child1)
		assert TModel.objects.count() == 3
		assert list(models.parent.children) == [replacer.reload(-1)]

		models.child2.replaceWith(replacer)
		replacer.reload(-1)
		assert models.parent.children.count() == 0
		assert replacer.name == 'child2'


import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.utils import check_X_y
from sklearn.linear_model import LogisticRegression
from scipy.sparse import csr_matrix
from scipy.sparse import lil_matrix

class SelfTraining(BaseEstimator, TransformerMixin):
    def __init__(self, learner=LogisticRegression(), iterations_nb=5, pool_size=100, training_way=0, replacement=True, random_state=None):
        self.learner = learner
        self.iterations_nb = iterations_nb
        self.pool_size = pool_size
        self.training_way = training_way #0: Hard, 1: Soft, 2: Soft multi
        self.replacement = replacement
        self.random_state = random_state
        self._labeled_data_size = -1
        self._unlabeled_data_size = -1
        self._categories_nb = -1
        self._accessible_index = None

    def get_params(self, deep=False): # deep=False due to a little issue in Scikit-Learn
        return {"learner": self.learner,
            "iterations_nb": self.iterations_nb,
            "pool_size": self.pool_size,
            "training_way": self.training_way,
            "replacement": self.replacement,
            "random_state": self.random_state}

    def set_params(self, **parameters):
        for parameter, value in parameters.items():
            setattr(self, parameter, value)

        return self

    def __x_y_preparation(self, X, y):
        X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'])

        self._categories_nb = len(np.unique(y))

        l_index = np.argwhere(y != -1)
        u_index = np.argwhere(y == -1)

        self._labeled_data_size = len(np.argwhere(y != -1))
        self._unlabeled_data_size = len(np.argwhere(y == -1))

        X_new_shape = ((self._labeled_data_size + self._categories_nb * self._unlabeled_data_size), X.shape[1])
        y_new_shape = ((self._labeled_data_size + self._categories_nb * self._unlabeled_data_size),)
        X_new = lil_matrix(X_new_shape, dtype=np.float64)
        y_new = np.zeros(y_new_shape)

        i_l = 0
        i_u = 0
        for i, y_row in enumerate(y):
            X_row = X[i]
            y_row = y_row

            # label
            if i in l_index:
                X_new[i_l] = X_row
                y_new[i_l] = y_row
                i_l += 1

            # unlabel
            elif i in u_index:
                # we need all possible category for each unlabeled data
                for cat in range(self._categories_nb):
                    local_i = (self._labeled_data_size) + (cat * self._unlabeled_data_size) + i_u
                    X_new[local_i] = X_row
                    y_new[local_i] = cat
                i_u += 1

        self._accessible_index = np.array(range(self._labeled_data_size, self._labeled_data_size + self._unlabeled_data_size))

        if not self.replacement and self._unlabeled_data_size < (self.iterations_nb * self.pool_size):
            print("Warning: Not enough unlabeled data for {} iterations with pool size of {} WITHOUT replacement!".format(self.iterations_nb, self.pool_size))
            self.iterations_nb = self._unlabeled_data_size // self.pool_size
            print("Number of iterations has been automaticly reduced to {}.".format(self.iterations_nb))

        return X_new, y_new

    def __m_step(self, X, y, weights):
        self.learner.fit(X, y, sample_weight=weights)

        return self

    def __e_step(self, X, weights):

        if not getattr(self.learner, "decision_function", None):
            raise ValueError('No decision_function in model.')

        selected_sample = np.random.choice(self._accessible_index, self.pool_size, replace=False)

        # Without replacement: we only keep labeled data as basis at each new step
        if not self.replacement:
            weights = np.array(self._labeled_data_size * [1] + self._categories_nb * self._unlabeled_data_size * [0], dtype=np.float64)

        for i in selected_sample:

            scores = self.learner.decision_function(X[i])
            predicted_cat = scores.argmax(axis=1)[0]

            # Computing weights
            new_weights = np.zeros(self._categories_nb)
            if self.training_way == 0: # Hard
                new_weights[predicted_cat] = 1

            elif self.training_way == 1: # Soft
                probs = np.exp(scores) / np.sum(np.exp(scores))
                new_weights[predicted_cat] = probs[0][predicted_cat] # only the best weight, not any others

            elif self.training_way == 2: # Soft multi
                probs = np.exp(scores) / np.sum(np.exp(scores))
                new_weights = probs[0] # all weights

            # Updating weights in each category for X[i]
            for cat in range(self._categories_nb):
                weights[i + cat * self._unlabeled_data_size] = new_weights[cat]

        # Without replacement: we remove used unlabeled data points index
        if not self.replacement:
            self._accessible_index = np.setdiff1d(self._accessible_index, selected_sample)

        return weights

    def fit(self, X, y):
        np.random.seed(self.random_state)

        X, y = self.__x_y_preparation(X, y)

        weights = np.array(self._labeled_data_size * [1] + self._categories_nb * self._unlabeled_data_size * [0], dtype=np.float64)

        self.__m_step(X, y, weights)
        for _ in range(self.iterations_nb):
            weights = self.__e_step(X, weights)
            self.__m_step(X, y, weights)

        return self

    def predict(self, X):
        return self.learner.predict(X)

    def decision_function(self, X):
        return self.learner.decision_function(X)

    def score(self, X, y):
        return self.learner.score(X, y)

if __name__ == '__main__':
    pass


from decimal import *
import time
from typing import List
from datetime import datetime
import json
from itertools import groupby
from dateutil.parser import parse

def _ElementJsonToOSMObject(jsonDict):
    """
    Parses an OpenStreetMap object into an :class:`OSMNode`, :class:`OSMWay` or :class:`OSMRelation`.

    Parameters
    ----------

    jsonDict : dict
        The dict object representing one of the OpenStreetMap objects

    Returns
    -------
    :class:`OSMNode`, :class:`OSMWay` or :class:`OSMRelation`

    """
    if jsonDict['type'] == 'node':
        return OSMNode(jsonDict['id'], jsonDict['type'], jsonDict['lat'], jsonDict['lon'], jsonDict.get('tags'))
    elif jsonDict['type'] == 'way':
        return OSMWay(jsonDict['id'], jsonDict['type'], jsonDict['nodes'], jsonDict.get('tags'))
    elif jsonDict['type'] == 'relation':
        return OSMRelation(jsonDict['id'], jsonDict['type'], _JsonMemberListToOSMRelationMember(jsonDict['members']), jsonDict.get('tags'))
    else:
            raise Exception("OSM Element type (%s) not implemented!" % jsonDict['type'])

def _JsonMemberListToOSMRelationMember(self, jsonMemberList):
    """
    Parses an OSMRelationMember dict into an OSMRelationMember object.

    Parameters
    ----------

    jsonMemberList : dict
        The dict object representing an :class:`OSMRelationMember`.

    Returns
    -------
    An arrays containing each parsed :class:`OSMRelationMember`.

    """
    members = []
    for member in jsonMemberList:
        members.append(OSMRelationMember(member['type'], member['ref'], member['role']))
    return members

class OSMObject(object):
    """Base class for other OSM objects:
        Attribute Name: default value
        version: 0
        visible: True
        timestamp: None
        changeset: 0
        user: '' #user's name
        uid: 0 #user's id
        """

    _optDefAttributes={"version": 0, "visible": True, "timestamp": None, "changeset": 0,
        "user": "", "uid": 0}

    ##  'tags' attribute is set as an empty dictionary so that it's possible to try
    ##  to look up for a 'name' tag even when an (faulty) object doesn't have any tags
    def __init__(self, id: int, type:str, tags={}, **kwargs):
        self.id = id
        self.type = type
        self.tags = tags
        for kw in OSMObject._optDefAttributes:
            setattr(self, kw, kwargs.get('kw') if kw in kwargs else OSMObject._optDefAttributes.get('kw'))

class OSMNode(OSMObject):
    """
    Class used as an OpenStreetMap Node object wrapper

    fields:
        - lat : Decimal
            Represents this OSMNode latitude
        - lon : Decimal
            Represents this OSMNode longitude
        - tags : dict
            Each :class:`OSMNode`, :class:`OSMWay` and :class:`OSMRelation` can
            contain an arbitratry number of tags, each of which corresponds to
            some attribute of that OSM object (e.g. an OSMNode could
            have a tag 'highway' with the value 'traffic_signals').
    """
    def __init__(self, id: int, type: str, lat: Decimal, lon: Decimal, tags={}):
        super().__init__(id, type, tags)
        self.lat = lat
        self.lon = lon

class OSMWay(OSMObject):
    """
    Class used as an OpenStreetMap Way object wrapper.
    In this abstraction an OSMWay is only considered to be a set
    of nodes contained in the field nodes.

    fields:
        - nodes : List[int]
            The set of nodes contained in this OSMWay. An OSMWay
            is used to represent usually linear features such
            as avenues and streets.
        - tags : dict
            See tags in :class:`OSMNode`
    """
    def __init__(self, id: int, type,  nodes: List[int], tags={}):
        super().__init__(id, type, tags)
        self.nodes = nodes

class OSMRelation(OSMObject):
    """
        A OpenStreetMap relation represents 'belongs to' relations (i.e. :class:`OSMNode` objects belonging
        to the same :class:`OSMWay` or a collection of :class:`OSMWay` objects that belongs to the same
        OSMRelation which represets for example the boundaries of some region.)

        fields:
            - id : int
                Represents the relation id as informed by OpenStreetMap database
            - type : str
                It can be anything for example 'boundary'.
            - members : List[OSMObject]
                An OSMRelation can be composed by any number and type of objects
                (e.g. an OSMRelation with type 'route' can contain different
                OSMWays that compose the route of a bus).
    """
    def __init__(self, id: int, type, members: List[OSMObject], tags={}):
        super().__init__(id, type, tags)
        self.members = members

class OSM3S:
    """Timestamp and copyright info"""
    def __init__(self, timestamp_osm_base = None, copyright = ""):
        if not timestamp_osm_base is None:
            self.timestamp_osm_base = parse(timestamp_osm_base)
        self.copyright = copyright

    def JsonToOSM3S(jsonString):
        """Parses an string to an OSM3S object"""
        return DictToOSM3S(json.loads(jsonString));

    def DictToOSM3S(jsonDict):
        """Parses an dict object into an OSM3S object"""
        return OSM3S(jsonDict['timestamp_osm_base'], jsonDict['copyright'])

class OSMRelationMember:
    """An element that 'belogs' to another (i.e. An :class:`OSMNode` that belongs to an :class:`OSMWay`)"""
    def __init__(self, type, ref, role):
        self.type = type
        self.ref = ref
        self.role = role

class OSMResult:
    """Represents a query result to an OpenStreetMap server"""
    def __init__(self, version: float, generator: str, osm3s: OSM3S, elements: list = []):
        self.version = version
        self.generator = generator
        self.osm3s = osm3s
        self.Nodes = {}
        self.Ways = {}
        self.Relations = {}
        for element in elements:
            osmObject = _ElementJsonToOSMObject(element)
            if type(osmObject) is OSMNode: self.Nodes[osmObject.id] = osmObject
            elif type(osmObject) is OSMWay:
                self.Ways[osmObject.id] = osmObject
            elif type(osmObject) is OSMRelation:
                self.Relations[osmObject.id] = osmObject

    def fromJsonString(jsonString):
        """Parses a json string into a OSMResult object"""
        return OSMResult.fromJsonDict(json.loads(jsonString))

    def fromJsonDict(jsonDict):
        """Parses a dict into a OSMResult"""
        version = jsonDict['version']
        generator = jsonDict['generator']
        osm3sDict = OSM3S.DictToOSM3S(jsonDict['osm3s'])
        return OSMResult(version, generator, osm3sDict, jsonDict['elements'])