from tkinter import *
from tkinter import Label, DoubleVar, Entry, Button
root = Tk()
root.geometry("400x240+50+50")
root.title("roi")
heading = Label(root, text="Return on Investment and Mark-up,   Enter data:", font=('calibri 13 bold'), fg="wheat4").pack()
entr_amt1 = Label(root, text="Full cost of a product pu:", font=('calibri 11 bold')).place(x=10, y=40)
cost_pu = DoubleVar()
ent_box1 = Entry(root, width=20, textvariable=cost_pu, font=('calibri 11 bold')).place(x=10, y=60)
entr_amt2 = Label(root, text="Total cost incurred: ", font=('calibri 11 bold')).place(x=10, y=80)
tcost = DoubleVar()
ent_box2 = Entry(root, width=20, textvariable=tcost, font=('calibri 11 bold')).place(x=10, y=100)
entr_amt3 = Label(root, text="Capital invested:", font=('calibri 11 bold')).place(x=10, y=120)
capital = DoubleVar()
ent_box3 = Entry(root, width=20, textvariable=capital, font=('calibri 11 bold')).place(x=10, y=140)
entr_amt4 = Label(root, text="Expected % ROI: ", font=('calibri 11 bold')).place(x=10, y=160)
perc_roi = DoubleVar()
ent_box4 = Entry(root, width=20, textvariable=perc_roi, font=('calibri 11 bold')).place(x=10, y=180)

def ret_roi():
    c = capital.get()
    p = perc_roi.get()
    ret = c*p/100
    ret_in = round(ret, 2)
    Label(root, text=("Required annual roi: "+str(ret_in)), font=('calibri 12 bold'), fg="gray26").place(x=180, y=80)

def roi_perc_cost():
    c = capital.get()
    p = perc_roi.get()
    t = tcost.get()
    rp = (c*p/100)/t
    roi_p = round(rp, 2)
    Label(root, text=("Roi as a % of total cost: "+str(roi_p)), font=('calibri 12 bold'), fg="gray26").place(x=180, y=100)

def mark_up():
    fc = cost_pu.get()
    c = capital.get()
    p = perc_roi.get()
    t = tcost.get()
    mp = ((c*p/100)/t)*fc
    mup = round(mp, 2)
    Label(root, text=("Profit Mark-up: "+str(mup)), font=('calibri 12 bold'), fg="gray26").place(x=180, y=180)

def s_price():
    fc = cost_pu.get()
    c = capital.get()
    p = perc_roi.get()
    t = tcost.get()
    s_pr = ((c*p/100)/t)*fc+fc
    sell_pr = round(s_pr, 2)
    Label(root, text=("Selling price: "+str(sell_pr)), font=('calibri 12 bold'), fg="gray26").place(x=180, y=200)

btn1 = Button(root, text="Expected ROI", font=('calibri 10 bold'), width=12, height=2, bg="sienna1", command=ret_roi).place(x=180, y=40)
btn2 = Button(root, text="ROI as a % \n of Total Cost", font=('calibri 10 bold'), width=12, height=2, bg="sienna1", command=roi_perc_cost).place(x=290, y=40)
btn3 = Button(root, text="Mark-up", font=('calibri 10 bold'), width=12, height=2, bg="sienna1", command=mark_up).place(x=180, y=140)
btn4 = Button(root, text="Selling price", font=('calibri 10 bold'), width=12, height=2, bg="sienna1", command=s_price).place(x=290, y=140)

root.mainloop()


import socketserver
import logging
import time
import sipfullproxy

HOST, PORT = '0.0.0.0', 5060

if __name__ == "__main__":
    logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', filename='proxy.log', level=logging.INFO,
                        datefmt='%H:%M:%S')
    logging.info(time.strftime("%a, %d %b %Y %H:%M:%S ", time.localtime()))
    ipaddress = '192.168.1.173'
    logging.info(ipaddress)
    sipfullproxy.recordroute = "Record-Route: <sip:%s:%d;lr>" % (ipaddress, PORT)
    sipfullproxy.topvia = "Via: SIP/2.0/UDP %s:%d" % (ipaddress, PORT)
    server = socketserver.UDPServer((HOST, PORT), sipfullproxy.UDPHandler)
    server.serve_forever()


# 1. Prompt user for numeric month/day/year
month = int(input('Enter a month numerically (1-12): '))
day = int(input('Enter a day of the month numerically (1-31): '))
year = int(input('Enter a four-digit year (1960): '))

# 2. Multiply month times day
magic = month * day
print('You entered a date of: ' + str(month) + '/' + str(day) + '/'\
      + str(year))

# 3. Test if month*day = year, display if magic
if magic == (abs(year) % 100):
    print('That is a magic date!')
else:
    print('That is not a magic date.')


from typing import (
    NamedTuple,
    Tuple,
)

from wasm.datatypes import (
    ValType,
)

class ControlFrame(NamedTuple):
    """
    Represents the equivalent of a label during expression validation.
    """
    label_types: Tuple[ValType, ...]
    end_types: Tuple[ValType, ...]
    height: int
    is_unreachable: bool

    def mark_unreachable(self) -> 'ControlFrame':
        """
        Recurn a copy of `self` with the `is_unreachable` flag set.
        """
        return type(self)(
            self.label_types,
            self.end_types,
            self.height,
            True,
        )


def combine_dic(dic_1, dic_2):
    # combine dic_1 et dic_2 en gardant la valeur max en cas de clef commune
    dic_3 = dic_1.copy()
    for key in dic_2:
        if key not in dic_3:
            dic_3[key] = dic_2[key]
        else:
            dic_3[key] = max(dic_1[key], dic_2[key])

    return dic_3

if __name__ == '__main__':
    # Combinaison de dictionnaire
    dic_1 = {'a': 5, 'b': 2, 'c': 9}
    dic_2 = {'a': 1, 'b': 8, 'd': 17}

    dic_3 = combine_dic(dic_1,dic_2)
    print(dic_3)


def solution(inputString) :
    gwalMap = {'(': ')', '{': '}', '[': ']', '<': '>'}
    gwalCount = 0
    for i in gwalMap:
        # 괄호 여는것 위치확인
        print(i)
        gwalIndex = inputString.find(i)
        gwalPairIndex = inputString.find(gwalMap[i])
        # 없으면 다음꺼 확인해보고 없으면 다음, 있으면 -1
        if (gwalIndex == -1):
            if gwalPairIndex != -1 : return -1
            else : continue
        else:

            if (gwalPairIndex == -1):
                return -1
            # 둘다 있을경우
            elif gwalPairIndex > gwalIndex:
                gwalCount += 1
                #        문자열지우기
                inputString = inputString[:gwalIndex] + inputString[gwalIndex + 1:]
                gwalPairIndex = inputString.find(gwalMap[i])
                inputString = inputString[:gwalPairIndex] + inputString[gwalPairIndex + 1:]
                print(inputString)
                gwalMapValue = gwalMap[i]
                del gwalMap[i]
                gwalMap[i] = gwalMapValue
                print(gwalMap)
            elif gwalIndex > gwalPairIndex:

                return -1

    return gwalCount

input = '>_<	'

print(solution(input))


from flask import g

from .config import *
from .telegram_api import TelegramApi

class TelegramExtension:
    """
    Flask application extension that adds telegram api to global context

    Attributes
    ----------
    api : TelegramApi
        TelegramApi instance
    """
    def __init__(self, app=None):
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        """
        Initialize app with TelegramApi context

        Note
        ----
        'token' key should be already in flask.g

        Raises
        ------
        RuntimeError
            if bot token not present in context
        """

        self.api = TelegramApi()

        @app.before_request
        def with_telegram_api():
            """
            Push TelegramApi as g.api
            """
            g.api = self.api
            bot_token = g.get('token')
            if bot_token is None:
                raise RuntimeError('bot token not found in context')
            g.api.bot_token = bot_token


# Goal 2

# Create a single iterable that combines all the columns from all the iterators.

# The iterable should yield named tuples containing all the columns.
# Make sure that the SSN's across the files match!
# All the files are guaranteed to be in SSN sort order, and every SSN is unique, and every SSN appears in every file.
# Make sure the SSN is not repeated 4 times - one time per row is enough!

from operator import add
from itertools import chain, islice
from functools import reduce
from collections import namedtuple
from goal1_iterators import file_iterator
from constants import data_types, files, tuple_names

def files_iterator(files, data_types):
    """My solution to goal2.  It is not efficient since it does not use the fact that
    namedtuples should be created once and reused.  It would thus be better to return the
    joined dictionary"""
    with open(files[0]) as f0, open(files[1]) as f1, open(files[2]) as f2, open(files[3]) as f3:
        file0_iterator = file_iterator(files[0], data_types[0], tuple_names[0])
        file1_iterator = file_iterator(files[1], data_types[1], tuple_names[1])
        file2_iterator = file_iterator(files[2], data_types[2], tuple_names[2])
        file3_iterator = file_iterator(files[3], data_types[3], tuple_names[3])

        iterators = list(zip(file0_iterator, file1_iterator,
                         file2_iterator, file3_iterator))

        # RESET THE ITERATORS if use get_joined_nt_names in improved post-study code
        f0.seek(0), f1.seek(0), f2.seek(0), f3.seek(0)
        return iterators

def print_n_rows_joint(num_rows_to_display=5):
    """Print n rows of the joint namedtuple"""
    iterators = files_iterator(files, data_types,)
    for row in islice(iterators, num_rows_to_display):
        dict0, dict1, dict2, dict3 = row[0]._asdict(
        ), row[1]._asdict(), row[2]._asdict(), row[3]._asdict()
        if dict0['ssn'] == dict1['ssn'] == dict2['ssn'] == dict3['ssn']:
            print("Next joined nametuple: ")
            joined_dict = {**dict0, **dict1, **dict2, **dict3}
            joined_nt = namedtuple('JoinedNT', joined_dict.keys())(
                *joined_dict.values())
            print("Joined NT: ", joined_nt)

if __name__ == "__main__":
    print_n_rows_joint(10)
    # OUTPUT
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='100-53-9824', first_name='Sebastiano', last_name='Tester', gender='Male', language='Icelandic', vehicle_make='Oldsmobile', vehicle_model='Bravada', model_year=1993, employer='Stiedemann-Bailey', department='Research and Development', employee_id='29-0890771', last_updated='2017-10-07T00:14:42Z', created='2016-01-24T21:19:30Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='101-71-4702', first_name='Cayla', last_name='MacDonagh', gender='Female', language='Lao', vehicle_make='Ford', vehicle_model='Mustang', model_year=1997, employer='Nicolas and Sons', department='Sales', employee_id='41-6841359', last_updated='2017-01-23T11:23:17Z', created='2016-01-27T04:32:57Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='101-84-0356', first_name='Nomi', last_name='Lipprose', gender='Female', language='Yiddish', vehicle_make='GMC', vehicle_model='Yukon', model_year=2005, employer='Connelly Group', department='Research and Development', employee_id='98-7952860', last_updated='2017-10-04T11:21:30Z', created='2016-09-21T23:04:07Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='104-22-0928', first_name='Justinian', last_name='Kunzelmann', gender='Male', language='Dhivehi', vehicle_make='Oldsmobile', vehicle_model='Intrigue', model_year=2000, employer='Upton LLC', department='Marketing', employee_id='56-9817552', last_updated='2017-03-28T12:38:29Z', created='2016-04-15T11:37:17Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='104-84-7144', first_name='Claudianus', last_name='Brixey', gender='Male', language='Afrikaans', vehicle_make='Ford', vehicle_model='Crown Victoria', model_year=2008, employer='Zemlak-Olson', department='Business Development', employee_id='46-2886707', last_updated='2018-02-19T01:34:33Z', created='2016-03-15T14:07:57Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='105-27-5541', first_name='Federico', last_name='Aggett', gender='Male', language='Chinese', vehicle_make='Ford', vehicle_model='Mustang', model_year=2001, employer='Kohler, Bradtke and Davis', department='Support', employee_id='80-0975518', last_updated='2017-07-24T08:58:52Z', created='2016-07-23T17:58:35Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='105-85-7486', first_name='Angelina', last_name='McAvey', gender='Female', language='Punjabi', vehicle_make='Chrysler', vehicle_model='300', model_year=2008, employer='Roberts, Torphy and Dach', department='Human Resources', employee_id='77-4895332', last_updated='2018-02-14T11:32:39Z', created='2016-12-15T05:46:43Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='105-91-5022', first_name='Moselle', last_name='Apfel', gender='Female', language='Latvian', vehicle_make='Isuzu', vehicle_model='Hombre Space', model_year=2000, employer='Lind-Jast', department='Marketing', employee_id='79-6418731', last_updated='2018-03-24T14:29:33Z', created='2016-03-24T03:43:03Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='105-91-7777', first_name='Audi', last_name='Roach', gender='Female', language='Estonian', vehicle_make='Chevrolet', vehicle_model='Silverado 3500', model_year=2004, employer='Bashirian-Lueilwitz', department='Engineering', employee_id='44-3328799', last_updated='2017-05-11T01:48:32Z', created='2016-05-31T00:38:13Z')
    # Next joined nametuple:
    # Joined NT:  JoinedNT(ssn='106-35-1938', first_name='Mackenzie', last_name='Nussey', gender='Male', language='Swedish', vehicle_make='GMC', vehicle_model='Sonoma Club', model_year=1992, employer='Windler, Marks and Haley', department='Services', employee_id='54-6271885', last_updated='2017-10-21T01:07:28Z', created='2016-09-08T04:02:12Z')

# OTHER SCRATCHWORK AND EXPERIMENTS

# SNIFF OUT THE NAME: MAKE THIS INTO A FUNCTION
# for row in islice(iterators, 1):
#     print("ROW 0 FIELDS: ", row[0]._fields)
#     print("ROW 1 FIELDS: ", row[1]._fields)
#     all_fields = row[0]._fields + row[1]._fields + \
#         row[2]._fields+row[3]._fields
#     all_fields = ('ssn',) + \
#         tuple(elt for elt in all_fields if elt != 'ssn')
#     joined_nt = namedtuple('JoinedNT', all_fields)
#     print(all_fields)

# You've pretty much nailed it as far as vanilla Python is concerned, but there's an extra simplification you can do if you're using Python 3.5+.

A = namedtuple("A", "a b c")
B = namedtuple("B", "d e")
a = A(10, 20, 30)
b = B(40, 50)
C = namedtuple("C", A._fields + B._fields)
# print(C(*(a + b)))
# C(a=10, b=20, c=30, d=40, e=50)
# Available in Python 3.5+
#print(C(*a, *b))
# C(a=10, b=20, c=30, d=40, e=50)
# print(A._fields)

def namedtuplemerge(*args):
    cls = namedtuple('_'.join(arg.__class__.__name__ for arg in args), reduce(
        add, (arg._fields for arg in args)))
    return cls(*chain(*args))

contents = [[] for _ in range(4)]
for i in range(4):
    #print(f"File {files[i]}")
    # contents[i] = list(chain(file_iterator(
    #     files[i], data_types=data_types[i], tuple_name=tuple_names[i])))
    contents[i] = file_iterator(
        files[i], data_types=data_types[i], tuple_name=tuple_names[i])
# print(contents)
#args = files
#all_names = list(nt[0]._fields for nt in contents)
# reduced = reduce(add, list(nt[0]._fields for nt in contents))
# # make 'ssn' appear as the first field
# fields_ = ('ssn',) + tuple(set(reduced)-{'ssn'})

# fields_ = ('ssn',)+tuple(set(contents[0][0]._fields + contents[1]
# [0]._fields+contents[2][0]._fields+contents[3][0]._fields)-{'ssn'})
# print(fields_)

# C = namedtuple("C", contents[0][0]._fields + contents[1][0]._fields)
# print(C(*a, *b))

# print(namedtuplemerge(contents[0], contents[1]))

# Loop through rows
# for each row, chain across row (ie, apply merge to each row) if ssn == for each arg

# print(len(contents))
#print(list(nt._fields for nt in contents))
#print(list((nt[0] for nt in contents)))
#print(list((nt[0] for nt in contents if not nt.ssn)))
def joined_fieldnames(contents):
    # To make this function self-contained, consider placing the import inside the function
    #from functools import reduce
    reduced = reduce(
        add, (nt[0]._fields for nt in contents))
    #print('REDUCED', reduced)
    reduced_ = ('ssn',)+tuple(elt for elt in reduced if elt != 'ssn')
    # print(reduced_)
    nt_joint = namedtuple("Joined", reduced_)
    #print("FIELDS: ", nt_joint._fields)
    return nt_joint

# joined_nt = joined_fieldnames(contents)
# count = 0
# for row in joined_nt(*chain(*files)):
#     print(joined_nt(row))
# # print(count)

# print(joined_nt(*chain(*files)))

# def files_iterator(files, data_types):
#     #FILES NOT OPENING CORRECTLY
#     with open(files[0]) as f0, open(files[1]) as f1, open(files[2]) as f2, open(files[3]) as f3:
#         reader0, reader1, reader2, reader3 = csv.reader(
#             f0), csv.reader(f1), csv.reader(f2), csv.reader(f3)
#         readers = reader0, reader1, reader2, reader3
#         nts = get_names(next(f0)), get_names(
#             next(f1)), get_names(next(f2)), get_names(next(f3))
#         yield from reader0

# print(files_iterator(files, data_types))

# def display_first_rows(num_rows=5):
#     contents = [[] for _ in range(len(files))]
#     for i in range(len(files)):
#         print(f"File {files[i]}")
#         contents[i] = files_iterator(
#             files[i], data_types=data_types[i])  # , tuple_name=tuple_names[i])
#         for ticket in islice(contents[i], num_rows):
#             print(ticket)

# display_first_rows(5)

def get_joined_nt_names():
    with open(files[0]) as f0, open(files[1]) as f1, open(files[2]) as f2, open(files[3]) as f3:
        file0_iterator = file_iterator(files[0], data_types[0], tuple_names[0])
        file1_iterator = file_iterator(files[1], data_types[1], tuple_names[1])
        file2_iterator = file_iterator(files[2], data_types[2], tuple_names[2])
        file3_iterator = file_iterator(files[3], data_types[3], tuple_names[3])

        iterators = list(zip(file0_iterator, file1_iterator,
                         file2_iterator, file3_iterator))

        # SNIFF OUT THE NAME: MAKE THIS INTO A FUNCTION
        for row in islice(iterators, 1):
            #print("ROW 0 FIELDS: ", row[0]._fields)
            #print("ROW 1 FIELDS: ", row[1]._fields)
            all_fields = row[0]._fields + row[1]._fields + \
                row[2]._fields+row[3]._fields
            all_fields = ('ssn',) + \
                tuple(elt for elt in all_fields if elt != 'ssn')
            joined_nt = namedtuple('JoinedNT', all_fields)
            # print(all_fields)
            return joined_nt

# def files_iterator(files, data_types):
#     with open(files[0]) as f0, open(files[1]) as f1, open(files[2]) as f2, open(files[3]) as f3:
#         reader0, reader1, reader2, reader3 = csv.reader(
#             f0), csv.reader(f1), csv.reader(f2), csv.reader(f3)
#         readers = zip(reader0, reader1, reader2, reader3)
#         #print("READERS: ", list(readers))
#         nt0, nt1, nt2, nt3 = get_names(next(f0), tuple_names[0]), get_names(
#             next(f1), tuple_names[1]), get_names(next(f2), tuple_names[2]), get_names(next(f3), tuple_names[3])
#         # print(nt0)
#         # print(nt1)
#         #print("LEN READERS: ", len(readers))

#         # for r0 in readers:
#         #     r0 = (type_caster(elt[0], elt[1])
#         #           for elt in zip(r0, data_types[0]))

#         #     print("NEXT", nt0(*r0).ssn)

#         for (r0, r1, r2, r3) in readers:
#             r0 = (type_caster(elt[0], elt[1])
#                   for elt in zip(r0, data_types[0]))
#             r1 = (type_caster(elt[0], elt[1])
#                   for elt in zip(r1, data_types[1]))
#             r2 = (type_caster(elt[0], elt[1])
#                   for elt in zip(r2, data_types[2]))
#             r3 = (type_caster(elt[0], elt[1])
#                   for elt in zip(r3, data_types[3]))
#             # yield list(chain(nt0(*r0), nt1(*r1), nt2(*r2), nt3(*r3)))
#             # yield list(chain(nt0(*r0), nt1(*r1), nt2(*r2), nt3(*r3)))
#             # print(nt0(*r0).ssn)
#             # print(nt1(*r1).ssn)
#             if nt0(*r0).ssn == nt1(*r1).ssn == nt2(*r2).ssn == nt3(*r3).ssn:
#                 print(list(zip(nt0(*r0), nt1(*r1), nt2(*r2), nt3(*r3))))

#             # print("R0", nt0(*r0))
#             # yield r0
#             # yield r1
#             # yield r2
#             # yield r3
# print("FILES ITERATOR: ", files_iterator(files, data_types=data_types))

# BACKUP
# def files_iterator(files, data_types):
#     with open(files[0]) as f0, open(files[1]) as f1, open(files[2]) as f2, open(files[3]) as f3:
#         reader0, reader1, reader2, reader3 = csv.reader(
#             f0), csv.reader(f1), csv.reader(f2), csv.reader(f3)
#         readers = reader0, reader1, reader2, reader3
#         nt0, nt1, nt2, nt3 = get_names(next(f0)), get_names(
#             next(f1)), get_names(next(f2)), get_names(next(f3))
#         print("LEN READERS: ", len(readers))
#         for r0, r1, r2, r3 in readers:
#             r0 = (type_caster(elt[0], elt[1])
#                   for elt in zip(r0, data_types[0]))
#             r1 = (type_caster(elt[0], elt[1])
#                   for elt in zip(r0, data_types[0]))
#             r2 = (type_caster(elt[0], elt[1])
#                   for elt in zip(r0, data_types[0]))
#             r3 = (type_caster(elt[0], elt[1])
#                   for elt in zip(r0, data_types[0]))
#             # yield list(chain(nt0(*r0), nt1(*r1), nt2(*r2), nt3(*r3)))
#             # yield list(chain(nt0(*r0), nt1(*r1), nt2(*r2), nt3(*r3)))
#             print("R0", nt0(*r0))
#             yield r0
#             yield r1
#             yield r2
#             yield r3

# print("FILES ITERATOR: ", files_iterator(files, data_types=data_types))

# for i in range(len(contents[0])):
#     print("NEXT: ")
#     if contents[0][i].ssn == contents[1][i].ssn == contents[2][i].ssn == contents[3][i].ssn:
#         print(joined_nt(contents[0][i], contents[1]
#               [i], contents[2][i], contents[3][i]))

# for j in range(len(files)):
#     # for k in range(len(contents[0][]))
#     # if contents[0]
#     #print(i, j)
#     elt = contents[j][i]
#     print(elt)
#     if
#     print(elt.ssn)
#print(f"File {files[i]}")
#contents[i] = file_iterator(files[i], data_types=filetypes[i])
# print(contents)


# All the software here is distributed under the terms of the GNU General Public License Version 3, June 2007.
# Trajalign is a free software and comes with ABSOLUTELY NO WARRANTY.
#
# You are welcome to redistribute the software. However, we appreciate is use of such software would result in citations of
# Picco, A., Kaksonen, M., _Precise tracking of the dynamics of multiple proteins in endocytic events_,  Methods in Cell Biology, Vol. 139, pages 51-68 (2017)
# http://www.sciencedirect.com/science/article/pii/S0091679X16301546
#
# Author: Andrea Picco (https://github.com/apicco)
# Year: 2017

import os
from trajalign.traj import Traj
import copy as cp
import numpy as np
import warnings as wr

from sklearn import linear_model

def header( version = 1.99 , year = 2018 , printit = True ) :

    if printit :

        print('|-----------------------------------------------------|')
        print('| Trajalign version ' + str( version ) +' Copyright ' + str( year ) + ' Andrea Picco. |')
        print('|   Url: www.apicco.github.io/trajectory_alignment/   |')
        print('|-----------------------------------------------------|')

    elif not printit :

        return version

    else :

        raise AttributeError('Please, if you want to print the header (printit = True) or if you want to return the verion number only (printit = False).')

def load_directory(path , pattern = '.txt' , sep = None , comment_char = '#' , dt = None , t_unit = '' , coord_unit = '' , intensity_normalisation = 'None' , **attrs ):

    """
    load_directory(path , pattern = '.txt' , sep = None , comment_char = '#' , dt = None , t_unit = '' , coord_unit = '' , intensity_normalisation = 'None' , **attrs ):
    loads all the trajectories listed in 'path', which have the same 'pattern'.
    columns are separated by 'sep' (default is None: a indefinite number of
    white spaces). Comments in the trajectory start with 'comment_char'.

    intensity_normalisation can be: 'None' (no normalisation, default), 'Integral' (normalise over the integral of the fluorescence intensity),
    or 'Absolute' (normalise the fluorescence intensity values between 0 and 1)"

    **attrs is used to assign columns to the trajectory attributes and to
    add annotations.
    If the time interval is added (and 't' is not called in the **attrs)
    then the time column 't' is added, and the 't_unit' can be set.
    If 'coord' is called then the unit must be added.
    """

    if ('coord' in attrs.keys()) & (len(coord_unit) == 0):
        raise AttributeError('Please, specify the coordinate unit \'coord_unit\'')
    if ('t' in attrs.keys()) & (len(t_unit) == 0):
        raise AttributeError('Please, specify the time unit \'t_unit\'')
    if (dt != None) & (len(t_unit) == 0):
        raise AttributeError('Please, specify the time unit \'t_unit\'')
    if (dt != None) & ('t' in attrs.keys()):
        raise AttributeError('Time is already loaded by the trajectories, you cannot also compute it from frames. Please, either remove the dt option or do not load the \'t\' column from the trajectories')

    trajectories = [] #the list of trajectories
    if ( pattern[ len( pattern ) - 1 ] == '$' ) :
        files = [ f for f in sorted( os.listdir(path) ) if f.endswith( pattern[ : - 1 ] ) ] #list all the files in path that have pattern
    else :
        files = [ f for f in sorted( os.listdir(path) ) if pattern in f] #list all the files in path that have pattern

    for file in files:
        trajectory = Traj(experiment = path, path = os.getcwd()+'/'+path, file = file)
        trajectory.load(path+'/'+file,sep = sep, comment_char = comment_char, **attrs)
        if (dt != None):
            trajectory.time(dt,t_unit)
        if ('coord' in attrs.keys()):

            trajectory.annotations('coord_unit',coord_unit)

        if intensity_normalisation == 'Integral' :

            trajectory.scale_f()

        elif intensity_normalisation == 'Absolute' :

            trajectory.norm_f()

        elif intensity_normalisation != 'None' :

            raise AttributeError( "load_directory: Please, choose a value for the variable intensity_normalisation between 'None' (no normalisation, default), 'Integral' (normalise over the integral of the fluorescence intensity), or 'Absolute' (normalise the fluorescence intensity values between 0 and 1)" )

        trajectory.annotations( 'intensity_normalisation' , intensity_normalisation )
        trajectory.fill()
        trajectories.append(trajectory)

    print( "\n >> load_directory: The 'intensity_normalisation' applied to the trajectories is '" + intensity_normalisation + "' <<\n" )

    return trajectories

def MSD(input_t1 , input_t2):

    """
    MSD(t1,t2): finds the rototranslation the  minimises the mean square displacement between the trajectories t1 and t2 and returns the rototranslation of t2.
    Adapted from Horn, 1987, to the 2D case with means weighted on the product of the fluorescence intensities.
    """

    msdt1 = cp.deepcopy(input_t1)
    msdt2 = cp.deepcopy(input_t2)

    if (len(msdt1.f()) == 0) | (len(msdt2.f()) == 0):
        raise AttributeError('MSD(msdt1,msdt2) requires that trajectories msdt1 and msdt2 have values for the fluorescence intensity')

    #the following code follow Horn's (1987) nomenclature. msdt1 is what is
    #called in the paper as 'right coordinates'.
    #msdt2 is what is called as 'left coordinates'

    with wr.catch_warnings():
        # if both f() are 0 or if their product is 0,  a warning about invalid true divide is output. Here we suppress such warnings.
        wr.simplefilter("ignore", category=RuntimeWarning)
        w = msdt1.f() * msdt2.f() / np.nansum( msdt1.f() * msdt2.f() )
    #computed the center of mass, weigthed on the fluorescence intensity product
    rc = np.array([ np.nansum( w * msdt1.coord()[0] ), np.nansum( w * msdt1.coord()[1] )])
    lc = np.array([ np.nansum( w * msdt2.coord()[0] ), np.nansum( w * msdt2.coord()[1] )])

    #translate the trajecotries to their weigthed center of mass
    msdt1.translate( -1 * rc )
    msdt2.translate( -1 * lc )

    Sxx = np.nansum( w * msdt2.coord()[0] * msdt1.coord()[0] )
    Sxy = np.nansum( w * msdt2.coord()[0] * msdt1.coord()[1] )
    Syx = np.nansum( w * msdt2.coord()[1] * msdt1.coord()[0] )
    Syy = np.nansum( w * msdt2.coord()[1] * msdt1.coord()[1] )

    A = ( Syx - Sxy )
    B = ( Sxx + Syy )

    if ( ( A == 0 ) & ( B == 0 ) ):
        theta = np.nan
    else :
        theta = np.arctan2( - A , B )
#        M = - A / B
#
#        theta1 = np.arctan( M )
#        theta2 = theta1 + np.pi
#
#        if B * np.cos(theta1) >= A * np.sin(theta1) :
#            theta = theta1
#        else:
#            theta = theta2

    msdt2.rotate( theta )

    #the 'score' is the mean square displacement weighted on the cross correlation of the fluorescence intensities
    score = np.nansum( w * ( msdt1.coord()[0] - msdt2.coord()[0] )**2 + w * ( msdt1.coord()[1] - msdt2.coord()[1] )**2 )

    #when the min_w is small and the software is sampling the start or end of trajectories,
    #which often have a large number of nan, then theta can become nan as M is the
    #fraction of 0.0/0.0. If that happens then the score is set to infinite.
    if theta != theta:
        score = np.inf

    return({
        'angle' : theta,
        'rc' : rc,
        'lc' : lc,
        'score' : score
        })

def nanMAD( x , axis = None , k = 1.4826):
    MAD = np.nanmedian( np.absolute( x - np.nanmedian( x , axis ) ) , axis )
    return( k * MAD )

def compute_average_start_and_end( trajectories_time_span , aligned_trajectories , max_frame ) :

    l = len( trajectories_time_span[ 'old_start' ] )
    # Define the START and the END of the average trajectory:
    #compute the start and the end of the average trajectory using the
    #start and end of the aligned trajectories whose frame numbers are
    # greater than 0 (i.e. that appeared after the movie recording was
    #started)
    traj_starts_to_average = [trajectories_time_span[ 'new_start' ][ j ] for j in range(l)\
            if trajectories_time_span[ 'old_start' ][ j ] > 0]
    if len( traj_starts_to_average ) > 0 :
        mean_starts = np.mean( traj_starts_to_average )
        n_starts =  len( traj_starts_to_average )
        std_starts = np.std( traj_starts_to_average )

    else  :
        #it can be that all trajectories start with 0 (old_start), which means they
        #started before the movie begun. If so the mean start is set as the latest
        #time between the two trajectories.
        mean_starts = max(trajectories_time_span[ 'new_start' ])
        n_starts = np.nan
        std_starts = np.nan
        print( 'Warning: all trajectory starts were trunkated' )

    #same as for nan mean_starts. However, for the selected mean_ends is the smallest
    traj_ends_to_average = [trajectories_time_span[ 'new_end' ][ j ] for j in range(l)\
            if trajectories_time_span[ 'old_end' ][ j ] < ( max_frame - 3 ) * float(aligned_trajectories[ 0 ].annotations()[ 'delta_t' ])]
    if len( traj_ends_to_average ) > 0 :
        mean_ends = np.mean( traj_ends_to_average )
        n_ends = len( traj_ends_to_average )
        std_ends = np.std( traj_ends_to_average )
    else  :
        mean_ends = min(trajectories_time_span[ 'new_end' ])
        n_ends = np.nan
        std_ends = np.nan
        print( 'Warning: all trajectory ends were trunkated' )

    return( mean_starts , std_starts , n_starts , mean_ends , std_ends , n_ends )

#-------------------------------------END-OF-DEFINITIONS-in-compute_average_start_and_end-----------------------------------
def trajectory_average( aligned_trajectories_to_average , r , median , fimax ) :

    #define the trajectory where the average will be stored
    t = Traj()

    #inherit the annotations from the reference trajectory
    for a in aligned_trajectories_to_average[ r ].annotations().keys():

        if a == 'file':
            t.annotations( 'reference_file' , aligned_trajectories_to_average[ r ].annotations()[ a ])
        else :
            t.annotations( a , aligned_trajectories_to_average[ r ].annotations()[ a ])

    if fimax :
        t.annotations( 'fimax' , 'TRUE' )

    #group all the attributes of the aligned trajectories...
    attributes = [ a for a in aligned_trajectories_to_average[ r ].attributes() if a not in ('t','frames')]
    #create an empy dictionary where all the attributes that will be then averaged are stored
    attributes_to_be_averaged = {}
    for a in attributes:
        attributes_to_be_averaged[a] = []

    #merge all the trajectory attributes into the dictionary attributes_to_be_averaged.
    for j in range( len( aligned_trajectories_to_average ) ):

        for a in attributes:

            attributes_to_be_averaged[a].append(getattr(aligned_trajectories_to_average[ j ],'_'+a))

    #all the aligned trajectories are set to start at the same  mean_start and finish at mean_end computed from
    #trajectories_time_span in compute_average().Hence, the time interval is the same
    t.input_values( 't' , aligned_trajectories_to_average[ r ].t())

    #average the attributes of the trajectories and assign
    #them to the average trajectory [ r ]
    with wr.catch_warnings():

        # if a line is made only of nan that are averaged, then a waring is outputed. Here we suppress such warnings.
        wr.simplefilter("ignore", category=RuntimeWarning)

        for a in attributes:

            if a[ len( a ) - 4 : len( a ) ] == '_err' :

                raise AttributeError('The trajectories to be averaged have already an non empty error element, suggeting that they are already the result of an average. These error currently are not propagated. Check that your trajectories are correct')

            #if _a_err is in the trajectories slots, it means that the attribute a is not
            #an error attribute (i.e. an attribute ending by _err; in fact if a would end by '_err'
            #then _a_err would have twice the appendix _err (i.e. _err_err) and would have
            #no equivalent in the trajectory __slots__. If _a_err is then in the trajectory
            #slots, then both the mean and the sem can be computed. There is no sem without mean.

            if '_' + a + '_err' in t.__slots__:

                if median :

                    t.input_values( a ,
                            np.nanmedian( attributes_to_be_averaged[ a ], axis = 0 )
                        )

                else :

                    t.input_values( a ,
                            np.nanmean( attributes_to_be_averaged[ a ], axis = 0 )
                        )

                #if there is no n defined yet, it computes it
                if not t.n().any() :
                    #compute the number of not-nan data points by dividing
                    #the nansum by the nanmean. The operation is performed
                    #on the last attribute in the loop that can either have
                    #two dimensions (as 'coord') or one. In case of two dims
                    #only one is used to compute '_n'.

                    x = np.nanmean( attributes_to_be_averaged[ a ] , axis = 0 )
                    y = np.nansum( attributes_to_be_averaged[a] , axis = 0 )

                    if len(x) == 2:
                        t.input_values( 'n' , y[0]/x[0] )
                    else:
                        with wr.catch_warnings():
                            # if both y[i] and x[i] are 0, then a waring is outputed. Here we suppress such warnings.
                            wr.simplefilter("ignore", category=RuntimeWarning)
                            t.input_values( 'n' , y/x )

                #compute the errors as standard errors of the mean/median
                try :
                    if median :

                        t.input_values( a + '_err' ,
                            nanMAD( attributes_to_be_averaged[ a ], axis = 0 ) / np.sqrt( t.n() )
                            )

                    else :

                        t.input_values( a + '_err' ,
                            np.nanstd( attributes_to_be_averaged[ a ], axis = 0 ) / np.sqrt( t.n() )
                            )

                except :

                    raise AttributeError( 'The attribute ' + a + ' cannot have its error assigned' )

            else :

                raise AttributeError( 'The attribute ' + a + ' is not recongnised as an attribute' )

    return( t )
#-------------------------------------END-OF-DEFINITION-of-trajectory_average-----------------------------------

def average_trajectories( trajectory_list , output_file = 'average' , median = False , unify_start_end = True , max_frame=[] , fimax = False , fimax_filter = [ -3/35 , 12/35 , 17/35 , 12/35 , -3/35 ] ):

    """
    average_trajectories( trajectory_list , max_frame = 500 , output_file = 'average' , median = False ): align all the
    trajectories in the list together, and average them. 'max_frame' is the max number of frames the movies from which
    the trajctories have been extracted has. It is used to check whether some trajectories are trunkated at the end.
    'output_file' is the name of the output. average_trajectories outputs a txt file with the average trajectroy and
    a directory with all the raw trajectories that have been used to compute the average aligned together in space and time.
    median is an option to compute the median instead of the average of the aligned trajectories. It is useful in case
    of noisy datasets.
    """

    if len(trajectory_list) == 0 :

        raise IndexError('There are not tajectories in the list; check that the trajectories were loaded correctly')

    if not max_frame :

        raise TypeError('You need to specify the max_frame, which is the frame number in your movies')

    def R(alpha):
        """
        R(alpha): returns the rotation matrix:

            /    cos(alpha)    -sin(alpha)        \
        R =    |                                |
            \    sin(alpha)    cos(alpha)        /
        """
        return(np.matrix([[np.cos(alpha),-np.sin(alpha)], [np.sin(alpha),np.cos(alpha)]]))

    def float_range(x,y,step):
        while x < y:
            yield x
            x += step
    def compute_score(average_t,t_list):

        s = []

        for t in t_list:

            w = average_t.f() * t.f() / np.nansum( average_t.f() * t.f() )
            s.append(np.nansum(w * np.nansum((average_t.coord() - t.coord())**2)))

        return(s)

    def triplicate_trajectory( t ):
        #triplicate t adding itself at its beginning and at its end

        output = cp.deepcopy( t )

        #anticipate the start of trajectory by the trajectory duration and a time interval (you need one time interval
        #between the beginning of the real trajectory and the last point of the "anticipated" bit.
        #as len(t) is the number of frames + 1, then len(t) *dt is the duration of the trajectory + one dt interval, which is needed to
        #separate the duplicate trajectory form the original trajectory
        output.start( t.start() - ( len( t ) * float(t.annotations()['delta_t']) ) )
        #delay the end of trajectory in the same way
        output.end( t.end() + ( len( t ) * float(t.annotations()['delta_t']) ) )

        #coord
        output.coord()[:,0:len(t)] = t.coord()
        output.coord()[:,( len(output) - len(t) ):len(output)] = t.coord()
        #f
        output.f()[0:len(t)] = t.f()
        output.f()[( len(output) - len(t) ):len(output)] = t.f()

        return(output)
    def meanangle(angle_estimates):

        #angles can be identical +- n * pi. Hence, averaging
        #absolute values for the mean angle is wrong. Example:
        # np.mean( ( 0, 2 * np.pi ) )
        #is not 0 but np.pi. However, cos and sin are invariant
        #and the mean angle can be computed back from the
        #mean cos and mean sin.
        #angle_estimates is an array (matrix) where the i-th
        #element has estimates of the angles that rotate the
        #i-th trajectory.

        mean_cos = np.mean(np.cos(angle_estimates),axis=1)
        mean_sin = np.mean(np.sin(angle_estimates),axis=1)

        mean_angle = np.arctan2( mean_sin , mean_cos )

        return( mean_angle )

    def refine_alignment( t1 , t2 , lag , alignments , WeightTrajOverlap = False ):

        t1_frames = t1.frames()
        t2_frames = t2.frames() + lag

        sel_t1 = [ i for i in range( len(t1_frames) ) if t1_frames[i] in t2_frames ]
        sel_t2 = [ i for i in range( len(t2_frames) ) if t2_frames[i] in t1_frames ]

        if ( len(sel_t1) > 0 ) & ( len(sel_t2) > 0 ) :

            alignments.append(
                    MSD( t1.extract( sel_t1 ) , t2.extract( sel_t2 ) )
                    )
            alignments[ len( alignments ) - 1][ 'lag' ] = lag

            if WeightTrajOverlap :
                #the scores are weighted with the number of datapoints of the two trajectoreis that
                #trajectories (for example, trajectories that overlap with two data points only).

                alignments[ len( alignments ) - 1 ][ 'score' ] =\
                        alignments[ len( alignments ) - 1 ][ 'score' ] / np.sqrt( len( sel_t1 ) )
        return()

    def lie_down( t ):
        translation_vector = ( - np.nanmedian( t.coord()[0] ) ,- np.nanmedian( t.coord()[1] ) )
        t.translate( translation_vector )

        I_xx = np.nansum( t.f() * t.coord()[1] ** 2 )
        I_yy = np.nansum( t.f() * t.coord()[0] ** 2 )
        I_xy = np.nansum( t.f() * t.coord()[0] * t.coord()[1] )

        theta = np.arctan2( 2 * I_xy , I_xx - I_yy ) / 2

        I_x = I_xx + I_xy * np.tan( theta )
        I_y = I_yy - I_xy * np.tan( theta )

        if I_x > I_y : theta = theta - np.pi/2
        t.rotate( theta )

        with wr.catch_warnings():
            # if a coord has nan then a waring is outputed when nan > 0 or nan < 0 is asked. Here we suppress such warnings.
            wr.simplefilter("ignore", category=RuntimeWarning)
            A = np.nanmedian( t.coord()[0][ t.coord()[0] > 0 ] ** 2 )
            B = np.nanmedian( t.coord()[0][ t.coord()[0] < 0 ] ** 2 )

            if B > A :
                t.rotate( np.pi )
                theta = theta + np.pi #to ouptput the angle

        model = linear_model.LinearRegression()
        model_RANSACR = linear_model.RANSACRegressor( model , random_state = 42 )

        l = len( t )

        j = 0
        for i in range( l ) :

            if ( not np.isnan( t.coord()[ 0 ][ i ] ) ) & ( not np.isnan( t.coord()[ 1 ][ i ] ) ) :
                if j == 0 :
                    X = np.array( [[ t.coord()[ 0 ][ i ] ]] )
                    y = np.array( [ t.coord()[ 1 ][ i ] ] )
                else :
                    X = np.insert( X , 0 , t.coord()[ 0 ][ i ] , axis = 0 )
                    y = np.insert( y , 0 , t.coord()[ 1 ][ i ] , axis = 0 )
                j += 1

        with wr.catch_warnings():
            # also a bug warning occurs from linear models, RANSACR.
            wr.simplefilter("ignore", category=RuntimeWarning)
            model_RANSACR.fit( X , y )

        t.rotate( - np.arctan( model_RANSACR.estimator_.coef_[0] ) )

        return( { 'translation' :  translation_vector , 'angle' : theta - np.arctan( model_RANSACR.estimator_.coef_[0] ) } )

    #-------------------------------------END-OF-DEFINITIONS-in-average_trajectories-----------------------------------

    def compute_transformations( t1 , t1_index , trajectory_list , fimax , fimax_filter ) :

            selected_alignments = []

            if ( fimax ) :

                t1 = t1.fimax( fimax_filter )

            for traj2 in [ traj2 for traj2 in trajectory_list ]:

                if trajectory_list.index(traj2) >= t1_index :

                    #list of trajectories called in the second loop; as the transformation matrices are
                    #symmetric, transformations are computed only in the upper diagonal.
                    selected_alignments.append(
                        {
                            'angle' : 0,
                            'rc' : np.array([0,0]),
                            'lc' : np.array([0,0]),
                            'lag' : 0,
                            'lag_unit' : 'frames',
                            'score' : np.NaN
                            })

                else :

                    t2 = cp.deepcopy( traj2 )
                    #t2.norm_f()

                    print( 'ref. traj.:\t' + t1.annotations()['file'] )
                    print( 'aligned traj.:\t' + t2.annotations()['file'] )

                    alignments = []

                    if ( fimax ) :

                        t2 = t2.fimax( fimax_filter )

                    #triplicate the longest trajectory by adding itself at its beginning and at its end
                    if ( len(t1)  >= len(t2) ) :
                        x = triplicate_trajectory(t1)
                        y = t2
                    else :
                        x = triplicate_trajectory(t2)
                        y = t1

                    convolution_steps = len(x) - len(y)
                    for i in range( 0 , convolution_steps ) :
                        #by triplicating the longest trajectory we can test all possible alignments in
                        #space and time starting with the entire trajectories x and y.
                        lag = int( x.frames( 0 ) - y.frames( 0 ) + i )
                        x_frames = x.frames()
                        y_frames = y.frames() + lag

                        #select the frames that are overlapping
                        sel_frames = [ i for i in range( len( x_frames) ) if x_frames[ i ] in y_frames ]

                        #which trajectory was triplicated decides the sign of the lag
                        if ( len(t1)  >= len(t2) ) :

                            alignments.append(
                                    MSD( x.extract( sel_frames ) , y )
                                    )
                            alignments[ len(alignments)-1 ][ 'lag' ] = lag

                        else :

                            alignments.append(
                                    MSD( y , x.extract( sel_frames ) )
                                    )
                            alignments[ len(alignments)-1 ][ 'lag' ] = - lag

                    s = [ a['score'] for a in alignments ]
                    lags = [ a['lag'] for a in alignments ]
                    sel_alignments = [ i for i in range(len(s)) if s[i] == min(s) ]

                    #check which of the selected alignments best fit the trajectory t1
                    #and not just its triplicate. Importantly, also recompute the alignment
                    #without repetitions of the trajectory, which alter the alignment output
                    refined_alignments_1 = []
                    t1_frames = t1.frames()
                    for sa in sel_alignments:

                        refine_alignment( t1 , t2 , lags[ sa ] , refined_alignments_1 , WeightTrajOverlap = True )

                    refined_s_1 = [  a['score'] for a in refined_alignments_1 ]

                    refined_alignments_2 =[]
                    lag = refined_alignments_1[ refined_s_1.index( min( refined_s_1 ) ) ][ 'lag' ]

                    #define a span, which is not too small, nor too big compared to the trajectory length
                    refine_span = int( min( len( t1 ) , len( t2 ) ) / 10 )
                    for refined_lag in range( lag - refine_span , lag + refine_span + 1 ):

                        refine_alignment( t1 , t2 , refined_lag , refined_alignments_2 , WeightTrajOverlap = False )

                    refined_s_2 = [  a['score'] for a in refined_alignments_2 ]

                    selected_alignments.append( refined_alignments_2[ refined_s_2.index( min( refined_s_2 ) ) ] )

            if ( fimax ) :

                print('\nfimax = True; Transformations were computed using only the trajectory information up to the max in fluorescence intensity.')

            print('________________')

            return( selected_alignments )

    #-------------------------------------END-OF-DEFINITIONS-in-compute_transformations-----------------------------------

    def compute_average( trajectory_list , tranformations , median , fimax , max_frame , unify_start_end ) :

        aligned_trajectories = [] #contains all the alignments in respect to each trajectory
        average_trajectory = [] #contains all averages in respect to each trajectory
        alignment_precision = [] #contains the alignment precision, measured as a score of the alignment

        #As each trajectory is aligned to a reference trajectory or
        #acts as a reference the rc and lc vectors are obtained
        #from rcs and its transpose (i.e. the aligning trajectory
        #becomes the aligned trajectory).
        rcs = transformations['rcs'] + np.transpose(transformations['lcs'],axes=(1,0,2))

        l = len(transformations['angles'])
        #reference trajectories are indexed with r
        for r in range( l ) :

            #define a dictionary used to store the starts and ends of the aligned
            #trajectories to compute the start of the average trajectory
            trajectories_time_span = \
                    { 'old_start' : [], 'new_start' : [], 'old_end' : [], 'new_end' : []}

            #compute the transformation of the trajectories
            #in respect to the r-th trajectory
            #--angles--
            angles_in_respect_of_r = transformations['angles'][ r , ] - transformations['angles']
            m_angles = meanangle(angles_in_respect_of_r)
            #--lags--
            #lags_in_respect_of_r = transformations['lags'] - transformations['lags'][ r ,]
            lags_in_respect_of_r = transformations['lags'][ r ,] - transformations['lags']
            m_lags = [ int(round(l)) for l in np.mean(lags_in_respect_of_r,axis=1)]
            #--translations--
            r_cm = np.mean([rcs[ r , j ] for j in range(l) if j != r ] , axis = 0 )

            #make a copy of the trajectory_list, whose trajectories need to be aligned
            aligned_trajectories.append( cp.deepcopy( trajectory_list ) )

            ##################################################
            #align the trajectoris together in space and time
            ##################################################
            for j in range(l):

                trajectories_time_span[ 'old_start' ].append(aligned_trajectories[ r ][ j ].start())
                trajectories_time_span[ 'old_end' ].append(aligned_trajectories[ r ][ j ].end())

                #compute the center of mass of the full trajectory

                l_cm = np.mean([rcs[ j , r ] for r in range(l) if r != j ] , axis=0 )

                # the following is equivalent to
                #
                # R( m_angles ) @ aligned_trajectories + T
                #
                # where R would be the rotation matrix computed from m_angles
                # and T is the translation computed as
                #
                # r_cm - R( m_angles ) @ l_cm
                #
                # see Horn 1987 for details.
                aligned_trajectories[ r ][ j ].translate( - l_cm )
                aligned_trajectories[ r ][ j ].rotate( m_angles[ j ] )

                aligned_trajectories[ r ][ j ].translate( r_cm )
                aligned_trajectories[ r ][ j ].lag( m_lags[ j ] )

                aligned_trajectories[ r ][ j ].annotations()[ 'l_cm' ] = tuple( l_cm )
                aligned_trajectories[ r ][ j ].annotations()[ 'r_cm' ] = tuple( r_cm )
                aligned_trajectories[ r ][ j ].annotations()[ 'm_angle' ] = m_angles[ j ]
                aligned_trajectories[ r ][ j ].annotations()[ 'm_lag' ] = m_lags[ j ]

                trajectories_time_span[ 'new_start' ].append(aligned_trajectories[ r ][ j ].start())
                trajectories_time_span[ 'new_end' ].append(aligned_trajectories[ r ][ j ].end())

            mean_start , std_start , n_start , mean_end , std_end , n_end = compute_average_start_and_end( trajectories_time_span , aligned_trajectories[ r ] , max_frame )

            if unify_start_end :

                #uniform start and end of aligned trajectories to mean_start and mean_end
                for j in range(l):

                    #if the unify_start_end is choosen, the average trajectory is started (and ended) from the average
                    #start (and end) of the trajectory minus (and plus) the 95% CI. This addition (or subtraction) as
                    #been choosen to counter the intrinsic underestimate of the trajectories lifetimes.
                    aligned_trajectories[ r ][ j ].start( mean_start - 1.96 * std_start / np.sqrt( n_start ) )
                    aligned_trajectories[ r ][ j ].end( mean_end + 1.96 * std_end / np.sqrt( n_end ) )

            else :

                for j in range(l):

                    aligned_trajectories[ r ][ j ].start( min( trajectories_time_span[ 'new_start' ] ) )
                    aligned_trajectories[ r ][ j ].end( max( trajectories_time_span[ 'new_end' ] ) )

            ########################################################################
            #compute the average of the trajectories aligned to the r-th trajectory
            #define the average trajectory and its time attribute
            ########################################################################

            ta = trajectory_average( aligned_trajectories[ r ] , r , median , fimax )

            #record the standard deviation of the average start and end, so that the user
            #knows how the start and end timepoints of the raw trajectories are distributed,
            #once alingned.

            ta.annotations( 'mean_starts' , str( mean_start ) )
            ta.annotations( 'std_starts' , str( std_start ) )
            ta.annotations( 'n_starts' , str( n_start ) )
            ta.annotations( 'mean_ends' , str( mean_end ) )
            ta.annotations( 'std_ends' , str( std_end ) )
            ta.annotations( 'n_ends' , str( n_end ) )

            if not unify_start_end :

                ta.annotations( 'unified_start' , mean_start - 1.96 * std_start / np.sqrt( n_start ) )
                ta.annotations( 'unified_end' , mean_end + 1.96 * std_end / np.sqrt( n_end ) )

            average_trajectory.append( ta )

            #store the transformations of the trajectories in respect of the trajectory r.
            if r == 0:
                all_m_angles = np.array([ m_angles ])
                all_m_lags = np.array([ m_lags ])
            else :
                all_m_angles = np.vstack([ all_m_angles , m_angles ])
                all_m_lags = np.vstack([ all_m_lags , m_lags ])

            mean_precision =  np.sqrt(
                    np.nanmean(
                        average_trajectory[ r ].coord_err()[ 0 ] ** 2 + average_trajectory[ r ].coord_err()[ 1 ] ** 2
                        )
                    )
            alignment_precision.append(mean_precision)

        print('ALIGNMENT PRECISIONS.\nMIN is the alignment\nselected for the average\n----------------------')
        for a in alignment_precision :

            if a == min( alignment_precision ) :

                print( 'MIN>>\t' + str( a ) )

            elif a == max( alignment_precision ) :

                print( 'MAX>>\t' + str( a ) )

            else :

                print( '\t' + str( a ) )

        print('----------------------')
        print( 'MEAN:\t' + str( np.mean( alignment_precision ) ) )

        return( aligned_trajectories , average_trajectory , alignment_precision )

    #-------------------------------------END-OF-DEFINITIONS-in-compute_average-----------------------------------

    header()

    #define the list where transformations are stored
    transformations = {
            'angles' : np.array( [] ),
            'rcs' : np.array( [ np.array( [] ) , np.array( [] ) ] ),#note that the matric rcs is the transpose of the lcs
            'lcs' : np.array( [ np.array( [] ) , np.array( [] ) ] ),
            'lags' : np.array( [] ),
            'lag_units' : np.array( [] )
            }

    for traj1 in trajectory_list:

        #t1 is the reference trajectory to which all the other trajectories are alinged
        #The loop goes on all trajectories as all of them are eligible to be used as reference

        #the index need to be computed now, becuase if fi_max is true, then t1 will be replaced
        #by the part of t1 trajectory that stops at the peak of fluorescence intensity. This
        #new trajectory cannot be found animore in trajectory_list. Hence, we must compute the
        #index before.

        t1 = cp.deepcopy( traj1 )
        t1_index = trajectory_list.index(traj1)
        #t1.norm_f()

        selected_alignments = compute_transformations( t1 , t1_index , trajectory_list , fimax , fimax_filter )

        #Create a matrix with all the transformations: angle, lag and center of masses.
        #As a convention the element i,j in the matrix contains the elements for the
        #rototranslation and temporal shift to align the trajectori i to j, j being the
        #reference.
        if t1_index == 0:

            transformations['angles'] = np.array(
                    [a['angle'] for a in selected_alignments]
                    )
            transformations['rcs'] = np.array([
                    np.array([a['rc'] for a in selected_alignments])
                    ])
            transformations['lcs'] = np.array([
                    np.array([a['lc'] for a in selected_alignments])
                    ])
            transformations['lags'] = np.array(
                    [a['lag'] for a in selected_alignments]
                    )
        else:

            transformations['angles'] = np.vstack([
                transformations['angles'],
                np.array([
                    np.array([a['angle'] for a in selected_alignments])
                    ])
                ])
            transformations['rcs'] = np.vstack([
                transformations['rcs'],
                    [[a['rc'] for a in selected_alignments]]
                ])
            transformations['lcs'] = np.vstack([
                transformations['lcs'],
                    [[a['lc'] for a in selected_alignments]]
                ])
            transformations['lags'] = np.vstack([
                transformations['lags'],
                np.array(
                    [a['lag'] for a in selected_alignments]
                    )
                ])

    transformations['angles'] = transformations['angles'] - np.transpose(transformations['angles'])
    transformations['lags'] = transformations['lags'] - np.transpose(transformations['lags'])

    l = len(transformations['angles'])
    for i in range( l ):
        transformations['lcs'][ i , i ] = [ 0 , 0 ]

    #compute the average transformation using each trajectory as possible reference
    aligned_trajectories , average_trajectory , alignment_precision = compute_average( trajectory_list , transformations , median , fimax , max_frame , unify_start_end )

    best_average = alignment_precision.index( np.nanmin( alignment_precision ) )
    worst_average = alignment_precision.index( np.nanmax( alignment_precision ) )

    if not unify_start_end :

        # compute the lie_down only on the part of the trajectory that represents
        # most of the average trajectories. That would be the part of average trajectory
        # chosen if unify_start_end = True, i.e. the part of trajectory comprised between
        # the annotations unified_start and unified_end
        average_trajectory_tmp = cp.deepcopy( average_trajectory[ best_average ] )
        average_trajectory_tmp.start( float( average_trajectory_tmp.annotations()[ 'unified_start' ] ) )
        average_trajectory_tmp.end( float( average_trajectory_tmp.annotations()[ 'unified_end' ] ) )
        lie_down_transform = lie_down( average_trajectory_tmp )

        # lie_down modified average_trajectory_tmp with the transformations in dict lie_down_transform
        # we apply these transformations to average_trajectory[ best_average ]
        average_trajectory[ best_average ].translate( lie_down_transform[ 'translation' ] )
        average_trajectory[ best_average ].rotate( lie_down_transform[ 'angle' ] )

    else :

        lie_down_transform = lie_down( average_trajectory[ best_average ] )

    average_trajectory[ best_average ].annotations()[ 'trajalign_version' ] = header( printit = False )
    average_trajectory[ best_average ].save( output_file )

    #save the trajectories use to compute the average, lied down as the average trajectory
    for i in range(l):
        aligned_trajectories[ best_average ][ i ].translate( lie_down_transform[ 'translation' ] )
        aligned_trajectories[ best_average ][ i ].rotate( lie_down_transform[ 'angle' ] )
        aligned_trajectories[ best_average ][ i ].annotations()[ 'trajalign_version' ] = header( printit = False )
        aligned_trajectories[ best_average ][ i ].annotations()[ 'lie_down_angle' ] = lie_down_transform[ 'angle' ]
        aligned_trajectories[ best_average ][ i ].annotations()[ 'lie_down_translation' ] = tuple( lie_down_transform[ 'translation' ] )

        filename = "./" + output_file + "/" + aligned_trajectories[ best_average ][ i ].annotations()[ 'file' ]
        if i == 0 :
            directory = os.path.dirname( filename )
            if not os.path.exists( directory ) :
                os.makedirs( directory )
        aligned_trajectories[ best_average ][ i ].save( filename )

    with open( "./" + output_file + "/alignment_precision.txt" , 'w' ) as f :

        for ap in alignment_precision :
            f.write( repr( ap ) + '\n' )

    f.close()

    return( average_trajectory[ best_average ] , average_trajectory[ worst_average ] , aligned_trajectories[ best_average ] )

def unified_start( t , add_CI = True , correct_lag = True ) :

    try :

        if add_CI :
            ustart =  float( t.annotations()[ 'mean_starts' ] ) - 1.96 * float( t.annotations()[ 'std_starts' ] ) / np.sqrt( float( t.annotations()[ 'n_starts' ] ) )
        else :
            ustart = float( t.annotations()[ 'mean_starts' ] )

        if ( 'alignment_lag' in t.annotations().keys() ) & correct_lag :
            return ustart + float( t.annotations()[ 'alignment_lag' ].split()[ 0 ] )
        else :
            return ustart

    except :

        print( 'Error: one or more of the annotations mean_starts, std_starts, and n_starts is/are missiong' )

def unified_end( t , add_CI = True , correct_lag = True ) :

    try :

        if add_CI :
            uend = float( t.annotations()[ 'mean_ends' ] ) + 1.96 * float( t.annotations()[ 'std_ends' ] ) / np.sqrt( float( t.annotations()[ 'n_ends' ] ) )
        else :
            uend = float( t.annotations()[ 'mean_ends' ] )

        if ( 'alignment_lag' in t.annotations().keys() ) & correct_lag :
            return uend + float( t.annotations()[ 'alignment_lag' ].split()[ 0 ] )
        else :
            return uend

    except :

        print( 'Error: one or more of the annotations mean_ends, std_ends, and n_ends is/are missiong' )


# -*- coding: utf-8 -*-

import bink
import pytest
from unittest.mock import patch #, mock_open

def test_args_happy_path():
    '''
    Pytest for the happy path args
    '''
    args = ['--file' , 'bink/poles.csv']
    parser = bink.parse_args()
    nargs = parser.parse_args(args)
    assert(nargs.file == 'bink/poles.csv')

def test_args_years():
    args = ['-y']
    parser = bink.parse_args()
    nargs = parser.parse_args(args)
    assert(nargs.years == True)

def test_args_invalid(capsys):
    args = ['-a', 'what is this']
    with pytest.raises(SystemExit):
        parser = bink.parse_args()
        parser.parse_args(args)
        out, err = capsys.readouterr()
        assert out == 2

data="test,a,new,file"
# mock_open_func = mock_open(read_data=data)
def mock_csv(f):
    print('I was called with {}'.format(f))
    return [ {'this':123, 'is':'boo', 'a':123,'dict':'really?'} ]

@patch('bink.Pole')
def test_file_open(mockPole):
    '''
    Will have to mock the filesystem for reading.
    '''
    test_data = [
        {'a':12, 'b':23, 'c':'elephant'},
        {'a':12, 'b':23, 'c':'elephant'},
        {'a':12, 'b':23, 'c':'elephant'},
        {'a':12, 'b':23, 'c':'elephant'},
    ]
    pole = mockPole()
    pole.read_data.return_value = True
    pole.csv_data = test_data

    res = pole.read_data('test')
    assert res == True

@patch('bink.csv.DictReader', side_effect=mock_csv)
def test_csv_reader(mock_csv):
    pole=bink.Pole('f')
    assert pole.csv_data == [ {'this':123, 'is':'boo', 'a':123,'dict':'really?'} ]


import argparse
from collections import OrderedDict
import glob
import json
import logging
import math
import os
import re
import sys
import git
from typing import Dict, List

import numpy as np
import pandas as pd
import tqdm

from merlion.evaluate.forecast import ForecastEvaluator, ForecastMetric, ForecastEvaluatorConfig
from merlion.models.ensemble.combine import CombinerBase, Mean, ModelSelector, MetricWeightedMean
from merlion.models.ensemble.forecast import ForecasterEnsembleConfig, ForecasterEnsemble
from merlion.models.factory import ModelFactory
from merlion.models.forecast.base import ForecasterBase
from merlion.transform.resample import TemporalResample, granularity_str_to_seconds
from merlion.utils import TimeSeries, UnivariateTimeSeries
from merlion.utils.resample import infer_granularity, to_pd_datetime

from ts_datasets.base import BaseDataset
from ts_datasets.forecast import *

import matplotlib.pyplot as plt

logger = logging.getLogger(__name__)

MERLION_ROOT = os.path.dirname(os.path.abspath(__file__))
CONFIG_JSON = os.path.join(MERLION_ROOT, "conf", "benchmark_forecast.json")
DATADIR = os.path.join(MERLION_ROOT, "data")
OUTPUTDIR = os.path.join(MERLION_ROOT, "results", "forecast")

def parse_args():
    with open(CONFIG_JSON, "r") as f:
        valid_models = list(json.load(f).keys())

    parser = argparse.ArgumentParser(
        description="Script to benchmark various Merlion forecasting models on "
        "univariate forecasting task. This file assumes that "
        "you have pip installed both merlion (this repo's main "
        "package) and ts_datasets (a sub-repo)."
    )
    parser.add_argument(
        "--dataset",
        default="M4_Hourly",
        help="Name of dataset to run benchmark on. See get_dataset() "
        "in ts_datasets/ts_datasets/forecast/__init__.py for "
        "valid options.",
    )
    parser.add_argument("--data_root", default=None, help="Root directory/file of dataset.")
    parser.add_argument("--data_kwargs", default="{}", help="JSON of keyword arguments for the data loader.")
    parser.add_argument(
        "--models",
        type=str,
        nargs="*",
        default=None,
        help="Name of forecasting model to benchmark.",
        choices=valid_models,
    )
    parser.add_argument(
        "--hash",
        type=str,
        default=None,
        help="Unique identifier for the output file. Can be useful "
        "if doing multiple runs with the same model but different "
        "hyperparameters.",
    )
    parser.add_argument(
        "--ensemble_type",
        type=str,
        default="selector",
        help="How to combine multiple models in an ensemble",
        choices=["mean", "err_weighted_mean", "selector"],
    )
    parser.add_argument(
        "--retrain_type",
        type=str,
        default="without_retrain",
        help="Name of retrain type, should be one of the three "
        "types, without_retrain, sliding_window_retrain"
        "or expanding_window_retrain.",
        choices=["without_retrain", "sliding_window_retrain", "expanding_window_retrain"],
    )
    parser.add_argument("--n_retrain", type=int, default=0, help="Specify the number of retrain times.")
    parser.add_argument(
        "--load_checkpoint",
        action="store_true",
        default=False,
        help="Specify this option if you would like continue "
        "training your model on a dataset from a "
        "checkpoint, instead of restarting from scratch.",
    )
    parser.add_argument("--debug", action="store_true", default=False, help="Whether to set logging level to debug.")
    parser.add_argument(
        "--visualize",
        action="store_true",
        default=False,
        help="Whether to plot the model's predictions after "
        "training on each example. Mutually exclusive "
        "with running any sort of evaluation.",
    )
    parser.add_argument(
        "--summarize",
        action="store_true",
        default=False,
        help="Specify this option if you want to summarize "
        "all results for a particular dataset. Note "
        "that this option only summarizes the results "
        "that have already been computed! It does not "
        "run any algorithms, aside from the one(s) given "
        "to --models (if any).",
    )

    args = parser.parse_args()
    args.data_kwargs = json.loads(args.data_kwargs)
    assert isinstance(args.data_kwargs, dict)

    # If not summarizing all results, we need at least one model to evaluate
    if args.summarize and args.models is None:
        args.models = []
    elif not args.summarize:
        if args.models is None:
            args.models = ["ARIMA"]
        elif len(args.models) == 0:
            parser.error("At least one model required if --summarize not given")

    return args

def get_dataset_name(dataset: BaseDataset) -> str:
    name = type(dataset).__name__
    if hasattr(dataset, "subset") and dataset.subset is not None:
        name += "_" + dataset.subset
    if isinstance(dataset, CustomDataset):
        root = dataset.rootdir
        name = os.path.join(name, os.path.basename(os.path.dirname(root) if os.path.isfile(root) else root))
    return name

def resolve_model_name(model_name: str):
    with open(CONFIG_JSON, "r") as f:
        config_dict = json.load(f)

    if model_name not in config_dict:
        raise NotImplementedError(
            f"Benchmarking not implemented for model {model_name}. Valid model names are {list(config_dict.keys())}"
        )

    while "alias" in config_dict[model_name]:
        model_name = config_dict[model_name]["alias"]

    return model_name

def get_model(model_name: str, dataset: BaseDataset, **kwargs) -> ForecasterBase:
    """Gets the model, configured for the specified dataset."""
    with open(CONFIG_JSON, "r") as f:
        config_dict = json.load(f)

    if model_name not in config_dict:
        raise NotImplementedError(
            f"Benchmarking not implemented for model {model_name}. Valid model names are {list(config_dict.keys())}"
        )

    while "alias" in config_dict[model_name]:
        model_name = config_dict[model_name]["alias"]

    # Load the model with default kwargs, but override with dataset-specific
    # kwargs where relevant, as well as manual kwargs
    model_configs = config_dict[model_name]["config"]
    model_type = config_dict[model_name].get("model_type", model_name)
    model_kwargs = model_configs["default"]
    model_kwargs.update(model_configs.get(type(dataset).__name__, {}))
    model_kwargs.update(kwargs)

    # Override the transform with Identity
    if "transform" in model_kwargs:
        logger.warning(
            f"Data pre-processing transforms currently not "
            f"supported for forecasting. Ignoring "
            f"transform {model_kwargs['transform']} and "
            f"using Identity instead."
        )
    model_kwargs["transform"] = TemporalResample(
        granularity=None, aggregation_policy="Mean", missing_value_policy="FFill"
    )

    return ModelFactory.create(name=model_type, **model_kwargs)

def get_combiner(ensemble_type: str) -> CombinerBase:
    if ensemble_type == "mean":
        return Mean(abs_score=False)
    elif ensemble_type == "selector":
        return ModelSelector(metric=ForecastMetric.sMAPE)
    elif ensemble_type == "err_weighted_mean":
        return MetricWeightedMean(metric=ForecastMetric.sMAPE)
    else:
        raise KeyError(f"ensemble_type {ensemble_type} not supported.")

def get_dirname(model_names: List[str], ensemble_type: str) -> str:
    dirname = "+".join(sorted(model_names))
    if len(model_names) > 1:
        dirname += "_" + ensemble_type
    return dirname

def train_model(
    model_names: List[str],
    dataset: BaseDataset,
    ensemble_type: str,
    csv: str,
    config_fname: str,
    retrain_type: str = "without_retrain",
    n_retrain: int = 10,
    load_checkpoint: bool = False,
    visualize: bool = False,
):
    """
    Trains all the model on the dataset, and evaluates its predictions for every
    horizon setting on every time series.
    """
    model_names = [resolve_model_name(m) for m in model_names]
    dirname = get_dirname(model_names, ensemble_type)
    dirname = dirname + "_" + retrain_type + str(n_retrain)
    results_dir = os.path.join(MERLION_ROOT, "results", "forecast", dirname)
    os.makedirs(results_dir, exist_ok=True)
    dataset_name = get_dataset_name(dataset)

    # Determine where to start within the dataset if there is a checkpoint
    if os.path.isfile(csv) and load_checkpoint:
        i0 = pd.read_csv(csv).idx.max()
    else:
        i0 = -1
        os.makedirs(os.path.dirname(csv), exist_ok=True)
        with open(csv, "w") as f:
            f.write("idx,name,horizon,retrain_type,n_retrain,RMSE,sMAPE\n")

    model = None
    # loop over dataset

    is_multivariate_data = dataset[0][0].shape[1] > 1

    for i, (df, md) in enumerate(tqdm.tqdm(dataset, desc=f"{dataset_name} Dataset")):
        if i <= i0:
            continue
        trainval = md["trainval"]

        # Resample to an appropriate granularity according to metadata
        if "granularity" in md:
            dt = md["granularity"]
            df = df.resample(dt, closed="right", label="right").mean().interpolate()

        vals = TimeSeries.from_pd(df)
        dt = infer_granularity(vals.time_stamps)

        # Get the train/val split
        t = trainval.index[np.argmax(~trainval)].value // 1e9
        train_vals, test_vals = vals.bisect(t, t_in_left=False)

        # Compute train_window_len and test_window_len
        train_start_timestamp = train_vals.univariates[train_vals.names[0]].time_stamps[0]
        test_start_timestamp = test_vals.univariates[test_vals.names[0]].time_stamps[0]
        train_window_len = test_start_timestamp - train_start_timestamp

        train_end_timestamp = train_vals.univariates[train_vals.names[0]].time_stamps[-1]
        test_end_timestamp = test_vals.univariates[test_vals.names[0]].time_stamps[-1]
        test_window_len = test_end_timestamp - train_end_timestamp

        # Get all the horizon conditions we want to evaluate from metadata
        if any("condition" in k and isinstance(v, list) for k, v in md.items()):
            conditions = sum([v for k, v in md.items() if "condition" in k and isinstance(v, list)], [])
            logger.debug("\n" + "=" * 80 + "\n" + df.columns[0] + "\n" + "=" * 80 + "\n")
            horizons = set()
            for condition in conditions:
                horizons.update([v for k, v in condition.items() if "horizon" in k])

        # For multivariate data, we use a horizon of 3
        elif is_multivariate_data:
            horizons = [3 * dt]

        # For univariate data, we predict the entire test data in batch
        else:
            horizons = [test_window_len]

        # loop over horizon conditions
        for horizon in horizons:
            horizon = granularity_str_to_seconds(horizon)
            try:
                max_forecast_steps = int(math.ceil(horizon / dt.total_seconds()))
            except:
                window = TimeSeries.from_pd(test_vals.to_pd()[: to_pd_datetime(train_end_timestamp + horizon)])
                max_forecast_steps = len(TemporalResample(granularity=dt)(window))
            logger.debug(f"horizon is {pd.Timedelta(seconds=horizon)} and max_forecast_steps is {max_forecast_steps}")
            if retrain_type == "without_retrain":
                retrain_freq = None
                train_window = None
                n_retrain = 0
            elif retrain_type == "sliding_window_retrain":
                retrain_freq = math.ceil(test_window_len / int(n_retrain))
                train_window = train_window_len
                horizon = min(retrain_freq, horizon)
            elif retrain_type == "expanding_window_retrain":
                retrain_freq = math.ceil(test_window_len / int(n_retrain))
                train_window = None
                horizon = min(retrain_freq, horizon)
            else:
                raise ValueError(
                    "the retrain_type should be without_retrain, sliding_window_retrain or expanding_window_retrain"
                )

            # Get Model
            models = [get_model(m, dataset, max_forecast_steps=max_forecast_steps) for m in model_names]
            if len(models) == 1:
                model = models[0]
            else:
                config = ForecasterEnsembleConfig(combiner=get_combiner(ensemble_type))
                model = ForecasterEnsemble(config=config, models=models)

            evaluator = ForecastEvaluator(
                model=model,
                config=ForecastEvaluatorConfig(train_window=train_window, horizon=horizon, retrain_freq=retrain_freq),
            )

            # Get Evaluate Results
            train_result, test_pred = evaluator.get_predict(train_vals=train_vals, test_vals=test_vals)

            rmses = evaluator.evaluate(ground_truth=test_vals, predict=test_pred, metric=ForecastMetric.RMSE)
            smapes = evaluator.evaluate(ground_truth=test_vals, predict=test_pred, metric=ForecastMetric.sMAPE)

            # Log relevant info to the CSV
            with open(csv, "a") as f:
                f.write(f"{i},{df.columns[0]},{horizon},{retrain_type},{n_retrain},{rmses},{smapes}\n")

            # generate comparison plot
            if visualize:
                name = train_vals.names[0]
                train_time_stamps = train_vals.univariates[name].time_stamps
                fig_dir = os.path.join(results_dir, dataset_name + "_figs")
                os.makedirs(fig_dir, exist_ok=True)
                fig_dataset_dir = os.path.join(fig_dir, df.columns[0])
                os.makedirs(fig_dataset_dir, exist_ok=True)
                if train_result[0] is not None:
                    train_pred = train_result[0]
                else:
                    train_pred = TimeSeries({name: UnivariateTimeSeries(train_time_stamps, None)})
                fig_name = dirname + "_" + retrain_type + str(n_retrain) + "_" + "horizon" + str(int(horizon)) + ".png"
                plot_unrolled_compare(
                    train_vals,
                    test_vals,
                    train_pred,
                    test_pred,
                    os.path.join(fig_dataset_dir, fig_name),
                    dirname + f"(sMAPE={smapes:.4f})",
                )

            # Log relevant info to the logger
            logger.debug(f"{dirname} {retrain_type} {n_retrain} sMAPE : {smapes:.4f}\n")

    # Save full experimental config
    if model is not None:
        full_config = dict(
            model_config=model.config.to_dict(),
            evaluator_config=evaluator.config.to_dict(),
            code_version_info=get_code_version_info(),
        )

        with open(config_fname, "w") as f:
            json.dump(full_config, f, indent=2, sort_keys=True)

def get_code_version_info():
    return dict(time=str(pd.Timestamp.now()), commit=git.Repo(search_parent_directories=True).head.object.hexsha)

def plot_unrolled_compare(train_vals, test_vals, train_pred, test_pred, outputpath, title):
    truth_pd = (train_vals + test_vals).to_pd()
    truth_pd.columns = ["ground_truth"]
    pred_pd = (train_pred + test_pred).to_pd()
    pred_pd.columns = ["prediction"]
    result_pd = pd.concat([truth_pd, pred_pd], axis=1)
    plt.figure()
    plt.rcParams["savefig.dpi"] = 500
    plt.rcParams["figure.dpi"] = 500
    result_pd.plot(linewidth=0.5)
    plt.axvline(train_vals.to_pd().index[-1], color="r")
    plt.title(title)
    plt.savefig(outputpath)
    plt.clf()

def join_dfs(name2df: Dict[str, pd.DataFrame]) -> pd.DataFrame:
    """
    Joins multiple results dataframes into a single dataframe describing the
    results from all models.
    """
    full_df, lsuffix = None, ""
    shared_cols = ["idx", "name", "horizon", "retrain_type", "n_retrain"]
    for name, df in name2df.items():
        df.columns = [c if c in shared_cols else f"{c}_{name}" for c in df.columns]
        if full_df is None:
            full_df = df
        else:
            full_df = full_df.merge(df, how="outer", left_on=shared_cols, right_on=shared_cols)
    unique_cols = [c for c in full_df.columns if c not in shared_cols]
    return full_df[shared_cols + unique_cols]

def summarize_full_df(full_df: pd.DataFrame) -> pd.DataFrame:
    # Get the names of all algorithms which have full results
    algs = [col[len("sMAPE") :] for col in full_df.columns if col.startswith("sMAPE") and not full_df[col].isna().any()]
    summary_df = pd.DataFrame({alg.lstrip("_"): [] for alg in algs})

    # Compute pooled (per time series) mean/median sMAPE, RMSE
    mean_smape, med_smape, mean_rmse, med_rmse = [[] for _ in range(4)]

    for ts_name in np.unique(full_df.name):
        ts = full_df[full_df.name == ts_name]
        # append smape
        smapes = ts[[f"sMAPE{alg}" for alg in algs]]
        mean_smape.append(smapes.mean(axis=0).values)
        med_smape.append(smapes.median(axis=0).values)
        # append rmse
        rmses = ts[[f"RMSE{alg}" for alg in algs]]
        mean_rmse.append(rmses.mean(axis=0).values)
        med_rmse.append(rmses.median(axis=0).values)

    # Add mean/median loglifts to the summary dataframe
    summary_df.loc["mean_sMAPE"] = np.mean(mean_smape, axis=0)
    summary_df.loc["median_sMAPE"] = np.median(med_smape, axis=0)
    summary_df.loc["mean_RMSE"] = np.mean(mean_rmse, axis=0)
    summary_df.loc["median_RMSE"] = np.median(med_rmse, axis=0)
    return summary_df

def main():
    args = parse_args()
    logging.basicConfig(
        format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
        stream=sys.stdout,
        level=logging.DEBUG if args.debug else logging.INFO,
    )
    dataset = get_dataset(args.dataset, rootdir=args.data_root, **args.data_kwargs)
    dataset_name = get_dataset_name(dataset)

    if len(args.models) > 0:
        # Determine the name of the results CSV
        model_names = [resolve_model_name(m) for m in args.models]
        dirname = get_dirname(model_names, args.ensemble_type)
        dirname = dirname + "_" + args.retrain_type + str(args.n_retrain)
        results_dir = os.path.join(MERLION_ROOT, "results", "forecast", dirname)
        basename = dataset_name
        if args.hash is not None:
            basename += "_" + args.hash
        config_fname = f"{dataset_name}_config"
        csv = os.path.join(results_dir, f"{basename}.csv")
        config_fname = os.path.join(results_dir, f"{config_fname}.json")

        train_model(
            model_names=args.models,
            dataset=dataset,
            ensemble_type=args.ensemble_type,
            retrain_type=args.retrain_type,
            n_retrain=args.n_retrain,
            csv=csv,
            config_fname=config_fname,
            load_checkpoint=args.load_checkpoint,
            visualize=args.visualize,
        )

        # Pool the mean/medium sMAPE, RMSE for all evaluation
        # settings for each time series, and report summary
        # pooled statistics.
        df = pd.read_csv(csv)
        summary = summarize_full_df(df)
        summary.to_csv(os.path.join(results_dir, f"{basename}_summary.csv"), index=True)
        summary = summary[summary.columns[0]]
        logger.info(f"Pooled mean   sMAPE: {summary['mean_sMAPE']:.4f}")
        logger.info(f"Pooled median sMAPE: {summary['median_sMAPE']:.4f}")
        logger.info(f"Pooled mean   RMSE: {summary['mean_RMSE']:.4f}")
        logger.info(f"Pooled median RMSE: {summary['median_RMSE']:.4f}")

    # Now we summarize all results. Get all the individual CSV's as dataframes
    name2df = OrderedDict()
    prefix = f"{MERLION_ROOT}/results/forecast/*/{dataset_name}"
    csvs = glob.glob(f"{prefix}.csv") + glob.glob(f"{prefix}_*.csv")
    csvs = [c for c in csvs if not c.endswith(f"_summary.csv")]
    if len(csvs) == 0:
        raise RuntimeError(
            f"Did not find any pre-computed results files "
            f"for dataset {dataset_name}. Please run this "
            f"script on the dataset with specific algorithms "
            f"before trying to summarize their results."
        )
    for csv in sorted(csvs):
        basename = re.search(f"{dataset_name}.*\\.csv", csv).group(0)
        model_name = os.path.basename(os.path.dirname(csv[: -len(basename)]))
        suffix = re.search(f"(?<={dataset_name}).*(?=\\.csv)", basename).group(0)
        try:
            name2df[model_name + suffix] = pd.read_csv(csv)
        except Exception as e:
            logger.warning(f'Caught {type(e).__name__}: "{e}". Skipping csv file {csv}.')
            continue

    # Join all the dataframes into one & summarize the results
    dirname = os.path.join(MERLION_ROOT, "results", "forecast")
    full_df = join_dfs(name2df)
    summary_df = summarize_full_df(full_df)
    if args.summarize:
        print(summary_df)

    full_fname, summary_fname = [os.path.join(dirname, f"{dataset_name}_{x}.csv") for x in ["full", "summary"]]
    os.makedirs(os.path.dirname(full_fname), exist_ok=True)
    full_df.to_csv(full_fname, index=False)
    summary_df.to_csv(summary_fname, index=True)

if __name__ == "__main__":
    main()


from django.views import View
from django.shortcuts import render, redirect, get_object_or_404

from iplapp.models import *

class SeasonView(View):

    def get(self, request, *args, **kwargs):
        seasons = Season.objects.all().order_by('-season')
        if  request.user.is_anonymous:
            current_user = None
        else:
            current_user = request.user

        if kwargs:
            season = Season.objects.get(season=kwargs.get('season'))
            page = int(kwargs.get('page'))

            start_page = 8 * (page - 1) + 1
            end_page = 8 * page

            match = Match.objects.filter(season=season)[start_page: end_page + 1]
            count = Match.objects.filter(season=season).count()

            next = page + 1
            prev = page - 1

            if start_page == 1:
                prev = None

            if start_page == (count // 8) * 8 + 1:
                next = None

        return render(
            request,
            template_name='seasons.html',
            context={
                'user': current_user,
                'year': season,
                'seasons': seasons,
                'next': next,
                'prev': prev,
                'matches': enumerate(match, start=start_page),
                'title': 'IPL Clone App',
            }
        )