import sys
import plotly.offline as py
import plotly.graph_objs as go
import plotly.figure_factory as ff
import pandas as pd
import os

import numpy as np

class DataWrapper:
    def __init__(self, data_file):
        self.df = self.fetch_data(data_file)
        self.compounds = self.get_compounds()
        self.sample_matrix = abs(self.get_sample_matrix())

    def get_compounds_list(self):
        '''returns the compounds attribute contents as a list'''
        return self.compounds.tolist()

    def get_sample_names(self):
        '''returns the sample names as a list'''
        columns = self.df.columns.tolist()
        return columns[6:]

    def get_sample_lists(self):
        '''returns the samples as a list of lists'''
        return self.sample_matrix.values.tolist()

    def fetch_data(self, f):
        df = pd.read_csv(f, sep='\t|,', lineterminator='\n', header=0, error_bad_lines=False, engine='python')
        return df

    def get_compounds(self):
        '''returns the compounds that are listed in the data file'''
        return self.df['compound']

    def get_sample_matrix(self):
        sample = self.df.ix[:,6:]
        sample = (sample - sample.mean()) / (sample.max() - sample.min())
        return sample

class GraphUtil:
    def __init__(self):
        self.num_compounds = None
        self.row_height = 40
        self.num_samples = None
        self.layout = None
        self.data = None

    def generate_heatmap_trace(self, compounds, sample_names, sample_matrix):
        self.num_compounds = len(compounds.values.tolist())
        self.num_samples = len(sample_names)

        self.data = [go.Heatmap(z=sample_matrix.values.tolist(),
                                x=sample_names,
                            y=compounds.values.tolist(),
                            colorscale='Viridis',
                            )
                    ]

        self.layout = go.Layout(
            title='Sample Heatmap Comparisons',
            autosize=True,
            xaxis=dict(type='category', categoryarray=sample_names, categoryorder='array', showgrid=True),
            yaxis=dict(type='category', categoryorder='index', categoryarray=compounds.values.tolist(), dtick=1),

            shapes=self.draw_heatmap_shapes(),

            height=self.num_compounds * self.row_height,
        )

    def generate_annotated_heatmap_trace(self, compounds, sample_names, sample_matrix):
        self.num_compounds = len(compounds.values.tolist())
        self.num_samples = len(sample_names)

        z = sample_matrix.values.tolist()
        x = sample_names
        y = compounds.values.tolist()
        z2 = []
        for s in z:
            items=[]
            for item in s:
                if item > 0:
                    items.append(1)
                elif item < 0:
                    items.append(-1)
                else:
                    items.append(0)
            z2.append(items)
        self.data = [go.Heatmap(z=z2,
                                x=x,
                                y=y,
                                colorscale='RdBu',
                                text=z,
                                hoverinfo='x+y+text',
                                reversescale=True
                            )
                    ]

        self.layout = go.Layout(
            title='Sample Production/Consumption Comparisons',
            autosize=True,
            xaxis=dict(type='category', categoryarray=sample_names, categoryorder='array', showgrid=True),
            yaxis=dict(type='category', categoryorder='index', categoryarray=compounds.values.tolist(), dtick=1),
            shapes=self.draw_heatmap_shapes(),

            height=self.num_compounds * self.row_height,
        )

    def generate_3dsurface_trace(self, compounds, sample_names, sample_matrix):
        self.num_compounds = len(compounds.values.tolist())
        self.num_samples = len(sample_names)

        min_mz = self.min_value(sample_matrix)
        max_mz = self.max_value(sample_matrix)

        self.data = [
            go.Surface(
                z=sample_matrix.values.tolist(),
                x=sample_names,
                y=compounds.values.tolist(),
                colorscale='Viridis',
            )
        ]

        self.layout = go.Layout(
            scene=dict(
                xaxis=dict(type='category', categoryarray=sample_names, categoryorder='array', dtick=1, title='Sample'),
                yaxis=dict(type='category', categoryorder='index', categoryarray=compounds.values.tolist(), dtick=4, title='Compound ({})'.format(self.num_compounds)),
                zaxis=dict(title='MZ')
            ),
            margin=dict(
            r=20, l=10,
            b=10, t=10
            ),
        )

    @staticmethod
    def max_value(inputlist):
        return max([sublist[-1] for sublist in inputlist])

    @staticmethod
    def min_value(inputlist):
        return min([sublist[-1] for sublist in inputlist])

    def generate_graph(self, filename):
        fig = go.Figure(data=self.data, layout=self.layout)
        py.plot(fig, filename=filename)

    def draw_heatmap_shapes(self):
        shapes_v = [self.make_vertical_line(x) for x in range(0, self.num_samples+1)[::2]]
        seps_v = [self.make_seperator_line(x) for x in range(1, self.num_samples)[::2]]
        shapes_h = [self.make_horizontal_line(y) for y in range(0, self.num_compounds+1)]
        shapes_v.extend(shapes_h)
        shapes_v.extend(seps_v)
        return shapes_v

    def make_vertical_line(self, x):
        return dict({'type': 'line',
              'x0': x-0.5,
              'y0': -0.5,
              'x1': x-0.5,
              'y1': self.num_compounds-0.5,
              'line': {
                'color': 'black',
                'width': 2,
            }})

    def make_seperator_line(self, x):
        return dict({'type': 'line',
              'x0': x-0.5,
              'y0': -0.5,
              'x1': x-0.5,
              'y1': self.num_compounds-0.5,
              'line': {
                'color': 'dark grey',
                'width': 2,
                'dash': 'dashdot'
            }})

    def make_horizontal_line(self, y):
        return dict({
            'type': 'line',
            'x0': -0.5,
            'y0': y-0.5,
            'x1': self.num_samples-0.5,
            'y1': y-0.5,
            'line': {
                'color': 'black',
                'width': 2,
            },
        })

    def generate_scatter_plot(self, compounds, sample_names, sample_matrix):

        self.data = []
        index = 0
        for sample in sample_matrix.T.values.tolist():
            name = sample_names[index]
            for value in sample:
                self.data.append(
                    go.Scatter(x=compounds.values.tolist(),
                               y=value,
                               mode='markers+lines',
                               opacity=0.7,
                               marker=dict(
                                   size=15,
                                   line=dict(width=0.5, color='white')
                               ),
                               name=name,
                               )
                )
            index = index + 1

        self.data = [
            go.Scatter(x=compounds.values.tolist(),
                       y=sample,
                       mode='markers',
                       opacity=0.7,
                       marker=dict(
                           size=15,
                           line=dict(width=0.5, color='white')
                       ),
                       name=name,
                       ) for sample, name in zip(sample_matrix.T.values.tolist(), sample_names)
            ]

        self.layout=go.Layout(
            title='Sample Compound Intensities',
            showlegend=True,
            xaxis=dict(type='category', categoryarray=compounds.values.tolist(), categoryorder='array', dtick=1, title='Sample'),
            yaxis=dict(type='range', title='MZ'),
        )

def tbone_main(output_file, output_dir, condensed_file=None, condensed_dir=None):
    o_dw = DataWrapper(output_file)

    gu = GraphUtil()
    gu.generate_heatmap_trace(o_dw.compounds, o_dw.get_sample_names(), o_dw.get_sample_matrix())
    gu.generate_graph(os.path.join(output_dir, 'output_heatmap.html'))

    gu.generate_scatter_plot(o_dw.compounds, o_dw.get_sample_names(), o_dw.get_sample_matrix())
    gu.generate_graph(os.path.join(output_dir, 'output_scatterplot.html'))

    gu.generate_3dsurface_trace(o_dw.compounds, o_dw.get_sample_names(), o_dw.get_sample_matrix())
    gu.generate_graph(os.path.join(output_dir, 'output_3dsurface.html'))

    if condensed_file is not None and condensed_dir is not None:
        c_dw = DataWrapper(condensed_file)
        gu = GraphUtil()

        gu.generate_annotated_heatmap_trace(c_dw.compounds, c_dw.get_sample_names(), c_dw.get_sample_matrix())
        gu.generate_graph(os.path.join(condensed_dir, 'condensed_heatmap.html'))

        gu.generate_scatter_plot(c_dw.compounds, c_dw.get_sample_names(), c_dw.get_sample_matrix())
        gu.generate_graph(os.path.join(condensed_dir, 'condensed_scatterplot.html'))

        gu.generate_3dsurface_trace(c_dw.compounds, c_dw.get_sample_names(), c_dw.get_sample_matrix())
        gu.generate_graph(os.path.join(condensed_dir, 'condensed_3dsurface.html'))

if __name__ == '__main__':
    if len(sys.argv) == 3:
        tbone_main(sys.argv[1], os.path.dirname(sys.argv[1]), sys.argv[2], os.path.dirname(sys.argv[2]))
    elif len(sys.argv) == 2:
        tbone_main(sys.argv[1], os.path.dirname(sys.argv[1]))
    else:
        exit(0)


import re
with open("test.txt", "r") as f:
    lines = f.readlines()

    # initialize structures to hold paren stack and constituents found within VP
    tl = []
    d = {}
    v_stack = []
    constituent_stack = []
    in_VP = False

    # loop over lines in file; tokenize
    for line in lines:
        verb = re.compile(r'\(VP [^\(]*\)')
        if re.search(verb, line):
            d["IVP"] = d.get('IVP', 0) + 1
        line = str(line)
        line = line.replace("(", " ( ")
        line = line.replace(")", " ) ")
        for l in line.split():
            tl.append(l)

    # loop over tokens
    for line in tl:
        if in_VP:
            if line == '(':
                v_stack.append(line)
            elif line == ')':
                v_stack = v_stack[0:-1]
                if len(v_stack) == 0:  # then we've reached end of VP
                    # check constituent_stack
                    if len(constituent_stack) == 2:
                        if constituent_stack[0] == "NP":
                            if constituent_stack[1] == "NP":
                                d["DVP"] = d.get("DVP", 0) + 1
                    else:
                        # didn't find a DVP or IVP
                        pass
                    constituent_stack = []
                    in_VP = False
            elif len(v_stack) == 2:
            # if theres only the opening brace of VP and potential NP in v_stack
                if line == "NP":
                    constituent_stack.append(line)
                elif line == "VP":
                    constituent_stack = []
                else:
                # if it doesn't match, fooey
                    pass
            elif line == "VP":
                constituent_stack = []

        if line == 'NP':
            d['NP'] = d.get('NP', 0) + 1
        if line == 'S':
            d['S'] = d.get('S', 0) + 1
        if line == 'VP':
            v_stack = []
            v_stack.append('(')
            in_VP = True
            d['VP'] = d.get('VP', 0) + 1

    for k in d.keys():
        print(k, "\t" + str(d[k]))


import numpy as np
from matplotlib import pyplot as plt
import tables
import re
import collections
import scipy

def createAccuratefQMatrix(bmaptitle, title1,titlefits1,title2,titlefits2,closefcutoff=3):

    freqs0=np.loadtxt('/home/sean/data/%s0.txt'%title1)
    f0=freqs0[:,0]
    f0=f0[1:]
    freqs1=np.loadtxt('/home/sean/data/%s1.txt'%title1)
    f1=freqs1[:,0]
    f1=f1[1:]
    freqs2=np.loadtxt('/home/sean/data/%s2.txt'%title1)
    f2=freqs2[:,0]
    f2=f2[1:]
    freqs3=np.loadtxt('/home/sean/data/%s3.txt'%title1)
    f3=freqs3[:,0]
    f3=f3[1:]
    FL1fits=np.loadtxt('/home/sean/data/%s'%titlefits1)
    approxf1=FL1fits[:,1]
    QFL1=FL1fits[:,2]
    QiFL1=FL1fits[:,4]

    freqs4=np.loadtxt('/home/sean/data/%s4.txt'%title2)
    f4=freqs4[:,0]
    f4=f4[1:]
    freqs5=np.loadtxt('/home/sean/data/%s5.txt'%title2)
    f5=freqs5[:,0]
    f5=f5[1:]
    freqs6=np.loadtxt('/home/sean/data/%s6.txt'%title2)
    f6=freqs6[:,0]
    f6=f6[1:]
    freqs7=np.loadtxt('/home/sean/data/%s7.txt'%title2)
    f7=freqs7[:,0]
    f7=f7[1:]
    FL2fits=np.loadtxt('/home/sean/data/%s'%titlefits2)
    approxf2=FL2fits[:,1]
    QFL2=FL2fits[:,2]
    QiFL2=FL2fits[:,4]

    accuratefs=[f0,f1,f2,f3,f4,f5,f6,f7]

    accuratefQ=[]
    indexofpickedapproxf1=[]
    indexofpickedapproxf2=[]
    indexOfDelAccuratef=[]
    FL1j=0
    FL2j=0

    for roach in xrange(0,4):
        fQroach=[]
        for pixf in xrange(0,len(accuratefs[roach])):
                foundmatch=0
                i=0
                while foundmatch==0:
                    if FL1j+i+1< len(approxf1):
                        if abs(accuratefs[roach][pixf]-approxf1[FL1j+i]) < closefcutoff:
                            fQroach.append([accuratefs[roach][pixf],QFL1[FL1j+i],QiFL1[FL1j+i]])
                            indexofpickedapproxf1.append(FL1j+i)
                            foundmatch=1
                            FL1j=FL1j+i+1
                        else:
                            i=i+1
                    else:
                        foundmatch=1
                        indexOfDelAccuratef.append([roach,pixf])
        accuratefQ.append(fQroach)
    for roach in xrange(4,8):
        fQroach=[]
        for pixf in xrange(0,len(accuratefs[roach])):
                foundmatch=0
                i=0
                while foundmatch==0:
                    if FL2j+i+1< len(approxf2):
                        if abs(accuratefs[roach][pixf]-approxf2[FL2j+i]) < closefcutoff:
                            fQroach.append([accuratefs[roach][pixf],QFL2[FL2j+i],QiFL2[FL2j+i]])
                            indexofpickedapproxf2.append(FL2j+i)
                            foundmatch=1
                            FL2j=FL2j+i+1
                        else:
                            i=i+1
                    else:
                        foundmatch=1
                        indexOfDelAccuratef.append([roach,pixf])
        accuratefQ.append(fQroach)

    approxfPicked2xFL1=[x for x, y in collections.Counter(indexofpickedapproxf1).items() if y > 1]
    approxfPicked2xFL2=[x for x, y in collections.Counter(indexofpickedapproxf2).items() if y > 1]

    #print indexOfDelAccuratef
    #print len(approxf1),len(f0)+len(f1)+len(f2)+len(f3),len(accuratefQ[0])+len(accuratefQ[1])+len(accuratefQ[2])+len(accuratefQ[3])
    #print len(accuratefQ[0]),len(f0),len(accuratefQ[1]),len(f1),len(accuratefQ[2]),len(f2),len(accuratefQ[3]),len(f3)

    #print len(approxf2),len(f4)+len(f5)+len(f6)+len(f7),len(accuratefQ[4])+len(accuratefQ[5])+len(accuratefQ[6])+len(accuratefQ[7])
    #print len(accuratefQ[4]),len(f4),len(accuratefQ[5]),len(f5),len(accuratefQ[6]),len(f6),len(accuratefQ[7]),len(f7)

    return accuratefQ

    #now have accuratefQ. The rows correspond to the roaches (1st row is r0) and the entries contain the f and Q for that pixel (accuratefQ[0][0]= [f,Q] for r0p0)

def getfQforgoodpix(bmaptitle, title1,titlefits1,title2,titlefits2):

    #title1/2 gives the base of the filename for the frequency files for Feedline 1/2 (Don't include '.txt').
    #titlefist1/2 gives the filename of the fits file (include '.txt')
    #bmaptitle gives the filename of the beammap (include '.h5')

    closefcutoff=.0007  #when the program compares the approximate freq in frequency files to the accurate frequency in the fit file, it will accept a difference of the closefcutoff (GHz)

    accuratefQ = createAccuratefQMatrix(bmaptitle, title1,titlefits1,title2,titlefits2,closefcutoff)

    #now have accuratefQ. The rows correspond to the roaches (1st row is r0) and the entries contain the f and Q for that pixel (accuratefQ[0][0]= [f,Q] for r0p0)

    fid=tables.openFile(bmaptitle)
    b=fid.root.beammap.beamimage

    pixlist = []
    for row in xrange(0,46):
        for column in xrange(0,44):
            string=b[row][column]
            a=re.findall(r'\d+',string)
            roachNum=int(a[0])
            pixelNum=int(a[1])
            if roachNum<8:
                if pixelNum < len(accuratefQ[roachNum]):
                    pixlist.append([[roachNum,pixelNum],accuratefQ[roachNum][pixelNum]])

    fid.close()

    pixlist=sorted(pixlist)

    return pixlist


def solution(picks, minerals):
    ans = 155

    def dfs(idx, dia, iron, stone, fatigue):
        nonlocal ans
        # 다 캤거나 or 연장 다 씀
        if idx >= len(minerals) or not (dia + iron + stone):
            ans = min(fatigue, ans)

        dia_pick, iron_pick, stone_pick = 0, 0, 0
        for i in range(idx, min(idx+5, len(minerals))):
            dia_pick += 1
            iron_pick += 5 if minerals[i] == 'diamond' else 1
            stone_pick += 25 if minerals[i] == 'diamond' else 5 if minerals[i] == 'iron' else 1

        if dia:
            dfs(idx+5, dia-1, iron, stone, fatigue + dia_pick)
        if iron:
            dfs(idx+5, dia, iron-1, stone, fatigue + iron_pick)
        if stone:
            dfs(idx+5, dia, iron, stone-1, fatigue + stone_pick)

    dfs(0, picks[0], picks[1], picks[2], 0)
    return ans

# solution([1, 3, 2], ["diamond", "diamond", "diamond", "iron", "iron", "diamond", "iron", "stone"])
# solution([0, 1, 1], ["diamond", "diamond", "diamond", "diamond", "diamond", "iron", "iron", "iron", "iron", "iron", "diamond"])


import datetime
import dropbox
import logging

#---------------------------------------------------------------------------
# Modify
#---------------------------------------------------------------------------
MAX_CONTENT_AGE = 100
DISPLAY_ONLY_MODE = True
TOKEN = "YOUR-DROPBOX-BUSINESS-TEAM-MEMBER-FILE-ACCESS-TOKEN"
LOG_NAME = "dropboxpurge.log"
#---------------------------------------------------------------------------

dbx = dropbox.DropboxTeam(TOKEN)
MAX_AGE = datetime.timedelta(days=MAX_CONTENT_AGE)
FORMAT = '%(asctime)s %(message)s'
logging.basicConfig(filename=LOG_NAME, format=FORMAT, level=logging.INFO)

class bcolors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'

def log_init():
    logging.info("Begin pass...")
    logging.info("MAX_CONTENT_AGE: {}".format(MAX_CONTENT_AGE))
    logging.info("DISPLAY_ONLY_MODE: {}".format(str(DISPLAY_ONLY_MODE)))

def retrieve_member_list(recursive=True):
    logging.info("retrieve_member_list (recursive={}".format(str(recursive)))

    all_members = []
    members = dbx.team_members_list()
    all_members.extend(members.members)
    if recursive:
        while members.has_more:
            print("(retrieving another 1000 users)")
            logging.info("(retrieving another 1000 users)")
            members = dbx.team_members_list_continue(members.cursor)
            all_members.extend(members.members)

    logging.info("member list obtained - {} members".format(len(all_members)))
    return all_members

def dropbox_content(member_id):
    logging.info("retrieving content for {}".format(member_id))
    content_listing = []
    result = dbx.as_user(member_id).files_list_folder(path="",
                                                        recursive=True,
                                                        include_media_info=False,
                                                        include_deleted=False,
                                                        include_has_explicit_shared_members=False)
    content_listing.extend(result.entries)
    while result.has_more:
        result = dbx.as_user(member_id).files_list_folder_continue(result.cursor)
        content_listing.extend(result.entries)

    logging.info("retrieved content for {}".format(member_id))
    return content_listing

def display_content(content):
    for item in content:
        if type(item) == dropbox.files.FileMetadata:
            last_touch_date = item.client_modified if item.client_modified > item.server_modified else item.server_modified
            if last_touch_date.date() > (datetime.date.today() - MAX_AGE):
                print("\t{0} ({1})".format(item.path_lower, last_touch_date))
                logging.info("{} would not have been deleted".format(item.path_lower))
            else:
                print(bcolors.FAIL + "\t{0} ({1})".format(item.path_lower, last_touch_date) + bcolors.ENDC)
                logging.info("{} would have been deleted".format(item.path_lower))

def delete_content(content, member_id):
    for item in content:
        if type(item) == dropbox.files.FileMetadata:
            last_touch_date = item.client_modified if item.client_modified > item.server_modified else item.server_modified
            if last_touch_date.date() > (datetime.date.today() - MAX_AGE):
                print("Ignoring:\t{}".format(item.path_lower))
                logging.info("Ignored: {}".format(item.path_lower))
            else:
                print(bcolors.FAIL + "Deleting:\t{}".format(item.path_lower) + bcolors.ENDC)
                logging.info("Attempting to delete: {}".format(item.path_lower))
                delete(member_id, item.path_lower)

def delete(member_id, path):
    try:
        result = dbx.as_user(member_id).files_delete(path)
        if type(result) == dropbox.files.DeleteError:
            print("Problem deleting {}".format(path))
            logging.info("Tried to delete: {} but couldn't".format(path))
        else:
            print("deleted {}".format(path))
            logging.info("Deleted: {}".format(path))
    except dropbox.exceptions.ApiError:
        print("Problem deleting {}".format(path))
        logging.info("Tried to delete: {} but couldn't".format(path))

if __name__ == "__main__":
    log_init()
    for team_member in retrieve_member_list():
        if team_member.profile.status == dropbox.team.TeamMemberStatus.active:
            print(bcolors.BOLD + 'Team Member:\t{}'.format(team_member.profile.email) + bcolors.ENDC)
            content = dropbox_content(team_member.profile.team_member_id)
            if DISPLAY_ONLY_MODE:
                display_content(content)
            elif not DISPLAY_ONLY_MODE:
                delete_content(content, team_member.profile.team_member_id)


import os
import json
import argparse
import datetime
import pickle

import numpy as np
import torch
from torch.optim import Adam, SGD, RAdam
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from tensorboardX import SummaryWriter
from torchvision import transforms
from sklearn.model_selection import train_test_split
from warmup_scheduler import GradualWarmupScheduler

from model import *
from betaVAE import *
from read_data import *
from utils import *

parser = argparse.ArgumentParser(description='betaVAE training over RNA-Seq data')
parser.add_argument('--config', type=str, help='JSON config file')
parser.add_argument('--checkpoint', type=str, default=None,
        help='File with the checkpoint to start with')
parser.add_argument('--seed', type=int, default=99,
        help='Seed for random generation')
parser.add_argument('--log', type=int, default=0,
        help='Use tensorboard for experiment logging')
parser.add_argument('--parallel', type=int, default=None,
        help='If data parallel wants to be used for training')

args = parser.parse_args()

#p.random.seed(args.seed)
#torch.manual_seed(args.seed)

with open(args.config) as f:
    config = json.load(f)

print(10*'-')
print('Config for this experiment \n')
print(config)
print(10*'-')

if 'flag' in config:
    args.flag = config['flag']
else:
    args.flag = 'train_{date:%Y-%m-%d %H:%M:%S}'.format(date=datetime.datetime.now())

if not os.path.exists(config['save_dir']):
    os.mkdir(config['save_dir'])

path_csv = config['path_csv']
rna_features = config['rna_features']
batch_size = config.get('batch_size', 64)
encoder_checkpoint = config.get('encoder_checkpoint', None)
beta = config.get('beta', 2)
quick = config.get('quick', 0)
opt = config.get('optimizer', 'Adam')

print('Loading dataset...')

datasets = {
    'train': [],
    'test': [],
    'val': []
}

test_labels = []
for id, dataset in enumerate(path_csv):
    print(dataset)
    df = pd.read_csv(dataset)
    train_df, test_df = train_test_split(df, test_size=0.2)

    train_df, val_df = train_test_split(train_df, test_size=0.2)

    #train_df, val_df, test_df, scaler = normalize_dfs(train_df, val_df, test_df, norm_type='minmax')

    datasets['train'].append(train_df)
    datasets['test'].append(test_df)
    datasets['val'].append(val_df)

    test_labels = test_labels + ([id] * test_df.shape[0])

if(len(datasets['train']) >=2):
    train_df = pd.concat([datasets['train'][0], datasets['train'][1]])
    val_df = pd.concat([datasets['val'][0], datasets['val'][1]])
    test_df = pd.concat([datasets['test'][0], datasets['test'][1]])
    for i in range(2, len(datasets['train'])):
        train_df = pd.concat([train_df, datasets['train'][i]])
        val_df = pd.concat([val_df, datasets['val'][i]])
        test_df = pd.concat([test_df, datasets['test'][i]])
else:
    train_df = datasets['train'][0]
    val_df = datasets['val'][0]
    test_df = datasets['test'][0]

print('Train shape {}'.format(train_df.shape))
print('Val shape {}'.format(val_df.shape))
print('Test shape {}'.format(test_df.shape))
train_df, val_df, test_df, scaler = normalize_dfs(train_df, val_df, test_df, norm_type='standard')

train_dataset = RNADataset([train_df], quick=quick)
val_dataset = RNADataset([val_df])
test_dataset = RNADataset([test_df])

train_dataloader = DataLoader(train_dataset,batch_size=batch_size,
               num_workers=4, shuffle=True)
val_dataloader = DataLoader(val_dataset,batch_size=batch_size,
               num_workers=4,
               shuffle=False)
test_dataloader = DataLoader(test_dataset,batch_size=1,
               num_workers=4,
               shuffle=False)

# training

print('Finished loading dataset and creating dataloader')

print('Initializing models')

if encoder_checkpoint:
    model = betaVAE(rna_features, 2048, [12000, 4096, 2048], [4096, 12000],
                      encoder_checkpoint=encoder_checkpoint)
    if args.checkpoint is not None:
        print('Restoring from checkpoint')
        print(args.checkpoint)
        model.load_state_dict(torch.load(args.checkpoint))
        print('Loaded model from checkpoint')
    else:
        model.z_mu.apply(init_weights_uniform)
        model.decoder.apply(init_weights_uniform)
        model.z_logvar.apply(init_weights_uniform)
else:
    model = betaVAE(rna_features, 2048, [6000, 4000, 2048], [4000, 6000], beta=beta)
    if args.checkpoint is not None:
        print('Restoring from checkpoint')
        print(args.checkpoint)
        model.load_state_dict(torch.load(args.checkpoint))
        print('Loaded model from checkpoint')
    else:
        model.apply(init_weights_xavier)

#torch.nn.utils.clip_grad_norm_(model.parameters(), 0.25)
print('Model initialized')

if args.parallel:
    print('Using more than one gpu...')
    model = nn.DataParallel(model)

if torch.cuda.is_available():
    model = model.cuda()

lr = config.get('lr', 3e-3)

if opt == 'RAdam':
    optimizer = RAdam(model.parameters(), weight_decay = config['weights_decay'], lr=lr)
elif opt == 'SGD':
    optimizer = SGD(model.parameters(), weight_decay = config['weights_decay'], lr=lr)
else:
    optimizer = Adam(model.parameters(), weight_decay = config['weights_decay'], lr=lr)

#scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=125, gamma=0.1)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 500)
scheduler_warmup = GradualWarmupScheduler(optimizer, multiplier=1, total_epoch=1000, after_scheduler=scheduler)
# train model

if args.log:
    summary_writer = SummaryWriter(
            os.path.join(config['summary_path'],
                datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") + "_{0}".format(args.flag)))

    summary_writer.add_text('config', str(config))
else:
    summary_writer = None

dataloaders = {
    'train': train_dataloader,
    'val': val_dataloader
}
model, results = train_betaVAE(model, optimizer, dataloaders,
                               save_dir=config['save_dir'],
                               device=config['device'],
                               log_interval=config['log_interval'],
                               summary_writer=summary_writer,
                               num_epochs=config['num_epochs'],
                               scheduler=scheduler_warmup)

results_test, predictions, real = evaluate_betaVAE(model, test_dataloader)

# Reversing the transformation
predictions = np.vstack(predictions)
real = np.vstack(real)
test_results = {'predictions': 0, 'real': 0}
test_results['predictions'] = scaler.inverse_transform(predictions)
test_results['real'] = scaler.inverse_transform(real)
test_results['test_ids'] = test_df['wsi_file_name'].values
test_results['test_labels'] = np.asarray(test_labels)
f = open(os.path.join(config['save_dir'], 'test_results.pkl'), "wb")
pickle.dump(test_results, f)
f.close()


# -*- coding: utf-8 -*-

class MaxSlidingWindowPy(object):

    def maxSlidingWindow(nums, k):
        if not nums: return []
        window, res = [], []
        for i, x in enumerate(nums):
            if i>= k and window[0] <= i - k:
                window.pop(0)
            while window and nums[window[-1]] <= x:
                window.pop()
            window.append(i)
            if i >= k - 1:
                res.append(nums[window[0]])
        return res

    if __name__ == '__main__':
        nums = [1,3,1,2,0,5]
        k = 3
        maxSlidingWindow(nums, k)


import requests
import aiohttp
import asyncio

from bs4 import BeautifulSoup

import pandas as pd

import re
import time

ROOT = "https://www.coursera.org/"

async def async_get_course(session, url_course):
    async with session.get(url_course) as res:
        response = await res.content.read()
        return response

async def async_get_course_infos(courses):

  # Use this line to control number of courses to fetch during debugging and development
  #courses = courses[:3]

  actions = []
  urls = []
  course_infos = []
  async with aiohttp.ClientSession() as session:
      for course in courses:
          url_course = f"{ROOT}{course['course_link']}"
          urls.append(url_course)
          actions.append(asyncio.ensure_future(async_get_course(session, url_course)))
      results = await asyncio.gather(*actions)

      for idx, res in enumerate(results):
          course_infos.append(get_info_from_course((courses[idx], urls[idx], res)))

  return course_infos

def get_summary_page(category):

    course_html_containers = []
    url_browse = f"{ROOT}browse/{category}"
    request_counter = 0

    # Try requesting up to 30 times, otherwise be a respectful scraper and give up
    while not course_html_containers and request_counter < 31:
      request_browse = requests.get(url_browse)
      browse_soup = BeautifulSoup(request_browse.content, 'lxml', from_encoding='utf-8')
      course_html_containers = browse_soup.find_all('div', {'class': 'rc-ProductCard'})
      time.sleep(0.5)
      print("Coursera is not responding. I will try " + str(30 - request_counter) + " more times.")
      request_counter += 1

    browse_soup = BeautifulSoup(request_browse.content, 'lxml', from_encoding='utf-8')

    # Scraping 'Course Name' and 'Course Link' for each course found in Coursera's summary page
    courses = []
    for i, j in enumerate(course_html_containers):
        course_type = j.find('label', {'class': 'rc-CardText css-1feobmm'}).get_text()
        if course_type == 'Course':
            course_features = dict(course_name='', course_link='')
            course_features['course_name'] = j.find('a', {'class': 'CardText-link'}).get_text()
            course_features['course_link'] = j.find('a', {'class': 'CardText-link'})['href']
            courses.append(course_features)

    return courses

def get_info_from_course(response):
  # Scraping course page for each course. Updating 'Course Description', 'Enrollments' and 'Ratings'
  #for course in courses[:3]:

  course, _, course_response = response

  course_soup = BeautifulSoup(course_response, 'lxml', from_encoding='utf-8')
  course['instructor'] = get_course_instructor(course_soup)
  try:
    course['description'] = get_course_description(course_soup)
    course['students_enrolled'] = get_enrollments(course_soup)
    course['ratings'] = get_ratings(course_soup)
  except:
    course['description'] = "The course hasn't started yet."
    course['students_enrolled'] = "0"
    course['ratings'] = "0"

  return course

def get_course_instructor(soup):
    """
    Gets full description (all paragraphs) for a single course.
    """
    try:
      instructor = soup.find('div', {'class': 'rc-BannerInstructorInfo'}).find('span').get_text()
      instructor = re.sub(r'\s\+\d+\s[\w\s]+', '', instructor)
      if instructor[-1] == ' ':
          instructor = instructor[:-1]
    except:
      instructor = "NA"

    return instructor

def get_course_description(soup):
    """
    Gets full description (all paragraphs) for a single course.
    """
    descriptions_html_block = soup.find_all('div', {'class': 'content-inner'})
    html_paragraphs = descriptions_html_block[0].find_all('p')

    string_paragraphs = []
    for i in html_paragraphs:
        string_paragraphs.append(i.get_text())

    course_description = ''.join(string_paragraphs)

    return course_description

def get_enrollments(soup):
    """
    Gets number of enrolled students in a single course.
    """
    enrollments_as_str = soup.find('div', {'class': 'rc-ProductMetrics'}).find('strong').find('span').get_text()
    enrollments = int(enrollments_as_str.replace(',', ''))
    return enrollments

def get_ratings(soup):
    """
    Gets ratings of a single course.
    """
    ratings_as_str = soup.find('span', {'data-test': 'ratings-count-without-asterisks'}).find('span').get_text()
    ratings = int(ratings_as_str.strip(' ratings').replace(',', ''))
    return ratings

def scrape(category):
    """
    Gets a single course category and saves all courses from this category as .xslx file. Returns a list of courses
    with features.

    Excel file contains following categories: Course Name, Course Category, Instructor, Description, # of Students
    Enrolled, # of Ratings.

    In case, the course hasn't started yet (therefore has no ratings and no students enrolled), this information
    is shown in the .xlsx file in the following way: "Description": "The course hasn't started yet", "# of Students
    Enrolled": 0, "# of Ratings": 0

    DISCLAIMER: This function is designed to work with the server back end. In order to run it locally, pass the
    category argument in the following format: scrape('data-science'). You can choose from the following categories:
    ['data-science', 'business', 'personal-development', 'language-learning', 'math-and-logic', 'physical-science-and-engineering]
    """

    courses = get_summary_page(category)

    courses_final = asyncio.run(async_get_course_infos(courses))

    # Converting a list of courses into pandas DataFrame; Cleaning data structure, formatting column headers
    df = pd.DataFrame(courses_final)

    df['Category Name'] = category
    #df.drop('course_link', axis=1, inplace=True)
    df.rename(columns={'course_name': 'Course Name', 'instructor': 'First Instructor Name',
                       'description': 'Course Description',
                       'students_enrolled': '# of Students Enrolled', 'ratings': '# of Ratings'}, inplace=True)
    df = df.loc[:, ['Category Name', 'Course Name', 'First Instructor Name', 'Course Description',
                    '# of Students Enrolled', '# of Ratings']]
    df = df.drop_duplicates()

    # Printing summary information
    print(f'DONE! There are {len(courses_final)} courses in the category {category}.')
    print(df)

    return df

def get_dropdown_choices():
    """
    Gets each course category available in Coursera and returns a list which can be used as a dropdown list on the
    front end.
    DISCLAIMER: 'data-science' as a single value is hardcoded in the 'categories' value, as this element wasn't
    possible to get without using Selenium.
    """
    url_browse = "https://www.coursera.org/browse"
    request = requests.get(url_browse)
    soup = BeautifulSoup(request.content, 'lxml', from_encoding='utf-8')

    categories = soup.find_all('div', {'class': 'topic-column'})

    # data-science is hardcoded
    categories_list = ['data-science']
    for c in categories:
        categories_list.append(c.find('a')['to'][8:])

    categories_dict = {}
    for c in categories_list:
        categories_dict[c.replace('-', ' ').capitalize()] = c

    return categories_dict

#scrape_and_close = scrape('data-science')


#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# ================================================================================================ #
# Project    : Atelier AI: Studio for AI Designers                                                 #
# Version    : 0.1.4                                                                               #
# Python     : 3.10.4                                                                              #
# Filename   : /atelier/persistence/odb.py                                                         #
# ------------------------------------------------------------------------------------------------ #
# Author     : John James                                                                          #
# Email      : john.james.ai.studio@gmail.com                                                      #
# URL        : https://github.com/john-james-ai/atelier-ai                                         #
# ------------------------------------------------------------------------------------------------ #
# Created    : Thursday March 2nd 2023 05:49:40 am                                                 #
# Modified   : Thursday March 2nd 2023 03:55:14 pm                                                 #
# ------------------------------------------------------------------------------------------------ #
# License    : MIT License                                                                         #
# Copyright  : (c) 2023 John James                                                                 #
# ================================================================================================ #
import os
import shelve
from typing import Any

from atelier.persistence.database import Database
from atelier.persistence.exceptions import (
    ObjectExistsError,
    ObjectNotFoundError,
    ObjectDatabaseConnectionError,
)

# ------------------------------------------------------------------------------------------------ #
#                                       OBJECT DB                                                  #
# ------------------------------------------------------------------------------------------------ #
class ObjectDB(Database):
    """Object Database"""

    def __init__(self, name: str, filepath: str) -> None:
        super().__init__()
        self._name = name
        self._filepath = filepath
        self._is_connected = False
        self._connection = None
        os.makedirs(os.path.dirname(self._filepath), exist_ok=True)

    @property
    def name(self) -> str:
        return self._name

    @property
    def filepath(self) -> str:
        return self._filepath

    @property
    def is_connected(self) -> bool:
        return self._is_connected

    def __enter__(self):
        self.connect()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self._is_connected:
            self.close()
        if exc_type is not None:
            self._logger.error(f"\nExecution Type: {exc_type}")
            self._logger.error(f"\nExecution Value: {exc_value}")
            self._logger.error(f"\nTraceback: {traceback}")

    def connect(self) -> None:
        """Connects to the database."""
        self._connection = shelve.open(self._filepath)
        self._is_connected = True

    def close(self) -> None:
        """Closes the underlying database connection."""
        self._connection.close()
        self._is_connected = False

    def insert(self, key: str, value: Any) -> None:
        """Inserts a key/value pair into the database."""
        if self.exists(key):
            msg = f"Object with key {key} already exists in the database {self._name}."
            self._logger.error(msg)
            raise ObjectExistsError(msg)

        self._connection[key] = value

    def select(self, key: str) -> Any:
        """Retrieves data from the database"""
        try:
            return self._connection[key]
        except KeyError:
            msg = f"Object with key {key} not found in database {self._name}."
            self._logger.error(msg)
            raise ObjectNotFoundError(msg)
        except ValueError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)
        except TypeError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)

    def selectall(self) -> Any:
        """Retrieves all data from the database"""
        objects = {}
        try:
            keys = self._connection.keys()
            for key in keys:
                objects[key] = self._connection[key]
            return objects
        except ValueError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)
        except AttributeError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)

    def update(self, key: str, value: Any) -> None:
        """Updates an existing object in the database."""
        if self.exists(key):
            self._connection[key] = value

        else:
            msg = f"Object with key {key} not found in database {self._name}."
            self._logger.error(msg)
            raise ObjectNotFoundError(msg)

    def delete(self, key: str) -> None:
        """Deletes existing data."""
        try:
            del self._connection[key]
        except KeyError:
            msg = f"Object with key {key} doesn't exist in the database {self._name}."
            self._logger.error(msg)
            raise ObjectNotFoundError(msg)
        except ValueError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)
        except TypeError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)

    def exists(self, key: str) -> bool:
        """Checks existence of an item in the database."""
        try:
            return key in self._connection.keys()
        except ValueError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)
        except AttributeError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)

    def clear(self) -> None:
        """Clears cache of all objects."""
        try:
            self._connection.clear()
        except ValueError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)
        except AttributeError:
            msg = f"Database connection {self._name} is closed."
            self._logger.error(msg)
            raise ObjectDatabaseConnectionError(msg)


"""
    Paper: Vision Transformer with Deformable Attention
    Link: https://arxiv.org/abs/2201.00520
"""

import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import einops
from timm.models.layers import to_2tuple, trunc_normal_

class LayerNormProxy(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.norm = nn.LayerNorm(dim)

    def forward(self, x):
        x = einops.rearrange(x, 'b c h w -> b h w c')
        x = self.norm(x)
        return einops.rearrange(x, 'b h w c -> b c h w')

class DAttentionBaseline(nn.Module):

    def __init__(
            self, q_size, kv_size, n_heads, n_head_channels, n_groups=3,
            attn_drop=0., proj_drop=0., stride=1,
            offset_range_factor=2, use_pe=True, dwc_pe=False,
            no_off=False, fixed_pe=False, stage_idx=2
    ):

        super().__init__()
        self.dim = n_head_channels * n_heads
        self.dwc_pe = dwc_pe
        self.n_head_channels = n_head_channels
        self.scale = self.n_head_channels ** -0.5
        self.n_heads = n_heads
        self.q_h, self.q_w = q_size
        self.kv_h, self.kv_w = kv_size
        self.nc = n_head_channels * n_heads
        self.n_groups = n_groups
        self.n_group_channels = self.nc // self.n_groups
        self.n_group_heads = self.n_heads // self.n_groups
        self.use_pe = use_pe
        self.fixed_pe = fixed_pe
        self.no_off = no_off
        self.offset_range_factor = offset_range_factor

        ksizes = [9, 7, 5, 3]
        kk = ksizes[stage_idx]

        self.conv_offset = nn.Sequential(
            nn.Conv2d(self.n_group_channels, self.n_group_channels, kk, stride, kk // 2, groups=self.n_group_channels),
            LayerNormProxy(self.n_group_channels),
            nn.GELU(),
            nn.Conv2d(self.n_group_channels, 2, 1, 1, 0, bias=False)
        )

        self.proj_q = nn.Conv2d(
            self.nc, self.nc,
            kernel_size=1, stride=1, padding=0
        )

        self.proj_k = nn.Conv2d(
            self.nc, self.nc,
            kernel_size=1, stride=1, padding=0
        )

        self.proj_v = nn.Conv2d(
            self.nc, self.nc,
            kernel_size=1, stride=1, padding=0
        )

        self.proj_out = nn.Conv2d(
            self.nc, self.nc,
            kernel_size=1, stride=1, padding=0
        )

        self.proj_drop = nn.Dropout(proj_drop, inplace=True)
        self.attn_drop = nn.Dropout(attn_drop, inplace=True)

        if self.use_pe:
            if self.dwc_pe:
                self.rpe_table = nn.Conv2d(self.nc, self.nc,
                                           kernel_size=3, stride=1, padding=1, groups=self.nc)
            elif self.fixed_pe:
                self.rpe_table = nn.Parameter(
                    torch.zeros(self.n_heads, self.q_h * self.q_w, self.kv_h * self.kv_w)
                )
                trunc_normal_(self.rpe_table, std=0.01)
            else:
                self.rpe_table = nn.Parameter(
                    torch.zeros(self.n_heads, self.kv_h * 2 - 1, self.kv_w * 2 - 1)
                )
                trunc_normal_(self.rpe_table, std=0.01)
        else:
            self.rpe_table = None

    @torch.no_grad()
    def _get_ref_points(self, H_key, W_key, B, dtype, device):

        ref_y, ref_x = torch.meshgrid(
            torch.linspace(0.5, H_key - 0.5, H_key, dtype=dtype, device=device),
            torch.linspace(0.5, W_key - 0.5, W_key, dtype=dtype, device=device)
        )
        ref = torch.stack((ref_y, ref_x), -1)
        ref[..., 1].div_(W_key).mul_(2).sub_(1)
        ref[..., 0].div_(H_key).mul_(2).sub_(1)
        ref = ref[None, ...].expand(B * self.n_groups, -1, -1, -1)  # B * g H W 2

        return ref

    def forward(self, x, H=14, W=14):
        B, N, C = x.shape
        x = x.permute(0, 2, 1).reshape(B, C, H, W)
        B, C, H, W = x.size()
        dtype, device = x.dtype, x.device

        q = self.proj_q(x)
        q_off = einops.rearrange(q, 'b (g c) h w -> (b g) c h w', g=self.n_groups, c=self.n_group_channels)
        offset = self.conv_offset(q_off)  # B * g 2 Hg Wg
        Hk, Wk = offset.size(2), offset.size(3)
        n_sample = Hk * Wk

        if self.offset_range_factor > 0:
            offset_range = torch.tensor([1.0 / Hk, 1.0 / Wk], device=device).reshape(1, 2, 1, 1)
            offset = offset.tanh().mul(offset_range).mul(self.offset_range_factor)

        offset = einops.rearrange(offset, 'b p h w -> b h w p')
        reference = self._get_ref_points(Hk, Wk, B, dtype, device)

        if self.no_off:
            offset = offset.fill(0.0)

        if self.offset_range_factor >= 0:
            pos = offset + reference
        else:
            pos = (offset + reference).tanh()

        x_sampled = F.grid_sample(
            input=x.reshape(B * self.n_groups, self.n_group_channels, H, W),
            grid=pos[..., (1, 0)],  # y, x -> x, y
            mode='bilinear', align_corners=True)  # B * g, Cg, Hg, Wg

        x_sampled = x_sampled.reshape(B, C, 1, n_sample)

        q = q.reshape(B * self.n_heads, self.n_head_channels, H * W)
        k = self.proj_k(x_sampled).reshape(B * self.n_heads, self.n_head_channels, n_sample)
        v = self.proj_v(x_sampled).reshape(B * self.n_heads, self.n_head_channels, n_sample)

        attn = torch.einsum('b c m, b c n -> b m n', q, k)  # B * h, HW, Ns
        attn = attn.mul(self.scale)

        if self.use_pe:

            if self.dwc_pe:
                residual_lepe = self.rpe_table(q.reshape(B, C, H, W)).reshape(B * self.n_heads, self.n_head_channels,
                                                                              H * W)
            elif self.fixed_pe:
                rpe_table = self.rpe_table
                attn_bias = rpe_table[None, ...].expand(B, -1, -1, -1)
                attn = attn + attn_bias.reshape(B * self.n_heads, H * W, self.n_sample)
            else:
                rpe_table = self.rpe_table
                rpe_bias = rpe_table[None, ...].expand(B, -1, -1, -1)

                q_grid = self._get_ref_points(H, W, B, dtype, device)

                displacement = (
                            q_grid.reshape(B * self.n_groups, H * W, 2).unsqueeze(2) - pos.reshape(B * self.n_groups,
                                                                                n_sample,
                                                                                2).unsqueeze(1)).mul(
                    0.5)

                attn_bias = F.grid_sample(
                    input=rpe_bias.reshape(B * self.n_groups, self.n_group_heads, 2 * H - 1, 2 * W - 1),
                    grid=displacement[..., (1, 0)],
                    mode='bilinear', align_corners=True
                )  # B * g, h_g, HW, Ns

                attn_bias = attn_bias.reshape(B * self.n_heads, H * W, n_sample)

                attn = attn + attn_bias

        attn = F.softmax(attn, dim=2)
        attn = self.attn_drop(attn)

        out = torch.einsum('b m n, b c n -> b c m', attn, v)

        if self.use_pe and self.dwc_pe:
            out = out + residual_lepe
        out = out.reshape(B, C, H, W)

        y = self.proj_drop(self.proj_out(out))

        # return y, pos.reshape(B, self.n_groups, Hk, Wk, 2), reference.reshape(B, self.n_groups, Hk, Wk, 2)

        return y

if __name__ == '__main__':
    # 224 * 224 settting
    dim = 768
    num_heads = 12
    H = W = 14
    B = 64

    # special
    n_head_channels = dim // num_heads
    n_groups = 3

    model = DAttentionBaseline((H, W), (H, W), num_heads, n_head_channels)

    from utils import measure_flops_params, measure_throughput_cpu, measure_throughput_gpu

    x = torch.randn(1, H*W, dim)
    measure_flops_params(model, x)
    measure_throughput_cpu(model)
    measure_throughput_gpu(model)


from enum import Enum
from itertools import product
from typing import List, Tuple, Dict
import sys
import copy
import time

class HC(Enum):
    EMPTY = 1
    WALL = 2
    GUARD_N = 3
    GUARD_E = 4
    GUARD_S = 5
    GUARD_W = 6
    CIVIL_N = 7
    CIVIL_E = 8
    CIVIL_S = 9
    CIVIL_W = 10
    TARGET = 11
    SUIT = 12
    PIANO_WIRE = 13
    N = 14
    E = 15
    S = 16
    W = 17

test_case_0 = [
    [HC.PIANO_WIRE,HC.EMPTY,HC.TARGET],
]
test_case_1 = [
    [HC.EMPTY, HC.EMPTY, HC.SUIT],
    [HC.PIANO_WIRE, HC.TARGET,HC.EMPTY],
    [HC.EMPTY, HC.CIVIL_W, HC.EMPTY]
]

test_case_5 = [
    [HC.EMPTY, HC.EMPTY, HC.PIANO_WIRE, HC.SUIT],
    [HC.EMPTY, HC.WALL, HC.CIVIL_N, HC.EMPTY],
    [HC.TARGET, HC.WALL, HC.EMPTY, HC.EMPTY],
    [HC.CIVIL_E, HC.CIVIL_W, HC.EMPTY, HC.GUARD_E],
]

world_example = [
    [HC.EMPTY, HC.EMPTY, HC.EMPTY, HC.SUIT, HC.GUARD_S, HC.WALL, HC.WALL],
    [HC.EMPTY, HC.WALL, HC.EMPTY, HC.EMPTY, HC.EMPTY, HC.EMPTY, HC.EMPTY],
    [HC.TARGET, HC.WALL, HC.EMPTY, HC.EMPTY, HC.EMPTY, HC.CIVIL_N, HC.EMPTY],
    [HC.WALL, HC.WALL, HC.EMPTY, HC.GUARD_E, HC.EMPTY, HC.CIVIL_E, HC.CIVIL_W],
    [HC.EMPTY, HC.EMPTY, HC.EMPTY, HC.EMPTY, HC.EMPTY, HC.EMPTY, HC.EMPTY],
    [HC.EMPTY, HC.EMPTY, HC.WALL, HC.WALL, HC.EMPTY, HC.PIANO_WIRE, HC.EMPTY],
]

class HitmanReferee:
    def __init__(self, filename: str = ""):
        self.__filename = filename
        if filename == "":
            self.__world = world_example
            self.__m = len(self.__world)
            self.__n = len(self.__world[0])
        else:
            raise NotImplementedError("TODO")

        self.__civil_count = self.__compute_civil_count()
        self.__guard_count = self.__compute_guard_count()
        self.__civils = self.__compute_civils()
        self.__guards = self.__compute_guards()
        self.__phase = 0
        self.__phase1_penalties = 0
        self.__phase1_guess_score = 0
        self.__phase2_penalties = 0
        self.__pos = (0, 0)
        self.__orientation = HC.N
        self.__has_guessed = False
        self.__is_in_guard_range = False
        self.__is_in_civil_range = False
        self.__phase1_history: List[str] = []
        self.__phase2_history: List[str] = []
        self.__has_suit = False
        self.__suit_on = False
        self.__has_weapon = False
        self.__is_target_down = False

    def start_phase1(self):
        self.__phase = 1
        return self.__get_status_phase_1()

    def __get_status_phase_1(self, err: str = "OK"):
        return {
            "status": err,
            "phase": self.__phase,
            "guard_count": self.__guard_count,
            "civil_count": self.__civil_count,
            "m": self.__m,
            "n": self.__n,
            "position": self.__pos,
            "orientation": self.__orientation,
            "vision": self.__get_vision(),
            "hear": self.__get_listening(),
            "penalties": self.__phase1_penalties,
            "is_in_guard_range": self.__is_in_guard_range,
        }

    def send_content(self, map_info: Dict[Tuple[int, int], HC]) -> bool:
        if not self.__has_guessed:
            self.__has_guessed = True
            guess_is_right = True
            for (x, y), content in map_info.items():
                if (
                    x >= self.__n
                    or y >= self.__m
                    or x < 0
                    or y < 0
                    or content != self.__get_world_content(x, y)
                ):
                    guess_is_right = False
                else:
                    self.__phase1_guess_score += 2
            all_tiles = list(product(range(self.__n), range(self.__m)))
            unobserved_tiles = [
                (x, y) for (x, y) in all_tiles if (x, y) not in map_info.keys()
            ]
            return len(unobserved_tiles) == 0 and guess_is_right
        else:
            raise ValueError("Err: cand only send content once")

    def end_phase1(self) -> Tuple[bool, str, List, Dict]:
        if not self.__has_guessed:
            return False, "Err: Cannot end phase1 without guessing the map", [], {}
        self.__phase = 0
        all_tiles = list(product(range(self.__n), range(self.__m)))
        map_content = {(x, y): self.__get_world_content(x, y) for (x, y) in all_tiles}
        return (
            True,
            f"Your score is {self.__phase1_guess_score-self.__phase1_penalties}",
            self.__phase1_history,
            map_content,
        )

    def __get_world_content(self, x: int, y: int):
        if x >= self.__n or y >= self.__m or x < 0 or y < 0:
            return f"//error, out of bounds , x: {x}, y: {y}"
        return self.__world[self.__m - y - 1][x]

    def __update_world_content(self, x: int, y: int, new_content: HC):
        self.__world[self.__m - y - 1][x] = new_content
        self.__civils = self.__compute_civils()
        self.__guards = self.__compute_guards()
    def __get_listening(self, dist=2):
        count = 0
        possible_offset = range(-dist, dist + 1)
        offsets = product(possible_offset, repeat=2)
        x, y = self.__pos
        for i, j in offsets:
            pos_x, pos_y = x + i, y + j
            if pos_x >= self.__n or pos_y >= self.__m or pos_x < 0 or pos_y < 0:
                continue
            if self.__get_world_content(pos_x, pos_y) in [
                HC.CIVIL_N,
                HC.CIVIL_E,
                HC.CIVIL_S,
                HC.CIVIL_W,
                HC.GUARD_N,
                HC.GUARD_E,
                HC.GUARD_S,
                HC.GUARD_W,
            ]:
                count += 1
            if count == 5:
                break

        return count
    def __get_offset(self):
        if self.__orientation == HC.N:
            offset = 0, 1
        elif self.__orientation == HC.E:
            offset = 1, 0
        elif self.__orientation == HC.S:
            offset = 0, -1
        elif self.__orientation == HC.W:
            offset = -1, 0

        return offset

    def __get_vision(self, dist=3):
        offset_x, offset_y = self.__get_offset()
        pos = self.__pos
        x, y = pos
        vision = []
        for _ in range(0, dist):
            pos = x + offset_x, y + offset_y
            x, y = pos
            if x >= self.__n or y >= self.__m or x < 0 or y < 0:
                break
            vision.append((pos, self.__get_world_content(x, y)))
            if vision[-1][1] != HC.EMPTY:
                break
        return vision

    def move(self):
        offset_x, offset_y = self.__get_offset()
        x, y = self.__pos

        if self.__phase == 1:
            self.__phase1_penalties += 1
        elif self.__phase == 2:
            self.__phase2_penalties += 1
        else:
            raise ValueError("Err: invalid phase")

        self.__add_history("Move")

        if (
            not (0 <= x + offset_x < self.__n)
            or not (0 <= y + offset_y < self.__m)
            or self.__get_world_content(x + offset_x, y + offset_y)
            not in [
                HC.EMPTY,
                HC.PIANO_WIRE,
                HC.CIVIL_N,
                HC.CIVIL_E,
                HC.CIVIL_S,
                HC.CIVIL_W,
                HC.SUIT,
                HC.TARGET,
            ]
        ):
            if self.__phase == 1:
                self.__phase1_penalties += 5 * self.__seen_by_guard_num()
                return self.__get_status_phase_1("Err: invalid move")
            else:
                self.__phase2_penalties += (
                    0 if self.__suit_on else 5 * self.__seen_by_guard_num()
                )
                return self.__get_status_phase_2("Err: invalid move")

        self.__pos = x + offset_x, y + offset_y

        if self.__phase == 1:
            self.__phase1_penalties += 5 * self.__seen_by_guard_num()
            return self.__get_status_phase_1()
        else:
            self.__seen_by_civil_num()
            self.__phase2_penalties += (
                0 if self.__suit_on else 5 * self.__seen_by_guard_num()
            )
            return self.__get_status_phase_2()

    def turn_clockwise(self):
        if self.__phase == 1:
            self.__phase1_penalties += 1
            self.__phase1_penalties += 5 * self.__seen_by_guard_num()
        elif self.__phase == 2:
            self.__phase2_penalties += 1
            self.__phase2_penalties += (
                0 if self.__suit_on else 5 * self.__seen_by_guard_num()
            )
        else:
            raise ValueError("Err: invalid phase")

        self.__add_history("Turn Clockwise")

        if self.__orientation == HC.N:
            self.__orientation = HC.E
        elif self.__orientation == HC.E:
            self.__orientation = HC.S
        elif self.__orientation == HC.S:
            self.__orientation = HC.W
        elif self.__orientation == HC.W:
            self.__orientation = HC.N

        return (
            self.__get_status_phase_1()
            if self.__phase == 1
            else self.__get_status_phase_2()
        )

    def turn_anti_clockwise(self):
        if self.__phase == 1:
            self.__phase1_penalties += 1
            self.__phase1_penalties += 5 * self.__seen_by_guard_num()
        elif self.__phase == 2:
            self.__phase2_penalties += 1
            self.__phase2_penalties += (
                0 if self.__suit_on else 5 * self.__seen_by_guard_num()
            )
        else:
            raise ValueError("Err: invalid phase")

        self.__add_history("Turn Anti-Clockwise")

        if self.__orientation == HC.N:
            self.__orientation = HC.W
        elif self.__orientation == HC.E:
            self.__orientation = HC.N
        elif self.__orientation == HC.S:
            self.__orientation = HC.E
        elif self.__orientation == HC.W:
            self.__orientation = HC.S
        return (
            self.__get_status_phase_1()
            if self.__phase == 1
            else self.__get_status_phase_2()
        )

    def start_phase2(self):
        self.__phase = 2
        self.__pos = (0, 0)
        self.__orientation = HC.N
        self.__seen_by_guard_num()
        self.__seen_by_civil_num()
        return self.__get_status_phase_2()

    def __get_status_phase_2(self, err: str = "OK"):
        return {
            "status": err,
            "phase": self.__phase,
            "guard_count": self.__guard_count,
            "civil_count": self.__civil_count,
            "m": self.__m,
            "n": self.__n,
            "position": self.__pos,
            "orientation": self.__orientation,
            "vision": self.__get_vision(),
            "hear": self.__get_listening(),
            "penalties": self.__phase2_penalties,
            "is_in_guard_range": self.__is_in_guard_range,
            "is_in_civil_range": self.__is_in_civil_range,
            "has_suit": self.__has_suit,
            "is_suit_on": self.__suit_on,
            "has_weapon": self.__has_weapon,
            "is_target_down": self.__is_target_down,
        }

    def end_phase2(self):
        if not self.__is_target_down or not self.__pos == (0, 0):
            return False, "Err: finish the mission and go back to (0,0)", []
        self.__phase = 0
        return True, f"Your score is {- self.__phase2_penalties}", self.__phase2_history

    def kill_target(self):
        if self.__phase != 2:
            raise ValueError("Err: invalid phase")

        self.__add_history("Kill Target")
        self.__phase2_penalties += 1
        self.__phase2_penalties += (
            0 if self.__suit_on else 5 * self.__seen_by_guard_num()
        )
        x, y = self.__pos
        if self.__get_world_content(x, y) != HC.TARGET or not self.__has_weapon:
            return self.__get_status_phase_2("Err: invalid move")

        self.__update_world_content(x, y, HC.EMPTY)
        self.__is_target_down = True

        self.__phase2_penalties += 100 * (
            self.__seen_by_guard_num() + self.__seen_by_civil_num()
        )
        return self.__get_status_phase_2()

    def neutralize_guard(self):
        if self.__phase != 2:
            raise ValueError("Err: invalid phase")

        self.__add_history("Neutralize Guard")
        self.__phase2_penalties += 1
        self.__phase2_penalties += 5 * self.__seen_by_guard_num()

        offset_x, offset_y = self.__get_offset()
        x, y = self.__pos
        if self.__get_world_content(x + offset_x, y + offset_y) not in [
            HC.GUARD_N,
            HC.GUARD_E,
            HC.GUARD_S,
            HC.GUARD_W,
        ] or (x, y) in [
            pos for (pos, _) in self.__guards[(x + offset_x, y + offset_y)]
        ]:
            return self.__get_status_phase_2("Err: invalid move")
        self.__phase2_penalties += 20
        self.__update_world_content(x + offset_x, y + offset_y, HC.EMPTY)
        self.__guard_count -= 1
        self.__phase2_penalties += 100 * (
            self.__seen_by_guard_num() + self.__seen_by_civil_num()
        )

        return self.__get_status_phase_2()

    def neutralize_civil(self):
        if self.__phase != 2:
            raise ValueError("Err: invalid phase")

        self.__add_history("Neutralize Civil")
        self.__phase2_penalties += 1
        self.__phase2_penalties += 5 * self.__seen_by_guard_num()

        offset_x, offset_y = self.__get_offset()
        x, y = self.__pos
        if self.__get_world_content(x + offset_x, y + offset_y) not in [
            HC.CIVIL_N,
            HC.CIVIL_E,
            HC.CIVIL_S,
            HC.CIVIL_W,
        ] or (x, y) in [
            pos for (pos, _) in self.__civils[(x + offset_x, y + offset_y)]
        ]:
            return self.__get_status_phase_2("Err: invalid move")

        self.__phase2_penalties += 20
        self.__update_world_content(x + offset_x, y + offset_y, HC.EMPTY)
        self.__civil_count -= 1
        self.__phase2_penalties += 100 * (
            self.__seen_by_guard_num() + self.__seen_by_civil_num()
        )

        return self.__get_status_phase_2()

    def take_suit(self):
        if self.__phase != 2:
            raise ValueError("Err: invalid phase")

        self.__add_history("Take Suit")
        self.__phase2_penalties += 1
        self.__phase2_penalties += 5 * self.__seen_by_guard_num()

        x, y = self.__pos
        if self.__get_world_content(x, y) != HC.SUIT:
            return self.__get_status_phase_2("Err: invalid move")

        self.__has_suit = True
        self.__update_world_content(x, y, HC.EMPTY)

        return self.__get_status_phase_2()

    def take_weapon(self):
        if self.__phase != 2:
            raise ValueError("Err: invalid phase")

        self.__add_history("Take Weapon")
        self.__phase2_penalties += 1
        self.__phase2_penalties += 5 * self.__seen_by_guard_num()
        x, y = self.__pos
        if self.__get_world_content(x, y) != HC.PIANO_WIRE:
            return self.__get_status_phase_2("Err: invalid move")

        self.__has_weapon = True
        self.__update_world_content(x, y, HC.EMPTY)

        return self.__get_status_phase_2()

    def put_on_suit(self):
        if self.__phase != 2:
            raise ValueError("Err: invalid phase")

        self.__add_history("Put on Suit")
        self.__phase2_penalties += 1
        self.__phase2_penalties += 5 * self.__seen_by_guard_num()

        if not self.__has_suit:
            return self.__get_status_phase_2("Err: invalid move")

        self.__suit_on = True
        self.__phase2_penalties += 100 * (
            self.__seen_by_guard_num() + self.__seen_by_civil_num()
        )
        return self.__get_status_phase_2()

    def __repr__(self) -> str:
        return f"HitmanReferee({self.__filename})"

    def __str__(self) -> str:
        return ASCII_ART

    def __compute_civil_count(self) -> int:
        count = 0
        for l in self.__world:
            for c in l:
                if (
                    c == HC.CIVIL_N
                    or c == HC.CIVIL_E
                    or c == HC.CIVIL_S
                    or c == HC.CIVIL_W
                ):
                    count += 1
        return count

    def __compute_guard_count(self) -> int:
        count = 0
        for l in self.__world:
            for c in l:
                if (
                    c == HC.GUARD_N
                    or c == HC.GUARD_E
                    or c == HC.GUARD_S
                    or c == HC.GUARD_W
                ):
                    count += 1
        return count

    def __compute_civils(
        self,
    ) -> Dict[Tuple[int, int], List[Tuple[Tuple[int, int], HC]]]:
        locations = {}
        for l_index, l in enumerate(self.__world):
            for c_index, c in enumerate(l):
                if (
                    c == HC.CIVIL_N
                    or c == HC.CIVIL_E
                    or c == HC.CIVIL_S
                    or c == HC.CIVIL_W
                ):
                    civil_x, civil_y = (c_index, self.__m - l_index - 1)
                    locations[(civil_x, civil_y)] = self.__get_civil_vision(
                        civil_x, civil_y
                    )
        return locations

    def __get_civil_offset(self, civil):
        if civil == HC.CIVIL_N:
            offset = 0, 1
        elif civil == HC.CIVIL_E:
            offset = 1, 0
        elif civil == HC.CIVIL_S:
            offset = 0, -1
        elif civil == HC.CIVIL_W:
            offset = -1, 0

        return offset

    def __get_civil_vision(self, civil_x, civil_y):
        civil = self.__get_world_content(civil_x, civil_y)
        offset_x, offset_y = self.__get_civil_offset(civil)
        pos = (civil_x, civil_y)
        x, y = pos
        vision = [(pos, self.__get_world_content(x, y))]

        pos = x + offset_x, y + offset_y
        x, y = pos
        if self.__n > x >= 0 and self.__m > y >= 0:
            vision.append((pos, self.__get_world_content(x, y)))
        return vision

    def __seen_by_civil_num(self) -> int:
        count = 0
        x, y = self.__pos
        if self.__get_world_content(x, y) in [
            HC.CIVIL_N,
            HC.CIVIL_E,
            HC.CIVIL_S,
            HC.CIVIL_W,
        ]:
            count = 1
            self.__is_in_civil_range = True
            return count

        for civil in self.__civils.keys():
            civil_x, civil_y = civil
            if civil_x == x and civil_y == y:
                count += 1
            else:
                count += (
                    1
                    if len(
                        [0 for ((l, c), _) in self.__civils[civil] if l == x and c == y]
                    )
                    > 0
                    else 0
                )
        self.__is_in_civil_range = count > 0
        return count

    def __compute_guards(
        self,
    ) -> Dict[Tuple[int, int], List[Tuple[Tuple[int, int], HC]]]:
        locations = {}
        for l_index, l in enumerate(self.__world):
            for c_index, c in enumerate(l):
                if (
                    c == HC.GUARD_N
                    or c == HC.GUARD_E
                    or c == HC.GUARD_S
                    or c == HC.GUARD_W
                ):
                    guard_x, guard_y = (c_index, self.__m - l_index - 1)
                    locations[(guard_x, guard_y)] = self.__get_guard_vision(
                        guard_x, guard_y
                    )
        return locations

    def __get_guard_offset(self, guard):

        if guard == HC.GUARD_N:
            offset = 0, 1
        elif guard == HC.GUARD_E:
            offset = 1, 0
        elif guard == HC.GUARD_S:
            offset = 0, -1
        elif guard == HC.GUARD_W:
            offset = -1, 0
        return offset

    def __get_guard_vision(self, guard_x, guard_y, dist=2):
        guard = self.__get_world_content(guard_x, guard_y)
        offset_x, offset_y = self.__get_guard_offset(guard)
        pos = (guard_x, guard_y)
        x, y = pos
        vision = []
        for _ in range(0, dist):
            pos = x + offset_x, y + offset_y
            x, y = pos
            if x >= self.__n or y >= self.__m or x < 0 or y < 0:
                break
            vision.append((pos, self.__get_world_content(x, y)))
            if vision[-1][1] != HC.EMPTY:
                break
        return vision

    def __seen_by_guard_num(self) -> int:
        count = 0
        x, y = self.__pos
        if self.__get_world_content(x, y) not in [
            HC.CIVIL_N,
            HC.CIVIL_E,
            HC.CIVIL_S,
            HC.CIVIL_W,
        ]:
            for guard in self.__guards.keys():
                # Note : un garde ne peut pas voir au dela d'un objet,
                # mais si Hitman est sur l'objet alors il voit Hitman
                count += (
                    1
                    if len(
                        [0 for ((l, c), _) in self.__guards[guard] if l == x and c == y]
                    )
                    > 0
                    else 0
                )
        self.__is_in_guard_range = count > 0
        return count

    def __add_history(self, action):
        if self.__phase == 1:
            self.__phase1_history.append(action)
        elif self.__phase == 2:
            self.__phase2_history.append(action)
        else:
            raise ValueError("Err: invalid phase")

    def succ(self):
        ActionMethods_list=['kill_target','neutralize_guard','neutralize_civil','take_suit','take_weapon','put_on_suit','turn_clockwise','turn_anti_clockwise','move']
        list_of_next_states=[]
        for method_name in ActionMethods_list:
            next_state=copy.deepcopy(self)
            method = getattr(next_state, method_name)

            test=method()
            if test['status']=="OK":
                list_of_next_states.append(next_state)
        return list_of_next_states
    def print_score(self):
        return self.__phase2_penalties
    def BFS(self):  #remove_head, insert_tail,
        l=[self]
        while l:
            s,l=remove_head(l)
            list_succ=s.succ()
            for x in list_succ:
                if x.end_phase2()[0]:
                    print("Success !")

                    return x
                insert_tail(x,l)
        return None
    def BFS_backtracking(self):
        l=[self]
        save=[self]
        while l:
            s,l=remove_head(l)
            list_succ=s.succ()
            for x in list_succ:
                if not x.in_list_comparator_testing(save):
                    save.append(x)
                    if x.end_phase2()[0]:
                        #print("Success ! \n len de save : ",len(save))
                        print("Success !")
                        return x,save
                    insert_tail(x,l)
        return None,save
    def DFS_backtracking(self):
        l=[self]
        save=[self]
        while l:
            s,l=remove_tail(l)
            list_succ=s.succ()
            for x in list_succ:
                if not x.in_list_comparator_testing(save):
                    save.append(x)
                    if x.end_phase2()[0]:
                        #print("Success ! \n len de save : ",len(save))
                        print("Success !")
                        return x,save
                    insert_tail(x,l)
        return None,save

    def get__phase2_penalties(self):
        return self.__phase2_penalties

    def glouton(self):
        L=[(self,self.heuristic())]
        save=[self]
        while L:
            L.sort(key=lambda x : x[1])
            s,L=remove_head(L)
            list_succ=s[0].succ()
            for x in list_succ:
                if not x.in_list_comparator_testing(save):
                    save.append(x)
                    if x.end_phase2()[0]:
                        print("Success !")
                        return x,save
                    insert_tail((x,x.heuristic()),L)
        return None,save
    def Astar(self):
        L=[(self,self.heuristic())]
        save=[self]
        while L:
            L.sort(key=lambda x : x[1])
            s,L=remove_head(L)
            list_succ=s[0].succ()
            for x in list_succ:
                if not x.in_list_comparator_testing(save):
                    save.append(x)
                    if x.end_phase2()[0]:
                        print("Success !")
                        return x,save
                    diff_penalties=x.current_penalties()-s[0].current_penalties()
                    insert_tail((x,x.heuristic()+diff_penalties),L)
        return None,save
    def manhattan_distance(self):
        if self.target_position()==None:
            return abs(self.__pos[0])+abs(self.__pos[1])
        return abs(self.__pos[0]-self.target_position()[0])+abs(self.__pos[1]-self.target_position()[1])
    def heuristic(self):
        Priority_inversed=0
        if self.__is_target_down ==False:
            target_distance=abs(self.__pos[0]-self.target_position()[0])+abs(self.__pos[1]-self.target_position()[1])
            Priority_inversed+=target_distance
        if self.__has_weapon==False:
            weapon_distance=abs(self.__pos[0]-self.get_piano()[0])+abs(self.__pos[1]-self.get_piano()[1])
            Priority_inversed+=weapon_distance
        if self.__suit_on==False and self.get_suit() is not None:
            suit_distance=abs(self.__pos[0]-self.get_suit()[0])+abs(self.__pos[1]-self.get_suit()[1])
            Priority_inversed+=suit_distance
        return Priority_inversed
    def target_position(self):
        for i_index,i in enumerate(self.__world):
            for j_index,j in enumerate(i)   :
                if j==HC.TARGET:
                    R=(j_index,self.__m-i_index-1)
                    return R
        return None
    def get_pos(self):
        return self.__pos
    def get_piano(self):
        for i_index,i in enumerate(self.__world):
            for j_index,j in enumerate(i)  :
                if j==HC.PIANO_WIRE:
                    R=(j_index,self.__m-i_index-1)
                    return R
        return None
    def get_suit(self):
        for i_index,i in enumerate(self.__world):
            for j_index,j in enumerate(i)  :
                if j==HC.SUIT:
                    R=(j_index,self.__m-i_index-1)
                    return R
        return None
    def object_comparator_personalized(self,other):
        if   (self.__civil_count == other.__civil_count and self.__guard_count == other.__guard_count
        and self.__civils == other.__civils
        and self.__guards == other.__guards
        and self.__pos == other.__pos
        and self.__orientation == other.__orientation
        and self.__is_in_guard_range == other.__is_in_guard_range
        and self.__is_in_civil_range == other.__is_in_civil_range
        and self.__has_suit == other.__has_suit
        and self.__suit_on == other.__suit_on
        and self.__has_weapon == other.__has_weapon
        and self.__is_target_down == other.__is_target_down):
            return True
        else :
            return False
    def in_list_comparator_testing(self,List):
        for i in List:
            if self.object_comparator_personalized(i):
                return True
        return False
    def get_status_phase_2(self):
        return self.__get_status_phase_2()
    def current_penalties(self):
        return self.__phase2_penalties

def insert_tail(s, l):
    l.append(s)
    return l
def remove_head(l):
    return l.pop(0), l
def remove_tail(l):
    return l.pop(), l

#hitman=HitmanReferee("test.txt")
#list_algorithms=['Astar','BFS_backtracking','DFS_backtracking','glouton','BFS']
list_algorithms=['Astar','glouton']
for algo in list_algorithms:
    hitman=HitmanReferee()
    hitman.start_phase2()
    start_time = time.time()
    method = getattr(hitman, algo)
    ending_object,saved=method()
    if ending_object is None:
        print("no solution")
    else :
        R=ending_object.end_phase2()
        print("algorithm used : ",algo," and ",R[1])
        #print("\nlist of actions : ",R[2])
    end_time = time.time()
    execution_time = end_time - start_time
    print(f"Execution time: {execution_time} seconds")

#ending_object,saved=hitman.Astar()
#ending_object,saved=hitman.DFS_backtracking()
#ending_object,saved=hitman.glouton()
#ending_object,saved=hitman.BFS_backtracking()
#ending_object,saved=hitman.BFS()
"""
if ending_object is None:
    print("no solution")
else :
    R=ending_object.end_phase2()
    print(R[1],"\nlist of actions : ",R[2])

end_time = time.time()
execution_time = end_time - start_time
print(f"Execution time: {execution_time} seconds")
"""


import pytest
from collections import Counter

from src.encoding import Encryptor

@pytest.mark.parametrize("word,cant_be_shuffled", [
    ('b', True),
    ('big', True),
    ('biiig', True),
    ('home', False),
    ('aloha', False)
])
def test_shuffle_word(word, cant_be_shuffled):
    b = Encryptor('')
    if cant_be_shuffled:
        assert b._shuffle_word(word) == word
    else:
        assert b._shuffle_word(word) != word

@pytest.mark.parametrize("text,encrypted_words", [
    ('This is really big', ['really', 'This']),
    ('This, is !!! really ## big **,', ['really', 'This']),
    (
        'This is a long looong test sentence,\nwith some big (biiiiig) words!',
        ['long', 'looong', 'sentence', 'some', 'test', 'This', 'with', 'words']
    ),
    (
        'Lets say we have a sentence like that one',
        ['have', 'Lets', 'like', 'sentence', 'that']
    )
])
def test_encode(text, encrypted_words):
    b = Encryptor(text)
    b.encode()
    result = b.result_text.split('\n-weird-\n')[1]
    assert b.encrypted_words == encrypted_words
    for word in encrypted_words:
        assert word not in result


import os
import socket
import time

def checkSum(_data):
    checksum = 0
    # Convert a list of bytes into a list of integers by list comprehension
    # Takes two bytes from the data list and converts it to an integer with little-endian order.
    #   Least significant byte comes firts
    _data = [int.from_bytes(_data[i:i+2], 'little') for i in range(0, len(_data), 2)]

    # Iterate over the new list of int.
    for data16 in _data:
        checksum += data16

        # Run if the checksum is larger than 2^16
        #   right-shifting checksum by 16 and checking if the result is greater than 0
        while ((checksum >> 16) > 0):
            # Bitwise AND of checksuma and 0xFFFF (65535 or 2^16)
            checksum = (checksum & 0xFFFF) + (checksum >> 16)

    # ~ bitwise NOT operator to flip the bits of checksum\
    # This with 0xFFFF to keep only the lowest 16 bits of checksum
    checksum = ~checksum & 0xFFFF

    return checksum

# Calculate the checksum on the data segment, and append it and the seq_num to the data
# to create our data packet: both are encoded using little endian
def createPacket(_data, _seqNum):
    checksum = checkSum(_data)
    return _seqNum.to_bytes(2, 'little') + checksum.to_bytes(2, 'little') + _data

# Break the file into 1458 byte chunks of data create individual packets with this
# Data and append themp to a window buffer.
def fillWindowBuffer(_socket, filename):
    try:
        seqNum = 0                      # Initialize the sequence number
        windowBuffer = []               # Create a window buffer to sotre the packets
        _socket.settimeout(5.0)         # Set the timeout to 5 seconds

        with open(filename, "rb") as fi:
            # Send the filename to the server if we are able to open the file
            _socket.send(filename.encode())

            while True:
                data = fi.read(1458)    # Read the file in chucks of 1458 bytes

                # If data is NULL (all bytes are read) break from the while loop
                if not data:
                    # Append the EOF flag to the end of the window buffer
                    windowBuffer.append("EOF".encode())
                    break

                # Create our packet
                # Append the data to the window buffer
                packet = createPacket(data, seqNum)
                windowBuffer.append(packet)

                # Increment seqNum
                seqNum += 1

            return windowBuffer
    # Return None if we have an invalid Filename
    except IOError:
        print("invalid Filename")
        return None

# printACK to the terminal for debugging purposes
def printACK(_ack):
    # if the server did not return a file, exit client program
    # As the connection is likely lost
    if not _ack:
        print("Server did not recieve file, or ACK didn't return")
        print("Connection to the server has been lost. Restart of the")
        print("Client is required")
        exit(1)
    else:
        # Print the reply
        print("%s\n"%_ack.decode())

# Logic to send the data in the window buffer to the sever using a 'sliding
# window' and the 'go-back-n' protocol
def sendFile(_socket, _windowBuffer, _base):
    # If there isn't any data in our _windowBuffer return
    if not _windowBuffer:
        print("The buffer is empty! Returning to main")
        return

# Main loop for client application
if __name__ == '__main__':
    file = ""
    host = "192.168.1.25"
    port = 32007

    # Create a socket using the internet address family IPv4, and specify we wish to
    #   Use the TCP protocol
    client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    try:
        # connect to the server
        client.connect((host, port))

        while(True):
            file = input("What is the files name: ")

            # interprate the given cmd, if exit close the client, if kill close both server and client
            #   Otherwise call the handleFile method and pass the socket and filename
            if(file == "exit"):
                client.close()
                exit(0)
            elif(file == "kill"):
                client.send(file.encode())      # send the kill cmd
                ack = client.recv(1024)         # wait for ACK
                printACK(ack)                   # print the ACK

                exit(0)

            # Calculate the total number of packets based on file_size
            # Then determin the maximum window size
            file_size = os.path.getsize(file)
            total_packets = (file_size + 1457) // 1458
            WINDOW_SIZE = min(total_packets, 50)

            client.send(str(WINDOW_SIZE).encode())

            dataBuffer = fillWindowBuffer(client, file)
            sendFile(client, dataBuffer, 0)
    # Return error file if and error occured creating or using the socket
    except socket.error as e:
        print(f"Could not connect to the server: {e}")


from numpy import zeros, ones, sin, cos, uint8, pi
from sys import argv
from tqdm import tqdm
from png import Writer
try: from PIL.Image import open as iopen
except ImportError: from Image import open as iopen
class DoublePendulum():
    def __init__(self, thetas): self.t, self.td, self.l, self.m = thetas, zeros(thetas.shape), ones(thetas.shape), ones(thetas.shape)
    def step(self):
        self.td[:, 0] += (-10 * (2 * self.m[:, 0] + self.m[:, 1]) * sin(self.t[:, 0]) - self.m[:, 1] * 10 * sin(self.t[:, 0] - 2 * self.t[:, 1]) - 2 * sin(self.t[:, 0] - self.t[:, 1]) * self.m[:, 1] * (self.td[:, 1] ** 2 * self.l[:, 1] + self.td[:, 0] ** 2 * self.l[:, 0] * cos(self.t[:, 0] - self.t[:, 1]))) / (2000 * self.l[:, 0] * (2 * self.m[:, 0] + self.m[:, 1] * (1 - cos(2 * (self.t[:, 0] - self.t[:, 1])))))
        self.td[:, 1] += (2 * sin(self.t[:, 0] - self.t[:, 1]) * (self.td[:, 0] ** 2 * self.l[:, 0] * (self.m[:, 0] + self.m[:, 1]) + 10 * (self.m[:, 0] + self.m[:, 1]) * cos(self.t[:, 0]) + self.td[:, 1] ** 2 * self.l[:, 1] * self.m[:, 1] * cos(self.t[:, 0] - self.t[:, 1]))) / (2000 * self.l[:, 1] * (2 * self.m[:, 0] + self.m[:, 1] * (1 - cos(2 * (self.t[:, 0] - self.t[:, 1])))))
        self.t += self.td / 2000
color = lambda x, y: (127 * cos(x / 4 - y / 4), 127 * (cos(x / 4 - y / 4) - sin(x / 4 + y / 4)), 127 * (sin(x / 4 - y / 4) + cos(x / 4 + y / 4)))
thetas = ones((893025, 2))
for i in range(893025): thetas[i, 0], thetas[i, 1] = pi * (2 * (i % 945) / 945 - 1), pi * (2 * (i // 945) / 945 - 1)
p, frames, FRAMES = DoublePendulum(thetas), [], int(argv[1])
for frame in tqdm(range(FRAMES)):
    fractal, file, writer = uint8([[a for i in range(945) for a in color(p.t[j * 945 + i, 0], p.t[j * 945 + i, 1])] for j in range(945)]), open(f'frames/frame{frame}.png', 'wb+'), Writer(945, 945, greyscale = False)
    writer.write(file, fractal)
    frames += [iopen(file)]
    for _ in range(33): p.step()
frames[0].save('output.gif', format = 'GIF', append_images = frames[1:], save_all = True, duration = FRAMES / 60)


from django import forms
from models import *

class Messageform(forms.ModelForm):
    class Meta:
        model=Messages_Orgs
        exclude = ('sender_org_id',)
class Notificationform(forms.ModelForm):
    class Meta:
        model=Notifications_Org
        fields=('target_org_id','is_message_from_org','message_from_org_id','is_message_from_admin','message_from_admin_id','is_request_from_admin','disaster_id')