import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import scale, StandardScaler, MaxAbsScaler
import os, urllib
from torch.utils.data import Dataset

import json, os, string, random, time, pickle, gc, pdb
from PIL import Image
from PIL import ImageFilter
import numpy as np

import torch
import torch.nn as nn
import torch.utils.data as data
import torchvision.transforms as transforms
import random
from sklearn.preprocessing import OneHotEncoder

class CelebALoader(data.Dataset):
    def __init__(self, args, annotation_dir, split = 'train', transform = None):
        print("CelebA dataloader")

        self.split = split
        self.annotation_dir = annotation_dir
        self.transform = transform

        print("loading %s annotations.........." % self.split)
        self.df = pd.read_csv(open(os.path.join(annotation_dir, split+".csv"), 'rb'))

        self.labels = np.eye(2)[self.df['Attractive'].values]
        self.gender = np.eye(2)[self.df['Male'].values]

        print("dataset size: %d" % len(self.labels))

        self.image_dir = self.df['Path'].values
        self.image_ids = [i.split('/')[-1] for i in self.df['Path'].values]

        print("man size : {} and woman size: {}".format(len(np.nonzero( \
                self.gender[:, 1])[0]), len(np.nonzero(self.gender[:, 0])[0])))

    def __getitem__(self, index):
        image_dir = self.image_dir[index]

        image_ids = self.image_ids[index]

        img_ = Image.open(image_dir).convert('RGB')

        if self.transform is not None:
            img_ = self.transform(img_)

        return img_, torch.Tensor(self.labels[index]), \
                torch.LongTensor(self.gender[index]), [self.image_ids[index]]
    def __len__(self):
        return len(self.gender)

class tabular_dataset(Dataset):
    def __init__(self, X, Y, A, std = None, noise = True, M=2):
        self.X_raw = X
        self.Y = Y
        self.A = A
        self.noise = noise
        self.std = std
        self.M = M

#         if noise:
#             self.X = []
#             self.X.append((self.X_raw + np.random.normal(0,std,self.X_raw.shape), 0, 1))
#             self.X.append((self.X_raw + np.random.normal(0,std,self.X_raw.shape), 0, 1))
#         else:
#             self.X = self.X_raw

    def __len__(self):
        return len(self.Y)

    def __getitem__(self, idx):

        if self.noise:
            return [self.X_raw[idx] + np.random.normal(0, self.std) for i in range(self.M)], self.Y[idx], self.A[idx]
        else:
            return self.X_raw[idx], self.Y[idx], self.A[idx]

def get_dataset(name, save=False, corr_sens=False, seed=42, verbose=False):
    """
    Retrieve dataset and all relevant information
    :param name: name of the dataset
    :param save: if set to True, save the dataset as a pickle file. Defaults to False
    :return: Preprocessed dataset and relevant information
    """
    def get_numpy(df):
        new_df = df.copy()
        cat_columns = new_df.select_dtypes(['category']).columns
        new_df[cat_columns] = new_df[cat_columns].apply(lambda x: x.cat.codes)
        return new_df.values

    if name == 'adult':
        # Load data
        feature_names = ['age', 'workclass', 'fnlwgt', 'education', 'education-num', \
                         'marital-status', 'occupation', 'relationship', 'race', 'sex', \
                         'capital-gain', 'capital-loss', 'hours-per-week', 'native-country', 'label']
        df = pd.read_csv('/data/adult/adult.data', names=feature_names)

        if verbose:
            print('Raw Dataset loaded.')
        num_train = df.shape[0]
        pos_class_label = ' >50K'
        neg_class_label = ' <=50K'
        y = np.zeros(num_train)
        y[df.iloc[:,-1].values == pos_class_label] = 1
#         df = df.drop(['fnlwgt', 'education-num'], axis=1)
        df = df.drop(['fnlwgt'], axis=1)
        num_var_names = ['age', 'capital-gain', 'capital-loss', 'hours-per-week']
#         cat_var_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race', 'sex', 'native-country']
        cat_var_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race', 'sex']
        feature_names = num_var_names + cat_var_names
        df = df[feature_names]
        df = pd.get_dummies(df, columns=['workclass', 'education', 'marital-status', 'occupation', 'relationship'], prefix_sep='=')
        if verbose:
            print('Selecting relevant features complete.')

        for col in df:
            if df[col].dtype == 'object':
                df[col] = df[col].astype('category')
            else:
                df[col] = df[col].astype(float)

        dtypes = df.dtypes

        X = get_numpy(df)
        if verbose:
            print('Numpy conversion complete.')

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=seed)
        if verbose:
            print('Dataset split complete.')

        # sens idx
        race_idx = df.columns.get_loc('race')
        sex_idx = df.columns.get_loc('sex')
#         print( df.columns.get_loc('sex'))
        sens_idc = [race_idx, sex_idx]

        race_cats = df.iloc[:, race_idx].cat.categories
        sex_cats = df.iloc[:, sex_idx].cat.categories

        if verbose:
            print(race_cats, sex_cats)

        # Remove sensitive information from data
        X_train_removed = np.delete(X_train, sens_idc , 1)
        X_test_removed = np.delete(X_test, sens_idc , 1)
        dtypes_ = dtypes[np.delete(np.arange(X.shape[1]), sens_idc, 0)]

        if verbose:
            print('Senstive attribute removal complete.')

    elif name == 'bank':
        # Load data

        df = pd.read_csv('/data/bank/bank-additional-full.csv', sep = ';', na_values=['unknown'])

        df['age'] = df['age'].apply(lambda x: x >= 25)
        df = df[np.array(df.default == 'no') + np.array(df.default == 'yes')]

        #         num_var_names = ['age', 'capital-gain', 'capital-loss', 'hours-per-week']
        cat_var_names = ['job', 'marital', 'education', 'default',
                         'housing', 'loan', 'contact', 'month', 'day_of_week',
                     'poutcome']
        #         feature_names = num_var_names + cat_var_names
        #         df = df[feature_names]

#         df = df.drop(['default'], axis=1)
        df = pd.get_dummies(df, columns=cat_var_names, prefix_sep='=')

        for col in df:
            if df[col].dtype == 'object':
                df[col] = df[col].astype('category')
            else:
                df[col] = df[col].astype(float)

        Xy = get_numpy(df)

        idx = np.zeros(Xy.shape[-1]).astype(bool)
        idx[df.columns.get_loc('y')] = 1

        X = Xy[:, ~idx]
        y = Xy[:, idx].reshape(-1)

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=seed)

        dtypes = df.dtypes[~idx]

        # sens idx
        sex_idx = df.columns.get_loc('age')
        race_idx = df.columns.get_loc('age')
        sens_idc = [sex_idx]

        # Remove sensitive information from data
        X_train_removed = np.delete(X_train, sens_idc , 1)
        X_test_removed = np.delete(X_test, sens_idc , 1)
        dtypes_ = dtypes[np.delete(np.arange(X.shape[1]), sens_idc, 0)]

        if verbose:
            print('Senstive attribute removal complete.')

    elif name == 'compas':

        def quantizeLOS(x):
            if x<= 7:
                return '<week'
            if 8<x<=93:
                return '<3months'
            else:
                return '>3 months'

                # Load data
        df = pd.read_csv('/data/compas/compas-scores-two-years.csv', index_col='id', na_values=[])

        df = df[['age', 'c_charge_degree', 'race', 'age_cat', 'score_text', 'juv_fel_count',
                  'juv_misd_count', 'juv_other_count', 'sex', 'priors_count', 'days_b_screening_arrest', 'decile_score','c_charge_desc',
                 'two_year_recid', 'c_jail_in', 'c_jail_out']]

        # Indices of data samples to keep
        ix = df['days_b_screening_arrest'] <= 30
        ix = (df['days_b_screening_arrest'] >= -30) & ix
        # ix = (df['is_recid'] != -1) & ix
        ix = (df['c_charge_degree'] != "O") & ix
        ix = (df['score_text'] != 'N/A') & ix
        df = df.loc[ix,:]
        df['length_of_stay'] = abs(pd.to_datetime(df['c_jail_out'])-
                                pd.to_datetime(df['c_jail_in'])).apply(
                                                        lambda x: x.days)

        # Restrict races to African-American and Caucasian
        dfcut = df.loc[~df['race'].isin(['Native American','Hispanic','Asian','Other']),:]

        # Restrict the features to use
        dfcutQ = dfcut[['sex','race','age_cat','c_charge_degree','score_text','priors_count',
                'length_of_stay', 'two_year_recid','c_charge_desc']].copy()
        dfcutQ['length_of_stay'] = dfcutQ['length_of_stay'].apply(lambda x: quantizeLOS(x))

        num_train = dfcutQ.shape[0]

        num_var_names = ['two_year_recid', 'sex','race', 'score_text','priors_count', 'length_of_stay','c_charge_desc' ]
        categorical_features = ['age_cat','c_charge_degree']

        dfcutQ = pd.get_dummies(dfcutQ, columns=categorical_features, prefix_sep='=')

        for col in dfcutQ:
            if dfcutQ[col].dtype == 'object':
                dfcutQ[col] = dfcutQ[col].astype('category')
            else:
                dfcutQ[col] = dfcutQ[col].astype(float)

        pos_class_label = 1
        neg_class_label = 0

        idx = np.zeros(dfcutQ.shape[1]).astype(bool)
        y_idx = dfcutQ.columns.get_loc('two_year_recid')
        idx[y_idx] = True

        Xy = get_numpy(dfcutQ)

        X = Xy[:, ~idx]
        y = Xy[:, idx].reshape(-1)

        #remove bad quality sample
        idx = X[:, 5] == -1

        X = X[~idx, :]
        y = y[~idx]

        dfcutQ = dfcutQ.drop(['two_year_recid'], axis = 1)

        dtypes = dfcutQ.dtypes

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=seed)

        if verbose:
            print('Dataset split complete.')

        race_idx = dfcutQ.columns.get_loc('race')
        sex_idx = dfcutQ.columns.get_loc('sex')

        sens_idc = [race_idx, sex_idx]

        race_cats = dfcutQ.iloc[:,race_idx].cat.categories
        sex_cats = dfcutQ.iloc[:,sex_idx].cat.categories

        if verbose:
            print(race_cats, sex_cats)

        X_train_removed = np.delete(X_train, sens_idc , 1)
        X_test_removed = np.delete(X_test, sens_idc , 1)
        dtypes_ = dtypes[np.delete(np.arange(X.shape[1]), sens_idc, 0)]

        if verbose:
            print('Senstive attribute removal complete.')

    elif name == 'meps':

        df = pd.read_csv('/data/meps/h181.csv', sep=',', na_values=[])

        def race(row):
            if ((row['HISPANX'] == 2) and (row['RACEV2X'] == 1)):  #non-Hispanic Whites are marked as WHITE; all others as NON-WHITE
                return 'White'
            return 'Non-White'

        def sex(row):
            if row['SEX'] == 1:
                return 'female'
            return 'male'

        df['RACEV2X'] = df.apply(lambda row: race(row), axis=1)
        df['SEX'] = df.apply(lambda row: sex(row), axis=1)

        df = df.rename(columns = {'RACEV2X' : 'RACE'})

        df = df[df['PANEL'] == 19]

        df = df.rename(columns = {'FTSTU53X' : 'FTSTU', 'ACTDTY53' : 'ACTDTY', 'HONRDC53' : 'HONRDC', 'RTHLTH53' : 'RTHLTH',
                              'MNHLTH53' : 'MNHLTH', 'CHBRON53' : 'CHBRON', 'JTPAIN53' : 'JTPAIN', 'PREGNT53' : 'PREGNT',
                              'WLKLIM53' : 'WLKLIM', 'ACTLIM53' : 'ACTLIM', 'SOCLIM53' : 'SOCLIM', 'COGLIM53' : 'COGLIM',
                              'EMPST53' : 'EMPST', 'REGION53' : 'REGION', 'MARRY53X' : 'MARRY', 'AGE53X' : 'AGE',
                              'POVCAT15' : 'POVCAT', 'INSCOV15' : 'INSCOV'})

        df = df[df['REGION'] >= 0] # remove values -1
        df = df[df['AGE'] >= 0] # remove values -1

        df = df[df['MARRY'] >= 0] # remove values -1, -7, -8, -9

        df = df[df['ASTHDX'] >= 0] # remove values -1, -7, -8, -9

        df = df[(df[['FTSTU','ACTDTY','HONRDC','RTHLTH','MNHLTH','HIBPDX','CHDDX','ANGIDX','EDUCYR','HIDEG',
                                 'MIDX','OHRTDX','STRKDX','EMPHDX','CHBRON','CHOLDX','CANCERDX','DIABDX',
                                 'JTPAIN','ARTHDX','ARTHTYPE','ASTHDX','ADHDADDX','PREGNT','WLKLIM',
                                 'ACTLIM','SOCLIM','COGLIM','DFHEAR42','DFSEE42','ADSMOK42',
                                 'PHQ242','EMPST','POVCAT','INSCOV']] >= -1).all(1)]  #for all other categorical features, remove values < -1

        def utilization(row):
            return row['OBTOTV15'] + row['OPTOTV15'] + row['ERTOT15'] + row['IPNGTD15'] + row['HHTOTD15']

        df['TOTEXP15'] = df.apply(lambda row: utilization(row), axis=1)
        lessE = df['TOTEXP15'] < 10.0
        df.loc[lessE,'TOTEXP15'] = 0.0
        moreE = df['TOTEXP15'] >= 10.0
        df.loc[moreE,'TOTEXP15'] = 1.0

        df = df.rename(columns = {'TOTEXP15' : 'UTILIZATION'})

        features_to_keep=['REGION','AGE','SEX','RACE','MARRY',
                                 'FTSTU','ACTDTY','HONRDC','RTHLTH','MNHLTH','HIBPDX','CHDDX','ANGIDX',
                                 'MIDX','OHRTDX','STRKDX','EMPHDX','CHBRON','CHOLDX','CANCERDX','DIABDX',
                                 'JTPAIN','ARTHDX','ARTHTYPE','ASTHDX','ADHDADDX','PREGNT','WLKLIM',
                                 'ACTLIM','SOCLIM','COGLIM','DFHEAR42','DFSEE42','ADSMOK42','PCS42',
                                 'MCS42','K6SUM42','PHQ242','EMPST','POVCAT','INSCOV','UTILIZATION']

        categorical_features=['REGION','SEX', 'MARRY',
             'FTSTU','ACTDTY','HONRDC','RTHLTH','MNHLTH','HIBPDX','CHDDX','ANGIDX',
             'MIDX','OHRTDX','STRKDX','EMPHDX','CHBRON','CHOLDX','CANCERDX','DIABDX',
             'JTPAIN','ARTHDX','ARTHTYPE','ASTHDX','ADHDADDX','PREGNT','WLKLIM',
             'ACTLIM','SOCLIM','COGLIM','DFHEAR42','DFSEE42','ADSMOK42',
             'PHQ242','EMPST','POVCAT','INSCOV']

        df = df[features_to_keep]
        df = pd.get_dummies(df, columns=categorical_features, prefix_sep='=')

        num_train = df.shape[0]

        pos_class_label = 1
        neg_class_label = 0
        y = np.zeros(num_train)

        verbose = True

         # sens idx
        race_idx = df.columns.get_loc('RACE')
        sex_idx = df.columns.get_loc('RACE')

        sens_idc = [race_idx]

        for col in df:
            if df[col].dtype == 'object':
                df[col] = df[col].astype('category')
            else:
                df[col] = df[col].astype(float)

        idx = np.zeros(df.shape[1]).astype(bool)
        y_idx = df.columns.get_loc('UTILIZATION')
        idx[y_idx] = True

#         min_max_scaler = MaxAbsScaler()
        Xy = get_numpy(df)
        X = Xy[:, ~idx]
        y = Xy[:, idx].reshape(-1)

        df = df.drop(['UTILIZATION'], axis=1)
        dtypes = df.dtypes

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=seed)
        if verbose:
            print('Dataset split complete.')

        # Remove sensitive information from data

        race_cats = df['RACE'].cat.categories
#         sex_cats = df[feature_names[sex_idx]].cat.categories
        if verbose:
            print(race_cats)

        X_train_removed = np.delete(X_train, sens_idc , 1)
        X_test_removed = np.delete(X_test, sens_idc , 1)
        dtypes_ = dtypes[np.delete(np.arange(X.shape[1]), sens_idc, 0)]

        if verbose:
            print('Senstive attribute removal complete.')

    elif name == 'german':
        # Download data if needed
        _german_loan_attribute_map = dict(
            A11='< 0 DM',
            A12='0-200 DM',
            A13='>= 200 DM',
            A14='no checking',
            A30='no credits',
            A31='all credits paid back',
            A32='existing credits paid back',
            A33='delayed past payments',
            A34='critical account',
            A40='car (new)',
            A41='car (used)',
            A42='furniture/equipment',
            A43='radio/television',
            A44='domestic appliances',
            A45='repairs',
            A46='education',
            A47='(vacation?)',
            A48='retraining',
            A49='business',
            A410='others',
            A61='< 100 DM',
            A62='100-500 DM',
            A63='500-1000 DM',
            A64='>= 1000 DM',
            A65='unknown/no sav acct',
            A71='unemployed',
            A72='< 1 year',
            A73='1-4 years',
            A74='4-7 years',
            A75='>= 7 years',
            #A91='male & divorced',
            #A92='female & divorced/married',
            #A93='male & single',
            #A94='male & married',
            #A95='female & single',
            A91='male',
            A92='female',
            A93='male',
            A94='male',
            A95='female',
            A101='none',
            A102='co-applicant',
            A103='guarantor',
            A121='real estate',
            A122='life insurance',
            A123='car or other',
            A124='unknown/no property',
            A141='bank',
            A142='stores',
            A143='none',
            A151='rent',
            A152='own',
            A153='for free',
            A171='unskilled & non-resident',
            A172='unskilled & resident',
            A173='skilled employee',
            A174='management/self-employed',
            A191='no telephone',
            A192='has telephone',
            A201='foreigner',
            A202='non-foreigner',
        )

        filename = '/data/german/german.data'
        if not os.path.isfile(filename):
            print('Downloading data to %s' % os.path.abspath(filename))
            urllib.request.urlretrieve('https://archive.ics.uci.edu/ml/machine-learning-databases/statlog/german/german.data',
                                       filename)

        # Load data and setup dtypes
        col_names = [
            'checkin_acc', 'duration', 'credit_history', 'purpose', 'amount',
            'saving_acc', 'present_emp_since', 'inst_rate', 'personal_status',
            'other_debtors', 'residing_since', 'property', 'age',
            'inst_plans', 'housing', 'num_credits',
            'job', 'dependents', 'telephone', 'foreign_worker', 'status']

#         AIF360
        column_names = ['status', 'month', 'credit_history',
            'purpose', 'credit_amount', 'savings', 'employment',
            'investment_as_income_percentage', 'personal_status',
            'other_debtors', 'residence_since', 'property', 'age',
            'installment_plans', 'housing', 'number_of_credits',
            'skill_level', 'people_liable_for', 'telephone',
            'foreign_worker', 'credit']

        df = pd.read_csv(filename, delimiter=' ', header=None, names=column_names)

        def group_credit_hist(x):
            if x in ['A30', 'A31', 'A32']:
                return 'None/Paid'
            elif x == 'A33':
                return 'Delay'
            elif x == 'A34':
                return 'Other'
            else:
                return 'NA'

        def group_employ(x):
            if x == 'A71':
                return 'Unemployed'
            elif x in ['A72', 'A73']:
                return '1-4 years'
            elif x in ['A74', 'A75']:
                return '4+ years'
            else:
                return 'NA'

        def group_savings(x):
            if x in ['A61', 'A62']:
                return '<500'
            elif x in ['A63', 'A64']:
                return '500+'
            elif x == 'A65':
                return 'Unknown/None'
            else:
                return 'NA'
        def group_status(x):
            if x in ['A11', 'A12']:
                return '<200'
            elif x in ['A13']:
                return '200+'
            elif x == 'A14':
                return 'None'
            else:
                return 'NA'

        def group_purpose(x):
            if x in ['A40', 'A41', 'A42', 'A43', 'A47', 'A410']:
                return 'non-essential'
            elif x in ['A44', 'A45', 'A46', 'A48', 'A49']:
                return 'essential'

        status_map = {'A91': 'male', 'A93': 'male', 'A94': 'male',
                      'A92': 'female', 'A95': 'female'}
        df['sex'] = df['personal_status'].replace(status_map)

        # group credit history, savings, and employment
        df['credit_history'] = df['credit_history'].apply(lambda x: group_credit_hist(x))
        df['savings'] = df['savings'].apply(lambda x: group_savings(x))
        df['employment'] = df['employment'].apply(lambda x: group_employ(x))
    #     df['age'] = df['age'].apply(lambda x: np.float(x >= 26))
        df['purpose'] = df['purpose'].apply(lambda x:group_purpose(x))
        df['status'] = df['status'].apply(lambda x: group_status(x))

        cat_features = ['credit_history', 'savings', 'employment',  'purpose', 'other_debtors', 'property', 'housing', 'skill_level', \
                'investment_as_income_percentage', 'status', 'installment_plans', 'foreign_worker']

        df = pd.get_dummies(df, columns=cat_features, prefix_sep='=')
        df = df.drop(['telephone', 'personal_status',], axis = 1)

        for col in df:
            if df[col].dtype == 'object':
                df[col] = df[col].astype('category')
            else:
                df[col] = df[col].astype(float)

        df['age'] = df['age'].apply(lambda x: x >= 25).astype('category')

        def get_numpy(df):
            new_df = df.copy()
            cat_columns = new_df.select_dtypes(['category']).columns
            new_df[cat_columns] = new_df[cat_columns].apply(lambda x: x.cat.codes)
            return new_df.values

        y_idx = df.columns.get_loc('credit')
        idx = np.zeros(df.shape[1]).astype(bool)
        idx[y_idx] = True

        Xy = get_numpy(df)
        X = Xy[:,~idx]
        y = Xy[:,idx].reshape(-1)

        # Make 1 (good customer) and 0 (bad customer)
        # (Originally 2 is bad customer and 1 is good customer)
        sel_bad = y == 2
        y[sel_bad] = 0
        y[~sel_bad] = 1
        feature_labels = df.columns.values[:-1]  # Last is prediction
        dtypes = df.dtypes[~idx]

        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=seed)

        # Senstivie attribute
#         foreign = 19
        age_idx = df.columns.get_loc('age')
        sex_idx = df.columns.get_loc('sex')
        sens_idc = [sex_idx, age_idx]

        age_cats = df.iloc[:, age_idx].cat.categories
        sex_cats = df.iloc[:, sex_idx].cat.categories
        print([age_cats, sex_cats])

        # Remove sensitive information from data
        X_train_removed = np.delete(X_train, sens_idc , 1)
        X_test_removed = np.delete(X_test, sens_idc , 1)
        dtypes_ = dtypes[np.delete(np.arange(X.shape[1]), sens_idc, 0)]

        race_idx = age_idx

    else:
        raise ValueError('Data name invalid.')

    return X_train, y_train, X_test, y_test, X_train_removed, X_test_removed, dtypes, dtypes_, sens_idc, race_idx, sex_idx

def get_csv_eqodd(fm, data_name='adult'):
    assert(fm.model is not None)
    # get csv files required for the eq_odds code to run
    label = fm.y_test
    group = fm.X_test[:, fm.sens_idx]
    prediction = fm.model.predict_proba(fm.X_test_removed)[:,1] # positive label prediction
    # make csv file
    f = open('%s_predictions.csv'%data_name, 'w')
    f.write(',label,group,prediction\n')
    for i, e in enumerate(zip(label, group, prediction)):
        line = '%d,%0.2f,%0.2f,%f\n'%(i, e[0],  e[1], e[2])
        f.write(line)
    f.close()

from torch.utils.data import Dataset

class tabular_dataset(Dataset):
    def __init__(self, X, Y, A):
        self.X_raw = X
        self.Y = Y
        self.A = A
    def __len__(self):
        return len(self.Y)

    def __getitem__(self, idx):
        return self.X_raw[idx], self.Y[idx], self.A[idx]

class CelebALoader_JTT(data.Dataset):
    def __init__(self, args, annotation_dir, error_lst, split = 'train', lamda = 20, transform = None):
        print("CelebA dataloader")

        self.split = split
        self.annotation_dir = annotation_dir
        self.transform = transform

        print("loading %s annotations.........." % self.split)
        self.df = pd.read_csv(open(os.path.join(annotation_dir, split+".csv"), 'rb'))

        with open(error_lst) as f:
            self.error_lst = json.load(f)

        new_lst = []
        for err in self.error_lst:
            sample = self.df['/data/celebA/CelebA/Img/img_align_celeba/' +  err == self.df['Path']]
            for i in range(lamda):
                new_lst.append(sample)

        error_df = pd.concat(new_lst, ignore_index = True)
        self.df = pd.concat([self.df, error_df], ignore_index=True)

        self.labels = np.eye(2)[self.df['Attractive'].values]
        self.gender = np.eye(2)[self.df['Male'].values]

        print("dataset size: %d" % len(self.labels))

        self.image_dir = self.df['Path'].values
        self.image_ids = [i.split('/')[-1] for i in self.df['Path'].values]

        print("man size : {} and woman size: {}".format(len(np.nonzero( \
                self.gender[:, 1])[0]), len(np.nonzero(self.gender[:, 0])[0])))

    def __getitem__(self, index):
        image_dir = self.image_dir[index]

        image_ids = self.image_ids[index]

        img_ = Image.open(image_dir).convert('RGB')

        if self.transform is not None:
            img_ = self.transform(img_)

        return img_, torch.Tensor(self.labels[index]), \
                torch.LongTensor(self.gender[index]), [self.image_ids[index]]
    def __len__(self):
        return len(self.gender)

class ImSituVerbGender_JTT(data.Dataset):
    def __init__(self, args, annotation_dir, error_lst, image_dir, split = 'train', lamda = 20,  transform = None, \
        balanced_val=False, balanced_test=False):
        print("ImSituVerbGender dataloader")

        self.split = split
        self.image_dir = image_dir
        self.annotation_dir = annotation_dir
        self.transform = transform
        self.args = args

        verb_id_map = pickle.load(open('./verb_classification/data/verb_id.map', 'rb'))
        self.verb2id = verb_id_map['verb2id']
        self.id2verb = verb_id_map['id2verb']

        print("loading %s annotations.........." % self.split)
        self.ann_data = pickle.load(open(os.path.join(annotation_dir, split+".data"), 'rb'))

        print("dataset size: %d" % len(self.ann_data))
        self.verb_ann = np.zeros((len(self.ann_data), len(self.verb2id)))
        self.gender_ann = np.zeros((len(self.ann_data), 2), dtype=int)

        args.num_verb = len(self.verb2id)

        for index, ann in enumerate(self.ann_data):
            self.verb_ann[index][ann['verb']] = 1
            self.gender_ann[index][ann['gender']] = 1

        self.image_ids = list(range(len(self.ann_data)))

        with open(error_lst) as f:
            self.error_lst = json.load(f)

        for err in self.error_lst:
            self.verb_ann = np.concatenate([self.verb_ann, self.verb_ann[err].reshape(1,-1).repeat(lamda, axis = 0)])
            self.gender_ann = np.concatenate([self.gender_ann, self.gender_ann[err].reshape(1,-1).repeat(lamda, axis = 0)])
            self.ann_data.append(self.ann_data[err])
            self.image_ids.append(err)

        print("man size : {} and woman size: {}".format(len(np.nonzero( \
                self.gender_ann[:, 0])[0]), len(np.nonzero(self.gender_ann[:, 1])[0])))

    def __getitem__(self, index):

        img = self.ann_data[index]
        image_name = img['image_name']
        image_path_ = os.path.join(self.image_dir, image_name)

        img_ = Image.open(image_path_).convert('RGB')

        if self.transform is not None:
            img_ = self.transform(img_)

        return img_, torch.Tensor(self.verb_ann[index]), \
                torch.LongTensor(self.gender_ann[index]), torch.LongTensor([self.image_ids[index]])

    def getGenderWeights(self):
        return (self.gender_ann == 0).sum(axis = 0) / (1e-15 + \
                (self.gender_ann.sum(axis = 0) + (self.gender_ann == 0).sum(axis = 0) ))

    def getVerbWeights(self):
        return (self.verb_ann == 0).sum(axis = 0) / (1e-15 + self.verb_ann.sum(axis = 0))

    def __len__(self):
        return len(self.ann_data)


from Perceptron import *
import matplotlib.pyplot as plt
import random
import numpy as np

dimensions = 10
points = 1000
pointslist = []
pointlabel = []
max = 1.0
w_init = []

w_init.append(random.uniform(-1.0, 1.0))

for i in range(dimensions):
    pointslist.append([])
    w_init.append(random.uniform(-1.0, 1.0))
    for j in range(points):
        pointslist[i].append(random.uniform(0.0, max))

#w0 + w1x1 + w2x2 + w3x3 + ... + w10x10 = 0

temp = 0
for i in range(points):
    pointsum = w_init[0]
    for j in range(dimensions):
        pointsum += w_init[j + 1] * pointslist[j][i]
    if pointsum >= 0:
        pointlabel.append(1)
        temp += 1
    else:
        pointlabel.append(0)

print(temp)

randarray = []
for i in range(points):
    randarray.append(i)

if temp >= 250 and temp <= 750:
    f = open("histogram_data.txt", "w")
    for _ in range(100):
        random.shuffle(randarray)
        temppointlabel = []
        temppointslist = []
        for i in range(dimensions):
            temppointslist.append([])
        for i in range(points):
            temppointlabel.append(pointlabel[randarray[i]])
            for j in range(dimensions):
                temppointslist[j].append(pointslist[j][randarray[i]])
        p = Perceptron(lr = 0.5, max_iterations = 10000)
        p = p.fit(temppointslist, temppointlabel, max)
        print(p.updates)
        f.write(str(p.updates) + "\n")
    f.close()


#coding:utf-8
import time

from lib.protocol.report.GPSReport_protocol import GPSReport_protocol
from lib.protocol.report.ProtocolBase import ProtocolBase
from lib.protocol.appendix.EventClass import EventClass
from lib.protocol.report.SecurityStatusReport_protocol import SecurityStatusReport_protocol

'''
终端上事件数据包
'''

class EventReport_protocol(ProtocolBase):
    #data = {"WATER_CODE":"0003","DEV_ID":"M121501010001","gpsInfo":{"UTCTime":"2020-04-14 11:03:20","latitude":"40.22077","longitude":"116.23128","speed":"80.8","directionAngle":"80.8","elevation":"2999.9","positionStar":"3","Pdop":"0.3","Hdop":"0.4","Vdop":"0.5","statusBit":162,"valtage":"36.9","OBDSpeed":"60.9","engineSpeed":"3000","GPSTotalMileage":"12800","totalOil":"100000","totalTime":"2020002","GPSTimestamp":"1586833400"},"securityData":{"securityStatus":107,"doorStatus":0,"lockStatus":0,"windowStatus":0,"lightStatus":0,"onoffStatusA":0,"onoffStatusB":112,"dataByte":249},"event":{}}
    data = {}
    def __init__(self,msgCount = 1,WATER_CODE = 26,DEV_ID = "M202003060520",locationType=1,eventType="0030",data={}):
        super().__init__()
        self.data = data
        if len(data) == 0:
            self.msgCount = msgCount                             #数据包个数

            self.WATER_CODE = int(WATER_CODE)                    #消息流水号
            self.DEV_ID = DEV_ID                                 #设备id

            self.locationType = int(locationType)                # 定位类型
            #self.GPSPkg = GPSpkg & BaseStationPkg  # GPS包或者基站包
            timeStamp = time.time()
            timeArray = time.localtime(int(timeStamp - 8 * 3600))
            dateTimeM = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
            timeHex = self.getUTCTime(dateTimeM)
            self.GPSPkg = timeHex + "0265b86206ed8c70026103280000752f03030405af017102610bb800003200000186a0001ed2a25e16fe3a"
            # self.GPSPkg = "1401091213260265b86206ed8c70026103280000752f03030405af017102610bb800003200000186a0001ed2a25e16fe3a"
            self.BaseStationPkg = "1401140a0c050207e407e607e807ea07ec4eea4eec4eee4ef04efc4efe4f004f024f040024025e07d00007a125000927c60000ea610100"
            self.eventType = eventType                           #事件类别
        else:
            self.msgCount = 1  # 数据包个数

            self.WATER_CODE = int(data["WATER_CODE"])               # 消息流水号
            self.DEV_ID = data["DEV_ID"]                            # 设备id

            self.locationType = 1                                   # 定位类型
            gpsInfo = data["gpsInfo"]
            self.gpsInfo = gpsInfo
            securityData = data["securityData"]
            self.securityData = securityData
            event = data["event"]
            self.event = event
            self.GPSPkg = GPSReport_protocol(1,self.WATER_CODE,self.DEV_ID,gpsInfo["UTCTime"],gpsInfo["latitude"],gpsInfo["longitude"]  \
                          ,gpsInfo["speed"],gpsInfo["directionAngle"],gpsInfo["elevation"],gpsInfo["positionStar"],gpsInfo["Pdop"]  \
                          ,gpsInfo["Hdop"],gpsInfo["Vdop"],gpsInfo["statusBit"],gpsInfo["valtage"],gpsInfo["OBDSpeed"],gpsInfo["engineSpeed"]  \
                          ,gpsInfo["GPSTotalMileage"],gpsInfo["totalOil"],gpsInfo["totalTime"],gpsInfo["GPSTimestamp"]).generateGpsData()

            self.securityPkg = SecurityStatusReport_protocol(1,self.WATER_CODE,self.DEV_ID,1,self.GPSPkg,"ffffffffffffffffffff" \
                               ,securityData["securityStatus"],securityData["doorStatus"],securityData["lockStatus"]  \
                               ,securityData["windowStatus"],securityData["lightStatus"],securityData["onoffStatusA"]  \
                               ,securityData["onoffStatusB"],securityData["dataByte"]).generateSecurityStatusData()
            self.BaseStationPkg = "1401140a0c050207e407e607e807ea07ec4eea4eec4eee4ef04efc4efe4f004f024f040024025e07d00007a125000927c60000ea610100"

    def setLatitude(self,data):
        self.latitude = data
    def setLongtitude(self,data):
        self.longitude = data
    def setEventType(self,data):
        self.eventType = data
    def setLocationType(self,data):
        self.locationType = data
    def setGPSPkg(self,data):
        self.GPSPkg = data
    def setBaseStationPkg(self,data):
        self.BaseStationPkg = data

    #####################################################
    #               生成事件信息消息
    #####################################################
    def generateEventMsg(self):
        self.getProtocalHeader()
        info = ""
        #消息头
        HEADER = "4040"
        #消息流水号
        WATER_CODE = self.getWaterCode(self.WATER_CODE)
        #设备id
        DEV_ID = self.devid2hexString(self.DEV_ID)
        # 功能id(GPS功能id)
        FUN_ID = "0021"
        #数据段
        data = ""
        if len(self.data) == 0:
            for i in range(0,self.msgCount):
                data += self.generateEventPkg(self.msgCount,self.generateEventData())
        else:
            for i in range(0,self.msgCount):
                data += self.generateEventPkg(self.msgCount,self.generateEventData_GUI())
            # data += self.generateEventData_GUI()
        # 消息长度
        LENGTH = self.getMsgLength(int(len(WATER_CODE + DEV_ID + FUN_ID + data)/2))

        info += HEADER
        info += LENGTH
        info += WATER_CODE
        info += DEV_ID
        info += FUN_ID
        info += data
        # 校验字段
        CHECK_CODE = self.getCheckCode(info)
        info += CHECK_CODE
        return info

    #####################################################
    #               创建GPS数据包，包含包个数
    #               data:传入GPS数据包的多条数据段
    #####################################################
    def generateEventPkg(self,pkgNum,data):
        pkgNumHex = self.int2hexString(int(pkgNum))
        pkg = pkgNumHex + data
        return pkg

    #####################################################
    #               创建事件信息数据段
    #####################################################
    def generateEventData(self):
        data = ""
        locationType = self.int2hexString(self.locationType)               #定位类型
        locationData = ""                                                  #GPS包或基站包
        if self.locationType == 1:
            locationData = self.GPSPkg
        elif self.locationType == 2:
            locationData = self.BaseStationPkg
        eventType = self.eventType
        extraInfo = self.getExtraData(eventType)                                      #附带信息
        extraInfoLen = self.int2hexStringByBytes(int((len(extraInfo) / 2)),2)                #附带信息长度
        data = locationType + locationData + eventType + extraInfoLen + extraInfo
        return data
    def generateEventData_GUI(self):
        data = ""
        locationType = self.int2hexString(self.locationType)               #定位类型
        locationData = ""                                                  #GPS包或基站包
        if self.locationType == 1:
            locationData = self.GPSPkg
        elif self.locationType == 2:
            locationData = self.BaseStationPkg
        eventData = self.getExtraData_GUI(self.event)
        data = locationType + locationData + eventData
        return data

    #####################################################
    #               获取附带信息
    #####################################################
    def getExtraData(self,eventType):
        if eventType == "0001":                                          #终端插入报警
            return EventClass().terminalInsertionAlarmExtraInfo()
        elif eventType == "0002":                                        #终端拔出报警
            return EventClass().terminalPulloutAlarmExtraInfo()
        elif eventType == "0003":                                        #汽车电瓶低电压报警
            return ""
        elif eventType == "0004":                                        #终端主电断电报警
            return ""
        elif eventType == "0005":                                        #GPS模块故障报
            return ""
        elif eventType == "0006":                                        #GPS天线开路报警
            return ""
        elif eventType == "0007":                                        #GPS天线短路报警
            return ""
        elif eventType == "0008":                                        #GPS定位时间过长报警
            return ""
        elif eventType == "0009":                                        #FLASH故障报警
            return ""
        elif eventType == "000A":                                        #CAN模块故障报警
            return ""
        elif eventType == "000B":                                        #3D传感器故障报警
            return ""
        elif eventType == "000C":                                        #RTC模块故障报警
            return ""
        elif eventType == "000D":                                        #TF卡故障报警
            return ""
        elif eventType == "000E":                                        #刷卡器故障报警
            return ""
        elif eventType == "000F":                                        #汽车预点火上报
            return EventClass().preFiringExtraInfo()
        elif eventType == "0010":                                        #汽车点火上报
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            return eventObj.fireExtraInfo()
        elif eventType == "0011":                                        #汽车熄火上报
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            return eventObj.misFireExtraInfo()
        elif eventType == "0012":                                        #汽车设防上报
            return EventClass().setUpDefencesExtraInfo()
        elif eventType == "0013":                                        #汽车撤防上报
            return EventClass().setDownDefencesExtraInfo()
        elif eventType == "0014":                                        #锁车未成功提醒
            return EventClass().lockCarFaillExtraInfo()
        elif eventType == "0015":                                        #超时未设防提醒
            return EventClass().noDefencesWithTimeoutExtraInfo()
        elif eventType == "0016":                                        #设防玻璃未关提醒
            return EventClass().defencesGlassNoCloseExtraInfo()
        elif eventType == "0017":                                        #设防非法开门报警
            return EventClass().defencesIllegalCloseDoorExtraInfo()
        elif eventType == "0023":                                        #碰撞告警
            return EventClass().collisionAlarmExtraInfo()
        elif eventType == "0027":                                        #超速报警
            return EventClass().overSpeedAlarm()
        elif eventType == "0028":                                        #疲劳驾驶报警
            return EventClass().tiredDrivingAlarm()
        elif eventType == "0030":                                        #水温报警
            return EventClass().waterTemperatureAlarm()
        elif eventType == "0031":                                        #低水温高转速报警
            return EventClass().lowWaterTemperatureHighEngineSpeed()
        elif eventType == "0032":                                        #怠速时间过长报警
            return EventClass().idlingOverTime()
        elif eventType == "0033":                                        #高速空挡滑行报警
            return EventClass().highSpeedNeutralGearSlide()
        elif eventType == "0036":                                        #低档高速报警
            return EventClass().lowGearHighSpeedAlarm()
        elif eventType == "0037":                                        #高档低速报警
            return EventClass().highGearLowSpeedAlarm()
        elif eventType == "004A":                                        #剩余油量异常告警
            return EventClass().surplusOilAlarm()

    #####################################################
    #               获取附带信息(针对图形界面)
    #####################################################
    def getExtraData_GUI(self,eventData):
        data = ""
        if ("0001" in eventData.keys()):                                           # 终端插入报警
            eventObj = EventClass()
            theData = eventObj.terminalInsertionAlarmExtraInfo()
            data = data + "0001" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0002" in eventData.keys()):                                           # 终端拔出报警
            eventObj = EventClass()
            theData = eventObj.terminalInsertionAlarmExtraInfo()
            data = data + "0002" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if("0003" in eventData.keys()):                                            #汽车电瓶低电压报警
            theData = ""
            data = data + "0003" + self.int2hexStringByBytes(int((len(theData) / 2)),2) + theData
        if ("0004" in eventData.keys()):                                          #终端主电断电报警
            theData = ""
            data = data + "0004" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0010" in eventData.keys()):                                          #汽车点火上报
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            eventObj.setSecurityData(self.securityPkg)
            theData = eventObj.fireExtraInfo(int(eventData["0010"]["allRapidlyAccelerateCount"]), int(eventData["0010"]["allSharpSlowdownCount"]), int(eventData["0010"]["allSharpTurn"]))
            data = data + "0010" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0011" in eventData.keys()):                                          #汽车熄火上报
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            eventObj.setSecurityData(self.securityPkg)
            theData = eventObj.misFireExtraInfo(int(eventData["0011"]["allRapidlyAccelerateCount"]), int(eventData["0011"]["allSharpSlowdownCount"]), int(eventData["0011"]["allSharpTurn"]))
            data = data + "0011" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0012" in eventData.keys()):                                          #汽车设防上报
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            eventObj.setSecurityData(self.securityPkg)
            theData = eventObj.setUpDefencesExtraInfo()
            data = data + "0012" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0013" in eventData.keys()):                                          #汽车撤防上报
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            eventObj.setSecurityData(self.securityPkg)
            theData = eventObj.setDownDefencesExtraInfo()
            data = data + "0013" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0020" in eventData.keys()):                                          #急加速
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            eventObj.setSecurityData(self.securityPkg)
            theData = eventObj.rapidlyAccelerateExtraInfo(int(eventData["0020"]["allRapidlyAccelerateCount"]),int(eventData["0020"]["allSharpSlowdownCount"]),int(eventData["0020"]["allSharpTurn"]),int(eventData["0020"]["dataProperty"]))
            data = data + "0020" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0021" in eventData.keys()):                                          #急减速
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            eventObj.setSecurityData(self.securityPkg)
            theData = eventObj.sharpSlowdownExtraInfo(int(eventData["0021"]["allRapidlyAccelerateCount"]),
                                                          int(eventData["0021"]["allSharpSlowdownCount"]),
                                                          int(eventData["0021"]["allSharpTurn"]),
                                                          int(eventData["0021"]["dataProperty"]))
            data = data + "0021" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0022" in eventData.keys()):                                          #急转弯
            eventObj = EventClass()
            eventObj.setGPSpkg(self.GPSPkg)
            eventObj.setSecurityData(self.securityPkg)
            theData = eventObj.sharpTurnExtraInfo(int(eventData["0022"]["allRapidlyAccelerateCount"]),
                                                          int(eventData["0022"]["allSharpSlowdownCount"]),
                                                          int(eventData["0022"]["allSharpTurn"]),
                                                          int(eventData["0022"]["direction"]),
                                                          int(eventData["0022"]["dataProperty"]))
            data = data + "0022" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0023" in eventData.keys()):                                          # 碰撞告警
            eventObj = EventClass()
            theData = eventObj.collisionAlarmExtraInfo(int(eventData["0023"]["totalCount"]),
                                                  int(eventData["0023"]["dataProperty"]),)
            data = data + "0023" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0027" in eventData.keys()):                                          # 超速告警
            eventObj = EventClass()
            theData = eventObj.overSpeedAlarm(int(eventData["0027"]["alarmType"]),
                                                  int(eventData["0027"]["durationTime"]))
            data = data + "0027" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0028" in eventData.keys()):                                          # 疲劳驾驶告警
            eventObj = EventClass()
            theData = eventObj.tiredDrivingAlarm(int(eventData["0028"]["alarmType"]),
                                                  int(eventData["0028"]["durationTime"]),)
            data = data + "0028" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("002A" in eventData.keys()):                                          # 急变道告警
            eventObj = EventClass()
            theData = eventObj.rapidChangeLanes(int(eventData["002A"]["nums"]),
                                                int(eventData["002A"]["direction"]),
                                                float(eventData["002A"]["lng"]),
                                                float(eventData["002A"]["lat"]))
            data = data + "002A" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0032" in eventData.keys()):                                          # 怠速时间过长报警
            eventObj = EventClass()
            theData = eventObj.idlingOverTime(int(eventData["0032"]["alarmType"]),int(eventData["0032"]["durationTime"]),int(eventData["0032"]["oilExpend"]))
            data = data + "0032" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0036" in eventData.keys()):                                          # 低档高速报警
            eventObj = EventClass()
            theData = eventObj.lowGearHighSpeedAlarm(int(eventData["0036"]["alarmType"]),int(eventData["0036"]["durationTime"]))
            data = data + "0036" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("0037" in eventData.keys()):                                          # 高档低速报警
            eventObj = EventClass()
            theData = eventObj.lowGearHighSpeedAlarm(int(eventData["0037"]["alarmType"]),int(eventData["0037"]["durationTime"]))
            data = data + "0037" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        if ("004A" in eventData.keys()):                                          # 高档低速报警
            eventObj = EventClass()
            theData = eventObj.surplusOilAlarm(int(eventData["004A"]["surplusOilType"]),int(eventData["004A"]["value"]))
            data = data + "004A" + self.int2hexStringByBytes(int((len(theData) / 2)), 2) + theData
        return data

    #####################################################
    #               将UTC时间转换为16进制，
    #        例如：2020-01-02   20:30:00 （年取后面2字节）则将20,01，02,20,30,00 转换为对应的6个字节
    #        theTime:传入一个类似：2020-01-03 13:05:13的一个字符串
    #####################################################
    def getUTCTime(self,theTime):
        # 获取当前时间，时间格式为：2020-01-03 13:05:13
        # now_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        # 将2020-01-03 13:05:13时间格式转换为一个数组
        # timeStr = "2020-01-03 13:05:13"
        timeStr = theTime
        timeArr = []
        timeArr.append(timeStr[2:4])
        timeArr.append(timeStr[5:7])
        timeArr.append(timeStr[8:11])
        timeArr.append(timeStr[11:13])
        timeArr.append(timeStr[14:16])
        timeArr.append(timeStr[17:19])
        UTCTime = ""
        for i in range(0, len(timeArr)):
            UTCTime += self.int2hexString(int(timeArr[i]))
        return UTCTime

if __name__ == "__main__":
    data = {"a":1,"b":2,"c":{"d":3}}
    data = {}
    print(len(data))


from ExpressionTree import ExpressionTree

def build_expression_tree(expression):
    """ The algorithm uses a stack S while scanning tokens of the input expression E to
        find values, operators, and right parentheses. (Left parentheses are ignored.)
        • When we see an operator ◦, we push that string on the stack.
        • When we see a literal value v, we create a single-node expression tree T
          storing v, and push T on the stack.
        • When we see a right parenthesis, ')', we pop the top three items from the
          stack S, which represent a subexpression (E1 ◦ E2). We then construct a
          tree T using trees for E1 and E2 as subtrees of the root storing ◦, and push
          the resulting tree T back on the stack."""
    s = []                                                           # use list as stack

    for token in expression:
        if token == '(':
            continue
        elif token in '+-/x*':
            s.append(token)
        elif token != '(' and token not in '+-/x' and token != ')':          # literal
            T = ExpressionTree(token)
            s.append(T)
        elif token == ')':                                          # pop top three items
            E1 = s.pop()
            operator = s.pop()
            E2 = s.pop()
            T = ExpressionTree(operator, E1, E2)
            s.append(T)

    return s.pop()


#!/user/bin/env python3
"""
Author: Nicolás Barrón Sour
Date Created: 09/10/2023
Description: Allows you to track and update your ELO rating as you compete with yourself in daily habit-tracking games, providing a numerical measure of your progress and skill in maintaining these habits.
"""

# Imports
import sqlite3
import climage
from datetime import datetime

# Global Values
numberHabits = 5
elo_deduction = 50  # ELO deduction for each missed day

# Function declarations
def db_innit():
    conn = sqlite3.connect("elo_ratings.db")
    cursor = conn.cursor()

    cursor.execute(
        """
		CREATE TABLE IF NOT EXISTS games (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			player_name TEXT,
			opponent_name TEXT,
			player_result INTEGER,
			game_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
		)
	"""
    )

    cursor.execute(
        """
		CREATE TABLE IF NOT EXISTS players (
			id INTEGER PRIMARY KEY AUTOINCREMENT,
			name TEXT UNIQUE,
			elo INTEGER
		)
	"""
    )

    conn.commit()
    conn.close()

def calculate_elo_rating(player_rating, opponent_rating, result):
    k = 10  # K-factor, determines how much ratings change after a game
    expected_result = 1 / (1 + 10 ** ((opponent_rating - player_rating) / 400))
    new_rating = player_rating + k * (result - expected_result)
    return round(new_rating)

def record_game(player_name, opponent_name, player_result, game_date=None):
    conn = sqlite3.connect("elo_ratings.db")
    cursor = conn.cursor()

    if game_date is None:
        game_date = datetime.now()

    # Insert the game record into the "games" table with the provided or current timestamp
    cursor.execute(
        "INSERT INTO games (player_name, opponent_name, player_result, game_date) VALUES (?, ?, ?, ?)",
        (player_name, opponent_name, player_result, game_date),
    )

    # Get the ID of the last inserted record
    record_id = cursor.lastrowid

    # Get the current ratings for the players
    cursor.execute("SELECT elo FROM players WHERE name = ?", (player_name,))
    player_rating = cursor.fetchone()[0]

    cursor.execute("SELECT elo FROM players WHERE name = ?", (opponent_name,))
    opponent_rating = cursor.fetchone()[0]

    # Calculate the new ratings
    new_player_rating = calculate_elo_rating(
        player_rating, opponent_rating, player_result
    )
    new_opponent_rating = calculate_elo_rating(
        opponent_rating, player_rating, 1 - player_result
    )

    # Update the ratings in the database
    cursor.execute(
        "UPDATE players SET elo = ? WHERE name = ?", (new_player_rating, player_name)
    )
    cursor.execute(
        "UPDATE players SET elo = ? WHERE name = ?",
        (new_opponent_rating, opponent_name),
    )

    conn.commit()
    conn.close()
    return record_id

def get_player_rating(player_name):
    conn = sqlite3.connect("elo_ratings.db")
    cursor = conn.cursor()

    cursor.execute("SELECT elo FROM players WHERE name = ?", (player_name,))
    rating = cursor.fetchone()[0]

    conn.close()
    return rating

def deductMissingDays():
    conn = sqlite3.connect("elo_ratings.db")
    cursor = conn.cursor()

    cursor.execute("SELECT game_date FROM games ORDER BY game_date DESC LIMIT 1")
    result = cursor.fetchone()

    if not result:
        print("No records found in the games table.")
        return

    last_day = datetime.strptime(result[0], "%Y-%m-%d %H:%M:%S.%f")
    elapsed_time = datetime.today() - last_day
    if elapsed_time.days > 1:
        elo_deduction_total = elapsed_time.days * elo_deduction
        new_elo = get_player_rating("NBS") - elo_deduction_total
        print("More than 1 day has passed since the last record.")
        print(
            f"You missed {elapsed_time.days} days. Deducted {elo_deduction_total} ELO."
        )
        print(f"Your new rank is {get_rank_name(new_elo)} with {new_elo}")
    else:
        # Perform other logic when the difference in days is 1 or less
        print(f"{elapsed_time} has passed since the last record.")
        # Your other logic here

    conn.close()

def get_rank_name(elo):
    if elo < 800:
        return "Iron"
    elif elo < 1000:
        return "Bronze"
    elif elo < 1200:
        return "Silver"
    elif elo < 1400:
        return "Gold"
    elif elo < 1600:
        return "Platinum"
    elif elo < 1800:
        return "Diamond"
    elif elo < 2000:
        return "Master"
    elif elo < 2200:
        return "Grandmaster"
    else:
        return "Challenger"

def print_rank(elo):
    if elo < 800:
        output = climage.convert("RankImages/iron.png")
    elif elo < 1000:
        output = climage.convert("RankImages/bronze.png")
    elif elo < 1200:
        output = climage.convert("RankImages/silver.png")
    elif elo < 1400:
        output = climage.convert("RankImages/gold.png")
    elif elo < 1600:
        output = climage.convert("RankImages/platinum.png")
    elif elo < 1800:
        output = climage.convert("RankImages/diamond.png")
    elif elo < 2000:
        output = climage.convert("RankImages/master.png")
    elif elo < 2200:
        output = climage.convert("RankImages/grandmaster.png")
    else:
        output = climage.convert("RankImages/challenger.png")

    print(output)

def main():
    # Initialize the database
    db_innit()

    # Welcome message
    prev_elo = get_player_rating("NBS")
    print(
        f'Hello welcome to ELO habit tracker, your current rank is {get_rank_name(get_player_rating("NBS"))} with {get_player_rating("NBS")}'
    )
    print_rank(get_player_rating("NBS"))

    num_habits = int(input("How many habits did you do:"))

    # Deduct ELO for missed days
    deductMissingDays()

    for _ in range(num_habits):
        # Record a game where you won against yourself
        record_game("NBS", "NBS", 0)

    new_elo = get_player_rating("NBS")

    if prev_elo < new_elo:
        print(f"You won some ELO, your rank is {get_rank_name(new_elo)} with {new_elo}")
    else:
        print(
            f"You lost some ELO, your rank is {get_rank_name(new_elo)} with {new_elo}"
        )

    print_rank(get_player_rating("NBS"))
    exit()

# Main body
if __name__ == "__main__":
    main()


import os
from ament_index_python import get_package_share_directory
from launch import LaunchDescription
from launch.actions import DeclareLaunchArgument
from launch.actions import IncludeLaunchDescription
from launch.launch_description_sources import PythonLaunchDescriptionSource
from launch.substitutions import LaunchConfiguration
from launch.conditions import IfCondition

def generate_launch_description():
    pkg_dir = get_package_share_directory('nif_launch')
    pkg_dir_robot_description = get_package_share_directory('av21_description')
    # pkg_dir_localization = get_package_share_directory('nif_localization_nodes')
    pgk_dir_lgsvl_simulation = get_package_share_directory('nif_lgsvl_simulation')

    IMS = 0
    LOR = 1

    # track = IMS
    track = LOR

    global_params_file = None

    if track == LOR:
        global_params_file = 'params_LOR.global.yaml'
    elif track == IMS:
        global_params_file = 'params_IMS.global.yaml'
    else:
        raise RuntimeError("ERROR: invalid track provided: {}".format(track))

    nif_global_parameters_file = os.path.join(
        get_package_share_directory('nif_launch'),
        'config',
        'deploy',
        global_params_file
    )

    global_parameters_launch = IncludeLaunchDescription(
        PythonLaunchDescriptionSource(
            get_package_share_directory('nif_common_nodes') + '/launch/parameters.launch.py'
        ),
        launch_arguments={
            'nif_global_parameters_file': nif_global_parameters_file
        }.items()
    )

    system_status_manager_launch = IncludeLaunchDescription(
        PythonLaunchDescriptionSource(
            get_package_share_directory('nif_system_status_manager_nodes') + '/launch/default.launch.py'
        ),
    )

    robot_description_launch = IncludeLaunchDescription(
        PythonLaunchDescriptionSource(
            pkg_dir_robot_description + '/launch/default.launch.py'
        )
    )

    launch_description = [
        global_parameters_launch,
        system_status_manager_launch,
        robot_description_launch,
    ]

    return LaunchDescription(launch_description)


"""
This is an example for how to use a generator directly and get all of its values
through the python API and not through the iterator abstraction (for loop).

Note that the following are equivalent:
- element = next(generator)
- element = generator.__next__()
- element = generator.send(None)
We use the first 1 (most simple) in the example.
"""

g = (x ** 2 for x in range(5))

while True:
    try:
        x = next(g)
        print(x)
    except StopIteration:
        g.close()
        break


import socket

HOST = '192.168.0.110'    # "" : 문자열   '':
PORT = 2000

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print("binding..")
s.bind((HOST,PORT))
print("success to bind")

s.listen()
print("Listen...")

c, addr = s.accept()
print("Connected by", addr)

s_M = b'server accept'   # 서버와 연결됐을때 클라이언트에게 전해줄 메시지
c.sendall(s_M)            # 서버에게 메시지 전달

while True:
	data = c.recv(65536)
	if not data:
		break
	print('Received from',addr, data.decode())
	c.sendall(data)

c.close()
s.close()


# buckets = []
C, M = list(), list()
p_cnt = 0

def pour(M, p_cnt):
    global C
    fr = M[p_cnt % 3]
    to = C[(p_cnt+1) % 3] - M[(p_cnt+1) % 3]
    if fr < to:
        amt = fr
    else:
        amt = to

    M[p_cnt % 3] -= amt
    M[(p_cnt+1) % 3] += amt

    return M

for i in range(3):
    c, m = list(map(int, input().split()))
    C.append(c)
    M.append(m)

while p_cnt < 100:
    idx = p_cnt % 3
    nxt = (p_cnt + 1) % 3
    M[idx], M[nxt] = max(M[idx] - (C[nxt] - M[nxt]),
                         0), min(C[nxt], M[nxt] + M[idx])
    p_cnt += 1

for b in M:
    print(b)


import pandas as pd
from datetime import datetime

def date_to_orbit(date):
    #establish path to orbit reference file
    filename = "./VEX_data_files/orbit_ref.tab"
    #open orbit reference file into pandas dataframe
    with open(filename) as f:
        orbit_ref = pd.read_csv(f,sep='\t',header=None,names = ['orbit','start','end','BS_in','BS_out'])
    #find orbit start and end times from whole file
    orbit_ref['start'] = pd.to_datetime(orbit_ref['start'])
    orbit_ref['end'] = pd.to_datetime(orbit_ref['end'])
    #make desired date a datetime instead of a string
    date = datetime(int(date[0:4]),int(date[5:7]),int(date[8:10]))

    #for each orbit in orbit reference file
    for i in orbit_ref.index:
        #if desired date is within two orbits, break
        if (date >= orbit_ref['start'][i]) and (date <= orbit_ref['end'][i]):
            orbit_num = orbit_ref['orbit'][i]
            break
    f.close()
    #return orbit number of date
    return orbit_num


# This file contains the common graphical elements that can be used by both
# ChordSelectionDisplay and PlaybackDisplay.

import sys
from common.gfxutil import *

from kivy.graphics.instructions import InstructionGroup
from kivy.graphics import Color, Ellipse, Line, Rectangle
from kivy.graphics import PushMatrix, PopMatrix, Translate, Scale, Rotate
from kivy.uix.widget import Widget
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.relativelayout import RelativeLayout
from kivy.uix.button import Button
from kivy.uix.label import Label

import random
import math

# A menu button.
class MenuButton(Button):
    def __init__(self, pos_hint, size_hint, label):
        super(MenuButton, self).__init__(pos_hint=pos_hint, size_hint=size_hint, text=label)
        self.label = label
        self.background_normal = ''
        self.background_color = [0, .7, 0, .8]

    def set_background_color(self, rgba):
        self.background_color = rgba

    def set_text_color(self, rgb):
        sel.color = rgb

    def set_callback(self, callback):
        self.bind(on_press=callback)

# A button representing a node in the graph (either a chord or phrase.)
class NodeButton(Button):
    def __init__(self, pos_hint, size_hint, block, preview=False):
        super(NodeButton, self).__init__(pos_hint=pos_hint, size_hint=size_hint, text=block.get_name())
        self.block = block
        self.background_normal = ''
        if preview:
            self.background_color = [0, .2, .8, .8]
            # self.background_color = [.8, 0, .2, .8]
        else:
            self.background_color = [0, .2, .8, .8]

    def set_callback(self, callback):
        self.bind(on_press=callback)

class BubbleWiggleToCenter(InstructionGroup):
    def __init__(self, pos, r, color, duration = 3):
        super(BubbleWiggleToCenter, self).__init__()

        center_x = Window.width/2
        center_y = Window.height/2
        x = []
        x.append((0,r))
        # this creates a bunch of tuples so that we can make a kfanim across
        # a lot of different points. It basically adds a sin wave on top of the
        # regularly scheduled decrease in radius
        for i in range(100):
            time = .1 + i*duration/100.
            radius = 2 * r * (100. - i) / 100 * (1 + 0.2 * np.sin(i / 2.))
            x.append((time, radius))

        self.radius_anim = KFAnim(*x)

        # the position still goes linearly
        self.pos_anim = KFAnim((0, pos[0], pos[1]), (duration, center_x, center_y))

        self.color = Color(*color)
        self.add(self.color)

        self.circle = CEllipse(cpos = pos, size = (2*r, 2*r), segments = 40)
        self.add(self.circle)

        self.time = 0
        self.on_update(0)

    def on_update(self, dt):
        # animate radius
        rad = self.radius_anim.eval(self.time)
        self.circle.csize = (2*rad, 2*rad)

        # animate position
        pos = self.pos_anim.eval(self.time)
        self.circle.cpos = pos

        # advance time
        self.time += dt
        # continue flag
        return self.radius_anim.is_active(self.time)

# A generic bubble.
class Bubble(InstructionGroup):
    def __init__(self, pos, rgb, size):
        super(Bubble, self).__init__()
        self.pos = pos
        self.size = size
        self.color = Color(*rgb)

        self.add(self.color)
        self.ellipse = CEllipse(cpos=pos, csize=(self.size, self.size))
        self.add(self.ellipse)

    def set_color(self, rgb):
        self.color.rgb = rgb

    def set_alpha(self, a):
        self.color.a = a

    def set_pos(self, pos):
        self.pos = pos
        self.ellipse.set_pos(pos)

    def on_update(self, dt):
        pass

class LineDivider(InstructionGroup):
    def __init__(self,first_point,second_point,rgba):
        super(LineDivider, self).__init__()
        self.first_point = first_point
        self.second_point = second_point
        self.color = Color(*rgba)
        self.add(self.color)
        self.points = [first_point[0],first_point[1],second_point[0],second_point[1]]
        self.line = Line(points=self.points,width=5)
        self.add(self.line)

    def on_update(self, dt):
        return True

class RandomMovingBubble(InstructionGroup):
    def __init__(self, pos, size=40):
        super(RandomMovingBubble, self).__init__()
        self.pos = pos
        self.size = size
        h = random.random()
        self.color = Color(hsv=(h,1,1))
        self.add(self.color)
        self.ellipse = CEllipse(cpos=pos, csize=(1, 1))
        self.add(self.ellipse)
        self.angle = random.random() * 2 * math.pi
        total_distance = ((Window.height ** 2 + Window.width ** 2) ** 0.5) / 2.
        final_pos_x = self.pos[0] + total_distance * math.sin(self.angle)
        final_pos_y = self.pos[1] + total_distance * math.cos(self.angle)
        self.pos_anim = KFAnim((0,self.pos[0],self.pos[1]),(1.5,final_pos_x,final_pos_y))
        self.size_anim = KFAnim((0,0),(0.2, self.size),(1.5,0))
        self.time = 0

    def on_update(self, dt):
        self.time += dt
        new_pos = self.pos_anim.eval(self.time)
        new_size = self.size_anim.eval(self.time)
        self.ellipse.cpos = new_pos
        self.ellipse.csize = (new_size,new_size)
        return self.pos_anim.is_active(self.time)


import time
from paddle import Pad
from turtle import Turtle,Screen
from ball import Ball

screen=Screen()
screen.setup(width=800,height=600)
screen.bgcolor("black")

ball=Ball()

rpad=Pad(380,0)
lpad=Pad(-380,0)

screen.listen()
screen.onkey(rpad.goup,"Up")
screen.onkey(rpad.godown,"Down")

screen.onkey(lpad.goup,"w")
screen.onkey(lpad.godown,"s")

gameison=True
while gameison:
    time.sleep(0.05)
    screen.update()
    ball.move()
    if ball.ycor()>=290 or ball.ycor()< -290:
        ball.bounce()

    if ball.distance(rpad)<50 and ball.xcor()>350 or ball.distance(rpad)<50 or ball.xcor() < -350 :
        print("made contact")
        ball.bounce_back()

screen.exitonclick()


import allure
from assertpy import assert_that, soft_assertions

from bases.test_base import TestBase
from models.pet import Pet
from models.category import Category
from models.tag import Tag
from requests_api.post_new_pet_request import PostNewPetRequest
from utils.utils import Utils

@allure.suite("New Pet Feature")
class TestPostNewPet(TestBase):

    @allure.tag("API Test")
    @allure.title("Add new pet sucess with Pet object")
    @allure.description("test adding new pet successfully")
    def test_add_new_pet_success(self):
        # arranges
        payload = Pet()
        payload.id = "44"
        category = Category()
        category.id = "55"
        category.name = "Dogs"
        payload.category = category
        payload.name = "Ro"
        payload.photoUrls = ["fotinhafofinha.com.br/foto1", "fotinhafofinha.com.br/foto2"]
        payload.tags = []
        tag1 = Tag()
        tag1.id = 1
        tag1.name = "Pinscher"
        payload.tags.append(tag1)
        tag2 = Tag()
        tag2.id = 2
        tag2.name = "Bravo"
        payload.tags.append(tag2)
        payload.status = "sold"

        request = PostNewPetRequest()

        # actions
        request.set_json_body(payload)
        response = request.execute_request()
        response_body = Utils.get_json_in_response(response)

        # asserts
        assert_that(response.status_code).is_equal_to(200)
        with soft_assertions():
            assert_that(response_body["name"]).contains("Ro")
            assert_that(response_body["tags"][0]["name"]).is_equal_to("Pinscher")
            assert_that(response_body["tags"]).extracting("name").contains("Bravo")
            assert_that(response_body["tags"]).extracting("name").is_equal_to(['Pinscher', 'Bravo'])

    @allure.tag("API Test")
    @allure.title("Add new pet sucess with json file")
    @allure.description("test adding new pet successfully with json file")
    def test_add_new_pet_success_with_json_file(self):
        # arranges
        id = "55"
        category_id = "66"
        category_name = "Dogs"
        name = "Titi"
        photo_urls = "fotinhastiti.com.br"
        tag_id = "1"
        tag_name = "Basset"
        status = "sold"
        request = PostNewPetRequest()

        # actions
        request.set_json_body_from_file(id, category_id, category_name, name, photo_urls, tag_id, tag_name, status)
        response = request.execute_request()
        response_body = Utils.get_json_in_response(response)

        # asserts
        assert_that(response.status_code).is_equal_to(200)

    @allure.tag("API Test")
    @allure.title("Add new pet sucess with Pet text")
    @allure.description("test adding new pet successfully with Pet text")
    def test_add_new_pet_success_with_text(self):
        # arranges
        id = "77"
        category_id = "88"
        category_name = "Dogs"
        name = "Dogo"
        photo_urls = "fotinhasdogo.com.br"
        tag_id = "1"
        tag_name = "Srd"
        status = "sold"
        request = PostNewPetRequest()

        # actions
        request.set_json_body_from_text(id, category_id, category_name, name, photo_urls, tag_id, tag_name, status)
        response = request.execute_request()
        response_body = Utils.get_json_in_response(response)

        # asserts
        assert_that(response.status_code).is_equal_to(200)


import pygame
from pygame.locals import*
import sys

from scripts.config import _
from scripts.menu import *

def AireCraftTutorial(screen):

    running = True
    scroll_y = 0
    scroll_speed = 15

    while running:
        screen.fill((0,0,0))

        draw_text(screen, 'move mouse and click on left buttom for simple fire & right buttom for extra fire.', _.font, (255, 255, 255), 0, 20+scroll_y)
        drawImage(screen, "resources/images/airecraft/ac_2.png", 0, 80+scroll_y)
        drawImage(screen, "resources/images/mira.gif", 0, 250+scroll_y)
        draw_text(screen, 'press [Echap] to return', _.font, (255, 255, 255), 0, 420+scroll_y)

        for event in pygame.event.get():
            if event.type == QUIT:
                pygame.quit()
                sys.exit()
            if event.type == KEYDOWN:
                if event.key == K_ESCAPE:
                    running = False
                elif event.key == K_UP: # up
                    scroll_y += scroll_speed
                elif event.key == K_DOWN: # down
                    scroll_y -= scroll_speed
            if event.type == pygame.MOUSEBUTTONDOWN:
                if event.button == 4: # up
                    scroll_y += scroll_speed
                if event.button == 5: # down
                    scroll_y -= scroll_speed

            if scroll_y > 0:
                scroll_y = 0

        pygame.display.update()
        _.mainClock.tick(_.FPS)


import warnings

import torch

from vp_suite.measure import METRIC_CLASSES

class PredictionMetricProvider:
    r"""
        This class provides bundled access to multiple metrics. With this class's :meth:`get_metrics()` method,
        all specified metric scores are calculated on the same input prediction and target tensor.

        Attributes:
            device (str): A string specifying whether to use the GPU for calculations (`cuda`) or the CPU (`cpu`).
            available_metrics (dict): A dictionary containing the string identifiers and corresponding metrics that the metric provider should use when provided with input tensors.
            metrics (dict): The concrete instantiated metrics that the metric provider uses when provided with input tensors.
    """
    def __init__(self, config: dict):
        r"""
        Initializes the provider by extracting device and metric IDs from the provided config dict
        and instantiating the metrics that shall be used.

        Args:
            config (dict): A dictionary containing the devices and metrics to use.
        """
        self.device = config["device"]
        self.available_metrics = METRIC_CLASSES if config["metrics"] == "all" \
            else {k: METRIC_CLASSES[k] for k in config["metrics"]}
        if config["img_c"] not in [2, 3]:
            warnings.warn("'FVD' measure won't be used since image channels needs to be in [2, 3]")
            self.available_metrics.pop("fvd")
        self.metrics = {k: metric(device=self.device) for k, metric in self.available_metrics.items()}

    def get_metrics(self, pred: torch.Tensor, target: torch.Tensor, frames: int = None, all_frame_cnts: bool = False):
        r"""
        Takes in tensors of predicted frames and the corresponding ground truth and calculates the metric scores for
        the metrics instantiated previously.

        Args:
            pred (torch.Tensor): The predicted frame sequence as a 5D float tensor (batch, frames, c, h, w).
            target (torch.Tensor): The ground truth frame sequence as a 5D float tensor (batch, frames, c, h, w)

            frames (int): If frames is specified, only considers the first 'frames' frames.
            all_frame_cnts (bool): If set to true, elicits metrics for all prediction horizons from 1 up to the maximum number of frames. Otherwise, just elicits metrics for the specified number of frames

        Returns:
            A list of dictionaries, where each dictionary contains the metric ids
            and the corresponding result value for a specific number of prediction frames.
        """
        if pred.ndim != 5 or target.ndim != 5:
            raise ValueError("Input tensors expected to be 5-dimensional!")
        if pred.shape != target.shape:
            raise ValueError("Output images and target images are of different shape!")
        pred = pred.contiguous()
        target = target.contiguous()
        frames = frames or pred.shape[1]

        metrics = []
        frames = [frames] if not all_frame_cnts else range(1, frames + 1)
        for frame_cnt in frames:
            pred_ = pred[:, :frame_cnt]
            target_ = target[:, :frame_cnt]
            frame_cnt_metrics = dict()
            for key, metric in self.metrics.items():
                metric_val = metric(pred_, target_)
                if metric_val is not None:
                    frame_cnt_metrics[f"{key} ({'↑' if metric.BIGGER_IS_BETTER else '↓'})"] \
                        = metric.to_display(metric_val.item())
            # remove metrics that returned 'None' (e.g. because they don't support the current frame cnt
            frame_cnt_metrics = {k: v for k, v in frame_cnt_metrics.items() if v is not None}
            metrics.append(frame_cnt_metrics)

        return metrics


from collections import namedtuple
import torch as t
import torch.nn as nn
from const import DIM_POSITION_2x2, DIM_POSITION_3x3, DIM_NUMBER_2x2, DIM_NUMBER_3x3, DIM_TYPE, DIM_SIZE, DIM_COLOR
from nvsa.reasoning.vsa_block_utils import binding_circular, block_discrete_codebook
from models import MLP
from VSAConverter import VSAConverter

Scene = namedtuple("Scene", ["position", "number", "type", "size", "color"])

class AttributeLevelReasoner(nn.Module):

    def __init__(self, device, constellation, model, hidden_layers, dictionary, vsa_conversion=False, vsa_selection=False, context_superposition=False):
        super(AttributeLevelReasoner, self).__init__()
        self.device = device
        self.constellation = constellation
        self.compute_attribute_dims()
        self.d = dictionary.shape[1]*dictionary.shape[2]
        self.k = dictionary.shape[1]
        self.vsa_conversion = vsa_conversion
        self.vsa_selection = vsa_selection
        self.context_superposition = context_superposition
        self.vsa_converter = VSAConverter(device, self.constellation, dictionary)
        self.compute_attribute_models(model, hidden_layers)
        if self.context_superposition:
            context_keys, _ = block_discrete_codebook(device=device, d=self.d, k=self.k, scene_dim=8)
            self.context_keys = nn.Parameter(context_keys)
            position_number_keys, _ = block_discrete_codebook(device=device, d=self.d, k=self.k, scene_dim=2)
            self.position_key, self.number_key = nn.Parameter(position_number_keys[0]), nn.Parameter(position_number_keys[1])

    def compute_attribute_dims(self):
        if "distribute" in self.constellation:
            if "four" in self.constellation:
                DIM_POSITION = DIM_POSITION_2x2
                DIM_NUMBER = DIM_NUMBER_2x2
            else:
                DIM_POSITION = DIM_POSITION_3x3
                DIM_NUMBER = DIM_NUMBER_3x3
            self.position_dim = DIM_POSITION
            self.number_dim = DIM_NUMBER
        self.type_dim = DIM_TYPE + 1
        self.size_dim = DIM_SIZE + 1
        self.color_dim = DIM_COLOR + 1

    def compute_attribute_models(self, model, hidden_layers):
        if self.vsa_conversion:
            if "distribute" in self.constellation:
                self.pos_num_model = MLP(2*self.d, self.position_dim, self.d, hidden_layers=hidden_layers, softmax=True)
            self.type_model = MLP(self.d, self.type_dim, self.d, softmax=True)
            self.size_model = MLP(self.d, self.size_dim, self.d, softmax=True)
            self.color_model = MLP(self.d, self.color_dim, self.d, softmax=True)
        else:
            if "distribute" in self.constellation:
                self.pos_num_model = MLP(self.position_dim + self.number_dim, self.position_dim, self.d, hidden_layers=hidden_layers, softmax=True)
            self.type_model = MLP(self.type_dim, self.type_dim, self.d, softmax=True)
            self.size_model = MLP(self.size_dim, self.size_dim, self.d, softmax=True)
            self.color_model = MLP(self.color_dim, self.color_dim, self.d, softmax=True)

    def compute_vsa_outputs(self, outputs):
        if "distribute" in self.constellation:
            position_dist = outputs.position.unsqueeze(-1)
            position_dict = self.vsa_converter.position_dictionary.reshape((self.vsa_converter.position_dictionary.shape[0], -1)).unsqueeze(0)
            position = t.sum(position_dist*position_dict, dim=1)
        else:
            position = None

        type_dist = outputs.type.unsqueeze(-1)
        type_dict = self.vsa_converter.type_dictionary.reshape((self.vsa_converter.type_dictionary.shape[0], -1)).unsqueeze(0)
        type = t.sum(type_dist*type_dict, dim=1)

        size_dist = outputs.size.unsqueeze(-1)
        size_dict = self.vsa_converter.size_dictionary.reshape((self.vsa_converter.size_dictionary.shape[0], -1)).unsqueeze(0)
        size = t.sum(size_dist*size_dict, dim=1)

        color_dist = outputs.color.unsqueeze(-1)
        color_dict = self.vsa_converter.color_dictionary.reshape((self.vsa_converter.color_dictionary.shape[0], -1)).unsqueeze(0)
        color = t.sum(color_dist*color_dict, dim=1)

        return Scene(position, None, type, size, color)

    def compute_context_superposition(self, x):
        if self.context_superposition:
            x_with_superposition = binding_circular(x.reshape((-1, self.k, self.d//self.k)),
                                    self.context_keys.repeat(x.shape[0], 1, 1))
            x_with_superposition = x_with_superposition.reshape((x.shape[0], x.shape[1], -1))
            x_with_superposition = x_with_superposition.sum(dim=1)/x.shape[1]
        else:
            x_with_superposition = x
        return x_with_superposition

    def position_number_superposition(self, position, number):
        position_binded = binding_circular(position.reshape((-1, self.k, self.d//self.k)),
                         self.position_key.unsqueeze(0).repeat(position.shape[0]*position.shape[1], 1, 1))
        number_binded = binding_circular(number.reshape((-1, self.k, self.d//self.k)),
                         self.number_key.unsqueeze(0).repeat(number.shape[0]*number.shape[1], 1, 1))
        pos_num = (position_binded + number_binded)/2
        pos_num = pos_num.reshape((position.shape[0], position.shape[1], -1))
        return pos_num

    def forward(self, scene_prob):
        scene_vsa = self.vsa_converter(scene_prob)
        if "distribute" in self.constellation:
            scene_vsa = Scene(t.flatten(scene_vsa.position, start_dim=len(scene_vsa.position.shape)-2),
                              t.flatten(scene_vsa.number, start_dim=len(scene_vsa.number.shape)-2),
                              t.flatten(scene_vsa.type, start_dim=len(scene_vsa.type.shape)-2),
                              t.flatten(scene_vsa.size, start_dim=len(scene_vsa.size.shape)-2),
                              t.flatten(scene_vsa.color, start_dim=len(scene_vsa.color.shape)-2))
        else:
            scene_vsa = Scene(None,
                              None,
                              t.flatten(scene_vsa.type, start_dim=len(scene_vsa.type.shape)-2),
                              t.flatten(scene_vsa.size, start_dim=len(scene_vsa.size.shape)-2),
                              t.flatten(scene_vsa.color, start_dim=len(scene_vsa.color.shape)-2))

        scene_prob = Scene(scene_prob.position_prob, scene_prob.number_prob, scene_prob.type_prob, scene_prob.size_prob, scene_prob.color_prob)

        if self.vsa_conversion:
            scene = scene_vsa
        else:
            scene = scene_prob

        if "distribute" in self.constellation:
            if self.context_superposition:
                pos_num = self.position_number_superposition(scene.position[:, :8], scene.number[:, :8])
                pos_num_output = self.pos_num_model(self.compute_context_superposition(pos_num))
            else:
                pos_num_output = self.pos_num_model(t.cat((self.compute_context_superposition(scene.position[:, :8]),
                                                        self.compute_context_superposition(scene.number[:, :8])), dim=-1))
        else:
            pos_num_output = None
        type_output = self.type_model(self.compute_context_superposition(scene.type[:, :8]))
        size_output = self.size_model(self.compute_context_superposition(scene.size[:, :8]))
        color_output = self.color_model(self.compute_context_superposition(scene.color[:, :8]))
        outputs = Scene(pos_num_output, None, type_output, size_output, color_output)
        if self.vsa_selection:
            outputs = self.compute_vsa_outputs(outputs)
            scene = scene_vsa
        else:
            scene = scene_prob

        if "distribute" in self.constellation:
            candidates = Scene(scene.position[:, 8:], scene.number[:, 8:], scene.type[:, 8:], scene.size[:, 8:], scene.color[:, 8:])
        else:
            candidates = Scene(None, None, scene.type[:, 8:], scene.size[:, 8:], scene.color[:, 8:])

        return outputs, candidates


from firebase_admin import credentials, firestore, initialize_app, _apps, get_app

initialized = False

# Functions needed:
# - List clubs by school ID
# - List students in club by club ID
# - Add student to a club
# - Add administrator to a club
# - Add a club with a school id
def get_interactor():
    global initialized
    dbapp = None
    if initialized == False:
        cred = credentials.Certificate("key.json")
        dbapp = initialize_app(cred, name="Schaumburg Library Hackathon")
        initialized = True
    else:
        dbapp = get_app(name="Schaumburg Library Hackathon")
    print("before")
    db = firestore.client(dbapp)
    print("after")
    interactor = DatabaseInteractor(db)
    return interactor

class Collection:
    collection = None

    def __init__(self, db, collection):
        self.collection = db.collection(collection)

    def create(self, object):
        id = len(self.read_all())
        object["id"] = id
        self.collection.document(str(id)).set(object)
        return id

    def update(self, id, object):
        self.collection.document(str(id)).update(object)

    def read_all(self):
        items = [doc.to_dict() for doc in self.collection.stream()]
        return items

    def read(self, id):
        item = self.collection.document(str(id)).get()
        return item.to_dict()

    def delete(self, id):
        self.collection.document(id).delete()

class DatabaseInteractor:
    def __init__(self, db):
        self.school_collection = Collection(db, 'school')
        self.student_collection = Collection(db, 'student')
        self.club_collection = Collection(db, 'club')
        self.post_collection = Collection(db, 'post')
        self.sponsor_collection = Collection(db, 'sponsor')
        self.comment_collection = Collection(db, 'comment')

    def get_collection(self, collection):
        mapper = {
            'school': self.school_collection,
            'student': self.student_collection,
            'club': self.club_collection,
            'post': self.post_collection,
            'sponsor': self.sponsor_collection,
        }

        return mapper.get(collection)

    def create_comment(self, postId, authorName, content, timestamp):
        # Create school and return ID
        id = self.comment_collection.create({
            "post": postId,
            "author": authorName,
            "content": content,
            "timestamp": timestamp,
        })

        return id

    def create_school(self, name, emailSuffixes, adminEmail):
        # Check that school does not already exist
        schools = self.school_collection.read_all()
        for school in schools:
            if school["name"] == name:
                raise ValueError("School with that name already exists")

        # Create school and return ID
        id = self.school_collection.create({
            "name": name,
            "emailSuffixes": emailSuffixes,
            "adminEmail": adminEmail
        })

        return id

    def create_student(self, schoolId, name, email):
        # Check that school does not already exist
        students = self.student_collection.read_all()
        for student in students:
            if student["name"] == name:
                raise ValueError("Student with that name already exists")

        # Create student and return ID
        id = self.student_collection.create({
            "name": name,
            "email": email,
            "school": schoolId,
            "clubs": [],
        })

        return id

    def create_sponsor(self, schoolId, name, email, tags):
        # Check that school does not already exist
        sponsors = self.sponsor_collection.read_all()
        for sponsor in sponsors:
            if sponsor["name"] == name:
                raise ValueError("Sponsor with that name already exists")

        # Create student and return ID
        id = self.sponsor_collection.create({
            "name": name,
            "email": email,
            "school": schoolId,
            "tags": [],
            "clubs": []
        })

        return id

    def create_club(self, name, tags, studentId, description):
        student = self.student_collection.read(studentId)

        # Check that school does not already exist
        clubs = self.club_collection.read_all()
        for club in clubs:
            if club["name"] == name and club["school"] == student["school"]:
                raise ValueError("Club with that name at that school already exists")

        id = self.club_collection.create({
            "school": student["school"],
            "name": name,
            "tags": tags,
            "posts": 0,
            "administrator": None,
            "students": [studentId],
            "description": description,
        })

        return id

    def create_club_as_sponsor(self, name, tags, sponsorId, description):
        student = self.student_collection.read(sponsorId)

        # Check that school does not already exist
        clubs = self.club_collection.read_all()
        for club in clubs:
            if club["name"] == name and club["school"] == school["id"]:
                raise ValueError("Club with that name at that school already exists")

        id = self.club_collection.create({
            "school": student["school"],
            "name": name,
            "tags": tags,
            "posts": 0,
            "administrator": student,
            "students": [],
            "description": description,
        })

        return id

    def create_club_as_admin(self, name, tags, adminId, description):
        school = self.school_collection.read(adminId)

        # Check that school does not already exist
        clubs = self.club_collection.read_all()
        for club in clubs:
            if club["name"] == name and club["school"] == school["id"]:
                raise ValueError("Club with that name at that school already exists")

        id = self.club_collection.create({
            "school": school["id"],
            "name": name,
            "tags": tags,
            "posts": 0,
            "administrator": None,
            "students": [],
            "description": description,
        })

        return id

    def create_post(self, content, clubId, title, authorName, date):
        id = self.post_collection.create({
            "content": content,
            "title": title,
            "club": clubId,
            "author": authorName,
            "timestamp": date,
        })

        return id

    def add_student_to_club(self, studentId, clubId):
        prevStudent = self.student_collection.read(studentId)
        prevStudent["clubs"].append(clubId)
        self.student_collection.update(studentId, {
            "clubs": prevStudent["clubs"]
        })

        prevClub = self.club_collection.read(clubId)
        prevClub["students"].append(studentId)
        self.club_collection.update(clubId, {
            "students": prevClub["students"]
        })

    def add_sponsor_to_club(self, clubId, sponsorId):
        self.club_collection.update(clubId, {
            "administrator": sponsorId,
        })

    def clubs_by_school_id(self, schoolId):
        clubs = self.club_collection.read_all()
        clubResult = []
        for club in clubs:
            if club["school"] == schoolId:
                clubResult.append(club)

        return clubResult

    def posts_by_club_id(self, clubId):
        posts = self.post_collection.read_all()
        postArr = []
        for post in posts:
            if post["club"] == clubId:
                postArr.append(post)

        return postArr

    def get_student_by_email(self, email):
        students = self.student_collection.read_all()
        for student in students:
            if student["email"] == email:
                return student

        return None

    def get_sponsor_by_email(self, email):
        students = self.sponsor_collection.read_all()
        for student in students:
            if student["email"] == email:
                return student

        return None

    def get_school_by_admin_email(self, email):
        students = self.school_collection.read_all()
        for student in students:
            if student["adminEmail"] == email:
                return student

        return None

    def get_school_by_name(self, name):
        students = self.school_collection.read_all()
        for student in students:
            if student["name"] == email:
                return student

        return None

    def get_school_by_email_ending(self, email_ending):
        students = self.school_collection.read_all()
        for student in students:
            emailSuffixes = student["emailSuffixes"]
            for emailSuffix in emailSuffixes:
                if emailSuffix == email_ending:
                    return student

        return None

    def get_comments_by_post_id(self, postId):
        commResult = []
        comments = self.comment_collection.read_all()
        for comment in comments:
            if int(comment["post"]) == int(postId):
                commResult.append(comment)

        return commResult


from concurrent import futures
import logging

import grpc

import os
import lba2pba_pb2
import lba2pba_pb2_grpc

import shard

def getGlusterHost():
	cmd = 'gluster peer status | grep Hostname'
	err, res = subprocess.getstatusoutput(cmd)

	hosts = ['localhost']
	if res:
		for line in res.split('\n'):
			hosts.append(line.split(": ")[1])

		return hosts

def getFM(volume,fName):
	shardList = shard.getShardList()

#	re = {}

	for t in shardList.keys():
		if not volume in shardList[t].keys():
			continue

		for f in shardList[t][volume].keys():
			if fName in f:
				fName = f

		#if not volume in re.keys():
		#	re[volume] = {}

		#if not fList:
		#	flist = shardList[t][volume].keys()

		if t == 'replica':
		#	for fName in flist:
			FM = {'FileName':fName,'Type':t,'rPba':[]}
			for host in shardList[t][volume][fName].keys():
				rPba = {}
				pba = []
				for f in shardList[t][volume][fName][host]:
					with grpc.insecure_channel(f'{host}:23829') as channel:
						stub = lba2pba_pb2_grpc.WorkerStub(channel)
						res = stub.Get(lba2pba_pb2.WGetPbaRequest(FileName=f))

						for p in res.Pba:
							tdict = {
								'Disk': res.Disk,
								'Host': host,
								'Offset': p.Offset,
								'Length': p.Length,
							}

							pba.append(tdict)

				rPba['Pba'] = pba
				rPba['Node'] = host

				FM['rPba'].append(rPba)

			#re[volume][fName]=FM
			return FM

		elif t == 'distribute':
			#for fName in flist:
			FM = {'FileName':fName, 'Type':t, 'Pba':[]}

			for fInfo in shardList[t][volume][fName]:
				host, f = fInfo.split(' ')

				with grpc.insecure_channel(f'{host}:23829') as channel:
					stub = lba2pba_pb2_grpc.WorkerStub(channel)
					res = stub.Get(lba2pba_pb2.WGetPbaRequest(FileName=f))

					pba = []
					for p in res.Pba:
						tdict = {
							'Disk': res.Disk,
							'Host': host,
							'Offset': p.Offset,
							'Length': p.Length,
							'Major': p.Major,
							'Minor': p.Minor
						}

						FM['Pba'].append(tdict)

			return FM

	return

class Trace(lba2pba_pb2_grpc.TraceServicer):
	def Get(self,request,context):

		fName = request.FileName
		volume = request.VolName

		data = getFM(volume,fName)

		if data['Type'] == 'distribute':
			return lba2pba_pb2.TGetPbaResponse(Pba=data['Pba'])
		elif data['Type'] == 'replica':
			return lba2pba_pb2.TGetPbaResponse(RPba=data['rPba'])

def serve():
	server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
	lba2pba_pb2_grpc.add_TraceServicer_to_server(Trace(),server)
	server.add_insecure_port('[::]:23830')
	print("LBA2PBA Trace Start !!")
	server.start()
	server.wait_for_termination()

if __name__ == "__main__":
	logging.basicConfig()
	serve()