import pandas as pd
import numpy as np

def getDailyVol(close,
                span0=100):
    """SNIPPET 3.1 DAILY VOLATILITY ESTIMATES
    Daily vol reindexed to close
    """
    df0=close.index.searchsorted(close.index-pd.Timedelta(days=1))
    df0=df0[df0>0]
    df0=(pd.Series(close.index[df0-1],
                   index=close.index[close.shape[0]-df0.shape[0]:]))
    try:
        df0=close.loc[df0.index]/close.loc[df0.values].values-1 # daily rets
    except Exception as e:
        print(f'error: {e}\nplease confirm no duplicate indices')
    df0=df0.ewm(span=span0).std().rename('dailyVol')
    return df0

def applyPtSlOnT1(close,
                  events,
                  ptSl,
                  molecule):
    """SNIPPET 3.2 TRIPLE-BARRIER LABELING METHOD
    Apply stop loss/profit taking, if it takes place before t1 (end of event)
    """
    events_=events.loc[molecule]
    out=events_[['t1']].copy(deep=True)
    if ptSl[0]>0: pt=ptSl[0]*events_['trgt']
    else: pt=pd.Series(index=events.index) # NaNs
    if ptSl[1]>0: sl=-ptSl[1]*events_['trgt']
    else: sl=pd.Series(index=events.index) # NaNs
    for loc,t1 in events_['t1'].fillna(close.index[-1]).iteritems():
        df0=close[loc:t1] # path prices
        df0=(df0/close[loc]-1)*events_.at[loc,'side'] # path returns
        out.loc[loc,'sl']=df0[df0<sl[loc]].index.min() # earliest stop loss
        out.loc[loc,'pt']=df0[df0>pt[loc]].index.min() # earliest profit taking
    return out

def getEvents(close,
              tEvents,
              ptSl,
              trgt,
              minRet,
              numThreads,
              t1=False,
              side=None):
    """SNIPPET 3.3 GETTING THE TIME OF FIRST TOUCH
    SNIPPET 3.6 EXPANDING getEvents TO INCORPORATEMETA-LABELING
    """
    #1) get target
    trgt=trgt.loc[tEvents]
    trgt=trgt[trgt>minRet] # minRet
    #2) get t1 (max holding period)
    if t1 is False:t1=pd.Series(pd.NaT, index=tEvents)
    #3) form events object, apply stop loss on t1
    if side is None:side_,ptSl_=pd.Series(1.,index=trgt.index), [ptSl[0],ptSl[0]]
    else: side_,ptSl_=side.loc[trgt.index],ptSl[:2]
    events=(pd.concat({'t1':t1,'trgt':trgt,'side':side_}, axis=1)
            .dropna(subset=['trgt']))
    df0=mpPandasObj(func=applyPtSlOnT1,pdObj=('molecule',events.index),
                    numThreads=numThreads,close=close,events=events,
                    ptSl=ptSl_)
    events['t1']=df0.dropna(how='all').min(axis=1) # pd.min ignores nan
    if side is None:events=events.drop('side',axis=1)
    return events

def addVerticalBarrier(tEvents,
                       close,
                       numDays=1):
    """SNIPPET 3.4 ADDING A VERTICAL BARRIER
    """
    t1=close.index.searchsorted(tEvents+pd.Timedelta(days=numDays))
    t1=t1[t1<close.shape[0]]
    t1=(pd.Series(close.index[t1],index=tEvents[:t1.shape[0]]))
    return t1

def getBinsOld(events,
               close):
    """SNIPPET 3.5 LABELING FOR SIDE AND SIZE
    """
    #1) prices aligned with events
    events_=events.dropna(subset=['t1'])
    px=events_.index.union(events_['t1'].values).drop_duplicates()
    px=close.reindex(px,method='bfill')
    #2) create out object
    out=pd.DataFrame(index=events_.index)
    out['ret']=px.loc[events_['t1'].values].values/px.loc[events_.index]-1
    out['bin']=np.sign(out['ret'])
    # where out index and t1 (vertical barrier) intersect label 0
    try:
        locs = out.query('index in @t1').index
        out.loc[locs, 'bin'] = 0
    except:
        pass
    return out

def getBins(events,
            close):
    """SNIPPET 3.7 EXPANDING getBins TO INCORPORATE META-LABELING
    Compute event's outcome (including side information, if provided).
    events is a DataFrame where:
    -events.index is event's starttime
    -events['t1'] is event's endtime
    -events['trgt'] is event's target
    -events['side'] (optional) implies the algo's position side
    Case 1: ('side' not in events): bin in (-1,1) <-label by price action
    Case 2: ('side' in events): bin in (0,1) <-label by pnl (meta-labeling)
    """
    #1) prices aligned with events
    events_=events.dropna(subset=['t1'])
    px=events_.index.union(events_['t1'].values).drop_duplicates()
    px=close.reindex(px,method='bfill')
    #2) create out object
    out=pd.DataFrame(index=events_.index)
    out['ret']=px.loc[events_['t1'].values].values/px.loc[events_.index]-1
    if 'side' in events_:out['ret']*=events_['side'] # meta-labeling
    out['bin']=np.sign(out['ret'])
    if 'side' in events_:out.loc[out['ret']<=0,'bin']=0 # meta-labeling
    return out

def dropLabels(events,
               minPct=.05):
    """SNIPPET 3.8 DROPPING UNDER-POPULATED LABELS
    Apply weights, drop labels with insufficient examples
    """
    while True:
        df0=events['bin'].value_counts(normalize=True)
        if df0.min()>minPct or df0.shape[0]<3:break
        print('dropped label: ', df0.argmin(),df0.min())
        events=events[events['bin']!=df0.argmin()]
    return events


import pandas as pd
from shapely.geometry import Point
import geopandas as gpd

import matplotlib.pyplot as plt

plt.rcParams.update({'font.size': 20})

geometry = [Point(xy) for xy in zip(tmp['LONGNUM'], tmp['LATNUM'])]
gdf = gpd.GeoDataFrame(tmp, geometry=geometry)

#this is a simple map that goes with geopandas
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
africa = world.query('continent == "Africa"')
gdf.plot(ax=africa.plot(figsize=(20, 12), edgecolor='gray', color='#FFFDD0'), column = "Under5_Mortality_Rate", marker='o', markersize=3, cmap = "inferno_r", legend=True,
         legend_kwds={"label":"Under 5 Mortality Rate"})

plt.axis('off')
plt.show()

geometry = [Point(xy) for xy in zip(df['LONGNUM'], df['LATNUM'])]
gdf = gpd.GeoDataFrame(df, geometry=geometry)
gdf["Mean_BMI"] = np.where(gdf["Mean_BMI"]> 35, 35, gdf["Mean_BMI"])

#this is a simple map that goes with geopandas
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
africa = world.query('continent == "Africa"')
gdf.plot(ax=africa.plot(figsize=(20, 12), edgecolor='gray', color='#FFFDD0'), column = "Mean_BMI", marker='o', markersize=3, cmap = "inferno_r", legend=True,
         legend_kwds={"label":"Mean Body Mass Index"})

plt.axis('off')
plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Dec 20 08:30:08 2020

@author: Mattias Nilsson
"""
# Packages
import pandas as pd
import numpy as np

# 9 different functions
# Naming of the functions are not clear
# (2 are resolving some issue) One function for each purpose.
# (4 functions are fixing cooking event)
# (2-3 functions that are making output/dataframes) Naming and grouping
# Start & End should be in the same row
# Longer lines of code than possible
# I would like to have Unit Tests.
# Commits different names, not misleading, clear and precise. The longer the name for longer functions, because the parameters are usually more important.
# Standard for naming the commits with some detail. What did you do this and how did you do this. Added upper limit to cooking time for an event.
# Shorter extract_cooking_events function by a max

def resolve_spikes(df_raw,
                   minimum_size_of_spikes: float = 1,
                   time_resolution=None,
                   minimum_energy_per_cooking_event=None,
                   minimum_power_mean=None,
                   minimum_active_load=None,
                   power_capacity=None,
                   t_between=None,
                   minimum_event_current=None,
                   maximum_cooking_time=None,
                   energy_error_margin=None):
    """
    Check a Pandas Dataframe called 'df_raw' for data spikes larger than 1 kWh
    and handles them accordingly to return the updated dataframe.

    Parameters
    ----------
    df_raw : a dataframe that needs to be sorted by meter_number and timestamp.
    mininum_size_of_spikes : float, min. energy consumption of a data spike.
    Explained below in points 1-3.. The default is 1.

    (1) Why range(100)?: There are more than 50 recordings in some of the spikes.
    (2) Why not drop rows?: To save the dates of when the spikes occur.
    (3) Why mininum_size_of_spikes? = To avoid losing small energy movements, incl. timestamp issue.

    Returns
    -------
    df_raw : returns a Pandas DataFrame.

    """

    for _ in range(100):
        df_raw.loc[(df_raw.energy > df_raw.energy.shift(-1) + minimum_size_of_spikes) &
                   (df_raw.meter_number == df_raw.meter_number.shift(-1)), 'energy'] = df_raw.energy.shift(-1)
    return df_raw

def extract_cooking_events(df_raw,
                           minimum_energy_per_cooking_event: float = 0.05,
                           minimum_power_mean: float = 0.05,
                           minimum_event_current: float = 2,
                           maximum_cooking_time: float = 300,
                           time_resolution: int = 5,
                           mininum_size_of_spikes=None,
                           minimum_active_load: float = 0.15,
                           power_capacity: float = 1,
                           t_between: float = 15,
                           energy_error_margin=None):
    """
    Extract cooking events by defining start and end of each cooking event,
    disqualify cooking events that aren't using enough energy and returns
    a Pandas DataFrame.

    Parameters
    ----------
    df_raw : a Pandas Dataframe that needs to be sorted by meter_number and timestamp.
    minimum_energy_per_cooking_event : float, min. energy consumption of a cooking event,
    excl. start & end energy (recorded within the time resolution interval) that will be added later.
    The default is 0.05.
    minimum_power_mean : float, disqualify cooking events that have an average power
    level below e.g. 50 W which is below MCR 5% of an EPC. The default is 0.05.
    minimum_event_current : float, disqualify cooking events that don't have a recorded current above
    a certain threshold, e.g. above 1 A. An EPC normally uses approx. 4 A. The default is 0.5.
    maximum_cooking_time : float, the maximum length [minutes] that an EPC is believed to be cooking before
    it's not considered a normal EPC event. The default is 300.
    time_resolution : int, for DESCRIPTION see function 'apply_event_conditions'. The default is 5.
    minimum_active_load : float, for DESCRIPTION see function 'apply_event_conditions'. The default is 0.15.
    power_capacity : float, for DESCRIPTION see function 'apply_event_conditions'. The default is 1.
    t_between : float, for DESCRIPTION see function 'apply_event_conditions'. The default is 15.

    Returns
    -------
    df_processed : returns a Pandas DataFrame.

    """

    df_processed = df_raw.copy()

    # Recalculate the power level of the EPC using the current and voltage,
    # because this is originally calculated using an average value which can
    # have a delay.
    df_processed['power'] = (df_processed.current *
                             df_processed.voltage) / 1000  # kW

    # Format the 'timestamp' column
    format_timestamp(df_processed)
    df_processed.reset_index(inplace=True)

    # Create columns based on columns 'meter_number' and 'timestamp' by
    # selecting the time difference between rows for each meter_number to
    # conduct the further analysis.
    df_processed.loc[(df_processed.meter_number.diff() == 0),
                     'diff_prev_timestamp'] = df_processed.timestamp.diff()
    df_processed.loc[(df_processed.meter_number.diff(-1) == 0),
                     'diff_next_timestamp'] = df_processed.timestamp.shift(-1) - df_processed.timestamp

    # Create columns for Cooking 'start' & 'end'
    df_processed['cooking_start'] = False
    df_processed['cooking_end'] = False

    # Apply conditions for creating distinct cooking events
    df_processed = apply_event_conditions(df_processed,
                                          time_resolution,
                                          minimum_active_load,
                                          power_capacity,
                                          t_between)

    # Create a column called 'cooking_event' for accumulated numbering of
    # cooking events.
    df_processed['cooking_event'] = 0
    df_processed.cooking_event += df_processed['cooking_start']
    df_processed.cooking_event = df_processed['cooking_event'].cumsum()

    # Create columns to show start timestamp of each cooking event.
    start_cooking = df_processed.groupby('cooking_event').first()
    start_cooking.reset_index(inplace=True)
    df_processed['time_start'] = df_processed.cooking_event.map(
        start_cooking.set_index('cooking_event')['timestamp'].to_dict())
    df_processed['energy_start'] = df_processed.cooking_event.map(
        start_cooking.set_index('cooking_event')['energy'].to_dict())

    # Create columns to show end timestamp of each cooking event.
    end_cooking = df_processed.copy()
    end_cooking = df_processed.groupby('cooking_event').last()
    end_cooking.reset_index(inplace=True)
    df_processed['time_end'] = df_processed.cooking_event.map(
        end_cooking.set_index('cooking_event')['timestamp'].to_dict())
    df_processed['energy_end'] = df_processed.cooking_event.map(
        end_cooking.set_index('cooking_event')['energy'].to_dict())

    # Create columns for getting duration of cooking event and sequence time
    # during cooking event.
    df_processed['cooking_time'] = (
        df_processed.time_end - df_processed.time_start) / np.timedelta64(1, 'm')
    df_processed['seq_time'] = (
        df_processed.timestamp - df_processed.time_start) / np.timedelta64(1, 'm')

    # Disqualify cooking events of 'too low' average energy
    df_processed.loc[((df_processed.energy_end -
                       df_processed.energy_start < minimum_energy_per_cooking_event) | ((df_processed.energy_end -
                                                                          df_processed.energy_start) /
                                                                         (df_processed.cooking_time /
                                                                          60) < minimum_power_mean)), 'cooking_event'] = np.nan

    # Classify recordings that are not part of a cooking event with NaN.
    df_processed.loc[((df_processed.cooking_event.isnull())
                      ), 'cooking_time'] = np.nan
    df_processed.loc[((df_processed.cooking_event.isnull())
                      ), 'seq_time'] = np.nan

    # Check the max current, which is approx. 4 A for the EPC.
    # If it's much different, this can prove that other device is used.
    df_processed_current = df_processed.groupby(
        'cooking_event').agg({'current': 'max'})
    df_processed_current.reset_index(inplace=True)
    df_processed['event_max_current'] = df_processed.cooking_event.map(
        df_processed_current.set_index('cooking_event')['current'].to_dict())

    # Disqualify cooking events that doesn't meet criterium for current
    # threshold
    df_processed.loc[
        (
            ((df_processed.event_max_current < minimum_event_current)
             & (df_processed.event_max_current != 0))
            | (df_processed.cooking_time > maximum_cooking_time)
            | (df_processed.energy_end - df_processed.energy_start < minimum_energy_per_cooking_event)

        ), 'cooking_event'] = np.nan

    # Set timestamp in index to facilitate plotting with timeseries on the
    # x-axis.
    df_processed.set_index('timestamp', inplace=True)

    return df_processed

def apply_event_conditions(df_processed,
                           time_resolution: int = 5,
                           minimum_active_load: float = 0.15,
                           power_capacity: float = 1,
                           t_between: float = 15,
                           mininum_size_of_spikes=None,
                           minimum_energy_per_cooking_event=None,
                           minimum_power_mean=None,
                           minimum_event_current=None,
                           maximum_cooking_time=None,
                           energy_error_margin=None):
    """
    Apply event conditions to get True or False values in the 'cooking_start'
    and 'cooking_false' columns and returns a Pandas DataFrame.

    Parameters
    ----------
    df_processed : a Pandas Dataframe that is used to extract cooking events.
    time_resolution : int, 1\5 minutes. Indicates the setting for the time resolution
    of the smart meter in minutes. The default is 5.
    minimum_active_load : float, min. power load as a percentage of the power capacity to
    indicate that an EPC is active, i.e. 0.15 = 15 % of Max. Rated Capacity. The default is 0.15.
    power_capacity : the power capacity of the EPC in this project, which is 1 kW. The default is 1.
    t_between : float, the max. time interval between two recordings which indicates
    that they belong to the same cooking event. The default is 15.

    Returns
    -------
    df_processed : returns a Pandas DataFrame with the indications of start and end of cooking events.

    """

    # (i): create coefficients to indicate when an EPC is turned ON
    power_threshold = minimum_active_load * power_capacity
    energy_threshold = power_threshold * time_resolution / 60

    # (ii): Create column 'load' for when a load is applied as the energy
    # consumption [kWh] between the current and pervious recording.
    df_processed.loc[(
        (
            (df_processed.energy.diff() > energy_threshold)
            | (df_processed.power > minimum_active_load * power_capacity))
        & (df_processed.meter_number == df_processed.meter_number.shift())
    ), 'load'] = df_processed.energy.diff()

    # (iii): Create a column 'load_count' for accumulated numbering of when a load is applied
    df_processed['load_count'] = 0  # start
    df_processed.loc[(df_processed.load.isnull()
                      == False), 'load_count'] += 1
    df_processed.load_count = df_processed.load_count.cumsum()

    # (iv): Create a column 'timestamp_load' for a timestamp of each load instance
    load_instance = df_processed.groupby('load_count').first()
    load_instance.reset_index(inplace=True)
    df_processed['timestamp_load'] = df_processed.load_count.map(
        load_instance.set_index('load_count')['timestamp'].to_dict())

    # (v): Cooking_start = TRUE: if timestamp_load - current timestamp is more
    # than t_between and above energy_threshold OR new meter_number.
    df_processed.loc[
        (
            (
                (df_processed.timestamp -
                 df_processed.timestamp_load.shift() > pd.to_timedelta(
                     t_between,
                     unit='m'))
                & (df_processed.energy.diff() >= energy_threshold))
            | (
                df_processed.meter_number != df_processed.meter_number.shift())
        ), 'cooking_start'] = True

    # (vi): Cooking_start = FALSE: if energy increase is above energy threshold and diff_prev_timestamp is less than t_between.
    df_processed.loc[
        (
            (df_processed.energy.diff() >= energy_threshold)
            & (df_processed.diff_prev_timestamp < pd.to_timedelta(t_between, unit='m'))
        ), 'cooking_start'] = False

    # (vii): Cooking_start = TRUE: if previous to current timestamp_load difference is above t_between + time_resolution AND power level is above 'power threshold', i.e. minimum_active_load * power_capacity
    df_processed.loc[
        (
            (df_processed.timestamp_load.diff() > pd.to_timedelta(t_between + time_resolution, unit='m'))
            & (df_processed.power >= power_threshold)
        ), 'cooking_start'] = True

    # (viii): Cooking_start = FALSE: if a recording in the previous-to-previous row is within t_between from current row, make sure that this isn't start of a cooking event since it should be part of the same event.
    df_processed.loc[
        (
            (df_processed.timestamp.diff(2) <= pd.to_timedelta(t_between, unit='m'))
            & (df_processed.cooking_start.shift(2))
            & (df_processed.cooking_start)
            & (df_processed.meter_number == df_processed.meter_number.shift(2))
        ), 'cooking_start'] = False

    # (ix): Cooking_end = TRUE: if difference between current timestamp and timestamp_load is above t_between AND power is above threshold on current and previous row AND same meter_number are all TRUE.
    df_processed.loc[
        (
            (df_processed.timestamp - df_processed.timestamp_load > pd.to_timedelta(
                t_between, unit='m'))
            & ((df_processed.power < power_threshold)
               & (df_processed.power.shift() < power_threshold)
               & (df_processed.meter_number == df_processed.meter_number.shift())
               )
            | (df_processed.energy - df_processed.energy.shift(-1) == 0)
        ), 'cooking_end'] = True

    # (x): Cooking_end = TRUE: if cooking_start in next row is TRUE OR new meter_number
    df_processed.loc[
        (
            (df_processed.cooking_start.shift(-1))
            | (df_processed.meter_number != df_processed.meter_number.shift(-1))
        ), 'cooking_end'] = True

    # (xi): Cooking_end = TRUE: if cooking_end in next row is TRUE AND diff_next_timestamp is above t_between
    df_processed.loc[
        (
            (df_processed.cooking_end.shift(-1))
            & (df_processed.diff_next_timestamp > pd.to_timedelta(t_between, unit='m'))
        ), 'cooking_end'] = True

    # (xii): Cooking_start = TRUE: if cooking_end on prev row AND cooking_end on current row
    df_processed.loc[
        (
            (df_processed.cooking_end.shift())
            & (df_processed.cooking_end)
        ), 'cooking_start'] = True

    # (ix): Cooking_start = TRUE: if difference between current timestamp and timestamp_load is above t_between AND power above power_threshold
    df_processed.loc[
        (
            (df_processed.timestamp - df_processed.timestamp_load > pd.to_timedelta(
                t_between, unit='m'))
            & (df_processed.power >= power_threshold)
        ), 'cooking_start'] = True

    # (xiv): Cooking_end = FALSE: if cooking_end on prev row AND cooking_end in current row == TRUE AND diff_prev_timestamp is more than t_between AND diff_next_timestamp is less than t_between.
    df_processed.loc[
        (
            (df_processed.cooking_end.shift())
            & (df_processed.cooking_end)
            & (df_processed.diff_prev_timestamp > pd.to_timedelta(t_between, unit='m'))
            & (df_processed.diff_next_timestamp <= pd.to_timedelta(t_between, unit='m'))
        ), 'cooking_end'] = False

    # (xvi): Cooking_end = TRUE: if previous and next row isn't an end of a cooking event and the time to previous recording is more than t_between, then cooking_end = True.
    df_processed.loc[
        (
            (df_processed.cooking_end.shift(-1) == 0)
            & (df_processed.cooking_end.shift() == 0)
            & (df_processed.diff_next_timestamp > pd.to_timedelta(
                t_between, unit='m'))
        ), 'cooking_end'] = True

    # (xv): Cooking_start = FALSE: if cooking_start on prev row AND cooking_start = TRUE in current row AND diff_prev_timestamp is less than t_between AND diff_next_timestamp is more than t_between.
    df_processed.loc[
        (
            (df_processed.cooking_start.shift())
            & (df_processed.cooking_start)
            & (df_processed.diff_prev_timestamp <= pd.to_timedelta(
                t_between, unit='m'))
            & (df_processed.diff_next_timestamp > pd.to_timedelta(
                t_between, unit='m'))
        ), 'cooking_start'] = False

    # (xvi): Cooking_start = TRUE: if previous timestamp is more than t_between + time_resolution, energy consumption is above 0,
    # time to next timestamp is less than t_between and the meter number is
    # the same in the previous and next row, then cooking_start = TRUE.
    df_processed.loc[
        (
            (df_processed.diff_prev_timestamp > pd.to_timedelta(t_between + time_resolution, unit='m'))
            & (df_processed.energy.shift(-1) - df_processed.energy != 0)
            & (df_processed.diff_next_timestamp <= pd.to_timedelta(t_between, unit='m'))
            & (df_processed.meter_number == df_processed.meter_number.shift())
            & (df_processed.meter_number == df_processed.meter_number.shift(-1))
        ), 'cooking_start'] = True

    # (xvii): Cooking_start = FALSE: if cooking_start in prev row = TRUE AND cooking_start in current row = TRUE AND diff_prev_timestamp is less than t_between AND prev row has power above threshold.
    df_processed.loc[
        (
            (df_processed.cooking_start.shift())
            & (df_processed.cooking_start)
            & (df_processed.diff_prev_timestamp < pd.to_timedelta(
                t_between, unit='m'))
            & (df_processed.power.shift() >= power_threshold)
        ), 'cooking_start'] = False

    # (xviii): if new meter number Cooking_start = TRUE, Cooking_end = FALSE
    df_processed.loc[
        (df_processed.meter_number.diff() != 0), 'cooking_start'] = True
    df_processed.loc[
        (df_processed.meter_number.diff() != 0), 'cooking_end'] = False

    # (xix): Take out "unnecessary" cooking_end
    df_processed.loc[
        (
            (
                (df_processed.cooking_start.shift(-1) == 0)
                & (df_processed.cooking_start.shift() == 0)
                & (df_processed.cooking_start == 0)
            )
            | (df_processed.cooking_start == 1)

        ), 'cooking_end'] = False
    return df_processed

def resolve_timestamps(df_processed,
                       energy_error_margin: float = 0.04,
                       time_resolution=None,
                       minimum_active_load=None,
                       power_capacity=None,
                       t_between=None,
                       mininum_size_of_spikes=None,
                       minimum_energy_per_cooking_event=None,
                       minimum_power_mean=None,
                       minimum_event_current=None,
                       maximum_cooking_time=None):
    """
    Resolve timestamps is a function that is removing cooking events that have been duplicated.
    This occured because the smart meters needed to reconfigure the factory settings, which had the local timezone, to UTC time.
    The timestamp duplicates are identified by comparing the energy meter value [kWh] at the start and end of each cooking events per meter.
    If the difference between cooking events in the energy meter value [kWh] at the start or end is less than energy_error_margin, this indicates that
    this cooking event is already classified as a cooking event and that the recording is a duplicate.
    All smart meters are now in UTC by default to avoid this problem happening in the future, i.e. since Dec-2020.

    Parameters
    ----------
    df_processed : a Pandas Dataframe that has defined cooking events.
    energy_error_margin : float, the energy consumption difference [kWh] that the previous
    cooking event should have compared to current cooking event. The default is 0.04.

    Returns
    -------
    df_epc : returns a Pandas DataFrame without any duplicated cooking events.

    """

    # Make a copy of dataframe to differentiate between the Pandas DataFrame with
    # vis-a-vis without duplicated cooking events.
    df_epc = df_processed.copy()
    df_epc.reset_index(inplace=True)

    # Make an intermediate Cooking Event Count
    df_epc['event_count'] = 0
    df_epc.loc[(df_epc.cooking_event.diff()
                != 0), 'event_count'] += 1
    df_epc.event_count = df_epc['event_count'].cumsum()
    df_epc.loc[
        (df_epc.cooking_event.isnull()),
        'event_count'] = np.nan

    # Check start of events
    start_of_event = df_epc.copy()
    start_of_event = start_of_event.groupby(
        ['meter_number', 'event_count']).head(1)
    start_of_event.loc[
        ((start_of_event['energy'] -
          energy_error_margin <= start_of_event['energy'].shift()) & (
            start_of_event.event_count.isnull() == False) & (
            start_of_event.meter_number == start_of_event.meter_number.shift())),
        'timestamp_issue'] = True

    # Make an indication of timestamp issue at start of cooking event.
    df_epc['timestamp_issue'] = df_epc.event_count.map(
        start_of_event.set_index('event_count')['timestamp_issue'].to_dict())

    # Check end of events
    end_of_event = df_epc.copy()
    end_of_event = end_of_event.groupby(
        ['meter_number', 'event_count']).tail(1)
    end_of_event.loc[
        ((end_of_event['energy'] -
          energy_error_margin <= end_of_event['energy'].shift()) & (
            end_of_event.event_count.isnull() == False) & (
            end_of_event.meter_number == end_of_event.meter_number.shift())),
        'timestamp_issue'] = True

    # Make an indication of timestamp issue at end of cooking event.
    df_epc['timestamp_issue'] = df_epc.event_count.map(
        end_of_event.set_index('event_count')['timestamp_issue'].to_dict())

    # Discard recordings that are part of duplicated cooking events,
    # only leaving the cooking event's first occurance.
    df_epc.drop(df_epc[(df_epc['timestamp_issue'] == 1)].index, inplace=True)

    # Update the cooking event count
    df_epc['cooking_event'] = 0
    df_epc.loc[((df_epc.event_count.diff() != 0) & (
        df_epc.event_count.isnull() == False)), 'cooking_event'] += 1
    df_epc.cooking_event = df_epc['cooking_event'].cumsum()

    # Drop rows that do no longer have any function
    df_epc.drop(['event_count', 'timestamp_issue'], axis=1, inplace=True)

    # Set timestamp in index to facilitate plotting with timeseries on the
    # x-axis.
    df_epc.set_index('timestamp', inplace=True)
    return df_epc

def add_event_end(df_epc,
                  time_resolution: int = 5,
                  power_capacity: float = 1,
                  energy_error_margin=None,
                  minimum_active_load=None,
                  t_between=None,
                  mininum_size_of_spikes=None,
                  minimum_energy_per_cooking_event=None,
                  minimum_power_mean=None,
                  minimum_event_current=None,
                  maximum_cooking_time=None):
    """
    Add a row at the end of each cooking event that meets certain criteria.
    This new row will compensate for the energy consumed at the end of a cooking event, i.e. after the last recording of the "originally defined" cooking event.
    The added energy comes from the energy value difference [kWh] between the current row and the next row.
    This energy value difference occurs due to the EPC being turned off prematurely or because a data gap/loss-of-network occurs.

    Parameters
    ----------
    df_epc : a Pandas Dataframe that has defined cooking events.
    time_resolution : int, 1\5 minutes. Indicates the setting for the time resolution
    of the smart meter in minutes. The default is 5.
    power_capacity : float, the power capacity of the EPC in this project,
    which is 1 kW. The default is 1.

    Returns
    -------
    df_epc : returns a Pandas DataFrame which includes the prolonged start of
    certain cooking events.

    """

    # Make sure 'timestamp' column is not assigned as index
    df_epc = format_timestamp(df_epc)
    df_epc.reset_index(inplace=True)

    # Make a copy of the dataframe which will used to mark the end of each
    # cooking event
    df_epc_energy_gaps = df_epc.copy()

    # Quantify the energy gap at the end of each cooking event
    df_epc_energy_gaps.loc[
        (
            (df_epc_energy_gaps.cooking_start == False)
            & (df_epc_energy_gaps.cooking_end == True)
            & (
                (df_epc_energy_gaps.cooking_event != df_epc_energy_gaps.cooking_event.shift())
                | (df_epc_energy_gaps.cooking_event != df_epc_energy_gaps.cooking_event.shift(-1))
            )
            & (df_epc_energy_gaps.meter_number == df_epc_energy_gaps.meter_number.shift(-1))
        ), 'energy_gap_to_next'] = df_epc_energy_gaps.energy.shift(-1) - df_epc_energy_gaps.energy

    # Only keep the last row of each cooking event that has an energy_gap
    # which should be used to prolong the cooking event.
    df_epc_energy_gaps = df_epc_energy_gaps.groupby('cooking_event').last()
    df_epc_energy_gaps = df_epc_energy_gaps.loc[df_epc_energy_gaps['energy_gap_to_next'] > 0]

    # Reset the index to keep the 'cooking_event' in the columns
    df_epc_energy_gaps.reset_index(inplace=True)

    # Equalize the energy gap in kWh to the equivalent in minutes
    df_epc_energy_gaps['energy_gap_time'] = df_epc_energy_gaps.energy_gap_to_next / \
        power_capacity * 60

    # Format the value in minutes to allow timedelta calculations
    df_epc_energy_gaps['energy_gap_time_datetime'] = df_epc_energy_gaps['energy_gap_time'] * \
        60 * np.timedelta64(1, 's')

    # Check the occurences of energy gaps with a timedelta below time
    # resolution, e.g. 5 minutes
    energy_gap_less_than_time_resolution = (
        df_epc_energy_gaps.energy_gap_time <= time_resolution)

    # Check the occurences of energy gaps with a timedelta above time
    # resolution, e.g. 5 minutes
    energy_gap_above_time_resolution = (
        df_epc_energy_gaps.energy_gap_time > time_resolution)

    # Update row of extended cooking event at ending, comprising updating 6 columns
    # If the energy gap is less than the time resolution value.
    df_epc_energy_gaps.loc[energy_gap_less_than_time_resolution,
                           'timestamp'] += df_epc_energy_gaps.energy_gap_time_datetime
    df_epc_energy_gaps.loc[energy_gap_less_than_time_resolution,
                           'time_end'] += df_epc_energy_gaps.energy_gap_time_datetime
    df_epc_energy_gaps.loc[energy_gap_less_than_time_resolution,
                           'cooking_time'] += df_epc_energy_gaps.energy_gap_time
    df_epc_energy_gaps.loc[energy_gap_less_than_time_resolution,
                           'seq_time'] += df_epc_energy_gaps.energy_gap_time
    df_epc_energy_gaps.loc[energy_gap_less_than_time_resolution,
                           'energy'] += df_epc_energy_gaps.energy_gap_to_next
    df_epc_energy_gaps.loc[energy_gap_less_than_time_resolution,
                           'energy_end'] += df_epc_energy_gaps.energy_gap_to_next

    # If the energy gap is above than the time resolution value.
    df_epc_energy_gaps.loc[energy_gap_above_time_resolution,
                           'timestamp'] += pd.Timedelta(minutes=time_resolution)
    df_epc_energy_gaps.loc[energy_gap_above_time_resolution,
                           'time_end'] += pd.Timedelta(minutes=time_resolution)
    df_epc_energy_gaps.loc[energy_gap_above_time_resolution,
                           'cooking_time'] += time_resolution
    df_epc_energy_gaps.loc[energy_gap_above_time_resolution,
                           'seq_time'] += time_resolution
    df_epc_energy_gaps.loc[energy_gap_above_time_resolution,
                           'energy'] += time_resolution / 60
    df_epc_energy_gaps.loc[energy_gap_above_time_resolution,
                           'energy_end'] -= time_resolution / 60

    # Add the new rows to the main Pandas DataFrame
    df_epc = df_epc.append(df_epc_energy_gaps)

    # Remove "unnecessary" rows
    df_epc.drop(['energy_gap_time', 'energy_gap_time_datetime',
                 'energy_gap_to_next'], axis=1, inplace=True)

    # Update all rows of each cooking event that has been prolonged
    df_epc['time_end'] = df_epc.cooking_event.map(
        df_epc_energy_gaps.set_index('cooking_event')['time_end'].to_dict())
    df_epc['cooking_time'] = df_epc.cooking_event.map(
        df_epc_energy_gaps.set_index('cooking_event')['cooking_time'].to_dict())
    df_epc['energy_end'] = df_epc.cooking_event.map(
        df_epc_energy_gaps.set_index('cooking_event')['energy_end'].to_dict())
    df_epc.sort_values(by=['meter_number', 'timestamp'],
                       ascending=[True, True], inplace=True)

    # Set timestamp in index to facilitate plotting with timeseries on the
    # x-axis.
    df_epc.set_index('timestamp', inplace=True)
    return df_epc

def add_event_start(df_epc,
                    time_resolution: int = 5,
                    power_capacity: float = 1,
                    energy_error_margin=None,
                    minimum_active_load=None,
                    t_between=None,
                    mininum_size_of_spikes=None,
                    minimum_energy_per_cooking_event=None,
                    minimum_power_mean=None,
                    minimum_event_current=None,
                    maximum_cooking_time=None):
    """
    Add a row at the start of each cooking event that meets certain criteria.
    This new row will compensate for the energy consumed at the start of a cooking event that happens before the first "original" recording in a defined cooking event.

    Parameters
    ----------
    df_epc : a Pandas Dataframe that has defined cooking events.
    time_resolution : int, 1\5 minutes. Indicates the setting for the time resolution
    of the smart meter in minutes. The default is 5.
    power_capacity : float, the power capacity of the EPC in this project,
    which is 1 kW. The default is 1.

    Returns
    -------
    df_epc : returns a Pandas DataFrame which includes the prolonged start of
    certain cooking events.

    """

    # Make sure 'timestamp' column is not assigned as index
    df_epc = format_timestamp(df_epc)
    df_epc.reset_index(inplace=True)

    # Make a copy of the dataframe which will used to mark the start of each
    # cooking event
    df_epc_energy_gaps2 = df_epc.copy()

    # Quantify the energy gap at the start of each cooking event
    df_epc_energy_gaps2.loc[
        (
            (df_epc_energy_gaps2.cooking_end == False)
            & (df_epc_energy_gaps2.cooking_start == True)
            & (
                (df_epc_energy_gaps2.cooking_event != df_epc_energy_gaps2.cooking_event.shift())
                | (df_epc_energy_gaps2.cooking_event != df_epc_energy_gaps2.cooking_event.shift(-1))
            )
            & (df_epc_energy_gaps2.energy.shift().isnull())
            & (df_epc_energy_gaps2.meter_number == df_epc_energy_gaps2.meter_number.shift())
        ), 'energy_gap_to_prev'] = df_epc_energy_gaps2.energy.diff()

    # Only keep the first row of each cooking event that has an energy_gap
    # which should be used to prolong the cooking event.
    df_epc_energy_gaps2 = df_epc_energy_gaps2.groupby('cooking_event').first()
    df_epc_energy_gaps2 = df_epc_energy_gaps2.loc[df_epc_energy_gaps2['energy_gap_to_prev'] > 0]

    # Reset the index to keep the 'cooking_event' in the columns
    df_epc_energy_gaps2.reset_index(inplace=True)

    # Equalize the energy gap in kWh to the equivalent in minutes
    df_epc_energy_gaps2['energy_gap_time'] = df_epc_energy_gaps2.energy_gap_to_prev / \
        power_capacity * 60

    # Format the value in minutes to allow timedelta calculations
    df_epc_energy_gaps2['energy_gap_time_datetime'] = df_epc_energy_gaps2['energy_gap_time'] * \
        60 * np.timedelta64(1, 's')

    # Check the occurences of energy gaps with a timedelta below time
    # resolution, e.g. 5 minutes
    energy_gap_less_than_time_resolution = (
        df_epc_energy_gaps2.energy_gap_time <= time_resolution)

    # Check the occurences of energy gaps with a timedelta above time
    # resolution, e.g. 5 minutes
    energy_gap_above_time_resolution = (
        df_epc_energy_gaps2.energy_gap_time > time_resolution)

    # Update row of extended cooking event at beginning, comprising updating 6 columns
    # If the energy gap is less than the time resolution value.
    df_epc_energy_gaps2.loc[energy_gap_less_than_time_resolution,
                            'timestamp'] -= df_epc_energy_gaps2.energy_gap_time_datetime
    df_epc_energy_gaps2.loc[energy_gap_less_than_time_resolution,
                            'time_start'] -= df_epc_energy_gaps2.energy_gap_time_datetime
    df_epc_energy_gaps2.loc[energy_gap_less_than_time_resolution,
                            'cooking_time'] += df_epc_energy_gaps2.energy_gap_time
    df_epc_energy_gaps2.loc[energy_gap_less_than_time_resolution,
                            'seq_time'] -= df_epc_energy_gaps2.energy_gap_time
    df_epc_energy_gaps2.loc[energy_gap_less_than_time_resolution,
                            'energy'] -= df_epc_energy_gaps2.energy_gap_to_prev
    df_epc_energy_gaps2.loc[energy_gap_less_than_time_resolution,
                            'energy_start'] -= df_epc_energy_gaps2.energy_gap_to_prev

    # If the energy gap is above than the time resolution value.
    df_epc_energy_gaps2.loc[energy_gap_above_time_resolution,
                            'timestamp'] -= pd.Timedelta(minutes=time_resolution)
    df_epc_energy_gaps2.loc[energy_gap_above_time_resolution,
                            'time_start'] -= pd.Timedelta(minutes=time_resolution)
    df_epc_energy_gaps2.loc[energy_gap_above_time_resolution,
                            'cooking_time'] += time_resolution
    df_epc_energy_gaps2.loc[energy_gap_above_time_resolution,
                            'seq_time'] -= time_resolution
    df_epc_energy_gaps2.loc[energy_gap_above_time_resolution,
                            'energy'] -= time_resolution / 60
    df_epc_energy_gaps2.loc[energy_gap_above_time_resolution,
                            'energy_start'] -= time_resolution / 60

    # Add the new rows to the main Pandas DataFrame
    df_epc = df_epc.append(df_epc_energy_gaps2)

    # Remove "unnecessary" rows
    df_epc.drop(['energy_gap_time', 'energy_gap_time_datetime',
                 'energy_gap_to_prev'], axis=1, inplace=True)

    # Update all rows of each cooking event that has been prolonged
    df_epc['time_start'] = df_epc.cooking_event.map(
        df_epc_energy_gaps2.set_index('cooking_event')['time_start'].to_dict())
    df_epc['cooking_time'] = df_epc.cooking_event.map(
        df_epc_energy_gaps2.set_index('cooking_event')['cooking_time'].to_dict())
    df_epc['energy_start'] = df_epc.cooking_event.map(
        df_epc_energy_gaps2.set_index('cooking_event')['energy_start'].to_dict())
    df_epc.sort_values(by=['meter_number', 'timestamp'],
                       ascending=[True, True], inplace=True)

    # Set timestamp in index to facilitate plotting with timeseries on the
    # x-axis.
    df_epc.set_index('timestamp', inplace=True)
    return df_epc

def create_only_event_df(df_epc,
                         TZS_per_kWh: int = 100):
    """
    Create a Pandas DataFrame which only contains rows of cooking events by grouping
    all recordings that belongs to a qualified cooking event.

    Parameters
    ----------
    df_epc : a dataframe that has defined cooking events.
    TZS_per_kWh : int, the price of cooking with the EPC per kWh in Tanzanian Shilling.
    The default is 100.

    Returns
    -------
    df_only_events : returns a dataframe without any non-cooking events.

    """

    # Make a copy of dataframe to keep the non-cooking events in the dataframe
    # that is used as source file.
    df_only_events = df_epc.copy()
    df_only_events.reset_index(inplace=True)

    # Create a new column called 'event_energy' to do statistics of how much
    # energy consumption that each cooking event has.
    df_only_events['event_energy'] = df_only_events.energy

    # Group all cooking events according to columns 'meter_number' and
    # 'cooking_event'
    df_only_events = df_only_events.groupby(['meter_number',
                                             'cooking_event']).agg({'energy': 'max',
                                                                    'event_energy': 'min',
                                                                    'power': 'mean',
                                                                    'cooking_time': 'max',
                                                                    'timestamp': 'min',
                                                                    'current': 'max',
                                                                    'voltage': 'count',
                                                                    'id': 'mean'})

    # (a): Count the number of recordings in each cooking event
    df_only_events.rename(
        columns={
            'voltage': 'no_recordings'},
        inplace=True)

    # (b): Calculate the energy usage of each cooking event
    df_only_events.event_energy = df_only_events.energy - df_only_events.event_energy

    # (c): Calculate the average power level during a cooking event
    df_only_events['power_mean'] = df_only_events.event_energy / \
        (df_only_events.cooking_time / 60)

    # (d): Calculate the cost of cooking (Tanzanian Shilling)
    df_only_events['cooking_cost'] = df_only_events.event_energy * TZS_per_kWh

    # (e): Conduct renumbering the cooking event count
    df_only_events.reset_index(inplace=True)
    df_only_events['event_count'] = 0
    df_only_events.loc[(df_only_events.cooking_event.diff()
                        != 0), 'event_count'] += 1
    df_only_events.event_count = df_only_events['event_count'].cumsum()

    # Set timestamp in index to facilitate plotting with timeseries on the
    # x-axis.
    df_only_events.set_index('timestamp', inplace=True)
    df_only_events.drop({'cooking_event'},axis=1,inplace=True)
    return df_only_events

def format_timestamp(df):
    """
    Format column 'timestamp' for plotting timeseries in x-axis.

    Parameters
    ----------
    df : a dataframe that either has timestamp in the index column or will
    replace current index and with the timestamp.

    Returns
    -------
    df : Return a Pandas DataFrame with an timeseries index of localized time

    """
    def format_df():
        # Update numbering in index
        df.reset_index(inplace=True)

        # Make sure the timestamp is in type datetime
        df.timestamp = pd.to_datetime(df.timestamp)
        df.timestamp = np.int64(df.timestamp)
        df.timestamp = pd.to_datetime(df.timestamp)

        # Localize timestamp
        df['timestamp'] = df['timestamp'].dt.tz_localize(
            'utc').dt.tz_convert(df['timezone_region'].astype(str)[0])
        df.set_index('timestamp', inplace=True)

    if 'timestamp' in df.columns:

        df.set_index('timestamp', inplace=True)
        format_df()
    else:
        format_df()

    return df

def preprocess_epc_data(df_raw, **kwargs):
    """
    Preform a sequence of functions to produce the complete preprocessing
    of the raw EPC data to create a dataframe that can be used to conduct data
    analysis on how the households have used their EPCs.

    Parameters
    ----------
    df_raw : dataframe that is used as source file.
    **kwargs : add any parameters that are part of subfunctions to see the
    effect of parameter changes.

    Returns
    -------
    df_epc : returns a Pandas DataFrame.

    """

    df_raw = resolve_spikes(df_raw, **kwargs)
    df_processed = extract_cooking_events(df_raw, **kwargs)
    df_epc = resolve_timestamps(df_processed, **kwargs)
    df_epc = add_event_end(df_epc, **kwargs)
    df_epc = add_event_start(df_epc, **kwargs)
    df_epc = extract_cooking_events(df_epc, **kwargs)
    return df_epc

def clean_data_set(df_epc):
    """
    Clean up the dataframe after the preprocessing steps have been conducted.
    This is done by renumbering the cooking events and deleting "unnecessary columns"
    that shouldn't be part of the final dataframe.

    Parameters
    ----------
    df_epc : dataframe that has gone through the desired preprocessing steps.

    Returns
    -------
    df_epc : returns a Pandas DataFrame.

    """

    df_epc.reset_index(inplace=True)

    # Do a renumbering of the cooking event count
    df_epc['event_count'] = 0
    df_epc.loc[(
        (df_epc.cooking_event.diff()
         != 0)
        & (df_epc.cooking_event.isnull() == 0)
    ), 'event_count'] += 1
    df_epc.event_count = df_epc['event_count'].cumsum()
    df_epc.loc[(
        (df_epc.cooking_event.isnull())
    ), 'event_count'] += np.nan

    # Create column 'event_energy' for kWh per event.
    df_epc['event_energy'] = df_epc['energy_end'] - df_epc['energy_start']

    # Delete "unnecessary" columns
    df_epc.drop(['diff_next_timestamp', 'diff_prev_timestamp',
                 'timestamp_load', 'load_count', 'load',
                 'cooking_start', 'cooking_end', 'cooking_event',
                 'time_start', 'energy_start', 'time_end',
                 'energy_end', 'region', 'id'], axis=1, inplace=True)

    # Set meter_number in index to optimize the release file format for
    # combining source files relating to smart meter numbers.
    df_epc.set_index('meter_number', inplace=True)
    return df_epc

#Uncomment the lines below to execute the code directly in this file
'''

# Explaining parameters and their default values.
"""
'time_resolution' : 5 minutes is the frequency with which the smart meter is recording.

't_between' : 15 minutes is the time between two cooking events.

'energy_error_margin' : 0.04 kWh is how much two measurements that are taken
more or less simultaneously [+/- 2 minutes] can differ. This parameter is used
to resolve timestamp issues.

'mininum_size_of_spikes' : 1 kWh is the threshold to qualify a jump in the
energy as a corrupt value.

'minimum_energy_per_cooking_event' : 0.05 kWh is the least amount of energy consumption
that can be classified as a cooking event.

'minimum_power_mean' : 0.05 kW is the minimum average power level that a
cooking event needs to have to be classified as a cooking event.

'minimum_event_current' : 0.5 A is the minimum current that the EPC needs to have
recorded to qualify a cooking event.

'maximum_cooking_time' : 300 minutes is the maximum length that an EPC is believed to be cooking
before it's not considered a normal EPC event.

'minimum_active_load' : 0.15 is the min. average ratio of the Max. Rated Capacity
that a possible cooking event needs to have to be classified as a cooking event.

'power_capacity': 1 kW is the rated power capacity of the EPC in this study.
"""

# Change these parameters to influence all preprocessing steps through **kwargs.
params = {'time_resolution': 5,
         't_between': 15,
         'energy_error_margin': 0.04,
         'mininum_size_of_spikes': 1,
         'minimum_energy_per_cooking_event': 0.05,
         'minimum_power_mean': 0.05,
         'minimum_event_current': 0.5,
         'maximum_cooking_time':300,
         'time_resolution': 5,
         'minimum_active_load': 0.15,
         'power_capacity': 1}

# Source file
df_raw = pd.read_csv('dataframe_raw_jan14.csv', sep=',')

# Outputs
df_epc = preprocess_epc_data(df_raw, **params)
df_only_events = create_only_event_df(df_epc)
df_epc = clean_data_set(df_epc)
'''


#!/usr/bin/env python3

import os
import sys
import re

import json

from vmaf.core.result import Result

__copyright__ = "Copyright 2016-2020, Netflix, Inc."
__license__ = "BSD+Patent"

def print_usage():
    print("usage: python " + os.path.basename(sys.argv[0]) + " input_file\n")
    print("input_file contains a list of files for assembly (can be xml or json)")

class FileAssembler:

    SUPPORTED_FILE_TYPES = ['xml', 'json']

    def __init__(self, to_assemble_input):
        self.to_assemble_input = to_assemble_input

    @staticmethod
    def create_assembly_file_list(to_assemble_input):

        to_assemble_list = []
        if isinstance(to_assemble_input, list):
            to_assemble_list = to_assemble_input
        else:
            with open(to_assemble_input, "rt") as input_file:
                for line in input_file.readlines():

                    # match comment
                    mo = re.match(r"^#", line)
                    if mo:
                        print("Skip commented line: {}".format(line))
                        continue

                    # match whitespace
                    mo = re.match(r"[\s]+", line)
                    if mo:
                        continue

                    mo = re.match(r"([\S]+)", line)
                    if not mo:
                        print("Invalid file: {}".format(line))
                        print_usage()
                        return 1

                    to_assemble_list.append(line.strip())

        return to_assemble_list

    def _create_result_list(self, to_assemble_list):
        pass

    def assemble(self):
        """
        Main file assembly logic
        """
        to_assemble_list = self.create_assembly_file_list(self.to_assemble_input)
        self._assert(to_assemble_list)
        results = self._create_result_list(to_assemble_list)
        combined_result = Result.combine_result(results)

        return combined_result

    def _assert(self, to_assemble_list):
        """
        Perform necessary assertions before parsing any of the files.
        """

        # check that the number of files is greater than 0
        assert len(to_assemble_list) > 0
        # check that the file formats match
        assemble_format_list = [os.path.splitext(f)[1].split(".")[1] for f in to_assemble_list]
        assert len(set(assemble_format_list)) == 1, "The file formats for assembly do not much."
        # check that the file format is supported for assembly
        assert assemble_format_list[0] in self.SUPPORTED_FILE_TYPES, \
            "The assembly format is not consistent, use any of {fmts}".format(fmts=str(self.SUPPORTED_FILE_TYPES))

class XmlAssembler(FileAssembler):

    @staticmethod
    def _parse_files(to_assemble_list):

        to_assemble_xml_strings = []
        for to_assemble_xml in to_assemble_list:
            with open(to_assemble_xml, 'r') as f:
                to_assemble_xml_strings.append(f.read())

        return to_assemble_xml_strings

    def _create_result_list(self, to_assemble_list):

        to_assemble_xml_strings = self._parse_files(to_assemble_list)

        results = []
        for to_assemble_xml_string in to_assemble_xml_strings:
            results.append(Result.from_xml(to_assemble_xml_string))

        return results

class JsonAssembler(FileAssembler):

    @staticmethod
    def _parse_files(to_assemble_list):

        to_assemble_json_strings = []
        for json_file in to_assemble_list:
            with open(json_file, 'r') as f:
                to_assemble_json_strings.append(json.load(f))

        return to_assemble_json_strings

    def _create_result_list(self, to_assemble_list):

        to_assemble_jsons = self._parse_files(to_assemble_list)

        results = []
        for to_assemble_json in to_assemble_jsons:
            to_assemble_json_string = json.dumps(to_assemble_json)
            results.append(Result.from_json(to_assemble_json_string))

        return results

def main():

    if len(sys.argv) != 2:
        print_usage()
        return 2

    files_to_assemble_list = sys.argv[1]

    desired_file_list = FileAssembler.create_assembly_file_list(files_to_assemble_list)

    if ".xml" in desired_file_list[0]:
        xml_output = XmlAssembler(files_to_assemble_list).assemble().to_xml()
        print(xml_output)
    elif ".json" in desired_file_list[0]:
        json_output = JsonAssembler(files_to_assemble_list).assemble().to_json()
        print(json_output)
    else:
        print_usage()
        return 2

if __name__ == "__main__":
    ret = main()
    exit(ret)


import os
import math

import pygame as pg

from resources import Spritesheet

class Golfer(pg.sprite.Sprite):

    def __init__(self, screen, clock, club):
        """Initialize the golfer.

        :param screen: The pygame display
        :param clock: The pygame game clock
        :param club: The golfer's club
        """
        super(Golfer, self).__init__()
        self.screen = screen
        self.clock = clock
        self.size = 110, 200
        # Set the radius for detecting collisions
        self.radius = 50

        # Set the sprite images
        self.images, self.image_functions = self.get_images()
        self.image_index = 0
        self.set_image()

        # Set the rectangle, or the box where it is drawn
        self.rect = self.surf.get_rect()
        self.rect.bottom = self.screen.get_size()[1] - 60
        self.rect.right = self.screen.get_size()[0] / 2 - 50
        self.vector = [0, 0]
        self.speed = 0
        self.club = club
        # Set the animation stuff
        self.is_in_swing = False
        self.animation_frames = 5  # the frames per animation
        self.current_frame = 0

    def get_images(self):
        """Get all images defined by the spritesheet and return them.

        :return: A list pygame.Surface and index dict
        :rtype: list of pygame.Surface and dict of int: int
        """
        ss = Spritesheet(
            os.path.join(os.getcwd(), 'resources', 'golfer_sheet.png')
        )
        strip = ss.load_strip((0,0,220,400), 4)
        scaled_strip = []
        for sprite in strip:
            scaled_strip.append(
                pg.transform.scale(sprite, self.size)
            )
        return scaled_strip, {
                0: self.set_resting,
                1: self.set_mid_swing,
                2: self.set_full_swing,
                3: self.set_mid_swing,
                4: self.set_resting,
                5: self.set_followthru
            }

    def set_image(self, image_index=0):
        if image_index is not None and \
           image_index in range(0, len(self.images)):
            self.surf = self.images[image_index]

    def set_resting(self):
        self.set_image(image_index=0)
        self.club.set_resting()

    def set_mid_swing(self):
        self.set_image(image_index=1)
        self.club.set_mid_swing()

    def set_full_swing(self):
        self.set_image(image_index=2)
        self.club.set_full_swing()

    def set_followthru(self):
        self.set_image(image_index=3)
        self.club.set_followthru()

    def update_frame_dependent(self):
        """Update the sprite image every 5 frames during a swing."""
        power, speed = 0, 0
        self.current_frame += 1
        if self.current_frame >= self.animation_frames:
            self.current_frame = 0
            if self.image_index == len(self.image_functions) - 2:
                power, speed = self.club.power, 0
            if self.image_index == len(self.image_functions) - 1:
                self.is_in_swing = False
            self.image_index = (self.image_index + 1) % \
                               len(self.image_functions)
            self.image_functions[self.image_index]()
        self.club.update()
        self.display()
        return power, speed

    def update(self, pressed_keys, start_swing=False):
        """Update the golfer's location.

        :param pressed_keys: A dict of pressed keys this fram
        """
        if self.is_in_swing:
            return self.update_frame_dependent()
        if start_swing:
            self.is_in_swing = True
            self.update_frame_dependent()
            return None, None
        if pressed_keys[pg.K_UP]:
            self.set_full_swing()
            self.club.set_full_swing()
        if pressed_keys[pg.K_DOWN]:
            self.set_resting()
            self.club.set_resting()
        if pressed_keys[pg.K_LEFT]:
            self.set_followthru()
            self.club.set_followthru()
        if pressed_keys[pg.K_RIGHT]:
            self.set_mid_swing()
            self.club.set_mid_swing()
        self.club.update()
        self.display()
        return None, None

    def calcnewpos(self, rect, vector):
        (angle,z) = vector
        (dx,dy) = (z*math.cos(angle), z*math.sin(angle))
        return rect.move(dx,dy)

    @property
    def image(self):
        return self.surf

    def display(self):
        self.screen.blit(self.surf, self.rect)


from divaGui.viewImports import *

class CollectionView(View):
    def get(self, request, *args, **kwargs):
        url = diva+self.kwargs['url']
        result = requests.get(url)
        result = result.json()
        name = self.kwargs['url'].split('/')
        name = name[0]
        images = []
        numberOfFiles = 0
        statusCode = ''
        statusMessage = ''
        percentage = ''
        statusCode = result['statusCode']
        statusMessage = result['statusMessage']
        percentage = result['percentage']

        isXML = False

        for element in result['files']:
            temp = element['file']['url']
            images.append(temp)
            numberOfFiles = numberOfFiles + 1
            temp = temp.split(".")
            temp = temp[len(temp) - 1]
            if temp != "png" and temp != "jpeg" and temp != "jpg" and temp != "JPG" and temp != "JPEG" and temp !="PNG":
                isXML = True

        imgNames = []

        for element in result['files']:
            temp = element['file']['identifier']
            temp = temp.split("/")
            imgNames.append(temp[1])

        # KEEP IDENTATION!
        images = zip(images, imgNames)

        context = {
            "imgNames": imgNames,
            "isXML": isXML,
            "directLink": url,
            "name": name,
            "statusCode": statusCode,
            "statusMessage": statusMessage,
            "percentage": percentage,
            "numberOfFiles": numberOfFiles,
            "images": images,
            "result": result,
        }
        return render(request, "collection.html", context)

    def post(self, request, *args, **kwargs):
        filename = self.request.POST.get("filename")
        collection = self.request.POST.get("name")

        url = diva+collection+ '/'  + filename
        print(filename)
        print(url)
        payload = {'some': 'data'}
        headers = {'content-type': 'application/json'}
        response = requests.delete(
            url, data=json.dumps(payload), headers=headers)
        print(response)
        response = requests.get(diva+collection)
        print(response)
        context = {}
        return HttpResponseRedirect("/collection/"+collection)


import csv
import pandas as pd
from tokenizations import get_alignments
import torch

pd.set_option('mode.chained_assignment', None)

def get_texts(corpus):
	'''
	Get a list of texts in the corpus
	'''
	texts = []
	text_ids = pd.unique(corpus.speechId)
	for t in text_ids:
		text = corpus[corpus.speechId==t]
		texts.append(text)
	return texts

def split_text(text, max_words=700, min_words=50):
	'''
	LM can't take too many tokens at once. So less than 700 words are fed into
	the model each time.
	'''
	result = []
	paragraph_ids = pd.unique(text.paragraphId)
	df = pd.DataFrame()
	for p in paragraph_ids:
		paragraph = text[text.paragraphId==p]
		if df.shape[0] + paragraph.shape[0] <= max_words:
			df = pd.concat([df, paragraph])
		else:
			result.append(df)
			df = paragraph
	# deal with the last batch of subtext
	if df.shape[0] <= min_words:
		result[-1] = pd.concat([result[-1], df])
	else:
		result.append(df)

	return result

def get_utt_logp(utterance, tokenizer, model):
	'''
	Given utterance input, return Transformer results as a list of
	tuples (token, logprob).
	'''
	result = []
	tokens = tokenizer.encode(utterance)
	lm_output = model(torch.tensor(tokens))
	lm_logp = torch.log_softmax(lm_output.logits, -1)
	for idx, t in enumerate(tokens):
		token = tokenizer.decode(t).strip()
		if idx == 0:
			logprob = None
		else:
			logprob = lm_logp[idx-1, t].item()
		result.append((token, logprob))
	return result

def add_text_logp(text, tokenizer, model):
	'''
	Given a corpus text, get the logp for each word and add one column of logp
	to the original text df. No return.
	'''
	tokens_corpus = text.word.tolist()
	utt = ' '.join(tokens_corpus)
	model_output = get_utt_logp(utt, tokenizer, model)
	num_lm_tokens = len(model_output)
	aligned_logp = align_tokenization(tokens_corpus, model_output)
	text['logp'], text['lm_tokens'] = aligned_logp, num_lm_tokens

def align_tokenization(tokens_corpus, model_output):
	'''
	Align the tokenization between model output and the corpus. Specifically,
	we want the tokenization in the corpus, but align it with model output.
	'''
	result = []
	tokens_lm = [w[0] for w in model_output]
	logprobs = [w[1] for w in model_output]
	corpus2lm, _ = get_alignments(tokens_corpus, tokens_lm)
	for i in corpus2lm:
		if len(i) == 1:
			logp = logprobs[i[0]]
		elif len(i) == 0:    # irreconcilable alignment
			logp = None
		else:
			if logprobs[i[0]] == None:
				logp = None
			else:
				logp = sum(logprobs[i[0]:i[-1]+1])
		result.append(logp)
	return result

def main(RT_corpus, output_file, tokenizer, model, if_test=True):
	'''
	Given an RT corpus, write a csv file that includes a column of log probability
	aligned with each token in the RT corpus.
	'''
	danish = pd.read_csv(RT_corpus)
	texts = get_texts(danish)
	if if_test:
		texts = texts[:1]   # Only use the first two texts if testing
	df = pd.DataFrame()
	for text in texts:
		subtexts = split_text(text)
		for idx, subtext in enumerate(subtexts):
			add_text_logp(subtext, tokenizer, model)
			subtext['subtextId'] = idx
			subtext['subtext_wordId'] = subtext.groupby('subtextId').cumcount()
			df = pd.concat([df, subtext])
	df.to_csv(output_file, index=False)


import binascii
from enum import Enum
import functools
import os
import random
import socket
import struct
import subprocess
import sys
import tempfile
import time

from . import enumhelper
import eventio
from . import ip
from . import pki
from . import utils

log =  functools.partial(print, 'info   :')
logw = functools.partial(print, 'warning:')
loge = functools.partial(print, 'error  :')
logd = functools.partial(print, 'debug  :')

def set_logfns(i, w, e, d):

    global log
    global logw
    global loge
    global logd

    log = i
    logw = w
    loge = e
    logd = d

def urandom(num_bytes):

    return open('/dev/urandom', 'rb').read(num_bytes)
class Modes(enumhelper.EnumHelper):

    Server = 'server'
    Client = 'client'

def decode_base64(data):

    ret = None
    try:
        ret = binascii.a2b_base64(data)
    except binascii.Error as e:
        loge(f'error: base64 decode: {e}: {data}')
    except Exception as e:
        loge(f'error: base64 decode: {e}: {data}')

    return ret

def hexstr_to_int(s):

    ret = -1
    try:
        ret = int(s, 16)
    except ValueError as e:
        loge(f'hexstr_to_int: {e}: {s}')
    except Exception as e:
        loge(f'hexstr_to_int: {e}: {s}')

    return ret

def int_to_hexstr(num):

    return f'{num:04x}'.encode()

class KernelLoader(object):

    pkv_cmds = {b'hello', b'hello_ack', b'hello_done'}
    core_msg = {b'type', b'mac', b'nonce', b'signature', b'name'}

    non_base64 = {b'type', b'length'}

    other_types = {
        b'type': bytes,
        b'length': hexstr_to_int,
    }

    def __init__(self, name, transport):

        # to -> temporal key
        self.tkeys_to = dict()
        self.tkeys_from = dict()
        # (from, to) -> shared key
        self.skeys = dict()

        self.name = name
        # name -> transport
        self.my_transport = transport
        self.my_transport.set_kl(self)
        self.transports = {
            b'__bcast__': self.my_transport,
        }

        self.handlers = {}

        self.pingers = {}

    def add_handler(self, transport=None, handler=None):

        if handler is None:
            return
        if transport is None:
            transport = self.my_transport
        self.handlers[transport] = handler

    @property
    def to(self):

        return self.my_transport.to

    @property
    def is_bcaster(self):

        return not self.my_transport.connector

    @property
    def is_connected(self):

        return self.my_transport.connected

    def on_close(self, handler=None, transport=None):

        logw(f'kl: closing: {handler}: {transport}')

        for name, transport_ in list(self.transports.items()):
            if transport_ is transport:
                self.transports.pop(name)
            try:
                self.pingers.pop(name)
            except KeyError:
                pass

        try:
            self.handlers.pop(handler)
        except KeyError:
            pass

    def send_kv(self, cmd, tmsg):

        to_name = tmsg.get('to')
        if to_name is None:
            to_name = tmsg.get(b'to')

        if isinstance(to_name, str):
            to_name = to_name.encode()

        if to_name is None:
            loge(f'send_kv: send_kv: no "to"')
        else:
            log(f'send_kv: sending to: {to_name}')

        transport = None
        if to_name in self.transports:
            transport = self.transports.get(to_name)
        elif self.is_connected:
            transport = self.my_transport
        elif self.is_bcaster:
            transport = self.my_transport
        else:
            logw(f'send_kv: no transport: {to_name}')
            return None

        if isinstance(cmd, str):
            cmd = cmd.encode()

        bmsg = dict()
        bmsg[b'addr_info'] = f'{self.my_transport.other_info}'.encode()
        bmsg[b'nonce'] = urandom(12)
        bmsg[b'name'] = self.name
        bmsg[b'cmd'] = cmd
        log(f'send_kv: cmd: {cmd}')

        for k, v in tmsg.items():
            if isinstance(k, str):
                k = k.encode(k)
            if k not in bmsg and k not in self.core_msg and k != 'signature':
                if isinstance(v, str):
                    v = v.encode()
                bmsg[k] = v

        to = None
        hsign = False
        if b'to' in bmsg:
            to = bmsg[b'to']
            hmac_key = self.get_skey(to)
            if hmac_key is not None:
                logd(f'send_kv: hmac_key: {hmac_key}')
                hsign = True
            else:
                loge(f'send_kv: no "skey": {to.decode()}')
        else:
            loge(f'send_kv: no "to"')

        psign = cmd in self.pkv_cmds

        have_sign = hsign or psign
        one_sign = hsign ^ psign

        if not have_sign:
            logd(f'send_kv: no sign method')
            return None
        if not one_sign:
            logd(f'send_kv: two sign methods')

        parts = []
        parts.append(b'type:kv')
        for k, v in bmsg.items():
            try:
                base64 = binascii.b2a_base64(v, newline=False)
            except TypeError as e:
                log(f'send_kv: encoding: {k}, {v}')

            parts.append(k + b':' + base64)

        s = b' '.join(parts)

        length_len = len(' lenght:1234')
        s += b' length:' + int_to_hexstr(len(s) + length_len)

        logd(f'send_kv: {self.name}: {s}')
        signature_slug = b''
        if psign:
            signature_slug = b' signature:'
            signature = pki.sign(self.name, s)
        elif hsign:
            signature = pki.calc_hmac(hmac_key, s)
            signature_slug = b' hmac:'
        else:
            return None

        s += signature_slug + binascii.b2a_base64(signature, newline=False)

        log(f'send_kv: {self.name}: {s}')

        return transport.send(s)

    def on_cmd_null(self, from_name, cmd, msg):

        log(f'{self.name}: {from_name}: null: cmd: {msg}')

        return None

    def recv_kv(self, data, bcast, transport=None):

        logd(f'recv_kv: data: {data}')
        split = data.split(b' ')
        signature = split[-1]
        payload = b' '.join(split[:-1])

        kvs = dict()
        logd(f'recv_kv: split: {split}')
        for part in split:
            k, v = part.split(b':')
            kvs[k] = v

        length_ok = False
        length = kvs.get(b'length')
        if length is not None:
            length = hexstr_to_int(length)
            calc_length = length + 1 + len(signature)
            length_ok = length + 1 + len(signature) == len(data)
            if not length_ok:
                loge(f'recv_kv: bad length: {calc_length} versus {len(data)}')
        else:
            loge(f'recv_kv: no length')

        verified = False
        name = kvs.get(b'name')
        logd(f'recv_kv: name: {name}')
        if name is not None and length_ok:
            name = decode_base64(name)
            if signature.startswith(b'signature:'):
                signature = signature.split(b':')[-1]
                signature = decode_base64(signature)
                logd(f'recv_kv: signature: {signature}')
                verified = pki.verify(name.decode(), signature, payload)
                log(f'recv_kv: verified: {verified}')
            elif signature.startswith(b'hmac:'):
                skey = self.get_skey(name)
                verified = False
                if skey is not None:
                    signature = signature.split(b':')[-1]
                    signature = decode_base64(signature)
                    logd(f'recv_kv: hmac: {signature}')
                    verified = pki.verify_hmac(skey, payload, signature)
                log(f'recv_kv: hmac verified: {verified}')
        else:
            logd(f'recv_kv: sig: {signature}')
            signature = None

        omsg = dict()
        cmsg = dict()
        if verified:
            for k, v in kvs.items():
                if k not in self.non_base64:
                    v = decode_base64(v)
                    if v and k not in self.core_msg:
                        omsg[k] = v
                    elif v:
                        cmsg[k] = v
                    else:
                        log(f'recv_kv: bad v: {v}')

            if set(cmsg.keys()) - self.core_msg:
                logw(f'recv_kv: not all core msg: {set(cmsg.keys())}')
                omsg = dict()

        if name not in self.transports and transport is not None:
            logw(f'recv_kv: adding transport: {name}: {transport}: {transport.addr_info}')
            self.transports[name] = transport

        handler_obj = self.handlers.get(transport)
        cmd = omsg.get(b'cmd')
        logd(f'recv_kv: cmd: {cmd}: handler: {handler_obj}')
        if isinstance(cmd, bytes):
            ret = handler_obj.handle_event(cmd, name, omsg)
            if ret:
                return ret

            cmd = cmd.decode()
            name = name.decode()
            handler = getattr(handler_obj, f'on_cmd_{cmd}', self.on_cmd_null)
            return handler(name, cmd, omsg)

        return None

    def on_readable(self, fd):

        return self.transport.on_readable(fd)

    def idle(self):

        while True:

            self.run_one()

    def setup_tkey(self, to_name):

        logd(f'setup_tkey: setting up tkey: {to_name}')

        _cmd = 'setup_tkey'
        temporal_pkey = pki.make_temporal_me_to_them(self.name, to_name)
        if temporal_pkey is None:
            logw(f'client[{self.name}]: {_cmd}: no temporal pkey')
            return None
        else:
            log(f'client[{self.name}]: {_cmd}: temporal pkey: {temporal_pkey}')
            self.tkeys_to[to_name] = temporal_pkey

        return temporal_pkey

    def get_tkey(self, to_name):

        _cmd = 'get_tkey'
        temporal_pkey = self.tkeys_to.get(to_name)
        if temporal_pkey is None:
            logw(f'client[{self.name}]: {_cmd}: no temporal pkey')
        else:
            log(f'client[{self.name}]: {_cmd}: temporal pkey: {temporal_pkey}')

        return temporal_pkey

    def get_skey(self, to_name):

        _cmd = 'get_skey'

        skey = self.skeys.get(to_name)
        if skey is None:
            logw(f'client[{self.name}]: {_cmd}: no skey: {to_name}: {tuple(self.skeys.keys())}')
        else:
            logd(f'client[{self.name}]: {_cmd}: skey: {skey}')

        return skey

    def save_from_tkey(self, from_name, temporal_pkey_from):

        ret = False
        tkey_fname = f'{from_name.decode()}_{self.name.decode()}_public.pem'
        with open(tkey_fname, 'wb') as outf:
            if isinstance(temporal_pkey_from, str):
                temporal_pkey_from = temporal_pkey_from.encode()

            outf.write(temporal_pkey_from)
            outf.flush()
            self.tkeys_from[from_name] = temporal_pkey_from
            skey = pki.derive_temporal_key(from_name, self.name)
            if skey is None:
                logw(f'save_from_tkey: no skey')
                self.tkeys_from.pop(from_name)
            else:
                logd(f'save_from_tkey: skey: {skey}')
                self.skeys[from_name] = skey
                ret = True

        return True

    def remove_keys(self, from_name):

        log(f'remove_keys...')
        try:
            self.skeys.pop(from_name)
        except KeyError:
            pass

        try:
            self.tkeys_to.pop(from_name)
        except KeyError:
            pass

        try:
            self.tkeys_from.pop(from_name)
        except KeyError:
            pass

    def has_connection(self, from_name):

        return False

class KernelLoaderEvents(object):

    def __getattr__(self, name):

        if name != 'subject' and hasattr(self.subject, name):
            return getattr(self.subject, name)

        raise AttributeError(f'{self.__class__}: {name}')


import numpy as np
from matplotlib import image
import matplotlib.pyplot as plt
from tqdm import tqdm

def calculate_k_means(data, k):
    n, d = data.shape

    # random initialize means
    mew_old = mew = np.random.randn(k, d)

    # repeatedly perform assignment and mean update
    # hard limit is 50
    print('starting k-means')
    for x in tqdm(range(50)):

        # assignment
        assignment = np.argmin(np.asarray([np.sum((data - mew[a]) ** 2, axis=1) for a in range(k)]), axis=0)

        # mean update
        mew = np.asarray([np.sum((assignment == a) * (data.transpose()), axis=1) / (
            np.sum(assignment == a) + 1e-8) for a in range(k)])

        # until convergence
        if np.sum((mew - mew_old) ** 2) < 1e-10:
            break
        mew_old = mew

    print("converged at iteration: ", x)

    return mew

def k_means_clustering(data, k):
    # find out k means
    mew = calculate_k_means(data, k)
    print("converged mew: \n", mew)

    # assign clusters
    assignment = np.argmin(np.asarray([np.sum((data - mew[a]) ** 2, axis=1) for a in range(k)]), axis=0)

    return assignment

def main():
    img_mat = image.imread('images/cow.jpg')
    h, w, d = img_mat.shape
    k = 2
    img_mat = img_mat.reshape((h * w, d))
    assignment = k_means_clustering(img_mat, k).reshape(h, w)

    # duplicate to create image to RGB
    assignment = np.asarray(np.dstack((assignment, assignment, assignment)), dtype=np.float32) / (k - 1 + 1e-20)
    plt.imshow(assignment)
    plt.show()

if __name__ == '__main__':
    main()


import os
from colorama import init, Fore, Back, Style
from .localizations import LOCALS

init(autoreset=True)

class Method:
    """Wrapper for functions which used in app."""

    def __init__(self, descr, function, inline=False):
        self.descr = descr
        self.function = function
        self.inline = inline

class ConsoleApp:
    def __init__(
        self,
        title: str = "ConsoleApp",
        subTitle: str = None,
        printHead: bool = True,
        printMethods: bool = False,
        width: int = 45,
        maxMethodLen: int = 9,
        onStart = None,
        onExit = None,
        lang = "eng",
        **kwargs
    ):
        self.__title = title
        self.__subTitle = subTitle
        self.__printHead = printHead
        self.__printMethods = printMethods
        if width < 40:
            self.__width = 40
        else:
            self.__width = width
        if maxMethodLen < 5:
            self.__maxMethodLen = 6
        else:
            self.__maxMethodLen = maxMethodLen + 1
        self.__onStart = onStart
        self.__onExit = onExit
        if LOCALS.get(lang) is None:
            self.__lang = "eng"
        else:
            self.__lang = lang
        self.__methods = kwargs

    def changeSubTitle(self, newSub: str):
        self.__subTitle = newSub
        self.__clear()

    def __printLine(self):
        print("─" * self.__width)

    def __helpMethod(self):
        for title, method in self.__methods.items():
            print("{0:>{2}} - {1}".format(title, method.descr, self.__maxMethodLen))
        print("{0:>{2}} - {1}".format("help", LOCALS[self.__lang]["help"], self.__maxMethodLen))
        print("{0:>{2}} - {1}".format("clear", LOCALS[self.__lang]["clear"], self.__maxMethodLen))
        print("{0:>{2}} - {1}".format("exit", LOCALS[self.__lang]["exit"], self.__maxMethodLen))

    def __printMethodsFunc(self):
        if not self.__printHead:
            self.__printLine()

        methods = list(self.__methods.keys())
        methods.extend(["help", "clear", "exit"])
        for index, method in enumerate(methods):
            if not (index + 1) % (self.__width // self.__maxMethodLen) or index == len(methods) - 1:
                print(f"{method:<{self.__maxMethodLen}}")
            else:
                print(f"{method:<{self.__maxMethodLen}}", end="")

        self.__printLine()

    def __printHeadFunc(self):
        print(Fore.CYAN + f"{self.__title:^{self.__width}}")
        if self.__subTitle:
            print(Fore.GREEN + f"{self.__subTitle:^{self.__width}}")

        hint = LOCALS[self.__lang]["hint"]
        print(f"{hint:^{self.__width}}")
        self.__printLine()

    def __clear(self):
        os.system("cls")
        if self.__printHead:
            self.__printHeadFunc()
        if self.__printMethods:
            self.__printMethodsFunc()

    def run(self):
        os.system(f"mode con cols={self.__width} lines={self.__width//2}")

        if self.__printHead:
            self.__printHeadFunc()
        if self.__printMethods:
            self.__printMethodsFunc()
        if self.__onStart:
            try:
                self.__onStart()
            except Exception as exc:
                print(Fore.RED + LOCALS[self.__lang]["error"], exc)

        while True:
            userInput = str(input(">"))
            userInput = userInput.split()

            try:
                command = userInput[0]
            except IndexError as exc:
                command = ""

            method = self.__methods.get(command)
            if method is not None:
                self.__printLine()
                if not method.inline:
                    try:
                        method.function()
                    except Exception as exc:
                        print(Fore.RED + LOCALS[self.__lang]["error"], exc)
                else:
                    try:
                        method.function(userInput[1:])
                    except IndexError as exc:
                        print(Fore.YELLOW + LOCALS[self.__lang]["args_error"])
                    except Exception as exc:
                        print(Fore.RED + LOCALS[self.__lang]["error"], exc)
                self.__printLine()
            elif command == "exit":
                if self.__onExit:
                    try:
                        self.__onExit()
                    except Exception as exc:
                        print(Fore.RED + LOCALS[self.__lang]["error"], exc)
                break
            elif command == "help":
                self.__printLine()
                self.__helpMethod()
                self.__printLine()
            elif command == "clear":
                self.__clear()
            else:
                print(LOCALS[self.__lang]["unknown_command"])


# -*- coding: utf-8 -*-
# __author__ = 'qingmei'
# last modify 20180517

from biocluster.agent import Agent
from biocluster.tool import Tool
from biocluster.core.exceptions import OptionError
import os

class CircosRef2bitAgent(Agent):
    """
    wgs里基因组浏览器需要这个文件
    """
    def __init__(self,parent):
        super(CircosRef2bitAgent,self).__init__(parent)
        options = [
            {"name": "reffa", "type": "string"},
        ]
        self.add_option(options)
        self.step.add_steps('CircosRef2bit')
        self.on('start', self.step_start)
        self.on('end', self.step_end)

    def step_start(self):
        self.step.CircosRef2bit.start()
        self.step.update()

    def step_end(self):
        self.step.CircosRef2bit.finish()
        self.step.update()

    def check_options(self):
        if not self.option("reffa"):
            raise OptionError("请设置reffa", code="34501301") # 必须有

    def set_resource(self):
        """
        运行所需资源
        """
        self._cpu = 2
        self._memory = '3G'

    def end(self):
        super(CircosRef2bitAgent,self).end()
########

class CircosRef2bitTool(Tool):
    def __init__(self, config):
        super(CircosRef2bitTool, self).__init__(config)
        self.ref2bit_path = '/bioinfo/align/ucsc_tools/faToTwoBit'  # self.config.SOFTWARE_DIR 就是到app/的路径

    def CircosRef2bit(self):
        """
        要重新写下！！！
        :return:
        """
        cmd = "{} {} {}"\
            .format(self.ref2bit_path, self.option("reffa"), self.output_dir + "/ref.2bit")
        self.logger.info(cmd)
        self.logger.info("开始进行CircosRef2bit")
        command = self.add_command("circosref2bit", cmd).run()  # nranno必须小写，
        self.wait(command)
        if command.return_code == 0:
            self.logger.info(cmd)
            self.logger.info("CircosRef2bit完成！")
        else:
            self.set_error("CircosRef2bit出错！", code="34501301")
            self.set_error("CircosRef2bit出错！", code="34501304")

    def run(self):
        super(CircosRef2bitTool, self).run()
        self.CircosRef2bit()
        self.end()


# These problems have no tests in the judge system. The solution gives identical results like the tests from
# the description file

import re
import string

chars_pattern = rf'[{string.ascii_letters}]'
non_chars_pattern = rf'[{string.punctuation}]'

output = []

with open('02_Line_Numbers_INPUT.txt') as f:
    lines = f.readlines()
    for idx, line in enumerate(lines):
        line = line.strip()

        chars = len(re.findall(chars_pattern, line))
        non_chars = len(re.findall(non_chars_pattern, line))

        line_final = f'Line {idx + 1}: {line} ({chars}) ({non_chars})'

        output.append(line_final)

with open('02_Line_Numbers_OUTPUT.txt', 'w') as f:
    output_final = '\n'.join(output)
    f.write(output_final)


#!/usr/bin/env python
# coding=utf-8
'''
@author: yandeqing
@date:  2020/7/29 16:58
'''
import time

import requests

import Config
import industry_moneyflow_eastmoney

def upload_one(dest_info, type):
    dest_info['type'] = type
    response = requests.post(Config.url, json=dest_info, headers={'Connection': 'close'})
    print(f"insert {dest_info}{response.text}")

def is_trade_day(debug=False):
    if debug:
        return True
    date = industry_moneyflow_eastmoney.get_latest_date()
    time_strftime = time.strftime("%Y-%m-%d", time.localtime())
    if date != time_strftime:
        print(f"今天{time_strftime}不是交易日")
    return date == time_strftime

if __name__ == '__main__':
    print(is_trade_day())


import tensorflow as tf
from datetime import datetime
from transformers import BertTokenizerFast
import tensorflow.keras as keras
from tensorflow.keras.utils import to_categorical
from sklearn.feature_extraction.text import TfidfVectorizer

#Watch the tutorial first

class DNN:
    def __init__(self, input_shape, output,learning_rate):
        ''' Make sure the data is an Pandas Dataframe'''
        self.input_shape = input_shape
        self.output = output
        self.model = self.build_model() # Building Model

        #SET UP
        self.optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
        self.loss_fn = tf.keras.losses.CategoricalCrossentropy()
        #Set up TensorBoard callback
        log_dir = "tensorboard tutorial"
        self.tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)

    def build_model(self):
        ''' Construct a Basic Deep Neural Network
            The first layer has 128 input neurons, which take 1574 input_shape
            The hidden layer has 64 neurons,
            the output layer has 6 neurons
        '''
        model = keras.Sequential([
            keras.layers.Dense(128, activation='relu', input_shape=(self.input_shape,)),
            keras.layers.Dense(64, activation='relu'),
            keras.layers.Dense(self.output, activation='softmax')])
        loss_fn = tf.keras.losses.CategoricalCrossentropy()

        model.compile(optimizer='adam', loss=loss_fn, metrics=['accuracy'])

        return model


class Round(object):

    def __init__(self, roundId, tournamentId, startTime, ratingLevel):
        self.roundId = roundId
        self.tournamentId = tournamentId
        self.startTime = startTime
        self.ratingLevel = ratingLevel

    def __str__(self):
        return ("{roundId: " + str(self.roundId) +
                ", tournamentId: " + self.tournamentId +
                ", startTime: " + self.startTime +
                ", ratingLevel: " + self.ratingLevel + "}")


class Solution:
    def getIntersectionNode(self, headA: ListNode, headB: ListNode) -> Optional[ListNode]:
        if headA is None or headB is None:
            return None

        pa = headA
        pb = headB

        while pa is not pb:
            if pa is None:
                pa = headB
            else:
                pa=pa.next
            if pb is None:
                pb = headA
            else:
                pb=pb.next

        return pa


"""Test the refitting functionality of an automl instance"""
from typing import Callable, Union

from itertools import repeat

import numpy as np

from autosklearn.automl import AutoML
from autosklearn.data.validation import InputValidator

from pytest_cases import parametrize
from unittest.mock import Mock

@parametrize("budget_type", [None, "iterations"])
def test_shuffle_on_fail(
    budget_type: Union[None, str],
    make_automl: Callable[..., AutoML],
) -> None:
    """
    Parameters
    ----------
    budget_type : Union[None, str]
        The budget type to use

    Fixtures
    --------
    make_automl : Callable[..., AutoML]
        Factory to make an AutoML instance

    Expects
    -------
    * The automl should not be able to fit before `refit`
    * The model should be attempted to fitted `n_fails` times before successing once
     after
     * The automl should be able to fit after `refit`
    """
    n_fails = 3
    failing_model = Mock()
    failing_model.fit.side_effect = [ValueError()] * n_fails + [None]  # type: ignore
    failing_model.estimator_supports_iterative_fit.side_effect = repeat(False)

    ensemble_mock = Mock()
    ensemble_mock.get_selected_model_identifiers.return_value = [(1, 1, 50.0)]

    X = np.ones((3, 2))
    y = np.ones((3,))

    input_validator = InputValidator()
    input_validator.fit(X, y)

    auto = make_automl()
    auto.ensemble_ = ensemble_mock  # type: ignore
    auto.models_ = {(1, 1, 50.0): failing_model}
    auto._budget_type = budget_type
    auto.InputValidator = input_validator

    assert not auto._can_predict
    auto.refit(X, y)

    assert failing_model.fit.call_count == n_fails + 1
    assert auto._can_predict


# -*- coding: utf-8 -*-
from django import http
from django.utils.http import is_safe_url

from . import geo

def set_country(request):
    """
    Sets the chosen country in the session or cookie.

    If `next' query param is present, it redirects to a given url.
    """
    if request.method == 'POST':
        next = request.POST.get('next', request.GET.get('next'))
        if is_safe_url(url=next, host=request.get_host()):
            response = http.HttpResponseRedirect(next)
        else:
            response = http.HttpResponse()

        country_code = request.POST.get('country', '').upper()
        if country_code != geo.get_supported_country(country_code):
            return http.HttpResponseBadRequest()

        if hasattr(request, 'session'):
            request.session[geo.COUNTRY_SESSION_KEY] = country_code
        else:
            response.set_cookie(geo.COUNTRY_COOKIE_NAME,
                               country_code,
                               max_age=geo.COUNTRY_COOKIE_AGE,
                               path=geo.COUNTRY_COOKIE_PATH)
        return response
    else:
        return http.HttpResponseNotAllowed(['POST'])


#importare moduli
import math
import time

#chiede all'utente l'anno
anno = int(input("Inserire l'anno: "))

#the Anonymous Gregorian Computus algorithm
a = anno%19
b = math.floor(anno/100)
c = anno%100
d = math.floor(b/4)
e = b%4
f = math.floor((b+8)/25)
g = math.floor((b-f+1)/3)
h = (19*a+b-d-g+15)%30
i = math.floor(c/4)
k = c%4
l = (32+2*e+2*i-h-k)%7
m = math.floor((a+11*h+22*l)/451)
mese = math.floor((h+l-7*m+114)/31)
giorno = 1+((h+l-7*m+114)%31)

#Calcolare la data
t = (anno,mese,giorno,0,0,0,6,0,0)
data = time.asctime(t)

#Stampa la risposta:
print("Pasqua dell'anno inserito avviene il ", data)


import random
import time

print('''
---------------
.:  12 dices  :.
----------------
 Press enter to
    throw!
----------------
TOTAL | PREVIOUS''')

i = 0
sump = 0
sumd = 0

print('''****************
      YOU       ''')

while sump < 12:
    input(sump)
    d = random.randint(1,6)
    sump += d
    print("             " + '['+ str(d)+ ']')
print (str(sump))

print('''****************
      DEALER       ''')

while sumd < 12:
    print(sumd)
    time.sleep(1)
    d = random.randint(1, 6)
    sumd += d
    print("             " + '[' + str(d) + ']')
print(str(sumd))

print('''****************
        RESULT''')

print('player:', sump)
print('dealer:', sumd)
print('**************')

if sumd == sump:
    print('draw')
elif sump < sumd:
    print('dealer win')
else:
    print('player win')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from PIL import Image
import requests
import io
import os
import time

class SavingImage():
    def __init__(self, scan_info, path_save):
        self.scan_info = scan_info
        self.path_save = path_save

    # save according to list of links
    def SaveImageList(self, url_list):
        for img_url in url_list:
            img_cont = requests.get(img_url).content
            img_file = io.BytesIO(img_cont)
            image = Image.open(img_file)
            img_path = os.path.join( self.path_save, os.path.basename(img_url) )
            with open(img_path, 'wb') as f:
                image.save(f)
            print( f'Img successfully saved: {img_path}' )
            time.sleep( float(self.scan_info['sleep_save']) )


from collections import deque
import sys

N = int(sys.stdin.readline())

while (N > 0):
  N -= 1
  sign = 1
  commands = list(sys.stdin.readline().rstrip())
  k = int(sys.stdin.readline())
  arr = sys.stdin.readline().lstrip('[').rstrip().rstrip(']').split(',')
  if not arr[0] == '':
    arr = list(map(int, arr))
  else:
    arr = []
  _deque = deque(arr)
  for command in commands:
    if command == 'R':
      sign *= -1
    if command == 'D':
      if not _deque:
        _deque = 'error'
        break
      if sign == 1:
        _deque.popleft()
      else:
        _deque.pop()
  if _deque == 'error':
    print('error')
    continue
  _deque = list(_deque)
  if sign == -1:
    _deque.reverse()
  answer = ','.join(map(str, _deque))
  print('[' + answer + ']')

"""
1
D
0
[]
"""


import pickle
import face_encoding
import os
import cfg_manager

def write_pickle_to_disk(save_path, data):
    print('[INFO] save data to pickle')
    f = open(r"%s" % save_path, "wb")
    f.write(pickle.dumps(data))
    f.close()

def load_pickle_from_disk(load_path):
    print('[INFO] loading pickle from disk')
    file = open(r"%s" % load_path, "rb")
    f = pickle.load(file)
    return f

def rebuild_pickle():
    pickledic = get_path()
    success = True
    # 尝试从硬盘中读取pickle
    try:
        data = load_pickle_from_disk(pickledic['embedding'])
    # 捕获异常
    # 尝试通过已有图像重新生成pickle
    except FileNotFoundError:
        flag = face_encoding.do_embedding()
        if flag == 'FaceNum_Error':
            # 程序刹车
            success = False
        else:
            # 成功重新生成则再次读取pickle
            data = load_pickle_from_disk(pickledic['embedding'])
    return success

def get_path():
    pickle_directory = cfg_manager.read_cfg('Common', 'pickle_directory')
    empickle = os.path.sep.join([pickle_directory, "embeddings.pickle"])
    lepickle = os.path.sep.join([pickle_directory, "le.pickle"])
    recopickle = os.path.sep.join([pickle_directory, "recognizer.pickle"])
    pickledic = {'embedding': empickle, 'le': lepickle, 'recognizer': recopickle}
    return pickledic