# model-related
from django.db.models import Q
from django.utils import timezone
from django.conf import settings
from . import models as m

# functionality
import operator as op
import datetime, random, re, traceback
from autotapta.analyze.Cluster import clusterRangeVars
from autotapta.input.Template import translateCapability
from autotapta.analyze.Analyze import extractEpisodes, extractTriggerCases, extractRevertCases
from autotapta.analyze.Rank import checkIfEpisodeCovered
from autotap.util import update_trace_in_cache
from autotap.variable import update_separation_count

# st-related
from . import capmap
from . import st_util

import json
import webcolors
from celery_workers import tasks

######################################
######################################
## INDEX                            ##
## FES :: FRONTEND / SELECTORS      ##
## STV :: ST-END VIEWS              ##
## RC  :: RULE CREATION             ##
## SPC :: SAFETY PROPERTY CREATION  ##
## SLM :: STATELOG MANAGEMENT       ##
## DM  :: DEVICE MANAGEMENT         ##
######################################
######################################

################################################################################
## [FES] FRONTEND / SELECTORS
################################################################################

def get_or_make_user(code,mode):
    try:
        user = m.User_ICSE19.objects.get(code=code)
    except m.User_ICSE19.DoesNotExist:
        user = m.User_ICSE19(code=code,mode=mode)
        user.save()
    return user

### @TO_DELETE: Potentially dead code
# return id:name dict of all devices
# def all_devs(user):
#     json_resp = {}
#     for dev in m.Device.objects.filter(users__contains=user):
#         json_resp[dev.id] = dev.name
#     return json_resp

### @TO_DELETE: Potentially dead code
# def user_devs(user):
#     if isinstance(user, m.User):
#         mydevs = m.Device.objects.filter(users__contains=user)
#     else:
#         mydevs = m.Device.objects.filter(owner=user)
#     pubdevs = m.Device.objects.filter(public=True)
#     return mydevs.union(pubdevs)

# return id:name dict of all channels
def valid_chans(user,is_trigger):
    json_resp = {'chans' : []}
    # for integration of user and user_icse19
    if isinstance(user, m.User):
        chans = m.Channel.objects.filter(Q(device__users__in=[user])).distinct().order_by('name')
        devs = m.Device.objects.filter(Q(users__in=[user]))
    else:
        chans = m.Channel.objects.filter(Q(capability__device__public=True)).distinct().order_by('name')
        devs = m.Device.objects.filter(Q(public=True))
    chans = filter(lambda x : chan_has_valid_cap(x,devs,is_trigger),chans)
    for chan in chans:
        json_resp['chans'].append({'id' : chan.id, 'name' : chan.name, 'icon' : chan.icon})
    return json_resp

def dev_has_valid_cap(dev,channel,is_trigger):
    poss_caps = dev.caps.all().intersection(m.Capability.objects.filter(channels=channel))
    if is_trigger:
        return any(map(lambda x : x.readable,poss_caps))
    else:
        return any(map(lambda x : x.writeable,poss_caps))

def dev_has_valid_cap_no_channel(dev, is_trigger):
    poss_caps = dev.caps.all()
    if is_trigger:
        return any(map(lambda x : x.readable,poss_caps))
    else:
        return any(map(lambda x : x.writeable,poss_caps))

def chan_has_valid_cap(chan,devs,is_trigger):
    dev_caps = m.Capability.objects.none()
    for dev in devs:
        dev_caps = dev_caps.union(dev.caps.all())
    poss_caps = m.Capability.objects.filter(channels=chan).intersection(dev_caps)
    if is_trigger:
        return any(map(lambda x : x.readable,poss_caps))
    else:
        return any(map(lambda x : x.writeable,poss_caps))

def rwfilter_caps(caps,is_trigger):
    if is_trigger:
        return filter((lambda x : x.readable),caps)
    else:
        return filter((lambda x : x.writeable),caps)

def map_labels(caps,is_trigger,is_event):
    if is_trigger:
        if is_event:
            return list(map((lambda x : (x.id,x.name,x.eventlabel)),caps))
        else:
            return list(map((lambda x : (x.id,x.name,x.statelabel)),caps))
    else:
        return list(map((lambda x : (x.id,x.name,x.commandlabel)),caps))

def filtermap_caps(caps,is_trigger,is_event):
    return map_labels(rwfilter_caps(caps,is_trigger),is_trigger,is_event)

###############################################################################
## [STV] ST-END VIEWS
###############################################################################

def is_valid_value(param,val):
    # not case sensitive
    val = val.lower()

    if param.type == 'set':
        try:
            m.SetParamOpt.objects.get(param_id=param.id,value=val)
            return True
        except m.SetParamOpt.DoesNotExist:
            return False
    elif param.type == 'range':
        try:
            floval = float(val)
        except ValueError:
            return False
        return (param.rangeparam.min <= floval <= param.rangeparam.max)
    elif param.type == 'bin':
        # not case sensitive
        return (param.binparam.tval.lower() == val or param.binparam.fval.lower() == val)
    elif param.type == 'input':
        if param.inputparam.inputtype == 'int':
            try:
                intval = int(val)
            except ValueError:
                return False
            else:
                return True
        else:
            return True
    else:
        #this shouldn't happen
        return False

def test_cond(cond,value):
    # for solving the case sensitive problem,
    # probably there's a better way to enforce this
    processed_value = value
    processed_cond_value = cond.val
    if cond.par.type == 'range':
        processed_value = float(processed_value)
        processed_cond_value = float(processed_cond_value)

    # timer support begin   -- by Bo Wang
    if cond.par.type == 'time':
        processed_value = processed_value.replace(':', '')
        processed_cond_value = processed_cond_value.replace(':', '')
    # timer support end

    if isinstance(value, str) and isinstance(cond.val, str):
        processed_value = value.lower()
        processed_cond_value = cond.val.lower()

    if cond.comp == '=':
        return op.eq(processed_value, processed_cond_value)
    elif cond.comp == '!=':
        return op.neq(processed_value, processed_cond_value)
    elif cond.comp == '<':
        return op.lt(processed_value, processed_cond_value)
    elif cond.comp == 'lte':
        return op.lte(processed_value, processed_cond_value)
    elif cond.comp == '>':
        return op.gt(processed_value, processed_cond_value)
    elif cond.comp == 'gte':
        return op.gte(processed_value, processed_cond_value)
    else:
        return None

def test_cond_dict(cond_par, cond_val, value):
    # for solving the case sensitive problem,
    # probably there's a better way to enforce this
    processed_value = value
    processed_cond_value = cond_val['value']
    if cond_par['type'] == 'range':
        processed_value = float(processed_value)
        processed_cond_value = float(processed_cond_value)

    # timer support begin   -- by Bo Wang
    if cond_par['type'] == 'time':
        processed_value = processed_value.replace(':', '')
        processed_cond_value = processed_cond_value.replace(':', '')
    # timer support end

    if isinstance(value, str) and isinstance(cond_val['value'], str):
        processed_value = value.lower()
        processed_cond_value = cond_val['value'].lower()

    if cond_val['comparator'] == '=':
        return op.eq(processed_value, processed_cond_value)
    elif cond_val['comparator'] == '!=':
        return op.neq(processed_value, processed_cond_value)
    elif cond_val['comparator'] == '<':
        return op.lt(processed_value, processed_cond_value)
    elif cond_val['comparator'] == 'lte':
        return op.lte(processed_value, processed_cond_value)
    elif cond_val['comparator'] == '>':
        return op.gt(processed_value, processed_cond_value)
    elif cond_val['comparator'] == 'gte':
        return op.gte(processed_value, processed_cond_value)
    else:
        return None

### @TO_DELETE: Potentially dead code
# determine if all Conditions of a Trigger are currently satisfied
# def check_trigger(trigger):
#     state = m.State.objects.get(cap=trigger_cap,dev=trigger_dev)
#     for cond in m.Condition.objects.filter(trigger=trigger):
#         try:
#             pv = m.ParVal.objects.get(state=state,par=cond_par)
#             if not test_cond(cond,pv.val):
#                 return False
#         except m.ParVal.DoesNotExist:
#             return False

#     return True

def update_state(cap,dev,param,val):
    '''
    Update/create PV corresponding to state of a given cap/dev pair. \n
    The state here stands for the current state of the device. \n
    Input:
        cap:    the related capability
        dev:    the related device
        param:  the parameter that should be updated
        val:    the new value of the parameter
    Output:
        pv:     the updated/created ParVal object
    '''
    # when is_action is False, the state is the state of a device's capability
    state = get_or_make_state(cap, dev, val, is_action=False)
    # ParVal specifies that for one specific state, what param/value it currently has
    try:
        pv = m.ParVal.objects.get(state=state,par=param)
        if param.type == 'color':
            hue, saturation = pv.val.split(',')
            target_hue, target_saturation = val.split(',')
            hue = hue if target_hue == '*' else target_hue
            saturation = saturation if target_saturation == '*' else target_saturation
            val = '%s,%s' % (hue, saturation)
        pv.val = val
        pv.save()
    except m.ParVal.DoesNotExist:
        pv = m.ParVal.objects.create(state=state,par=param,val=val)

    return pv

# @TODO: Get rid of `is_current` or don't use `State` as current state of the device
def update_log(cap, dev, parvals):
    '''
    Log the new event in StateLog and inactivate the old state.\n
    Input:
        cap     : the capability to be logged
        dev     : the device to be logged
        parvals : a list of ParVal objects
    Output:
        None
    '''
    # mark old log inactive (if one exists), and create new log
    for parval in parvals:
        is_superifttt = False
        # if there is a state that is triggered by superifttt but not confirmed yet
        # if it's a to-occur state, it has to be within 2 minutes
        old_entries = m.StateLog.objects.filter(
            timestamp__gte=get_now() - datetime.timedelta(seconds=120),
            status=m.StateLog.TO_OCCUR,
            cap=cap,
            dev=dev,
            loc=dev.location,
            param=parval.par,
            value=parval.val,
            is_superifttt=True
        ).all()
        if len(old_entries) > 0:
            # if there are TO_OCCUR StateLog, then delete them all and set is_superifttt to True
            old_entries.delete()
            is_superifttt = True

        # search for the current value of the param, change all of them to HAPPENED
        entries = m.StateLog.objects.filter(
            status = m.StateLog.CURRENT,
            cap = cap,
            dev = dev,
            loc = dev.location,
            param = parval.par
        ).all()
        # set all the found entries (although there should either be only one or none) to HAPPENED
        for entry in entries:
            entry.status = m.StateLog.HAPPENED
            entry.save()

        # if the old state is inactivated or cannot find a matching statelog,
        # then create a new entry
        state_log = m.StateLog.objects.create(
            status=m.StateLog.CURRENT,
            cap=cap,
            dev=dev,
            loc=dev.location,
            param=parval.par,
            value=parval.val,
            is_superifttt=is_superifttt
        )

        update_separation_count(state_log)
        update_trace_in_cache(state_log)
        schedule_history_triggered(state_log)
    return

def stval_to_val(param: m.Parameter, st_val):
    if param.type in ('bin', 'set'):
        try:
            return m.ParValMapping.objects.get(param=param, st_val=st_val).val
        except m.ParValMapping.DoesNotExist:
            err_msg = "cannot find error mapping for %s of parameter %s" % (st_val, param.name)
            m.ErrorLog.objects.create(err=err_msg)
            raise Exception(err_msg)
    elif param.type in ('range', 'color'):
        return st_val
    else:
        err_msg = "param type %s not supported yet" % param.type
        m.ErrorLog.objects.create(err=err_msg)
        raise Exception(err_msg)

def val_to_stval(param: m.Parameter, val):
    if param.type in ('bin', 'set'):
        try:
            return m.ParValMapping.objects.get(param=param, val=val).st_val
        except m.ParValMapping.DoesNotExist:
            err_msg = "cannot find error mapping for %s of parameter %s" % (val, param.name)
            m.ErrorLog.objects.create(err=err_msg)
            raise Exception(err_msg)
    elif param.type in ('range', 'color'):
        return val
    else:
        err_msg = "param type %s not supported yet" % param.type
        m.ErrorLog.objects.create(err=err_msg)
        raise Exception(err_msg)

def check_s_triggers(s_trigger_list):
    for s_trig in s_trigger_list:
        # state = m.State.objects.get(cap=s_trig.cap, dev=s_trig.dev)
        # conditions = m.Condition.objects.filter(trigger=s_trig)
        # for cond in conditions:
        #     try:
        #         val = m.ParVal.objects.get(par=cond.par, state=state).val
        #         if not test_cond(cond, val):
        #             return False
        #     except m.ParVal.DoesNotExist:
        #         err_msg = "unable to get current state of %s" % str(state)
        #         m.ErrorLog.objects.create(err=err_msg)
        #         raise Exception(err_msg)
        check_trig_func = check_history_s_trig if check_trigger_is_history(s_trig) else check_regular_s_trig
        if not check_trig_func(s_trig):
            return False

    return True

def check_trigger_is_history(trig):
    cap = trig.cap
    # TODO: currently use hard-coding. later we should
    # add another field to indicate history trigger
    if cap.name == 'Time Since State':
        return True
    else:
        return False

def check_regular_s_trig(s_trig):
    state = m.State.objects.get(cap=s_trig.cap, dev=s_trig.dev)
    conditions = m.Condition.objects.filter(trigger=s_trig)
    for cond in conditions:
        try:
            val = m.ParVal.objects.get(par=cond.par, state=state).val
            if not test_cond(cond, val):
                return False
        except m.ParVal.DoesNotExist:
            err_msg = "unable to get current state of %s" % str(state)
            m.ErrorLog.objects.create(err=err_msg)
            raise Exception(err_msg)
    return True

def check_history_s_trig(s_trig):
    conditions = list(m.Condition.objects.filter(trigger=s_trig))  # should have a time condition and a trigger condition
    if len(conditions) != 2:
        err_msg = "Time since state should have 2 conditions!"
        raise Exception(err_msg)

    time_in_secs = None
    comparator = None
    trigger_clause = None
    for cond in conditions:
        if cond.par.type == "duration":
            time_clause = json.loads(cond.val.replace('\'', '\"'))
            time_in_secs = time_to_int(time_clause)
            comparator = cond.comp
        elif cond.par.type == "meta":
            trigger_clause = json.loads(cond.val.replace('\'', '\"'))

    trigger_parameters = trigger_clause['parameters']
    trigger_parameterVals = trigger_clause['parameterVals']
    trigger_cap_id = trigger_clause['capability']['id']
    trigger_dev_id = trigger_clause['device']['id']

    if len(trigger_parameters) != 1 or len(trigger_parameterVals) != 1:
        err_msg = "Trigger in history channel should only have one parameter"
        m.ErrorLog.objects.create(err=err_msg)
        raise Exception(err_msg)

    trigger_parameter = trigger_parameters[0]
    trigger_parameterVal = trigger_parameterVals[0]

    current_time_utc = datetime.datetime.utcnow()
    start_time = current_time_utc - datetime.timedelta(seconds=time_in_secs)

    # check current status
    state = m.State.objects.get(cap__id=trigger_cap_id, dev__id=trigger_dev_id)
    current_parval = m.ParVal.objects.get(par__id=trigger_parameter['id'], state=state)

    if not test_cond_dict(trigger_parameter, trigger_parameterVal, current_parval.val):
        return False

    # check history
    # if comp == '>', state should always be true from start_time to current_time_utc
    # if comp == '<', state should once be false from start_time to current_time_utc, but finally is true
    post_state_logs = m.StateLog.objects.filter(dev__id=trigger_clause['device']['id'],
                                                cap__id=trigger_clause['capability']['id'],
                                                timestamp__gte=start_time,
                                                status__in=(m.StateLog.CURRENT, m.StateLog.HAPPENED))

    if_state_switch = False
    for state_log in post_state_logs:
        sl_param = state_log.param
        sl_value = state_log.value
        for cond_par, cond_val in zip(trigger_parameters, trigger_parameterVals):
            if test_cond_dict(cond_par, cond_val, sl_value):
                # such thing will neglect the condition when comp == '>'
                if_state_switch = True
                break

    if if_state_switch == (comparator == '<'):
        return True
    else:
        return False

def check_if_rule_triggered(cap, dev, pvs, esrule):
    pvs_dict = dict(pvs)

    # check if Etrigger is met
    if cap != esrule.Etrigger.cap or dev != esrule.Etrigger.dev:
        return False
    conditions = m.Condition.objects.filter(trigger=esrule.Etrigger)
    for cond in conditions:
        val = pvs_dict[cond.par]
        if not test_cond(cond, val):
            return False

    # check if Striggers are met
    return check_s_triggers(esrule.Striggers.all())

def schedule_history_triggered(log_obj: m.StateLog):
    '''
    should be called by update_log, to check if history trigger activated.
    '''
    history_cap = capmap.lookup('Time Since State', 'vd')
    assert len(history_cap) == 1
    history_cap = history_cap[0]
    dev_obj = log_obj.dev
    loc_obj = dev_obj.location
    st_installed_app_id = loc_obj.st_installed_app_id
    history_esrules = m.ESRule.objects.filter(Etrigger__cap=history_cap, st_installed_app_id=st_installed_app_id)
    # log_obj .cap  .dev  .param   .value   .valuetype

    schedule_time_list = list()
    for history_rule in history_esrules:
        conds = m.Condition.objects.filter(trigger=history_rule.Etrigger)
        assert len(conds) == 2 # Etrigger should have only one cond.
        history_trigger_pvs = dict()

        for cond in conds:
            history_trigger_pvs[cond.par.name] = cond.val

        # trigger_pvs should be this now:
        # {"trigger": "...<trigger in json>...", "time": <duration value> }
        event_trigger_obj = json.loads(history_trigger_pvs['trigger'].replace('\'', '\"'))  # ugly fix of json format
        event_trigger_parameters = event_trigger_obj['parameters']
        event_trigger_parameterVals = event_trigger_obj['parameterVals']
        event_trigger_cap_id = event_trigger_obj['capability']['id']
        event_trigger_dev_id = event_trigger_obj['device']['id']
        if log_obj.cap.id != event_trigger_cap_id:
            continue # this rule is not what we want
        if log_obj.dev.id != event_trigger_dev_id:
            continue

        current_rule_match = True
        # param = log_obj.param
        value = log_obj.value
        for cond_par, cond_val in zip(event_trigger_parameters, event_trigger_parameterVals):
            if not test_cond_dict(cond_par, cond_val, value):
                current_rule_match = False
                break

        if current_rule_match:
            time_clause = json.loads(history_trigger_pvs['time'].replace('\'', '\"'))
            count_down_time = time_to_int(time_clause) # translate this into seconds
            if count_down_time not in schedule_time_list:
                schedule_time_list.append(count_down_time)

    # schedule time events
    for count_down_time in schedule_time_list:
        print("# [HISTORY] schedule_history_triggered schedule:" + str((log_obj.id, count_down_time)))
        tasks.triggering_history_event.apply_async((log_obj.id, count_down_time), countdown=count_down_time)

def st_process_update(kwargs):
    '''
    Update states and find out what action should be triggered.\n
    Input:
        - kwargs: a dict of the new event occurred
        {
            "st_installed_app_id": the id of the installed SmartThings app instance,
            "capid": capability id in this project,
            "deviceid": device id in this project,
            "values": { <st_param_id>: <parval_value>, ... }
        }
    Output:
        - json_resp: a dict of actions that should be triggered
        {
            "actions": [{
                    "capability": "<commandname>",
                    "devid": "<device proj id>",
                    "values": { "<st_param_id>": "<parval_value>", ...}
                }, ...],
            "errors": [{"msg": "<error_details>"}, ...]
        }
    Note: Only devices that are not fake will be returned
    '''
    json_resp = {"actions": [], "errors": []}

    cap = capmap.lookup(kwargs['capid'],'st')
    if len(cap) > 0:
        cap = cap[0] # currently let's assume there's only one capability...
    else:
        m.ErrorLog.objects.create(err="st_process_update > capmap: Cannot find the corresponding capability: " + kwargs['capid'])
        return json_resp
    try:
        dev = m.Device.objects.get(id=kwargs['deviceid'])
        pvdict = kwargs['values'] #{st_param_id : value}
        if kwargs['capid'] == 'colorControl':
            hue = pvdict['hue'] if 'hue' in pvdict else '*'
            saturation = pvdict['saturation'] if 'saturation' in pvdict else '*'
            color_val = '%s,%s' % (hue, saturation)
            pvdict = {'color': color_val}
        pvs = [(m.Parameter.objects.get(sysname=pv_id), stval_to_val(m.Parameter.objects.get(sysname=pv_id), pvdict[pv_id])) for pv_id in pvdict]
    except m.Device.DoesNotExist:
        err_detail = "Cannot find the device: {}.".format(kwargs['deviceid'])
        json_resp["errors"].append(get_error_msg("st_process_update", err_detail, status=404))
        return json_resp
    except m.Parameter.DoesNotExist:
        err_detail = "Cannot find a parameter that's corresponding to '{}'.".format(pv_id)
        json_resp["errors"].append(get_error_msg("st_process_update", err_detail, status=500))
        return json_resp

    for param, val in pvs:
        # update the related par-val pair (trigger)
        pv = update_state(cap,dev,param,val)
        # assume that superifttt will only be installed once per home
        update_log(cap, dev, [pv])

    # ------------------------------------------------------------------------
    # --------- below: consider using check_actions_and_get_json -------------
    st_installed_app_id = kwargs['st_installed_app_id']
    json_resp = check_actions_and_get_json(st_installed_app_id, cap, dev, pvs)
    return json_resp
    # -------- before: consider using check_actions_and_get_json -------------
    # ------------------------------------------------------------------------

# def get_state_parval(cap, dev, param) (in branch production-dev-timer. useless now.)

def vd_process_update(kwargs):
    '''
    Update states of virtual devices and find out what action should be triggered.\n
    Input:
        - kwargs: a dict of the new event occurred
        {
            "st_installed_app_id": the id of the installed SmartThings app instance,
            "capid": capability id in this project,
            "deviceid": device id in this project,
            "values": { <st_param_id>: <parval_value>, ... }
        }
    Output:
        - json_resp: a dict of actions that should be triggered
        {
            "actions": [{
                    "capability": "<commandname>",
                    "devid": "<device proj id>",
                    "values": { "<st_param_id>": "<parval_value>", ...}
                }, ...],
            "errors": [{"msg": "<error_details>"}, ...]
        }
    '''
    json_resp = {"actions": [], "errors": []}

    cap = capmap.lookup(kwargs['capid'],'vd')
    if len(cap) > 0:
        cap = cap[0] # currently let's assume there's only one capability...
    else:
        m.ErrorLog.objects.create(err="vd_process_update > capmap: Cannot find the corresponding capability." + kwargs['capid'])
        return json_resp
    try:
        dev = m.Device.objects.get(id=kwargs['deviceid'])
        pvdict = kwargs['values'] #{st_param_id : value}
        pvs = [(m.Parameter.objects.get(name=pv_id, cap=cap), pvdict[pv_id]) for pv_id in pvdict]
    except m.Device.DoesNotExist:
        err_detail = "Cannot find the device: {}.".format(kwargs['deviceid'])
        json_resp["errors"].append(get_error_msg("vd_process_update", err_detail, status=404))
        return json_resp
    except m.Parameter.DoesNotExist:
        err_detail = "Cannot find a parameter that's corresponding to '{}'.".format(pv_id)
        json_resp["errors"].append(get_error_msg("vd_process_update", err_detail, status=500))
        return json_resp

    for param, val in pvs:
        # update the related par-val pair (trigger)
        pv = update_state(cap,dev,param,val)
        # assume that superifttt will only be installed once per home
        update_log(cap, dev, [pv])

    # ------------------------------------------------------------------------
    # --------- below: consider using check_actions_and_get_json -------------
    st_installed_app_id = kwargs['st_installed_app_id']
    json_resp = check_actions_and_get_json(st_installed_app_id, cap, dev, pvs)

    if json_resp['actions']:
        st_util.execute_commands_on_core_vd(dev.location, json_resp)
    return json_resp
    # -------- before: consider using check_actions_and_get_json -------------
    # ------------------------------------------------------------------------

def vd_process_update_history(log: m.StateLog, delay_seconds: int):
    # for history events, just check if any rules are triggered. no need to update the state log/state
    device = log.dev
    location = device.location

    json_resp = check_history_actions_and_get_json(log, delay_seconds)

    if json_resp['actions']:
        st_util.execute_commands_on_core_vd(location, json_resp)
    return json_resp

def apply_esrule_action(esrule: m.ESRule):
    # trigger the action
    action = esrule.action # an action
    # exclude devices that are not real
    if action.dev.dev_type == m.Device.FAKEDEV:
        return None

    action_dict = {
        'capability' : capmap.reverse_lookup(action.cap, 'st').name,
        'devid' : action.dev.id,
        'values': {}
    }
    st_cap = capmap.reverse_lookup(action.cap, "st")

    action_pvs = {}
    # find the corresponding Par-Val pair for this action
    action_conditions = m.ActionCondition.objects.filter(action=action)
    for ac in action_conditions:
        action_pvs[ac.par.sysname] = val_to_stval(ac.par, ac.val)
        # create a new TO_OCCUR log and mark it as issued by superifttt
        m.StateLog.objects.create(
            status = m.StateLog.TO_OCCUR,
            cap = action.cap,
            dev = action.dev,
            param = ac.par,
            value = ac.val,
            is_superifttt = True
        )

    action_dict = st_util.st_action_handler[st_cap.command_type](action_pvs)
    action_dict = {**action_dict, **{"capability": st_cap.name, "devid": action.dev.id}}
    return action_dict

def check_actions_and_get_json(st_installed_app_id, cap, dev, pvs, old_pvs=None):
    '''
    add **olf_pvs**. in case that some Etrigger need last time information.
    '''
    json_resp = {"actions": [], "errors": []}
    rule_queryset = m.ESRule.objects.filter(st_installed_app_id=st_installed_app_id)
    for esrule in rule_queryset:
        if check_if_rule_triggered(cap, dev, pvs, esrule):
            # trigger the action
            action_dict = apply_esrule_action(esrule)
            if action_dict:
                json_resp['actions'].append(action_dict)

    return json_resp

def check_history_actions_and_get_json(state_log: m.StateLog, delay_seconds: int):
    # when events corresponding to history channel happens, check whether any rules triggered
    history_cap = capmap.lookup('Time Since State', 'vd')[0]  # should be only one history cap
    device = state_log.dev
    location = device.location
    st_installed_app_id = location.st_installed_app_id

    json_resp = {"actions": [], "errors": []}

    history_esrules = m.ESRule.objects.filter(Etrigger__cap=history_cap, st_installed_app_id=st_installed_app_id)
    for history_rule in history_esrules:
        conds = m.Condition.objects.filter(trigger=history_rule.Etrigger)
        assert len(conds) == 2 # Etrigger should have only one cond.
        history_trigger_pvs = dict()

        for cond in conds:
            history_trigger_pvs[cond.par.name] = cond.val

        event_trigger_obj = json.loads(history_trigger_pvs['trigger'].replace('\'', '\"'))  # ugly fix of json format
        event_trigger_time = json.loads(history_trigger_pvs['time'].replace('\'', '\"'))
        event_trigger_time = time_to_int(event_trigger_time)
        event_trigger_parameters = event_trigger_obj['parameters']
        event_trigger_parameterVals = event_trigger_obj['parameterVals']
        event_trigger_cap_id = event_trigger_obj['capability']['id']
        event_trigger_dev_id = event_trigger_obj['device']['id']

        if state_log.cap.id != event_trigger_cap_id:
            continue # this rule is not what we want
        if state_log.dev.id != event_trigger_dev_id:
            continue
        if event_trigger_time != delay_seconds:
            continue

        rule_triggered = True
        # First check if event trigger is triggered
        post_state_logs = m.StateLog.objects.filter(dev=device, cap__id=state_log.cap.id,
                                                    timestamp__gte=state_log.timestamp,
                                                    status__in=(m.StateLog.CURRENT, m.StateLog.HAPPENED))
        for post_sl in post_state_logs:
            post_sl_param = post_sl.param
            post_sl_value = post_sl.value
            for cond_par, cond_val in zip(event_trigger_parameters, event_trigger_parameterVals):
                if not test_cond_dict(cond_par, cond_val, post_sl_value):
                    rule_triggered = False
                    break

            if not rule_triggered:
                break

        # Next check if state triggers are triggered
        if rule_triggered:
             rule_triggered = check_s_triggers(history_rule.Striggers.all())

        if rule_triggered:
            # trigger the rule
            action_dict = apply_esrule_action(history_rule)
            if action_dict:
                json_resp['actions'].append(action_dict)

    return json_resp

###############################################################################
## [RC] RULE CREATION
###############################################################################

def get_or_make_state(cap, dev, value, is_action=False):
    '''
    Get the state of a cap/dev pair, or create one if none exists
    '''
    state, created = m.State.objects.get_or_create(cap=cap,dev=dev,action=is_action)
    state.text = "{}".format(value)
    state.save()
    return state

# @TO_DELETE: Potentially dead code
# def update_pv(state,par_id,val):
#     try:
#         pv = m.ParVal.objects.get(state=state,par_id=par_id)
#         pv.val = val
#         pv.save()
#     except m.ParVal.DoesNotExist:
#         pv = m.ParVal(state=state,par_id=par_id,val=val)
#         pv.save()

###############################################################################
## [SPC] SAFETY PROPERTY CREATION
###############################################################################

def time_to_int(time):
    return time['seconds'] + time['minutes']*60 + time['hours']*3600

def int_to_time(secs):
    time = {}
    time['hours'] = secs // 3600
    time['minutes'] = (secs // 60) % 60
    time['seconds'] = secs % 60
    return time

def clause_to_trigger(clause):
    if 'channel' in clause and 'id' in clause['channel']:
        t = m.Trigger(chan=m.Channel.objects.get(id=clause['channel']['id']),
                    dev=m.Device.objects.get(id=clause['device']['id']),
                    cap=m.Capability.objects.get(id=clause['capability']['id']),
                    pos=clause['id'],
                    text=clause['text'])
    else:
        t = m.Trigger(dev=m.Device.objects.get(id=clause['device']['id']),
                    cap=m.Capability.objects.get(id=clause['capability']['id']),
                    pos=clause['id'],
                    text=clause['text'])
    t.save()

    try:
        pars = clause['parameters']
        vals = clause['parameterVals']
        for par,val in zip(pars,vals):
            cond = m.Condition(par=m.Parameter.objects.get(id=par['id']),
                               val=val['value'],
                               comp=val['comparator'],
                               trigger=t)
            cond.save()
    except KeyError:
        pass

    return t

def clause_to_action(clause):
    if 'channel' in clause and 'id' in clause['channel']:
        a = m.Action(chan=m.Channel.objects.get(id=clause['channel']['id']),
                     dev=m.Device.objects.get(id=clause['device']['id']),
                     cap=m.Capability.objects.get(id=clause['capability']['id']),
                     text=clause['text'])
    else:
        a = m.Action(dev=m.Device.objects.get(id=clause['device']['id']),
                     cap=m.Capability.objects.get(id=clause['capability']['id']),
                     text=clause['text'])
    a.save()

    try:
        pars = clause['parameters']
        vals = clause['parameterVals']
        for par,val in zip(pars,vals):
            cond = m.ActionCondition(par=m.Parameter.objects.get(id=par['id']),
                                     val=val['value'],
                                     action=a)
            cond.save()
    except KeyError:
        pass

    return a

def trigger_to_clause(trigger,is_event):
    chan = {'id' : trigger.chan.id, 'name' : trigger.chan.name, 'icon' : trigger.chan.icon} if trigger.chan else {}
    c = {'channel' : chan,
         'device' : {'id' : trigger.dev.id,
                     'name' : trigger.dev.name,
                     'label': trigger.dev.label},
         'capability' : {'id' : trigger.cap.id,
                         'name' : trigger.cap.name,
                         'label' : (trigger.cap.eventlabel if is_event else trigger.cap.statelabel)},
         'text' : trigger.text,
         'id' : trigger.pos}
    conds = m.Condition.objects.filter(trigger=trigger).order_by('id')
    if conds != []:
        c['parameters'] = []
        c['parameterVals'] = []

        for cond in conds:
            if cond.par.type == 'bin':
                values = [cond.par.binparam.tval, cond.par.binparam.fval]
            else:
                values = []
            val = json.loads(cond.val.replace('\'', '\"')) if cond.par.type in ('meta', 'duration') else cond.val
            c['parameters'].append({'id' : cond.par.id,
                                    'name' : cond.par.name,
                                    'type' : cond.par.type,
                                    'values': values})
            c['parameterVals'].append({'comparator' : cond.comp,
                                       'value' : val})
    return c

def action_to_clause(action):
    chan = {'id' : action.chan.id, 'name' : action.chan.name, 'icon' : action.chan.icon} if action.chan else {}
    c = {'channel' : chan,
         'device' : {'id' : action.dev.id,
                     'name' : action.dev.name,
                     'label': action.dev.label},
         'capability' : {'id' : action.cap.id,
                         'name' : action.cap.name,
                         'label' : action.cap.commandlabel},
         'text' : action.text}
    action_conds = m.ActionCondition.objects.filter(action=action).order_by('id')
    if action_conds != []:
        c['parameters'] = []
        c['parameterVals'] = []

        for cond in action_conds:
            if cond.par.type == 'bin':
                values = [cond.par.binparam.tval, cond.par.binparam.fval]
            else:
                values = []
            val = json.loads(cond.val.replace('\'', '\"')) if cond.par.type in ('meta', 'duration') else cond.val
            c['parameters'].append({'id' : cond.par.id,
                                    'name' : cond.par.name,
                                    'type' : cond.par.type,
                                    'values': values})
            c['parameterVals'].append({'comparator' : '=',
                                       'value' : val})
    return c

def state_to_clause(state):
    chan = {'id' : state.chan.id, 'name' : state.chan.name, 'icon' : state.chan.icon} if state.chan else {}
    c = {'channel' : chan,
         'device' : {'id' : state.dev.id,
                     'name' : state.dev.name,
                     'label': state.dev.label},
         'capability' : {'id' : state.cap.id,
                         'name' : state.cap.name,
                         'label' : state.cap.commandlabel},
         'text' : state.text}

    # pvs = m.ParVal.objects.filter(state=state).order_by('id')
    # if pvs != []:
    #     c['parameters'] = []
    #     c['parameterVals'] = []
    #
    #     for pv in pvs:
    #         c['parameters'].append({'id' : pv.par.id,
    #                                 'name' : pv.par.name,
    #                                 'type' : pv.par.type})
    #         c['parameterVals'].append({'comparator' : '=',
    #                                    'value' : pv.val})
    # escapeFunc = lambda x:re.sub(r'([\.\\\+\*\?\[\^\]\$\(\)\{\}\!\<\>\|\:\-])', r'\\\1', x)
    cap = m.Capability.objects.get(id=state.cap.id)
    par_list = m.Parameter.objects.filter(cap_id=cap.id)
    par_dict = dict()
    for par in par_list:
        values = []
        par_c = dict()
        par_c['id'] = par.id
        par_c['type'] = par.type
        par_c['name'] = par.name
        if par.type == 'bin':
            bin_par = m.BinParam.objects.get(id=par.id)
            par_c['value_list'] = [bin_par.fval, bin_par.tval]
            values = [bin_par.tval, bin_par.fval]
            t_template = r'\{%s/T\|(?P<value>[\w &\-]+)\}' % par.name
            f_template = r'\{%s/F\|(?P<value>[\w &\-]+)\}' % par.name
            t_val = re.search(t_template, cap.commandlabel)
            f_val = re.search(f_template, cap.commandlabel)
            if t_val and f_val:
                par_c['value_list_in_statement'] = [f_val.group('value'), t_val.group('value')]
            else:
                par_c['value_list_in_statement'] = par_c['value_list'] = [bin_par.fval, bin_par.tval]
        elif par.type == 'range':
            par_c['value_list'] = []
        elif par.type == 'set':
            par_c['value_list'] = [opt.value for opt in m.SetParamOpt.objects.filter(param_id=par.id)]
        elif par.type == 'color':
            par_c['value_list'] = [webcolors.CSS3_NAMES_TO_HEX.keys()]
        else:
            raise Exception('var type %s not supported' % par.type)
        par_c['values'] = values

        par_dict[par.name] = par_c

    if state.dev.label is not None and state.dev.label:
        template_text = re.sub(r'\{DEVICE\}', state.dev.label, cap.commandlabel)
    else:
        template_text = re.sub(r'\{DEVICE\}', state.dev.name, cap.commandlabel)
    template_text = re.sub(r'\{(\w+)/(T|F)\|[\w &\-]+\}\{\1/(T|F)\|[\w &\-]+\}', r'(?P<\1>[\w &\-]+)', template_text)
    template_text = re.sub(r'\{(\w+)\}', r'(?P<\1>[\w &\-]+)', template_text)
    re_mat = re.match(template_text, state.text)

    for par, par_c in par_dict.items():
        par_dict[par_c['name']]['value'] = re_mat.group(par_c['name'])

    c['parameters'] = [{'type': par_c['type'], 'name': par_c['name'], 'id': par_c['id'], 'values': par_c['values']}
                       for par, par_c in sorted(par_dict.items())]
    c['parameterVals'] = list()
    for par, par_c in sorted(par_dict.items()):
        if par_c['type'] == 'bin':
            value = par_c['value_list'][par_c['value_list_in_statement'].index(par_c['value'])]
        else:
            value = par_c['value']
        c['parameterVals'].append({'comparator': '=', 'value': value})

    return c

def display_trigger(trigger):
    return {'channel' : {'icon' : trigger.chan.icon}, 'text' : trigger.text}

###############################################################################
## [SLM] STATE & LOG MANAGEMENT
###############################################################################

# @TO_DELETE: Potentially dead code
# create and save new StateLog entry in database
# def log_state(newstate):
#     pvs = m.ParVal.objects.filter(state=newstate)
#     for pv in pvs:
#         try:
#             oldlog = m.StateLog.objects.get(
#                     is_current=True,
#                     cap=newstate.cap,
#                     dev=newstate.dev,
#                     param=pv.par
#                     )
#             # if no update, continue
#             if oldlog.val == pv.val:
#                 continue

#             oldlog.is_current=False
#             oldlog.save()
#         except m.StateLog.DoesNotExist:
#             pass

#         newlog = m.StateLog(
#                 is_current=True,
#                 cap=newstate.cap,
#                 dev=newstate.dev,
#                 param=pv.par,
#                 val=pv.val
#                 )
#         newlog.save()

# get current state of dev/cap pair
def current_state(dev,cap):
    try:
        curstate = m.State.objects.get(cap=cap,dev=dev,action=False)
        return curstate
    except m.State.DoesNotExist:
        return None

# get value of state of dev/cap pair at a given time
def historical_state(dev,cap,targettime):
    state = m.State.objects.get(dev=dev,cap=cap,action=False)
    out = []
    for pv in m.ParVal.objects.filter(state=state):
        try:
            logs = m.StateLog.objects.filter(
                    status__in=[m.StateLog.HAPPENED, m.StateLog.CURRENT],
                    cap=state.cap,
                    dev=state.dev,
                    param=pv.par,
                    timestamp__lte=targettime
                )
            if logs:
                lastlog = max(logs,key=lambda log : log.timestamp)
                out.append((lastlog.param,lastlog.value))
            else:
                out.append((pv.par.id,"N/A"))
        except m.StateLog.DoesNotExist:
            out.append((pv.par.id,"N/A"))

    return out

# get record of all logged changes to a dev/cap pair
def device_cap_history(dev,cap):
    qset = m.StateLog.objects.filter(status=m.StateLog.HAPPENED,dev=dev,cap=cap)
    states = []
    for entry in sorted(qset,key=lambda entry : entry.timestamp,reverse=True):
        states.append((entry.timestamp.ctime(),entry.param.id,entry.value))
    return states

###############################################################################
## [DM] DEVICE MANAGEMENT
###############################################################################

def prep_command(action, group):
    '''
    Preparing the commands that will be sent to IoTCore

    Input:

    Output:
    '''
    if group == 'st':
        cap = action.cap
        pvs = m.ActionParVal.objects.filter(state=action)
        pars = [pv.par for pv in pvs]
        enumcommands = filter(lambda x: pv.par.is_command,pvs) #this shouldn't return >1
        if enumcommands:
            commname = enumcommands[0].par.sysname
            commvals = []
        else:
            commname = None #cap.commandname
            commvals = [{'name' : pv.par.name, 'value' : pv.val} for pv in pvs]
        return {'command'   : {'name' : commname,
                               'vals' : commvals}}
    else:
        return None

def send_command(command, group):
    if group == 'st':
        signed_command = {'signature' : 'superifttt',
                          'command' : command['command']}

def get_param_vals(param):
    if param.type == 'bin':
        param_val_list =  [param.binparam.tval, param.binparam.fval]
    elif param.type == 'set':
        options = m.SetParamOpt.objects.filter(param=param)
        param_val_list = [opt.value for opt in options]
    else:
        # TODO: should try to handle range/color
        # maybe we could use kmeans to cluster the values based on the past events?
        param_val_list = []
    return param_val_list

def get_param_vals_range(dev, cap, param):
    range_seps = m.RangeCounter.objects.filter(dev=dev, cap=cap, param=param, count__gt=0)
    return [rs.representative for rs in range_seps]

def get_param_vals_color(dev, cap, param):
    color_counts = m.ColorCounter.objects.filter(dev=dev, cap=cap, param=param, count__gt=0)
    return [cc.name for cc in color_counts]

def get_dev_commands(dev, trace, template_dict, boolean_map, orig_tap_list, plain_revert=False):
    """
    return a list of tuple (cap, param, value, count)
    """
    result = list()
    for cap in dev.caps.all():
        if cap.name == 'Color Temperature':
            continue
        if cap.writeable:
            # this capability has commands
            # should iterate through all possible commands
            params = list(m.Parameter.objects.filter(cap=cap))
            if len(params) == 1:
                param = params[0]
                if param.type in ('set', 'bin'):
                    param_val_list = get_param_vals(param)
                elif param.type == 'range':
                    param_val_list = get_param_vals_range(dev, cap, param)
                elif param.type == 'color':
                    param_val_list = get_param_vals_color(dev, cap, param)
                else:
                    param_val_list = []
                for val in param_val_list:
                    entry = {
                        'device_name': dev.name if dev.dev_type == 'v' else dev.label,
                        'capability': cap.name,
                        'attribute': param.name,
                        'current_value': val
                    }
                    dev_name, cap_name, value = translateCapability(entry, template_dict=template_dict, boolean_map=boolean_map)
                    if type(value) == int or type(value) == float:
                        value = str(value)
                    elif type(value) == bool:
                        value = 'true' if value else 'false'
                    else:
                        value = value
                    target_action = dev_name + '.' + cap_name + '=' + str(value)
                    tap_list = [tap for tap in orig_tap_list if tap.action == target_action]
                    episode_list = extractEpisodes(trace, target_action, post_time_span=datetime.timedelta(minutes=2))
                    if not plain_revert:
                        _, revert_list = extractTriggerCases(trace, target_action, tap_list=tap_list)
                    else:
                        revert_time_list = extractRevertCases(trace, target_action)
                    count = len(episode_list)
                    covered = 0
                    for episode in episode_list:
                        if checkIfEpisodeCovered(episode, tap_list, target_action):
                            covered += 1
                    reverted = sum(revert_list) if not plain_revert else len(revert_time_list)
                    result.append((cap, param, val, count, covered, reverted))
            else:
                pass

    return result

#######################
# Permission Checking #
#######################

def is_error(status_code):
    return status_code >=400

def has_access_to_location(user, loc_id):
    if user.is_superuser:
        return True
    else:
        return m.Location.objects.filter(pk=loc_id, users=user).exists()

def check_access_to_location(request, loc_id: int):
    '''
    Given a request (which contains the user) and a location id, check:
        1. if the user is logged in
        2. if the location exists
        3. if the user has access to the location
    Input:
        request: Django's request object
        loc_id:  the id of the location、
    Output:
        err_json: (dict) {"err": "xxxxx"}, defaults to {}, which means no error detected
        status:   (int) the status number, defaults to 200, which means no error detected
    '''
    err_msg = {}
    status = 200
    if not request.user.is_authenticated:
        err_msg["err"] = "Please log in!"
        status = 401
    if not m.Location.objects.filter(pk=loc_id).exists():
        err_msg["err"] = "The requested location doesn't exist."
        status = 404
    if not has_access_to_location(request.user, loc_id):
        err_msg["err"] = "You don't have access to this location."
        status = 403

    return err_msg, status

############
# Recorder #
############
def record_request(req_dict, typ, location):
    req_str = json.dumps(req_dict)
    return m.Record.objects.create(request=req_str, response="", typ=typ, location=location)

def record_response(res_dict, typ, location):
    res_str = json.dumps(res_dict)
    m.Record.objects.create(request="", response=res_str, typ=typ, location=location)

def get_location_from_user(user):
    locations = m.Location.objects.all()
    for loc in locations:
        if user in list(loc.users.all()):
            return loc
    return None

##########
# Helper #
##########

def get_error_msg(func_name, err, status=500):
    err_msg = "{}: {}\n{}".format(func_name, err, traceback.format_exc())
    m.ErrorLog.objects.create(err=err_msg)
    return {"msg": "{}".format(err)}

def get_now():
    return timezone.now() if settings.USE_TZ else datetime.datetime.now()


from __future__ import print_function

import os
import sys
import subprocess

from .srcinfo import SrcInfoPool, iter_packages
from .pacman import PacmanPackage
from .utils import version_is_newer_than, version_cmp

def sorted_with_cmp(sequence, cmp_func, **kwargs):
    # sort with a cmp func which works under both Python 2 and 3

    if sys.version_info[0] == 3:
        from functools import cmp_to_key

        key = cmp_to_key(cmp_func)

        def do_sort(x):
            return sorted(x, key=key, **kwargs)
    else:
        def do_sort(x):
            return sorted(x, cmp=cmp_func, **kwargs)

    return do_sort(sequence)

def cmp_(a, b):
    return (a > b) - (a < b)

def get_pkgbuilds_to_build_in_order(packages_todo):
    """Returns a list of PKGBUILD files in the order they need to be build"""

    pool = SrcInfoPool()
    pkgbuilds = {}
    for package in packages_todo:
        pool.add_package(package)
        pkgbuilds.setdefault(package.pkgbuild_path, set()).add(package)

    # now sort the pkgbuilds by the dependencies of the contained packages

    def cmp_func(aa, bb):
        """Takes two sets of packages ands sorts according to their
        dependencies.
        """

        at = set()
        for a in aa:
            a_name = a.pkgname
            at.update(pool.get_transitive_dependencies(a))

        bt = set()
        for b in bb:
            b_name = b.pkgname
            bt.update(pool.get_transitive_dependencies(b))

        # make the result deterministic,
        # packages with fewer dependencies first
        a_key = (len(at), a_name)
        b_key = (len(bt), b_name)

        if a_name in bt and b_name in at:
            # cyclic dependency!
            return cmp_(a_key, b_key)
        elif a_name in bt:
            return -1
        elif b_name in at:
            return 1
        else:
            return cmp_(a_key, b_key)

    return pool, sorted_with_cmp(
        pkgbuilds.items(), lambda a, b: cmp_func(a[1], b[1]))

class BuildError(Exception):
    pass

def build_source(pkgbuild, packages, targetdir):
    """Build source packages

    Returns:
        set(str): The paths to the resulting packages
    Raises:
        BuildError
    """

    targetdir = os.path.abspath(targetdir)
    pkgbuild = os.path.abspath(pkgbuild)

    try:
        output = subprocess.check_output(
            ["bash", "/usr/bin/makepkg", "--noconfirm", "--noprogressbar",
             "--skippgpcheck", "--allsource", "--config",
             "/etc/makepkg_mingw64.conf", "-f",
             "-p", os.path.basename(pkgbuild),
             "SRCPKGDEST=%s" % targetdir],
            cwd=os.path.dirname(pkgbuild),
            stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        output = e.output
        raise BuildError(e)
    else:
        tarballs = set()
        for entry in os.listdir(targetdir):
            for p in packages:
                name = "%s-%s" % (p.pkgbase, p.build_version)
                if name in entry and ".src." in entry:
                    tarballs.add(os.path.join(targetdir, entry))
        return tarballs
    finally:
        some_pkg = list(packages)[0]
        target_key = "%s-%s" % (some_pkg.pkgbase, some_pkg.build_version)
        logpath = os.path.join(targetdir, target_key + ".src.log")
        with open(logpath, "wb") as h:
            h.write(output)

def build_and_install_binary(pkgbuild, packages, targetdir):
    """Build binary packages

    Returns:
        set(str): The paths to the resulting packages
    Raises:
        BuildError
    """

    targetdir = os.path.abspath(targetdir)
    pkgbuild = os.path.abspath(pkgbuild)

    try:
        output = subprocess.check_output(
            ["bash", "/usr/bin/makepkg-mingw", "--noconfirm",
             "--noprogressbar", "--skippgpcheck", "--nocheck", "--syncdeps",
             "--rmdeps", "--cleanbuild", "--install", "-f", "--noconfirm",
             "-p", os.path.basename(pkgbuild),
             "PKGDEST=%s" % targetdir],
            cwd=os.path.dirname(pkgbuild),
            stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        output = e.output
        raise BuildError(e)
    else:
        tarballs = set()
        for entry in os.listdir(targetdir):
            for p in packages:
                name = "%s-%s" % (p.pkgname, p.build_version)
                if name in entry and ".pkg." in entry:
                    tarballs.add(os.path.join(targetdir, entry))
        return tarballs
    finally:
        some_pkg = list(packages)[0]
        target_key = "%s-%s" % (some_pkg.pkgbase, some_pkg.build_version)
        logpath = os.path.join(targetdir, target_key + ".pkg.log")
        with open(logpath, "wb") as h:
            h.write(output)

def build(pkgbuild, packages, targetdir):
    """Build packages

    Returns:
        set(str): The paths to the resulting packages
    Raises:
        BuildError
    """

    results = set()
    some_pkg = list(packages)[0]
    target_key = "%s-%s" % (some_pkg.pkgbase, some_pkg.build_version)

    targetdir = os.path.abspath(targetdir)
    pkgbuild = os.path.abspath(pkgbuild)
    fail_path = os.path.join(targetdir, "%s.failed" % target_key)

    if os.path.exists(fail_path):
        print("%s found, build aborted. Delete the file to not skip "
              "the build." % fail_path)
        raise BuildError

    try:
        results.update(build_source(pkgbuild, packages, targetdir))
        results.update(
            build_and_install_binary(pkgbuild, packages, targetdir))
    except BuildError:
        open(fail_path, "wb").close()

        # something failed, try to clean up
        for path in results:
            try:
                os.unlink(path)
            except EnvironmentError:
                pass
        raise
    else:
        return results

def add_parser(subparsers):
    parser = subparsers.add_parser("build",
        help="Auto builds PKGBUILD files where the packages in the database "
             "are out of date. Builds them in the right order according to "
             "their dependency relation.")
    parser.add_argument(
        "path", help="path to the directory containg PKGBUILD files or a "
                     "PKGBUILD file itself")
    parser.add_argument(
        "target", help="path to the directory where the build packages will "
                       "be saved to")
    parser.add_argument('--dry-run', action='store_true',
                        help="Only show which packages will be build")
    parser.set_defaults(func=main)

def main(args):
    repo_path = os.path.abspath(args.path)
    target_path = os.path.abspath(args.target)

    repo_packages = PacmanPackage.get_all_packages()
    repo_packages = dict((p.pkgname, p) for p in repo_packages)

    # Find packages which not are not VCS and which are out of date
    packages_todo = set()
    for package in iter_packages(repo_path):
        if package.is_vcs:
            continue
        if package.pkgname in repo_packages:
            repo_pkg = repo_packages[package.pkgname]
            if version_is_newer_than(package.build_version,
                                     repo_pkg.build_version):
                packages_todo.add(package)

    # Throw away PKGBUILDS which build a package which is available from
    # another PKGBUILD but newer there.
    per_pkgname = {}
    for p in packages_todo:
        per_pkgname.setdefault(p.pkgname, set()).add(p)
    pkgbuilds_to_skip = set()
    for pkgname, packages in per_pkgname.items():
        # last is the newest
        packages_by_version = sorted_with_cmp(
            packages, lambda a, b: version_cmp(a.build_version,
                                               b.build_version))
        for to_blacklist in packages_by_version[:-1]:
            if to_blacklist.pkgbuild_path != \
                    packages_by_version[-1].pkgbuild_path:
                pkgbuilds_to_skip.add(to_blacklist.pkgbuild_path)
    packages_todo = set([p for p in packages_todo
                         if p.pkgbuild_path not in pkgbuilds_to_skip])

    # TODO: Support MSYS2 packages
    for package in packages_todo:
        print(package.repo)
        if package.repo == "msys":
            raise Exception(
                "Only mingw builds supported atm, please add "
                "support! (%s isn't)" % (package.pkgbuild_path,))

    # Sort them according to their dependencies so no package is build
    # before any of its dependencies
    pool, pkgbuilds = get_pkgbuilds_to_build_in_order(packages_todo)

    print("%d PKGBUILDs to build" % len(pkgbuilds))

    if args.dry_run:
        for path, packages in pkgbuilds:
            print(path)
            for package in packages:
                print("    -> ", package.pkgname)
        return

    if pkgbuilds:
        try:
            os.makedirs(target_path)
        except EnvironmentError:
            pass

    failed_packages = set()
    skipped = set()
    for path, packages in pkgbuilds:
        # In case some build failed, check if this one needs to be
        # skipped because it depends on the failed one.
        can_build = True
        reason = set()
        for failed in failed_packages:
            for p in packages:
                if failed in pool.get_transitive_dependencies(p):
                    can_build = False
                    reason.add(failed)
        if not can_build:
            skipped.update(packages)
            print("SKIPPING %s because %s failed" % (path, ", ".join(reason)))
            continue

        # start the build
        print("STARTING %s" % path)
        try:
            build(path, packages, target_path)
        except BuildError:
            print("FAILED")
            failed_packages.update(packages)
        else:
            print("DONE")

    # Final report
    print("All done.")
    if failed_packages:
        print("The following packages failed to build:")
        for p in sorted(failed_packages):
            print(p.pkgname)
        print("As a result, the following packages got skipped:")
        for p in sorted(skipped):
            print(p.pkgname)


import numpy as np

class Evaluation:
    def __init__(self, object_counts, failed_grasps=None):
        self.object_counts = np.array(object_counts)
        self.failed_grasps = np.array(failed_grasps) if failed_grasps else failed_grasps

        self.time_length = 120  # [s]

    @classmethod
    def sem(cls, data):
        return data.std() / np.sqrt(data.size)

    def __repr__(self):
        print(f'Mean: {self.object_counts.mean():.2f} ({self.sem(self.object_counts):.2f})')

        pph = self.object_counts.astype(np.float32) * 3600 / self.time_length
        print(f'Mean: {pph.mean():.2f} ({self.sem(pph):.2f})')

        if self.failed_grasps is not None:
            grasp_rate = self.object_counts / (self.object_counts + self.failed_grasps)
            print(f'Grasp rate: {grasp_rate.mean():.3f} ({self.sem(grasp_rate):.3f})')
        return ''

if __name__ == '__main__':
    random = Evaluation(
        object_counts=[4, 1, 3, 3, 0],
        failed_grasps=[14, 15, 12, 15, 16],
    )
    print(random)

    single = Evaluation(
        object_counts=[12, 14, 12, 13, 13],
        failed_grasps=[0, 0, 1, 0, 1],
    )
    print(single)

    single_prediction = Evaluation(
        object_counts=[16, 15, 16, 18, 17],
        failed_grasps=[1, 2, 1, 0, 1],
    )
    print(single_prediction)

    multiple = Evaluation(
        object_counts=[17, 19, 22, 23, 21],
        failed_grasps=[1, 1, 0, 0, 2],
    )
    print(multiple)

    multiple_prediction = Evaluation(
        object_counts=[23, 25, 24, 23, 22],
        failed_grasps=[2, 1, 0, 1, 3],
    )
    print(multiple_prediction)


from __future__ import division
import cv2
import os
import numpy as np
import scipy
import pickle
import matplotlib.pyplot as plt
from itertools import islice

LIMIT = None

data_dictionary = {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '7': 7,
                   '8': 8, '9': 9, 'a': 10, 'b': 11,
                   'c': 12,
                   'd': 13, 'e': 14, 'f': 15, 'g': 16, 'h': 17, 'k': 18,
                   'l': 19,
                   'm': 20, 'n': 21, 'p': 22, 'q': 23,
                   'r': 24,
                   's': 25, 't': 26, 'u': 27, 'w': 28, 'x': 29, 'y': 30,
                   'z': 31}

GESTURE_FOLDER = 'gestures/'

def return_data():
    X = []
    y = []
    features = []

    for filename in os.listdir(os.path.join(GESTURE_FOLDER)):
        for file in os.listdir(os.path.join(GESTURE_FOLDER, filename)):
            if file.endswith('.jpg'):
                full_path = os.path.join(GESTURE_FOLDER, filename, file)
                X.append(full_path)
                y.append(data_dictionary[filename])

    for i in range(len(X)):
        img = plt.imread(X[i])
        features.append(img)

    features = np.array(features).astype('float32')
    labels = np.array(y).astype('float32')

    with open("features_numbers_letters", "wb") as f:
        pickle.dump(features, f, protocol=4)
    with open("labels_numbers_letters", "wb") as f:
        pickle.dump(labels, f, protocol=4)

return_data()


# importar bibliotecas
import time
import requests
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from bs4 import BeautifulSoup
from colorama import init, Fore, Back

# 2 inserir init() para que o colocarama funcione no windows
init(autoreset = True)

# 3 definir número de jogos
def num_jogos():
    global número_de_jogos
    número_de_jogos = int(input('Quantos jogos você gostaria de cadastrar? '))

# 4 definir os jogos
def jogos():
    jogo = list()
    global jogos_num
    jogos_num = list()
    posição = 1
    número = 1
    for c in range(1, número_de_jogos+1):
        for números in range(1, 16):
            jogo.append(int(input(f'Qual o {posição}º número do {número}º jogo: ')))
            posição += 1
        print(f'{"----------":>70}')
        jogos_num.append(jogo[:])
        jogo.clear()
        número += 1
        posição = 1
    return jogos_num

# 5 criar função para mostrar os jogos
def mostra_jogos():
    for pos, números in enumerate(jogos_num):
        time.sleep(0.3)
        print(f'jogo {pos+1}: {números}')

# 6 criar funções para as opções do Menu
def opção1():
    num_jogos()
    jogos()
    print(f'Total de {len(jogos_num)} jogos cadastrados')
    print('--'*35)
    print('Jogos cadastrados com sucesso.')
    print('No menu, vá para a opção desejada.')
    print('--'*35)

def opção1_menu():
    números_novos = int(input('Quantos jogos a mais você gostaria de cadastrar? '))
    jogo = list()
    número = 1
    posição = 1
    for c in range(1, números_novos+1):
        for números in range(1, 16):
            jogo.append(int(input(f'Qual o {posição}º número do {número}º novo jogo: ')))
            posição += 1
        print(f'{"----------":>70}')
        jogos_num.append(jogo[:])
        jogo.clear()
        número += 1
        posição = 1
    return jogos_num

def opção2():
    print('--'*35)
    print(f'{"JOGOS CADASTRADOS":^70}')
    print('--'*35)
    if len(jogos_num) > 0:
        mostra_jogos()
        print('--'*35)
    else:
        print(f'Você não possui jogos cadastrados.\nPor favor, volte ao menu anterior e selecione a opção 1.')

def opção3():
    pergunta_jogos = str(input('Gostaria que seus jogos fossem mostrados novamente? S/N  ')).upper()
    print(f'{"----------":>70}')
    if pergunta_jogos == 'S':
        mostra_jogos()
        print(f'{"----------":>70}')
    pergunta_num = int(input('Qual jogo você gostaria de alterar?  '))
    print('--'*35)
    if pergunta_num == 0 or pergunta_num > len(jogos_num):
        time.sleep(0.3)
        print('Opção inválida, por favor refaça.')
        print(f'{"----------":>70}')
        print(f'   O jogo {pergunta_num} não existe.')
        print('   Volte ao menu anterior.')
    else:
        jogos_num.pop(pergunta_num-1)
        substituir_jogo = list()
        for c in range(1, 16):
            substituir_jogo.append(int(input(f'Qual o {c}º número do novo {pergunta_num}º jogo: ')))
        jogos_num.insert(pergunta_num-1, substituir_jogo[:])
        substituir_jogo.clear()
        print(f'jogo {pergunta_num} alterado com sucesso.')

def opção4():
    time.sleep(0.3)
    pergunta_jogos = str(input('Gostaria que seus jogos fossem mostrados novamente? S/N  ')).upper()
    print(f'{"----------":>70}')
    time.sleep(0.3)
    if pergunta_jogos == 'S':
        mostra_jogos()
        print(f'{"----------":>70}')
    pergunta = int(input('Qual jogo você gostaria de excluir?  '))
    print(f'{"----------":>70}')
    if pergunta == 0 or pergunta > len(jogos_num):
        time.sleep(0.3)
        print('Opção inválida, por favor refaça.')
        print(f'{"----------":>70}')
        print(f'  * O jogo {pergunta} não existe.')
        print('  * Volte ao menu anteior')
    else:
        jogos_num.pop(pergunta-1)
        time.sleep(0.3)
        print(f'jogo {pergunta} excluído com sucesso.')
        print(f'{"----------":>70}')
        time.sleep(0.3)
        print('   Seus jogos receberam uma nova numeração.')
        time.sleep(0.3)
        print('   Selecione a OPÇÃO 2 para nova visualização.')

def opção5():
    url = 'http://loterias.caixa.gov.br/wps/portal/loterias/landing/lotofacil/'

    option = Options()
    option.headless = True
    driver = webdriver.Firefox()

    driver.get(url)
    time.sleep(15)

    def concurso():
        try:
            element = driver.find_element_by_xpath("//div[@class='title-bar clearfix']//h2//span")
            html_content = element.get_attribute('outerHTML')
            soup = BeautifulSoup(html_content, 'html.parser')
            return soup.get_text().split()
        except:
            return 'Tente mais tarde'

    print(f' {concurso()[0]}  {concurso()[1]}  {concurso()[2]}')

    def números_sorteio():
        try:
            element2 = driver.find_element_by_xpath("//table[@class='simple-table lotofacil']//tbody")
            html_content2 = element2.get_attribute('outerHTML')
            soup2 = BeautifulSoup(html_content2, 'html.parser')
            números = soup2.get_text().split()
            num = list()
            índice = 0
            # for c in range(0,3): # Forma maior
            #     for pos, núm in enumerate(números[índice]):
            #         num.append(núm)
            #     índice += 1
            for c in range(0, 3):
                [num.append(núm) for pos, núm in enumerate(números[índice])] # Forma contraída
                índice += 1
            soma = 0
            posi = 0
            posiç = 1
            resultado = list()
            for c in range(0, 15):
                resultado.append(num[posi] + num[posiç])
                soma += 1
                posi += 2
                posiç += 2
            resultado_inteiros = [int(num) for num in resultado]
            return resultado_inteiros
        except:
            return 'Tente mais tarde'

    print(f' Num sorteados: {números_sorteio()}')

    def acertos():
        global acertos_jogos
        acertos_jogos = list()
        for números in jogos_num:
            acertos_jogos.append(list(set(números)&set(números_sorteio())))
        return acertos_jogos
    acertos()
    print(f'{"-----------":>70}')

    def mostra_jogos():
        for pos, números in enumerate(acertos_jogos):
            print(Fore.GREEN + f' jogo {pos+1}:  ACERTOS  {números}  TOTAL = {len(números)}')

    mostra_jogos()

    time.sleep(0.3)

    print('--'*35)
    print(f'{"Confira o seu bilhete no site oficial da Loteria !":^70}')
    print(f'{"O resultado pode conter erros, não é um programa oficial !":^70}')
    print(f'{"NÃO RASGUE O SEU BILHETE, CONFIRA ANTES NO SITE DA LOTERIA !":^70}')
    print(f'{"O USO DESTE PROGRAMA É DE SUA ÚNICA E EXCLUSIVA RESPONSABILIDADE !":^70}')
    print(f'{"ESTE PROGRAMA É APENAS PARA FINS ACADÊMICOS !":^70}')
    print('--'*35)

    driver.quit()

# 7 criar Menu de opções
def menu():
    print('--'*35)
    print(f'{"BEM VINDO AO VERIFICADOR DE JOGOS DE 15 NÚMEROS":^70}')
    print(f'{"O resultado pode conter erros, não é um programa oficial !":^70}')
    print(f'{"NÃO RASGUE O SEU BILHETE, CONFIRA ANTES NO SITE DA LOTERIA !":^70}')
    print(f'{"O USO DESTE PROGRAMA É DE SUA ÚNICA E EXCLUSIVA RESPONSABILIDADE !":^70}')
    print(f'{"ESTE PROGRAMA É APENAS PARA FINS ACADÊMICOS !":^70}')
    print('--'*35)
    print('--'*35)
    print((Back.WHITE + Fore.BLACK + f'{"Menu de Opções":^70}'))
    print('--'*35)
    time.sleep(0.3)
    print(Fore.BLUE + '''    1 - Cadastrar mais jogos
    2 - Mostrar jogos cadastrados
    3 - Alterar jogo cadastrado
    4 - Excluir jogo cadastrado
    5 - Executar Programa
    6 - Interromper programa''')
    print('--'*35)
    print('As opções não podem ser acessadas.')
    print('ANTES CADASTRE PELO MENOS UM JOGO.')
    print('--'*35)
    opção1()
    resp = 'S'
    while resp == 'S':
        print('--'*35)
        print((Back.WHITE + Fore.BLACK + f'{"Menu de Opções":^70}'))
        print('--'*35)
        time.sleep(0.3)
        print('''    1 - Cadastrar mais jogos
    2 - Mostrar jogos cadastrados
    3 - Alterar jogo cadastrado
    4 - Excluir jogo cadastrado
    5 - Executar Programa
    6 - Interromper programa''')
        print('--'*35)
        time.sleep(0.3)
        print('--'*35)
        time.sleep(0.3)
        opção = int(input('    Qual a opção desejada:  '))
        print('--'*35)
        while opção < 1 or opção > 8:
            print('Desculpe a opção digitada não é válida, por favor, refaça.')
            resp = str(input('Deseja continuar? S/N  ')).upper()
            if resp != 'S':
                print(f'Programa finalizado, volte sempre.')
                break
            if resp == 'S':
                time.sleep(0.3)
                print('--'*35)
                print((f'{"Menu de Opções":^70}'))
                print('--'*35)
                print('''
    1 - Cadastrar mais jogos
    2 - Mostrar jogos cadastrados
    3 - Alterar jogo cadastrado
    4 - Excluir jogo cadastrado
    5 - Executar Programa
    6 - Interromper programa''')
            print('--'*35)
            opção = int(input('    Qual a opção desejada:  '))
            print('--'*35)

        if opção == 1:
            opção1_menu()
            print(f' Total de {len(jogos_num)} jogos cadastrados')

        if opção == 2:
            opção2()

        if opção == 3:
            opção3()

        if opção == 4:
            opção4()

        if opção == 5:
            if len(jogos_num) == 0:
                time.sleep(0.3)
                print(Back.WHITE + 'Você não pode executar o programa.')
                print(Fore.RED + '  * Ainda não foram cadastrados jogos.')
                print(Fore.RED + '  * Volte ao menu anterior')
            else:
                opção5()

        if opção == 6:
            print('--'*35)
            print(f'{"Obrigado, volte sempre !":^70} ')
            print(f'{"Programa encerrado com sucesso !":^70}')
            print(f'{"O resultado pode conter erros, não é um programa oficial !":^70}')
            print(f'{"NÃO RASGUE O SEU BILHETE, CONFIRA ANTES NO SITE DA LOTERIA !":^70}')
            print(f'{"O USO DESTE PROGRAMA É DE SUA ÚNICA E EXCLUSIVA RESPONSABILIDADE !":^70}')
            print(f'{"ESTE PROGRAMA É APENAS PARA FINS ACADÊMICOS !":^70}')
            print('--'*35)
            break

menu()


import sys

from torch.utils.data import DataLoader, random_split

import torchvision.transforms as transforms
from torchvision.datasets import MNIST

import pytorch_lightning as pl

class DataModule(pl.LightningDataModule) :
    def __init__(self,
                 data_path = '/media/jhnam19960514/68334fe0-2b83-45d6-98e3-76904bf08127/home/namjuhyeon/Desktop/LAB/common material/Dataset Collection',
                 data_type='MNIST',
                 batch_size=64):
        super(DataModule, self).__init__()

        self.data_path = data_path
        self.data_type = data_type
        self.batch_size = batch_size

    def setup(self, stage=None):
        transform = transforms.Compose([transforms.ToTensor()])

        if self.data_type == 'MNIST':
            train_dataset = MNIST(self.data_path, train=True, download=True, transform=transform)
            test_dataset = MNIST(self.data_path, train=False, download=True, transform=transform)
        else :
            sys.exit()

        # train/val split
        train_dataset, val_dataset = random_split(train_dataset, [55000, 5000])

        # assign to use in dataloaders
        self.train_dataset = train_dataset
        self.val_dataset = val_dataset
        self.test_dataset = test_dataset

    def train_dataloader(self):
        return DataLoader(self.train_dataset, batch_size=self.batch_size)

    def val_dataloader(self):
        return DataLoader(self.val_dataset, batch_size=self.batch_size)

    def test_dataloader(self):
        return DataLoader(self.test_dataset, batch_size=self.batch_size)


from django.contrib import admin
from django.urls import path, include
from auth_app import views
urlpatterns = [
    path('home', views.home, name='home'),
    path('', views.home, name='home'),
    path('registerpage', views.registerpage, name='registerpage'),
    path('loginpage', views.loginpage, name='loginpage'),
    path('logoutpage', views.logoutpage, name='logoutpage'),
    path('individualreport', views.individualreport, name='individualreport'),

]


import random
import re
from tsv_io import TSVDataset, tsv_reader, tsv_writer
from qd_common import write_to_file, basename_no_ext
from qd_common import load_list_file
from taxonomy import synset_to_noffset, LabelTree, LabelToSynset
import json
import os
import os.path as op
import logging

def ensure_dataset_sample(source_dataset, sample_label, sample_image, out_data):
    if op.exists(out_data):
        logging.info('skip to generate the samle data since it exists')
        return
    logging.info('start to generate the sample data')
    assert source_dataset.name == 'imagenet2012', 'only cls dataset is supp'
    assert sample_label > 0 and sample_label <= 1
    assert sample_image > 0 and sample_image <= 1
    random.seed(777)
    labels = source_dataset.load_labelmap()
    num_labels = len(labels)
    num_sampled_labels = int(num_labels * sample_label)
    assert num_sampled_labels > 0
    label_idx = range(num_labels)
    random.shuffle(label_idx)
    sampled_labels_idx = label_idx[:num_sampled_labels]
    train_rows = tsv_reader(source_dataset.get_train_tsv())
    train_images = [row for row in train_rows if int(row[1]) in sampled_labels_idx]
    for row in train_images:
        row[1] = str(sampled_labels_idx.index(int(row[1])))
    random.shuffle(train_images)
    tsv_writer(train_images[: int(len(train_images) * sample_image)],
            op.join(out_data, 'train.tsv'))
    # process the test set
    test_rows = tsv_reader(source_dataset.get_test_tsv_file())
    test_images = [row for row in test_rows if int(row[1]) in sampled_labels_idx]
    for row in test_images:
        row[1] = str(sampled_labels_idx.index(int(row[1])))
    tsv_writer(test_images, op.join(out_data, 'test.tsv'))
    # save teh label map
    sampled_labels = [labels[i] for i in sampled_labels_idx]
    write_to_file('\n'.join(sampled_labels), op.join(out_data, 'labelmap.txt'))

def dynamic_process_tsv(source_dataset, output_data_path,
        tree,
        **kwargs):
    if 'dataset_ops' not in kwargs:
        kwargs['dataset_ops'] = []
    return process_dataset(source_dataset, output_data_path,
            kwargs['dataset_ops'], tree)

    # deprecate the following (use process_dataset())
    source = source_dataset.get_train_tsv()
    source_label = None
    desc = ''
    if 'remove_bb_label' in kwargs and kwargs['remove_bb_label']:
        if 'remove_bb_label_prob' in kwargs and kwargs['remove_bb_label_prob']:
            desc = desc + '_rb_{}{}'.format(kwargs['remove_bb_label'],
                    kwargs['remove_bb_label_prob'])
            source_label = os.path.join(output_data_path,
                    '{}_label_remove_{}_bb_{}.tsv'.format(os.path.splitext(os.path.basename(source))[0],
                        kwargs['remove_bb_label'].replace(',','_'),
                        kwargs['remove_bb_label_prob']))
        else:
            desc = desc + '_rb_{}1'.format(kwargs['remove_bb_label'])
            source_label = os.path.join(output_data_path,
                    '{}_label_remove_{}_bb.tsv'.format(os.path.splitext(os.path.basename(source))[0],
                        kwargs['remove_bb_label'].replace(',','_')))
        if not os.path.isfile(source_label):
            remove_bb(source, source_label, **kwargs)

    source_shuffle = None
    if 'remove_image' in kwargs and kwargs['remove_image']:
        desc = desc + '_ri_{}'.format(kwargs['remove_image'].replace(',', '_'))
        if 'remove_image_prob' in kwargs and kwargs['remove_image_prob']:
            desc = desc + '_{}'.format(kwargs['remove_image_prob'])
        if kwargs['remove_image'] == 'all' and \
                ('remove_image_prob' not in kwargs or \
                kwargs['remove_image_prob'] == 1):
            source = None
            source_label = None
        else:
            if 'remove_image_prob' in kwargs and kwargs['remove_image_prob']:
                source_shuffle = os.path.join(output_data_path,
                        '{}_image_remove_{}_{}.shuffle'.format(os.path.splitext(os.path.basename(source))[0],
                            kwargs['remove_image'].replace(',', '_'),
                            kwargs['remove_image_prob']))
            if not os.path.isfile(source_shuffle):
                remove_image(source, source_shuffle, **kwargs)

    if 'add_image_rule' in kwargs and kwargs['add_image_rule']:
        add_image_rule = kwargs['add_image_rule']
        desc = desc + '_ai_{}'.format(kwargs['add_image_rule'].replace(':',
            '_'))
        out_source = os.path.join(output_data_path,
                '{}{}.tsv'.format(
                    source_dataset._name,
                    desc))
        if not os.path.exists(out_source):
            source_labelmap = source_dataset.get_labelmap_file()
            merge_image(source, source_label, source_labelmap, add_image_rule, out_source)
        source = out_source
        source_label = None
        source_shuffle = randomize_tsv_file(out_source)

    return source, source_label, source_shuffle

def construct_low_shot(source_dataset, labels, num_train_images, shuffle_file):
    assert len(labels) == 1
    label_imageIdx = {}
    target_images = []
    rows = tsv_reader(source_dataset.get_train_tsv())
    shuffle = []
    for i, row in enumerate(rows):
        curr_labels = json.loads(row[1])
        if any([l['class'] == labels[0] for l in curr_labels]):
            target_images.append((i, row[0]))
        else:
            shuffle.append(i)
    assert len(target_images) >= num_train_images
    random.seed(777)
    random.shuffle(target_images)
    selected = target_images[:num_train_images]
    num_duplicate = (len(target_images) + num_train_images - 1) / num_train_images
    assert num_duplicate >= 1
    for d in range(num_duplicate):
        shuffle.extend([s[0] for s in selected])

    random.shuffle(shuffle)
    write_to_file('\n'.join(map(str, selected)), shuffle_file + '.selected')
    write_to_file('\n'.join(map(str, shuffle)), shuffle_file)

def process_dataset(source_dataset, output_root_data_path, dataset_ops,
        synset_tree):
    sources, source_labels = [], []
    source_shuffles, data_batch_weights = [], []
    labelmap = None
    if synset_tree:
        noffsets = synset_tree.noffsets
        labelmap = op.join(output_root_data_path,
            op.basename(synset_tree.basefilename))
        write_to_file('\n'.join(noffsets), labelmap)
    else:
        synset_tree = None
        labelmap = source_dataset.get_labelmap_file()

    source = source_dataset.get_train_tsvs()
    sources.append(source)
    train_label_files = source_dataset.get_train_tsvs('label')
    if all(op.isfile(f) for f in train_label_files):
        source_labels.append(train_label_files)
    shuffle_file = source_dataset.get_train_shuffle_file()
    if op.isfile(shuffle_file):
        source_shuffles.append(shuffle_file)

    for i, operator in enumerate(dataset_ops):
        if operator['op'] == 'remove':
            # remove the current dataset. That is, we don't want to use the
            # current dataset at all
            sources = []
            source_labels = []
            data_batch_weights = []
            source_shuffles = []
        elif operator['op'] == 'select_top':
            # dataset_ops = [{'op':'select_top', 'num_top': 1}]
            # only select the first few training images. This is used to verify
            # if the training loss can be reduced if we only have a few
            # training samples.
            num_top = operator['num_top']
            shuffle_file = op.join(output_root_data_path, source_dataset.name,
                    'train_shuffle{}.txt'.format(num_top))
            write_to_file('\n'.join(map(str, range(num_top))), shuffle_file)
            assert len(sources) == 1
            assert len(source_shuffles) == 0
            source_shuffles.append(shuffle_file)
        elif operator['op'] == 'add':
            # add a new dataset.
            if 'path' not in operator:
                extra_dataset = TSVDataset(operator['name'])
                if operator['source'] == 'train':
                    curr_source = extra_dataset.get_train_tsvs()
                    curr_labels = extra_dataset.get_train_tsvs('label')
                elif operator['source'] == 'trainval':
                    raise NotImplemented
                    curr_source = extra_dataset.get_trainval_tsv()
                else:
                    assert False
                #curr_labelmap = extra_dataset.load_labelmap()
                if op.isfile(extra_dataset.get_train_shuffle_file()):
                    source_shuffles.append(extra_dataset.get_train_shuffle_file())
                sources.append(curr_source)
                if len(curr_labels) == len(curr_source):
                    source_labels.append(curr_labels)
                data_batch_weights.append(operator['weight'])
            else:
                assert False
                curr_source = op.join(operator['path'], operator['source'] + '.tsv')
                #curr_labelmap = load_list_file(op.join(operator['path'],
                    #'labelmap.txt'))
            #source_label = op.join(output_root_data_path, operator['name'],
                    #'{}_{}_{}'.format(basename_no_ext(synset_tree.basefilename),
                        #basename_no_ext(curr_source), 'label.tsv'))
            #if not op.isfile(source_label):
                #map_label(curr_source,
                        #curr_labelmap,
                        #synset_tree,
                        #source_label)
        elif operator['op'] == 'sample':
            # randomly sample the training data by a probability for all
            assert len(dataset_ops) == 1
            sample_label = operator['sample_label']
            sample_image = operator['sample_image']
            out_folder = op.join(output_root_data_path,
                    '{}_{}_{}'.format(source_dataset.name,
                        operator['sample_label'],
                        operator['sample_image']))
            ensure_dataset_sample(source_dataset, sample_label, sample_image, out_folder)
            sources = [op.join(out_folder, 'train.tsv')]
            labelmap = op.join(out_folder, 'labelmap.txt')
        elif operator['op'] == 'low_shot':
            assert len(dataset_ops) == 1
            labels = operator['labels'].split(',')
            num_train = operator['num_train']
            shuffle_file = op.join(output_root_data_path, source_dataset.name,
                    'low_shot_shuffle_{}_{}.txt'.format('_'.join(labels),
                        num_train))
            source_shuffles.append(shuffle_file)
            if not op.isfile(shuffle_file):
                construct_low_shot(source_dataset, labels,
                        num_train, shuffle_file)
        elif operator['op'] == 'mask_background':
            assert i == 0
            sources = []
            new_label_idx = operator['new_label_idx']
            target_folder = op.join(output_root_data_path,
                    '{}_{}_{}'.format(source_dataset.name,
                        '.'.join(map(str, operator['old_label_idx'])),
                        new_label_idx))
            if not op.exists(target_folder):
                old_labels = source_dataset.load_labelmap()
                num_label = len(old_labels)
                old_new = create_mask_label_map(operator['old_label_idx'],
                        new_label_idx, num_label)
                mask_background(source_dataset.get_train_tsv(),
                        op.join(target_folder, 'train.tsv'),
                        old_new)
                mask_background(source_dataset.get_test_tsv_file(),
                        op.join(target_folder, 'test.tsv'),
                        old_new)
                new_labels = [None] * (num_label - len(operator['old_label_idx']) + 1)
                new_labels[new_label_idx] = 'background'
                for curr_old_label_idx, curr_new_label_idx in enumerate(old_new):
                    if new_labels[curr_new_label_idx] is None:
                        if curr_new_label_idx != new_label_idx:
                            new_labels[curr_new_label_idx] = old_labels[curr_old_label_idx]
                write_to_file('\n'.join(new_labels), op.join(target_folder,
                    'labelmap.txt'))
            sources.append(op.join(target_folder, 'train.tsv'))
        else:
            assert False
    return sources, source_labels, source_shuffles, data_batch_weights, labelmap

def create_mask_label_map(old_label_idx,
        new_label_idx, num_label):
    old_new = [None] * num_label
    idx = 0
    for i in range(num_label):
        if i in old_label_idx:
            old_new[i] = new_label_idx
        else:
            if idx == new_label_idx:
                idx = idx + 1
            old_new[i] = idx
            idx = idx + 1

    return old_new

def mask_background(tsv_file,
                    out_file,
                    old_new):

    def gen_rows():
        rows = tsv_reader(tsv_file)
        for row in rows:
            row[1] = str(old_new[int(float(row[1]))])
            yield row

    tsv_writer(gen_rows(), out_file)

def is_noffset_list(labels):
    has = False
    for label in labels:
        if re.match('^n[0-9]{8}$', label) == None:
            return False
        else:
            return True

def map_label(source, labels, synset_tree, source_label):
    assert synset_tree
    all_idx = []

    if is_noffset_list(labels):
        noffsets = labels
    else:
        mapper = LabelToSynset()
        noffsets = [synset_to_noffset(mapper.convert(l)) for l in labels]
        for noffset in noffsets:
            assert len(synset_tree.root.search_nodes(name=noffset)) == 1

    label_noffset = { label:noffset for label, noffset in zip(labels, noffsets)}
    rows = tsv_reader(source)
    def convert_label(rows, labels, label_noffset):
        is_cls_set = False
        for i, row in enumerate(rows):
            if row[1].isdigit():
                infos = [{'class': label_noffset[labels[int(row[1])]],
                    'rect': [0,0,0,0]}]
                is_cls_set = True
            else:
                assert not is_cls_set
                infos = json.loads(row[1])
                for info in infos:
                    info['class'] = label_noffset[info['class']]
            if (i % 1000) == 0:
                logging.info(i)
            yield row[0], json.dumps(infos)
    tsv_writer(convert_label(rows, labels, label_noffset), source_label)


import datetime
import os
import uuid
import random
import boto3
from botocore.exceptions import ClientError

from lambda_python_powertools.logging import (
    logger_inject_process_booking_sfn,
    logger_setup,
    MetricUnit,
    log_metric,
)

from lambda_python_powertools.tracing import Tracer

logger = logger_setup()
tracer = Tracer()

session = boto3.Session()
dynamodb = session.resource("dynamodb")
table_name = os.getenv("BOOKING_TABLE_NAME", "undefined")
table = dynamodb.Table(table_name)
#

def get_config(config_id, values):
    extract_values = lambda items: [value for value in [values for values in items[0].values()][0].values()][0]
    client = boto3.client('dynamodb')
    items = client.query(TableName='configuration_table', KeyConditionExpression='configID = :config',
                         ExpressionAttributeValues={
                             ':config': {'S': config_id}
                         }, ProjectionExpression=values)['Items']
    return extract_values(items)

_cold_start = True

class BookingReservationException(Exception):
    def __init__(self, message=None, status_code=None, details=None):

        super(BookingReservationException, self).__init__()

        self.message = message or "Booking reservation failed"
        self.status_code = status_code or 500
        self.details = details or {}

def is_booking_request_valid(booking):
    return all(x in booking for x in ["outboundFlightId", "customerId", "chargeId"])

@tracer.capture_method
def reserve_booking(booking,dow_executeAnomaly,update_item_executeAnomaly,rid):
    """Creates a new booking as UNCONFIRMED
    Parameters
    ----------
    booking: dict
        chargeId: string
            pre-authorization charge ID
        stateExecutionId: string
            Step Functions Process Booking Execution ID
        chargeId: string
            Pre-authorization payment token
        customer: string
            Customer unique identifier
        bookingOutboundFlightId: string
            Outbound flight unique identifier
    Returns
    -------
    dict
        bookingId: string
    """
    try:
        booking_id = str(uuid.uuid4())
        state_machine_execution_id = booking["name"]
        outbound_flight_id = booking["outboundFlightId"]
        customer_id = booking["customerId"]
        payment_token = booking["chargeId"]

        booking_item = {
            "id": booking_id,
            "stateExecutionId": state_machine_execution_id,
            "__typename": "Booking",
            "bookingOutboundFlightId": outbound_flight_id,
            "checkedIn": False,
            "customer": customer_id,
            "paymentToken": payment_token,
            "status": "UNCONFIRMED",
            "createdAt": str(datetime.datetime.now()),
        }

        logger.debug(
            {"operation": "reserve_booking", "details": {"outbound_flight_id": outbound_flight_id}}
        )
        num_of_oper = 1

        if update_item_executeAnomaly:
            num_of_oper = 0
            print('ANOMALY! REQUEST_ID: {}, START: {}, SOURCE: {}, TARGET: {}, OPERATION: {}, ANOMALY_TYPE: {}'.format(rid,'START','Airline-ReserveBooking-master',table_name,'updateItem','UpdateItemInsteadOfPutItem'))
            ret = table.update_item(Key={'id':'bf313090-82f4-4698-8eb8-29489f242c7d'},
                                UpdateExpression="set checkedIn=:r",
                                ExpressionAttributeValues={
                                    ':r': True
                                },
                                ReturnValues="UPDATED_NEW")
        if dow_executeAnomaly:
            num_of_oper = 20
            print('ANOMALY! REQUEST_ID: {}, START: {}, SOURCE: {}, TARGET: {}, OPERATION: {}, ANOMALY_TYPE: {}'.format(rid,'START','Airline-ReserveBooking-master',table_name,'putObject','DenialOfWalletMany'))

        for i in range(num_of_oper):
            ret = table.put_item(Item=booking_item)

        if dow_executeAnomaly:
            print('ANOMALY! REQUEST_ID: {}, START: {}, SOURCE: {}, TARGET: {}, OPERATION: {}, ANOMALY_TYPE: {}'.format(rid,'START','Airline-ReserveBooking-master',table_name,'putObject','DenialOfWalletMany'))

        anomaly =  update_item_executeAnomaly

        logger.info({"operation": "reserve_booking", "details": ret})
        logger.debug("Adding put item operation result as tracing metadata")
        tracer.put_metadata(booking_id, booking_item, "booking")

        return {"bookingId": booking_id, "anomaly": anomaly}
    except ClientError as err:
        logger.debug({"operation": "reserve_booking", "details": err})
        raise BookingReservationException(details=err)

@tracer.capture_lambda_handler(process_booking_sfn=True)
@logger_inject_process_booking_sfn
def lambda_handler(event, context):
    """AWS Lambda Function entrypoint to reserve a booking
    Parameters
    ----------
    event: dict, required
        Step Functions State Machine event
        chargeId: string
            pre-authorization charge ID
        stateExecutionId: string
            Step Functions Process Booking Execution ID
        chargeId: string
            Pre-authorization payment token
        customerId: string
            Customer unique identifier
        bookingOutboundFlightId: string
            Outbound flight unique identifier
    context: object, required
        Lambda Context runtime methods and attributes
        Context doc: https://docs.aws.amazon.com/lambda/latest/dg/python-context-object.html
    Returns
    -------
    bookingId: string
        booking ID generated
    Raises
    ------
    BookingReservationException
        Booking Reservation Exception including error message upon failure
    """
    global _cold_start
    print("Reserve")
    anomaly_mode = get_config('anomaly_mode', 'Activate')
    dow_anomaly_prob = float(get_config('Airline-ReserveBooking-master-DOW', 'anomaly_prob'))
    dow_anomaluseExecution = random.random() < dow_anomaly_prob
        # if both ANOMALY_MODE and anomaluseExecution are true - execute anomaly
    dow_executeAnomaly = anomaly_mode == True & dow_anomaluseExecution == True

    update_item_anomaly_prob = float(get_config('Airline-ReserveBooking-master-UpdateItem', 'anomaly_prob'))
    update_item_anomaluseExecution = random.random() < update_item_anomaly_prob
        # if both ANOMALY_MODE and anomaluseExecution are true - execute anomaly
    update_item_executeAnomaly = anomaly_mode == True & update_item_anomaluseExecution == True

    # If two attacks were chosen  cancel one of them randomly:
    if update_item_executeAnomaly and dow_executeAnomaly:
        if random.random() < 0.5:
            update_item_executeAnomaly = False
        else:
            dow_executeAnomaly = False

    if not (update_item_executeAnomaly or dow_executeAnomaly):
        cancel_path = get_config('cancel_mode', 'Activate')
        cancel_prob = float(get_config('cancel_mode', 'Prob'))
        cancelExecution = random.random() < cancel_prob
        executeCancel = cancel_path == True & cancelExecution == True

        if executeCancel:
            raise ValueError("Cancel booking request")

    if _cold_start:
        log_metric(
            name="ColdStart", unit=MetricUnit.Count, value=1, function_name=context.function_name
        )
        _cold_start = False

    if not is_booking_request_valid(event):
        log_metric(
            name="InvalidBookingRequest",
            unit=MetricUnit.Count,
            value=1,
            operation="reserve_booking",
        )
        logger.error({"operation": "invalid_event", "details": event})
        raise ValueError("Invalid booking request")

    try:
        logger.debug(f"Reserving booking for customer {event['customerId']}")
        rid = context.aws_request_id
        ret = reserve_booking(event,dow_executeAnomaly,update_item_executeAnomaly,rid)

        log_metric(name="SuccessfulReservation", unit=MetricUnit.Count, value=1)
        logger.debug("Adding Booking Reservation annotation")
        tracer.put_annotation("Booking", ret["bookingId"])
        tracer.put_annotation("BookingStatus", "RESERVED")

        if update_item_executeAnomaly or dow_executeAnomaly:
            raise ValueError("Cancel booking request")

        # Step Functions use the return to append `bookingId` key into the overall output
        return ret["bookingId"]
    except BookingReservationException as err:
        log_metric(name="FailedReservation", unit=MetricUnit.Count, value=1)
        logger.debug("Adding Booking Reservation annotation before raising error")
        tracer.put_annotation("BookingStatus", "ERROR")
        logger.error({"operation": "reserve_booking", "details": err})
        raise BookingReservationException(details=err)


y = ''
while True:
    x = round(float(input()), 2)
    if (x < 0.00 or x > 10.00):
        print("nota invalida")
        continue
    try :
        print("media = {0:.2f}".format(round((x+y)/2, 2)))
        break
    except:
        y = x
        continue