from collections import deque, namedtuple
from functools import partial
from typing import Callable, Optional, Tuple, Union

import dm_env
import haiku as hk
import jax
import numpy as np
from bsuite.baselines.base import Agent
from dm_env import specs
from jax import numpy as jnp

Observation = np.ndarray
Action = int
Features = Union[np.ndarray, jnp.ndarray]
BasisFunction = Callable[[Observation, Action], Features]
AgentState = namedtuple('AgentState', "seq_weights")

@jax.jit
def _eval(
    agent_state: AgentState,
    batch_f: jnp.ndarray,
    batch_r: jnp.ndarray,
) -> AgentState:
    batch_rev_q = jnp.cumsum(batch_r[::-1]) / jnp.arange(1, 1 + len(batch_r))
    batch_q = batch_rev_q[::-1]
    weights = jnp.linalg.lstsq(batch_f, batch_q)[0]
    return agent_state._replace(seq_weights=agent_state.seq_weights +
                                [weights])

@partial(jax.jit, static_argnums=(4, ))
def _select_last(
    agent_state: jnp.ndarray,
    batch_f: jnp.ndarray,
    lr: float,
    key: np.ndarray,
    size: Optional[int] = None,
) -> Action:
    w_all = agent_state.seq_weights
    if size is not None:
        w_all = w_all[-(size + 1):]
    if len(w_all) == 1:
        q = jnp.matmul(batch_f, w_all[0])
        div = jnp.linalg.norm(q - jnp.mean(q), ord='inf')
        logits = 2 * q
        temperature = lr * jnp.sqrt(2) * div
    else:
        q_all = jnp.matmul(jnp.stack(w_all), batch_f.T)
        div = jnp.linalg.norm(q_all[1:] - q_all[:-1], ord='inf', axis=1)
        logits = q_all[1:].sum(axis=0) + q_all[-1]
        temperature = lr * jnp.sqrt(2 * jnp.power(div, 2).sum())
    action = jax.random.categorical(key, logits / temperature)
    return action

@partial(jax.jit, static_argnums=(4, ))
def _select_random(
    agent_state: jnp.ndarray,
    batch_f: jnp.ndarray,
    lr: float,
    key: np.ndarray,
    size: Optional[int] = None,
) -> Action:
    w_all = agent_state.seq_weights
    q = jnp.matmul(batch_f, w_all[-1])
    if (size is not None) and len(w_all) > size:
        idxs = jax.random.choice(key,
                                 jnp.arange(1, len(w_all)), (size, ),
                                 replace=False)
        scale = jnp.sqrt(len(w_all) / size)
    else:
        idxs = jnp.arange(1, len(w_all))
        scale = 1
    if len(w_all) == 1:
        div = jnp.linalg.norm(q - jnp.mean(q), ord='inf')
        logits = 2 * q
        temperature = lr * jnp.sqrt(2) * div
    else:
        w = jnp.stack(w_all)[idxs]
        w_ = jnp.stack(w_all)[idxs - 1]
        q_all = jnp.matmul(jnp.stack(w), batch_f.T)
        q_all_ = jnp.matmul(jnp.stack(w_), batch_f.T)
        div = jnp.linalg.norm(q_all - q_all_, ord='inf', axis=1)
        logits = q_all.sum(axis=0) + q
        temperature = lr * jnp.sqrt(2 * jnp.power(div, 2).sum())
        temperature *= scale
    action = jax.random.categorical(key, logits / temperature)
    return action

class Buffer:
    def __init__(self, capacity: Optional[int] = None):

        self._observations = deque(maxlen=capacity)
        self._actions = deque(maxlen=capacity)
        self._rewards = deque(maxlen=capacity)

    def append(self, timestep: dm_env.TimeStep, action: Action,
               new_timestep: dm_env.TimeStep):

        self._observations.append(timestep.observation)
        self._actions.append(action)
        self._rewards.append(new_timestep.reward)

        if new_timestep.last():
            raise AssertionError(
                'Environment should have infinite horizon (see paper).')

    def get(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
        batch_o = np.stack(self._observations)
        batch_a = np.stack(self._actions)
        batch_r = np.stack(self._rewards)
        return batch_o, batch_a, batch_r

class AAPI(Agent):
    def __init__(self,
                 obs_spec: specs.Array,
                 action_spec: specs.DiscreteArray,
                 basis_function: BasisFunction,
                 train_every: int,
                 learning_rate: float,
                 buffer_size: int,
                 rng: hk.PRNGSequence,
                 weight_select: str = 'last',
                 weight_size: Optional[int] = None):
        assert weight_select in [
            'random', 'last'
        ], "'weight_select' take values in ['random', 'last'] "
        self._obs_spec = obs_spec
        self._action_spec = action_spec
        self._basis_function = jax.vmap(basis_function)
        self._train_every = train_every
        self._learning_rate = learning_rate
        self._rng = rng
        self._weight_size = weight_size

        if weight_select == 'random':
            self._select = _select_random
        else:
            self._select = _select_last

        # initialize buffer
        capacity = buffer_size if buffer_size > 0 else None
        self._t = 0
        self._buffer = Buffer(capacity)

        # initialize parameters
        self._state = AgentState(seq_weights=[])

    def update(
        self,
        timestep: dm_env.TimeStep,
        action: Action,
        new_timestep: dm_env.TimeStep,
    ):
        self._t += 1
        self._buffer.append(timestep, action, new_timestep)
        if self._t % self._train_every == 0:
            batch_o, batch_a, batch_r = self._buffer.get()
            batch_f = self._basis_function(batch_o, batch_a)
            self._state = _eval(self._state, batch_f, batch_r)

    def select_action(
        self,
        timestep: dm_env.TimeStep,
    ) -> Action:
        nA = self._action_spec.num_values
        if len(self._state.seq_weights) == 0:
            action = jax.random.randint(next(self._rng), (), 0, nA)
            return int(action)
        lr = self._learning_rate
        o = timestep.observation
        batch_o = jnp.repeat(np.expand_dims(o, 0), nA, axis=0)
        batch_a = jnp.arange(nA)
        batch_f = self._basis_function(batch_o, batch_a)
        action = self._select(self._state, batch_f, lr, next(self._rng),
                              self._weight_size)
        return int(action)


"""
This is the main class for importing data from the CME SPAN FILES
"""
import os.path
import ntpath
from span_reader.instrument_info import InstrumentInfo
from span_reader.span_objects import *
from span_reader.mongo_queries import MongoQueries
import warnings
from tradingcore.signalapp import SignalApp, APPCLASS_DATA
from tradingcore.messages import *
from span_reader.settings import *
import numpy as np
#from span_reader.option_calcs import calculateOptionVolatilityNR

import pyximport; pyximport.install(setup_args={"include_dirs": np.get_include()})
#import span_reader.helloworld
from span_reader.option_calcs_fast import calculateOptionVolatilityNR

#C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build

class CmeSpanImport(object):
    """
        The object that contains methods to import span data into mongodb
    """

    def __init__(self, args = None):

        if args != None:
            self.args = args
            self.optionenabled = args['optionenabled']

        else:
            self.optionenabled = 2

        self.testing = TESTING

        self.mongo_queries = MongoQueries()

        self.instrumentInfo = InstrumentInfo(optionenabled=self.optionenabled)

        self.signalapp = SignalApp('historicaldata', APPCLASS_DATA, RABBIT_HOST, RABBIT_USER, RABBIT_PASSW)

    def load_span_file(self, filepath):
        """
            Reads and loads the span file into the mongodb
        """
        self.filepath = filepath
        self.short_file_name = ntpath.basename(self.filepath)

        #print(self.short_file_name)

        #print(self.filepath)

        if os.path.exists(self.filepath):

            if PUSH_SLACK_LOGGING:
                self.signalapp.send(MsgStatus('HISTORICAL_LOAD',
                                          'Initialized data load {0}'.format(self.short_file_name), notify=True))

            self.filled_risk_free_rate = False

            file_object = open(self.filepath, 'r')

            try:
                file_lines = file_object.readlines()
            except:
                file_lines = []

                while True:
                    try:
                        line = file_object.readline()

                        while line:
                            try:
                                file_lines.append(line)
                                line = file_object.readline()
                            except:
                                warnings.warn("Can't Import File Line")
                                continue

                        break
                    except:
                        continue

            #print(file_lines)

            #print('test ', self.filepath)

            #file_lines = self.mongo_queries.save_span_file(file_lines, file_name=self.short_file_name,
            #                                  full_file_name=self.filepath)

            data_row_type = SPAN_FILE_ROW_TYPES.TYPE_NULL

            for instrument in self.instrumentInfo.instrument_list:

                if RUN_SPECIFIC_INSTRUMENT_ID is None\
                        or instrument['idinstrument'] in RUN_SPECIFIC_INSTRUMENT_ID:

                    if self.testing:
                        self.test_df = []

                    rowListTypeB = [];
                    rowListTypeB_Future_dict = {}; #used as dictionary for finding future by month and year
                    rowListType8_F = [];

                    rowListTypeB_Option_dict = {};  # used as dictionary for finding future by month and year
                    rowListType8_OOF = [];
                    #rowListType8_P = [];

                    rowListTypeMAIN = [];

                    print('running',instrument['symbol'])

                    for line in file_lines:

                        #print(line)

                        line_row_type = self.get_cme_line_type(line)

                        #print(line_row_type)

                        if line_row_type == SPAN_FILE_ROW_TYPES.TYPE_0:
                            #print(line)

                            if len(line) > 0:

                                self.extract_rowtype_0(line_in = line, instrument_symbol = instrument)

                                '''
                                if PUSH_SLACK_LOGGING:
                                    self.signalapp.send(
                                        MsgStatus('HISTORICAL_LOAD',
                                              'Initialized Loading {0} {1}'.format(instrument['symbol'],
                                                                                self.span_file_date_time.strftime(
                                                                                '%b %d %Y')),
                                              notify=True))
                                              '''

                            '''get the interest rate from the database after date is extracted from rowtype_0'''

                            if not self.filled_risk_free_rate:
                                try:
                                    self.risk_free_rate = self.mongo_queries.get_risk_free_rate(self.span_file_date_time)
                                    self.filled_risk_free_rate = True
                                except:
                                    self.risk_free_rate = 0.01
                                    warnings.warn("Can't find risk free rate for: {0}".format(self.span_file_date_time))
                                    continue

                            '''update the instrument info specific to the current date'''
                            self.instrumentInfo.update_instrument_list(instrument,self.span_file_date_time)

                            data_row_type = self.decide_data_rowtype_based_on_file_prefix()

                            #print("data_row_type " + str(data_row_type))

                        elif line_row_type == SPAN_FILE_ROW_TYPES.TYPE_B:
                            """
                                Extracts the B row type data out of the line
                            """
                            if len(line) > 108:

                                dstBe = DataSpanType_B_Extract(line, instrument)
                                dstBe.extract_identifiers()

                                if dstBe.row_exchg == instrument['exchange']['spanexchangesymbol'] and \
                                    dstBe.underlying == instrument['spanfuturecode']:

                                    rowListTypeB.append(dstBe)

                                    #print("extract_rowtype_B ",
                                    #      instrument['exchange']['spanexchangesymbol'],
                                    #      instrument['spanfuturecode'],
                                    #     line)

                        elif line_row_type == data_row_type:

                            if len(line) > 0:#CME_SPAN_TYPE_8.product_type_end - 1:

                                dst8e = DataSpanType_8_Extract(line, self.span_file_date_time, instrument)
                                dst8e.extract_identifiers()

                                #print("@@@" + dst8e.row_exchg)

                                if dst8e.row_exchg == instrument['exchange']['spanexchangesymbol'] and \
                                        (dst8e.commodity_product_code == instrument['spanfuturecode'] or
                                            dst8e.commodity_product_code in \
                                                instrument['span_cqg_codes_dict']):
                                                 #dst8e.commodity_product_code == instrument['spanoptioncode']):

                                    if dst8e.product_type == SPAN_FILE_PRODUCT_TYPE_CODES.fut:
                                        rowListType8_F.append(dst8e)

                                        #print("@@@" + dst8e.row_exchg)

                                    elif dst8e.product_type == SPAN_FILE_PRODUCT_TYPE_CODES.oof:

                                        dst8e.extract_option_identifiers_type8()

                                        rowListType8_OOF.append(dst8e)

                                        #if dst8e.option_type == SPAN_FILE_CONTRACT_TYPE.call:

                                        #elif dst8e.option_type == SPAN_FILE_CONTRACT_TYPE.put:
                                        #    rowListType8_P.append(dst8e)

                        '''The row types have all been categorized. Now going through all the lists of the different line
                        types'''
                        '''
                        FILE_HEADER,
                        FUTURE_CONTRACT_INFO_IMPORT,
                        FUTURE_CONTRACT_DATA_IMPORT,
                        OPTION_CONTRACT_INFO_IMPORT,
                        OPTION_CALL_IMPORT,
                        OPTION_PUT_IMPORT
                        '''

                    '''below imports the future contract info'''
                    for row_dstBe_option_info in rowListTypeB:

                        row_dstBe_option_info.extract_commodity_product_code_identifiers()

                        if row_dstBe_option_info.product_type == SPAN_FILE_PRODUCT_TYPE_CODES.fut:

                            row_dstBe_option_info.extract_future_identifiers()

                            row_dstBe_option_info.extract_future_expiration()

                            rowListTypeB_Future_dict[row_dstBe_option_info.future_contract_month,row_dstBe_option_info.future_contract_year] = row_dstBe_option_info

                            if len(row_dstBe_option_info.future_expiration_str) > 0:
                                info_dict = \
                                {
                                    "contractname": row_dstBe_option_info.future_cqg_symbol,
                                    "expirationdate": row_dstBe_option_info.future_contract_expiration,
                                    "month": row_dstBe_option_info.future_contract_month_char,
                                    "idinstrument": row_dstBe_option_info.instrument['idinstrument'],
                                    "cqgsymbol": row_dstBe_option_info.future_cqg_symbol,
                                    "year": row_dstBe_option_info.future_contract_year,
                                    "monthint": row_dstBe_option_info.future_contract_month
                                    # "idcontract" : 1, # Not required
                                }

                                if self.testing:
                                    #pp = pprint.PrettyPrinter(indent=4)
                                    #pp.pprint(info_dict)
                                    row_dstBe_option_info.idcontract = 0

                                    #df = pd.DataFrame.from_dict(info_dict, orient="index")
                                    #if self.test_df is None:
                                    #    self.test_df = df
                                    #else:

                                    #x = str(info_dict)
                                    #print(self.test_df)

                                    self.test_df.append(str(info_dict))

                                    #print(self.test_df)

                                    #df.to_csv(instrument['symbol'] + "_data.csv")

                                else:
                                    contract_info_idcontract = self.mongo_queries.save_future_info(info_dict)

                                    row_dstBe_option_info.idcontract = contract_info_idcontract

                    '''gets future contract settlements'''
                    for row_dst_8_F_e_future_data in rowListType8_F:
                        #print('****** row_dst_8_F_e_future_data ' + row_dst_8_F_e_future_data.product_type)
                        #if row_dst_8_F_e_future_data.product_type == SPAN_FILE_PRODUCT_TYPE_CODES.fut:

                        row_dst_8_F_e_future_data.extract_future_identifiers()

                        #print('ticksize, display', instrument['spanticksize'], instrument['spantickdisplay'])

                        if data_row_type == SPAN_FILE_ROW_TYPES.TYPE_81:

                            row_dst_8_F_e_future_data.extract_settlement_filetype81( \
                                instrument['spanticksize'], instrument['spantickdisplay'])

                        elif data_row_type == SPAN_FILE_ROW_TYPES.TYPE_82:

                            row_dst_8_F_e_future_data.extract_settlement_filetype82( \
                                instrument['spanticksize'], instrument['spantickdisplay'])

                        row_dstBe_future = \
                            rowListTypeB_Future_dict[row_dst_8_F_e_future_data.future_contract_month, row_dst_8_F_e_future_data.future_contract_year]

                        row_dstBe_future.extracted_future_data_row = row_dst_8_F_e_future_data

                        #update future contract with settlement and date to mongo
                        #self.mongo_queries.fill_future_price(row_dst_8_F_e_future_data, row_dstBe_future)

                        info_dict = \
                            {'idcontract': row_dstBe_future.idcontract,
                             'settlement': row_dst_8_F_e_future_data.settlement_price,
                             'openinterest': 0,
                             'volume': 0,
                             'date': row_dst_8_F_e_future_data.span_file_date_time}

                        #self.mongo_queries.save_futures_settlement(info_dict)

                        if self.testing:
                            info_dict = \
                                {'idcontract': row_dstBe_future.idcontract,
                                 'settlement': row_dst_8_F_e_future_data.settlement_price,
                                 'settlement_price_str':row_dst_8_F_e_future_data.settlement_price_str,
                                 'openinterest': 0,
                                 'volume': 0,
                                 'date': row_dst_8_F_e_future_data.span_file_date_time}

                            self.test_df.append(str(info_dict))
                        else:
                            self.mongo_queries.save_futures_settlement(info_dict)

                    '''below imports the OPTION contract info'''
                    for row_dstBe_option_info in rowListTypeB:

                        row_dstBe_option_info.extract_commodity_product_code_identifiers()

                        #if row_dstBe_option_info.commodity_product in \
                        #        row_dstBe_option_info.instrument['span_cqg_codes_dict']:
                        #    print('!@#',row_dstBe_option_info.instrument['span_cqg_codes_dict'][row_dstBe_option_info.commodity_product])

                        if row_dstBe_option_info.product_type == SPAN_FILE_PRODUCT_TYPE_CODES.oof and \
                            row_dstBe_option_info.commodity_product in \
                                row_dstBe_option_info.instrument['span_cqg_codes_dict']:

                            row_dstBe_option_info.extract_future_identifiers()

                            row_dstBe_option_info.extract_option_identifiers_typeB()

                            row_dstBe_option_info.extract_option_expiration_and_timetoexp()

                            row_dstBe_option_info.span_underlying_future_contract_props = \
                                rowListTypeB_Future_dict[row_dstBe_option_info.future_contract_month, row_dstBe_option_info.future_contract_year]

                            rowListTypeB_Option_dict[
                                row_dstBe_option_info.option_contract_month, \
                                row_dstBe_option_info.option_contract_year, \
                                row_dstBe_option_info.commodity_product] = row_dstBe_option_info

                            #TimeSpan span = optionContractMonthDate - spanOptionContractProps.optionContractExpiration;
                            #if (span.TotalDays < 150)
                            #        spanOptionContractPropsList.Add(spanOptionContractProps)

                    for row_dst_8_OOF_e_option_data in rowListType8_OOF:
                        #print('****** row_dst_8_F_e_future_data ' + row_dst_8_F_e_future_data.product_type)
                        #if row_dst_8_F_e_future_data.product_type == SPAN_FILE_PRODUCT_TYPE_CODES.oof:

                        row_dst_8_OOF_e_option_data.extract_option_identifiers_type8()

                        if row_dst_8_OOF_e_option_data.commodity_product_code in \
                                row_dst_8_OOF_e_option_data.instrument['span_cqg_codes_dict']:

                            row_dst_8_OOF_e_option_data.extract_future_identifiers()

                            key = (row_dst_8_OOF_e_option_data.option_contract_month, \
                                    row_dst_8_OOF_e_option_data.option_contract_year, \
                                    row_dst_8_OOF_e_option_data.commodity_product_code)

                            if key in rowListTypeB_Option_dict:

                                row_dstBe_option_info = \
                                    rowListTypeB_Option_dict[key]

                                #if dst8e.option_type == SPAN_FILE_CONTRACT_TYPE.call:

                                optionTickSize = instrument['spanoptionticksize']
                                optionTickDisplay = instrument['spanoptiontickdisplay']

                                if instrument['secondaryoptionticksizerule'] > 0:

                                    ratio = instrument['secondaryoptionticksize'] / instrument['spanoptionticksize']

                                    optionTickDisplay = ratio * instrument['spanoptiontickdisplay']

                                    optionTickSize = instrument['secondaryoptionticksize']

                                if data_row_type == SPAN_FILE_ROW_TYPES.TYPE_81:

                                    row_dst_8_OOF_e_option_data.extract_settlement_filetype81( \
                                        optionTickSize, \
                                        optionTickDisplay)

                                elif data_row_type == SPAN_FILE_ROW_TYPES.TYPE_82:

                                    row_dst_8_OOF_e_option_data.extract_settlement_filetype82( \
                                        optionTickSize, \
                                        optionTickDisplay)

                                #row_dstBe_option_info.span_underlying_future_contract_props.extracted_future_data_row \
                                #    .settlement_price
                                '''
                                print('$$$$$$$$$$',row_dstBe_option_info.product_type, row_dst_8_OOF_e_option_data.option_type, \
                                                                   row_dstBe_option_info.span_underlying_future_contract_props.extracted_future_data_row \
                                                                    .settlement_price, \
                                                                   row_dst_8_OOF_e_option_data.option_strike_price, \
                                                                   row_dstBe_option_info.option_time_to_exp, \
                                                                   self.risk_free_rate, \
                                                                   row_dst_8_OOF_e_option_data.settlement_price, \
                                                                   optionTickSize)
                                                                   '''

                                #calculate implied vol

                                row_dst_8_OOF_e_option_data.implied_vol = \
                                    round(calculateOptionVolatilityNR(row_dst_8_OOF_e_option_data.option_type, \
                                                                   row_dstBe_option_info.span_underlying_future_contract_props.extracted_future_data_row \
                                                                    .settlement_price, \
                                                                   row_dst_8_OOF_e_option_data.option_strike_price, \
                                                                   row_dstBe_option_info.option_time_to_exp, \
                                                                   self.risk_free_rate, \
                                                                   row_dst_8_OOF_e_option_data.settlement_price, \
                                                                   optionTickSize),5)

                                #print('^^^^^^^^^^^',row_dst_8_OOF_e_option_data.implied_vol)

                                option_info_dict = \
                                {
                                    "expirationdate": row_dstBe_option_info.option_contract_expiration,
                                    "idinstrument": row_dst_8_OOF_e_option_data.instrument['idinstrument'],
                                    "strikeprice": row_dst_8_OOF_e_option_data.option_strike_price,
                                    # "idoption" : 3,  # Not required
                                    "callorput": row_dst_8_OOF_e_option_data.option_type,
                                    "optionname": row_dst_8_OOF_e_option_data.option_cqg_symbol,
                                    "optionmonthint": row_dst_8_OOF_e_option_data.option_contract_month,
                                    "cqgsymbol": row_dst_8_OOF_e_option_data.option_cqg_symbol,
                                    "idcontract": row_dstBe_option_info.span_underlying_future_contract_props.idcontract,
                                    "optionmonth": row_dst_8_OOF_e_option_data.option_contract_month_char,
                                    "optionyear": row_dst_8_OOF_e_option_data.option_contract_year,
                                    "optioncode": row_dst_8_OOF_e_option_data.option_span_cqg_code['optcod']
                                }

                                #id_option = self.mongo_queries.save_option_info(option_info_dict)

                                id_option = 0

                                if self.testing:
                                    #pp = pprint.PrettyPrinter(indent=4)
                                    #pp.pprint(option_info_dict)

                                    #df = pd.DataFrame.from_dict(option_info_dict, orient="index")
                                    #self.test_df = self.test_df.append(df)

                                    option_info_dict_test = \
                                        {
                                            "strike_price_str": row_dst_8_OOF_e_option_data.option_strike_price_str,
                                            "expirationdate": row_dstBe_option_info.option_contract_expiration,
                                            "idinstrument": row_dst_8_OOF_e_option_data.instrument['idinstrument'],
                                            "strikeprice": row_dst_8_OOF_e_option_data.option_strike_price,
                                            # "idoption" : 3,  # Not required
                                            "callorput": row_dst_8_OOF_e_option_data.option_type,
                                            "optionname": row_dst_8_OOF_e_option_data.option_cqg_symbol,
                                            "optionmonthint": row_dst_8_OOF_e_option_data.option_contract_month,
                                            "cqgsymbol": row_dst_8_OOF_e_option_data.option_cqg_symbol,
                                            "idcontract": row_dstBe_option_info.span_underlying_future_contract_props.idcontract,
                                            "optionmonth": row_dst_8_OOF_e_option_data.option_contract_month_char,
                                            "optionyear": row_dst_8_OOF_e_option_data.option_contract_year,
                                            "optioncode": row_dst_8_OOF_e_option_data.option_span_cqg_code['optcod']
                                        }

                                    self.test_df.append(str(option_info_dict_test))
                                else:
                                    id_option = self.mongo_queries.save_option_info(option_info_dict)

                                #self.mongo_queries.fill_option_info_and_data(row_dst_8_OOF_e_option_data, \
                                #                                             row_dstBe_option_info)

                                option_data_dict = \
                                    {
                                        "timetoexpinyears" : row_dstBe_option_info.option_time_to_exp,
                                        "idoption" : id_option,
                                        "price" : row_dst_8_OOF_e_option_data.settlement_price,
                                        "datetime" : row_dst_8_OOF_e_option_data.span_file_date_time,
                                        "impliedvol" : row_dst_8_OOF_e_option_data.implied_vol
                                    }

                                #id_option = self.mongo_queries.save_options_data(option_data_dict)

                                if self.testing:
                                    option_data_dict_test = \
                                        {
                                            "timetoexpinyears": row_dstBe_option_info.option_time_to_exp,
                                            "idoption": id_option,
                                            "price_str": row_dst_8_OOF_e_option_data.settlement_price_str,
                                            "price": row_dst_8_OOF_e_option_data.settlement_price,
                                            "datetime": row_dst_8_OOF_e_option_data.span_file_date_time,
                                            "impliedvol": row_dst_8_OOF_e_option_data.implied_vol
                                        }

                                    self.test_df.append(str(option_data_dict_test))
                                else:
                                    self.mongo_queries.save_options_data(option_data_dict)

                    #self.test_df.to_csv(instrument['symbol'] + "_data.csv")

                    #x = pd.DataFrame(self.test_df)
                    #x.to_csv(instrument['symbol'] + "_data.csv")

                    if self.testing:
                        thefile = open(instrument['symbol'] + "_data.txt", 'w')
                        for item in self.test_df:
                            print(item)
                            thefile.write("%s\n" % item)

                    print('finished', instrument['symbol'])

                    if PUSH_SLACK_LOGGING:
                        self.signalapp.send(
                            MsgStatus('HISTORICAL_LOAD',
                                  'Loaded {0} {1}'.format(instrument['symbol'],
                                                                    self.span_file_date_time.strftime('%b %d %Y')),
                                  notify=True))

    def get_cme_line_type(self, line = ''):
        """
            Gets the CME line type
            Parameters:
              line - the line of the file
        """
        rowType = SPAN_FILE_ROW_TYPES.TYPE_NULL

        try:

            if len(line) > 2:

                rowTypeString = line[0:2].strip()

                switcher = {
                    "0" : SPAN_FILE_ROW_TYPES.TYPE_0,
                    "81": SPAN_FILE_ROW_TYPES.TYPE_81,
                    "82": SPAN_FILE_ROW_TYPES.TYPE_82,
                    "B": SPAN_FILE_ROW_TYPES.TYPE_B,
                    "83": SPAN_FILE_ROW_TYPES.TYPE_83,
                }

                return switcher.get(rowTypeString, -1)

        except:

            print("rowtype error")

            return rowType

    def extract_rowtype_0(self, line_in = '', instrument_symbol = ''):
        """
            Extracts the 0 row type data out of the line
            Parameters:
              line - the line of the file
              instrument - the instrument that is importing
        """

        try:

            exchangeComplex = line_in[CME_SPAN_TYPE_0.exchange_complex_start:
                CME_SPAN_TYPE_0.exchange_complex_end].strip()

            # ccyyMMdd
            businessDate = line_in[CME_SPAN_TYPE_0.business_date_start:
                CME_SPAN_TYPE_0.business_date_end].strip()

            settlementOrIntraday = line_in[CME_SPAN_TYPE_0.settlement_or_intraday_start:
                CME_SPAN_TYPE_0.settlement_or_intraday_end].strip()

            fileIdentifier = line_in[CME_SPAN_TYPE_0.file_identifier_start:
                CME_SPAN_TYPE_0.file_identifier_end].strip()

            fileFormat = line_in[CME_SPAN_TYPE_0.file_format_start:
                CME_SPAN_TYPE_0.file_format_end].strip()

            self.span_file_date_time = datetime.strptime(businessDate,"%Y%m%d")

            #print(instrument_symbol['symbol'],
            #      self.span_file_date_time,
            #      exchangeComplex,
            #      businessDate,
            #      settlementOrIntraday,
            #      fileIdentifier,
            #      fileFormat)

        except:

            #print("extract_rowtype_0 error")
            logging.exception("extract_rowtype_0 error")

    def decide_data_rowtype_based_on_file_prefix(self):
        """
            Chooses the rowtype to use based on the prefix of the file name
        """

        if self.short_file_name[0:3] == "ccl":

            return SPAN_FILE_ROW_TYPES.TYPE_82

        elif self.short_file_name[0:3] == "cme":

            if self.span_file_date_time >= datetime(2010, 5, 5):

                return SPAN_FILE_ROW_TYPES.TYPE_81

            else:

                return SPAN_FILE_ROW_TYPES.TYPE_82

        elif self.short_file_name[0:3] == "nyb":

            return SPAN_FILE_ROW_TYPES.TYPE_82


import math, random, copy
import numpy as np
import os, sys

import torch
import torch.nn as nn
import torch.optim as optim
import torch.autograd as autograd
import torch.nn.functional as F

from DGN import DGN
from buffer_R import ReplayBuffer
from env import multi_agent_env as Surviving
from config import *

USE_CUDA = torch.cuda.is_available()
print('yes')
env = Surviving(n_agent=5)
print(env.landmarks)
n_ant = env.n_agent
observation_space = env.len_obs
n_actions = env.n_action
mask_dim = env.mask_dim

# buff = ReplayBuffer(capacity, observation_space, n_actions, n_ant)
buff = ReplayBuffer(capacity,hidden_dim, observation_space, n_actions, n_ant)
model = DGN(n_ant, observation_space, hidden_dim, n_actions,mask_dim)
model = model.cuda()
# model.load_state_dict(torch.load('model_cifar_BiGRU_hidden7070.pt'))
model.load_state_dict(torch.load('model_cifar.pt'))

best_score = 0
M_Null = torch.Tensor(np.array([np.eye(n_ant)] * batch_size)).cuda()
KL = nn.KLDivLoss()

i_episode += 1
steps = 0
obs = env.reset()
env.render()
terminated = False
hidden_states = None
epsilon = 0.0
# env.state_n= [np.array([71, 71,  0]) for _ in
#                         range(n_ant)]
    # print(adj.shape)
# while True:
#         # print(obs)
#     steps += 1
#     action = []
#     n_adj = adj + np.eye(n_ant)
#     q, a_w = model(torch.Tensor(np.array([obs])).cuda(), torch.Tensor(np.array([n_adj])).cuda())
#     q = q[0]
#     for i in range(n_ant):
#         if np.random.rand() < epsilon:
#             a = np.random.randint(n_actions)
#         else:
#             a = q[i].argmax().item()
#         action.append(a)
#
#     next_obs, next_adj, reward, terminated = env.step(action)
#     env.render()
#     # break
#     buff.add(np.array(obs), action, reward, np.array(next_obs), n_adj, next_adj, terminated)
#     obs = next_obs
#     adj = next_adj
#     score += sum(reward)
#     if terminated:
#         break

while True:
    # print(obs)
    steps += 1
    action = []
    if hidden_states is None:
        hidden_states = model.init_hidden()
    q, next_hidden_states = model(torch.Tensor(np.array([obs])).cuda(), hidden_states)
    env.render()
    # if hidden_states is not None:
    hidden_states = hidden_states.detach().cpu().numpy()
    next_hidden_states = next_hidden_states.detach().cpu().numpy()
    q = q[0]
    for i in range(n_ant):
        a = q[i].argmax().item()
        action.append(a)
    # print(action)
    next_obs, reward, terminated = env.step(action)
    # break
    buff.add(np.array(obs), action, reward, np.array(next_obs), hidden_states, next_hidden_states, terminated)
    hidden_states = torch.Tensor(next_hidden_states).cuda()
    obs = next_obs

    score += sum(reward)
    if terminated:
        break


#!/usr/bin/env python

import rospy

from precise_driver.srv import Gripper, GripperRequest, GripperResponse
from precise_driver.srv import Plate, PlateRequest, PlateResponse
from precise_driver.srv import SetFreeMode, SetFreeModeRequest, SetFreeModeResponse
from std_srvs.srv import Trigger, TriggerResponse, SetBool, SetBoolResponse
from cob_srvs.srv import SetString, SetStringResponse

class EmulatedPreciseDriver(object):
    def __init__(self):
        self.init_srv = rospy.Service('/arm/driver/init', Trigger, self.init_cb)
        self.recover_srv = rospy.Service('/arm/driver/recover', Trigger, self.recover_cb)
        self.teachmode_srv = rospy.Service('/arm/driver/teach_mode', SetFreeMode, self.teachmode_cb)
        self.home_srv = rospy.Service('/arm/driver/home', Trigger, self.home_cb)
        self.power_srv = rospy.Service('/arm/driver/power', SetBool, self.power_cb)
        self.command_srv = rospy.Service('/arm/driver/command', SetString, self.command_cb)
        self.grasp_plate_srv = rospy.Service('/arm/driver/grasp_plate', Plate, self.grasp_plate_cb)
        self.release_plate_srv = rospy.Service('/arm/driver/release_plate', Plate, self.release_plate_cb)
        self.gripper_srv = rospy.Service('/arm/driver/gripper', Gripper, self.gripper_cb)

        rospy.loginfo('Kevin EmulatedPreciseDriver is running')

    def init_cb(self, req):
        res = TriggerResponse()
        res.success = True
        return res

    def recover_cb(self, req):
        res = TriggerResponse()
        res.success = True
        return res

    def teachmode_cb(self, req):
        output_message = "Teach mode ACTIVATED!" if req.data else "Teach mode DEACTIVATED!"
        rospy.loginfo("{}".format(output_message))
        res = SetFreeModeResponse()
        res.success = True
        return res

    def home_cb(self, req):
        res = TriggerResponse()
        res.success = True
        return res

    def power_cb(self, req):
        output_message = "Power is ACTIVATED!" if req.data else "Power is DEACTIVATED!"
        rospy.loginfo("{}".format(output_message))
        res = SetBoolResponse()
        res.success = True
        return res

    def command_cb(self, req):
        output_message = "Command is {}".format(req.data)
        rospy.loginfo("{}".format(output_message))
        res = SetStringResponse()
        res.success = True
        return res

    def grasp_plate_cb(self, req):
        res = PlateResponse()
        res.success = True
        return res

    def release_plate_cb(self, req):
        res = PlateResponse()
        res.success = True
        return res

    def gripper_cb(self, req):
        res = GripperResponse()
        res.success = True
        return res

if __name__ == '__main__':
    rospy.init_node('emulated_precise_driver')

    EmulatedPreciseDriver()
    rospy.spin()


#!/usr/bin/env python3

from subprocess import call
import os
from Bio import SeqIO
import shutil
import multiprocessing
import datetime

import drep.d_filter
import drep.d_cluster

# def run_cmd(cmd,dry=False,shell=True,quiet= True):
#     devnull = open(os.devnull, 'w')
#     if shell:
#         if not dry:
#             if quiet:
#                 call(cmd,shell=True,stdout=devnull, stderr=devnull)
#             else:
#                 call(cmd,shell=True)
#         else:
#             print(cmd)
#     else:
#         if not dry:
#             if quiet:
#                 call(cmd,stdout=devnull, stderr=devnull)
#             else:
#                 call(cmd)
#         else: print(' '.join(cmd))
#     return

def run_cmd(cmd, dry=False, shell=True, logdir=False):
    # if dry, just print cmd
    if dry:
        if shell:
            print(cmd)
        else:
            print(' '.join(cmd))
        return

    # figure out what files you're going to store the output
    if logdir == False:
        devnull = open(os.devnull, 'w')
        sto = devnull
        ste = devnull
    else:
        uniq_filename = str(datetime.datetime.now().date()) + '_' + \
            str(datetime.datetime.now().time()).replace(':', '.')
        sto = open(os.path.join(logdir + uniq_filename + '.STDOUT'), 'w')
        ste = open(os.path.join(logdir + uniq_filename + '.STDERR'), 'w')

        # log the command
        now = open(os.path.join(logdir + uniq_filename + '.CMD'), 'w')
        if shell:
            now.write(str(cmd) + '\n')
        else:
            now.write(' '.join(cmd) + '\n')
        now.close()

    # run the command
    if shell:
        call(cmd,shell=True,stdout=sto, stderr=ste)
    else:
        call(cmd,stdout=sto, stderr=ste)
    return

def thread_cmd_wrapper(tup):
    run_cmd(*tup)

def thread_cmds(cmds, dry=False, shell=False, logdir=False, t=10):
    pool = multiprocessing.Pool(processes=t)
    tups = [(cmd, dry, shell, logdir) for cmd in cmds]
    pool.map(thread_cmd_wrapper, tups)
    pool.close()
    pool.join()
    return

def make_dir(outdirname,dry=False,overwrite=False):
    if dry:
        return
    if os.path.exists(outdirname):
        if overwrite:
            pass
            #shutil.rmtree(outdirname, ignore_errors=True)
            #os.makedirs(outdirname)
        else:
            assert False, "{0} already exsists! Will not overwrite with current settings".\
                            format(outdirname)
    else:
        os.makedirs(outdirname)

def clobber_dir(outdirname,dry=False,overwrite=False):
    if dry:
        return
    if os.path.exists(outdirname):
        if overwrite:
            shutil.rmtree(outdirname, ignore_errors=True)
            os.makedirs(outdirname)
        else:
            assert False, "{0} already exsists! Will not overwrite with current settings".\
                            format(outdirname)
    else:
        os.makedirs(outdirname)

def fasta_length(fasta):
    total = 0
    for seq_record in SeqIO.parse(fasta, "fasta"):
        total += len(seq_record)
    return total


import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from sklearn.metrics import f1_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
os.getcwd()
Mdata=pd.read_csv("Churn1.csv")
os.getcwd()
Mdata=pd.read_csv("Churn1.csv")
#copying the oregional set of data inanother variable
data=Mdata.copy(deep=True)
data.info()
data.describe()
data.isnull().sum()
#checking and filling out the missing values in the TotalCharges column.
print(np.unique(data['TotalCharges']))
x=data['TotalCharges'].mean()
print(x)
#replacing nan with the mean values
data['TotalCharges']=data['TotalCharges'].replace(np.nan,x)
#check the unique values of TotalCharges column
print(np.unique(data['TotalCharges']))
data.isnull().sum()

data_col=data.columns
data_col
le = preprocessing.LabelEncoder()
for i in data_col:
    data[i]=le.fit_transform(data[i])
    #after conversion , now trying to visualse the corelations and the trends formed by the data
#figuring out the relation between the coustomer data and churn
corr = data.corr()
sns.heatmap(corr, xticklabels=corr.columns.values, yticklabels=corr.columns.values, annot = True, annot_kws={'size':12})
heat_map=plt.gcf()
heat_map.set_size_inches(20,15)
plt.xticks(fontsize=10)
plt.yticks(fontsize=10)
plt.show()
data=data.drop(columns='TotalCharges')
data= data.drop(columns='gender')

data.info()

#lets see how many data points are there in the Churn
print(len(data['Churn']))
#figuring out the exact value of customers churned from the company
data['Churn'].value_counts()

data['Churn'].value_counts().plot(kind = 'bar', title = ' Non-Churners vs Churners by Count (Churn is a 1)', color = 'red', align = 'center')
plt.show()

# Finding the relationship between instances of Tech Support and Churn.
# Stacked Bar of Tech Support and Churn.
tech_support_churn = pd.crosstab(data['TechSupport'], data['Churn'])
tech_support_churn.plot(kind = 'bar', stacked = True)
plt.ylabel('Count')
plt.xlabel('Tech Support Count')
plt.title('Churn Rate Relative to Uses of Tech Support (Churned is a 1)')
plt.show()
#in x lable the tech support count - 0 =No, 1= No internet service and 2= Yes
#by looking at the graph below we can figure out that, the people who use tech support are less lickely to be Churner

# Churn rate relative to tenure.
# Stacked bar of tenure and churn.
tenure_churn = pd.crosstab(data['tenure'], data['Churn'])
tenure_churn.plot(kind = 'bar', stacked = True)
plt.ylabel('Count')
plt.xlabel('Tenure of Subscription')
plt.title('Churn Rate Relative to Tenure of Subscription (Churned is a 1)')
plt.show()
# We can see that long term subscribers are less lickly to be a cherner

# Looking at the relation beteew , how the customer pays column with the churn column.
_, axes = plt.subplots(1, 2, sharey=True, figsize=(10, 4))
sns.countplot(x='PaperlessBilling', hue='Churn',
              data=data, ax=axes[0]);
sns.countplot(x='PaymentMethod', hue='Churn',
              data=data, ax=axes[1]);
# We can see that customers that use paperless billing are much more likely to churn (0 = don't have paperless billing). That seems backwards I would go check that data with the team.
# We can see that customers that have the 0 payment method (electronic check) are much more likely to churn. Let's discourage that option.

# See if the other products they have from this company has to do with their churn.
_, axes = plt.subplots(1, 2, sharey=True, figsize=(10, 4))
sns.countplot(x='PhoneService', hue='Churn',
              data=data, ax=axes[0]);
sns.countplot(x='InternetService', hue='Churn',
              data=data, ax=axes[1]);
# If they don't have Phone Service, they are more likely to churn.
# If they don't have Internet Service, they are more likely to churn. Those customers with the highest Internet Service are least likely to churn.
# Conclusion: This makes sense. Customers with other products from the company, and premium products, churn less.
# Offer customers these additional products, perhaps even at a deep discount, so they take them and are less likely to churn. #preparing the data and fitting the test and train data
data.head()
y=data.iloc[:,-1]
X=data.iloc[:,0:-1]
from sklearn.feature_selection import SelectKBest
test = SelectKBest(score_func=chi2, k=4)
fit = test.fit(X, y)
features = fit.transform(X)
features
print(features[0:5,:])
np.set_printoptions(precision=3)
print(fit.scores_)
c=['Partner','InternetService', 'OnlineSecurity',
       'OnlineBackup','TechSupport',
       'Contract','PaymentMethod','tenure']

x=data.loc[:,c]
scaler = StandardScaler()
X=scaler.fit_transform(X)
x=scaler.fit_transform(x)

X
#APPLYING MODELS ON UNBALENCED CLASS DATA
#LOGISTIC
f1 = []
acc=[]
n = 150
print("Logistic:")
for i in range(n):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
    lr = LogisticRegression(random_state=0, solver='liblinear')
    lr.fit(X_train, y_train)
    ypred=lr.predict(X_test)
    f1.append(f1_score(y_test, ypred, average='weighted'))
    acc.append(accuracy_score(y_test, ypred))
print("Avg F1-Score",np.mean(f1))
print("Max F1-Score",np.max(f1))
print("Min F1-Score",np.min(f1))
print("Avg Accuracy",np.mean(acc))
print("Max Accuracy",np.max(acc))
print("Min Accuracy",np.min(acc))

#RANDOM FOREST
n = 50
print("Random Forest:")
for i in range(n):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
    rf = RandomForestClassifier(n_estimators = 100, random_state = 0)
    rf.fit(X_train, y_train)
    ypred=rf.predict(X_test)
    f1.append(f1_score(y_test, ypred, average='weighted'))
    acc.append(accuracy_score(y_test, ypred))
print("Avg F1-Score",np.mean(f1))
print("Max F1-Score",np.max(f1))
print("Min F1-Score",np.min(f1))
print("Avg Accuracy",np.mean(acc))
print("Max Accuracy",np.max(acc))
print("Min Accuracy",np.min(acc))

#RANDOM FOREST
score = metrics.accuracy_score(y_test, ypred)
print("accuracy:   %0.3f" % (score*100))
cm = metrics.confusion_matrix(y_test, ypred, labels=[0,1])
print(cm)
sns.set(font_scale=1.4)
sns.heatmap(cm,annot=True,annot_kws={'size':16})
#sns.set_axis_labels('true values','predicted')
plt.xlabel("predicted")
plt.ylabel("true values")

from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
auc = roc_auc_score(y_test,ypred)
print('AUC: %.3f' % auc)
fpr, tpr, thresholds = roc_curve(y_test, ypred)
# plot no skill
plt.plot([0, 1], [0, 1], linestyle='--')
# plot the roc curve for the model
plt.plot(fpr, tpr, marker='.')
# show the plot
plt.show()


#!/usr/bin/env python

import logging

from beepboop import resourcer
from beepboop import bot_manager

from bot.settings import config
from bot.slack_bot import SlackBot
from bot.slack_bot import spawn_bot

logger = logging.getLogger(__name__)

if __name__ == "__main__":

    log_level = config.log_level
    logging.basicConfig(format='%(asctime)s - %(levelname)s: %(message)s', level=log_level)

    slack_token = config.slack_token

    if slack_token == "":
        logging.info(
            "SLACK_TOKEN env var not set, expecting token to be provided by Resourcer events"
        )
        slack_token = None
        botManager = bot_manager.BotManager(spawn_bot)
        res = resourcer.Resourcer(botManager)
        res.start()
    else:
        # only want to run a single instance of the bot in dev mode
        bot = SlackBot(slack_token)
        bot.start({})


from tensorflow.keras.optimizers import Adam
init_lr = 1e-4
epochs = 100
opt = Adam(learning_rate=init_lr, decay=init_lr / epochs)

modelA.compile(optimizer=opt,
              loss = ["mse", # for age
                      "binary_crossentropy"], # for gender
              loss_weights = [4.,  # for age
                              0.1], # for gender
              metrics = ["mae",    # for age
                         "accuracy"]) # for gender

from keras.callbacks import ModelCheckpoint

callbacks = [
     ModelCheckpoint("./model_checkpoint", monitor='val_loss')
]

def generator_wrapper(generator):
  for batch_x,batch_y in generator:
    yield (batch_x,[batch_y[:,i] for i in range(2)])

STEP_SIZE_TRAIN=train_generator.n//train_generator.batch_size
STEP_SIZE_VALID=valid_generator.n//valid_generator.batch_size
STEP_SIZE_TEST=test_generator.n//test_generator.batch_size

history = modelA.fit_generator(
                    generator=generator_wrapper(train_generator),
                    steps_per_epoch=STEP_SIZE_TRAIN,
                    epochs = 1,
                    verbose= 2,
                    validation_data = generator_wrapper(valid_generator),
                    validation_steps=70)

model_folder = '/content/drive/MyDrive/Colab Notebooks/models'
modelA.save(model_folder+"age_gender_A.h5")


from sunpy.net import Fido, attrs as a, vso
import datetime
from astropy import units as u
import urllib
from bs4 import BeautifulSoup
import os
import time
import numpy as np

#start_time = time.time()

#get utc time rounded down to the previous hour
time_now = datetime.datetime.utcnow().replace(microsecond = 0, second = 0, minute = 0)

def get_xrt(time_now, out_dir):
	if not os.path.exists(out_dir):
		os.mkdir(out_dir)

	base_url = 'http://solar.physics.montana.edu/HINODE/XRT/QL/syn_comp_fits/'

	#find the list of available fits files on webpage
	test = urllib.request.urlopen(base_url)
	soup = BeautifulSoup(test, features="lxml")

	fits_links = []
	for link in soup.findAll('a'):
		if link.get('href')[-4:] == 'fits':
			fits_links.append(base_url  + link.get('href'))

	time_dif = []
	for i in range(len(fits_links)):

		time_dif.append((time_now - datetime.datetime.strptime(fits_links[i][-22:-7], '%Y%m%d_%H%M%S')).total_seconds())

	index = np.where(time_dif == np.min(time_dif))[0][0]
	latest_file = fits_links[index]

	print(latest_file)

	#output path directory
	output_path2 = out_dir + latest_file.split('/')[-1]

	urllib.request.urlretrieve(latest_file, output_path2)
	if os.path.exists(output_path2):
		print('XRT success!')

#end_time = time.time()
#print(end_time - start_time)


import contextlib
import collections
import copy

import advent_tools

def main():
    input_data = advent_tools.read_input_lines()
    dependencies = process_input(input_data)
    print('Part 1:', run_part_1(copy.deepcopy(dependencies)))
    print('Part 2:', run_part_2(dependencies, num_workers=5))

def process_input(data):
    result = collections.defaultdict(set)
    for line in data:
        words = line.split()
        result[words[-3]].add(words[1])
    return result

def run_part_1(dependencies):
    not_done = {val for dep in dependencies.values()
                for val in dep}.union(dependencies.keys())
    completed = []
    while not_done:
        for step in sorted(not_done):
            if not dependencies[step]:
                completed.append(step)
                not_done.remove(step)
                for dep in dependencies.values():
                    with contextlib.suppress(KeyError):
                        dep.remove(step)
                break
    return "".join(completed)

def run_part_2(dependencies, num_workers=2):
    not_started = {val for dep in dependencies.values()
                   for val in dep}.union(dependencies.keys())
    completion_times = collections.defaultdict(set)
    available = set()
    completion_times[0] = set()
    tick = 0
    while completion_times:
        tick = sorted(completion_times)[0]
        not_started = not_started.difference(completion_times[tick])
        for step, dep in dependencies.items():
            with contextlib.suppress(KeyError):
                dependencies[step] = dep.difference(completion_times[tick])
        available = available.union(
            {step for step in sorted(not_started) if not dependencies[step]}).difference(
            completion_times[tick]
        )
        del completion_times[tick]
        for worker in range(num_workers - len(completion_times)):
            if available:
                to_do = sorted(available)[0]
                available.remove(to_do)
                not_started.remove(to_do)
                completion_times[ord(to_do) - 4 + tick].add(to_do)
    return tick

if __name__ == '__main__':
    main()


import os
import random
from typing import Any, Dict, List, Text

import apache_beam as beam
import tensorflow as tf
from tfx import types
from tfx.dsl.components.base import base_beam_executor
from tfx.dsl.io import fileio
from tfx.types import artifact_utils
from tfx.utils import io_utils, json_utils

from tfx_addons.sampling import spec

class Executor(base_beam_executor.BaseBeamExecutor):
  """Executor for Sampler."""
  def _CreatePipeline(self, unused_transform_output: Text) -> beam.Pipeline:
    """Creates beam pipeline.
    Args:
      unused_transform_output: unused.
    Returns:
      Beam pipeline.
    """

    return self._make_beam_pipeline()

  def Do(
      self,
      input_dict: Dict[Text, List[types.Artifact]],
      output_dict: Dict[Text, List[types.Artifact]],
      exec_properties: Dict[Text, Any],
  ) -> None:
    """Sampler executor entrypoint.
    Args:
      input_dict: Input dict from input key to a list of artifacts, including:
        - examples: A list of type `standard_artifacts.Examples` which should
          contain custom splits specified in splits_config. If custom split is
          not provided, this should contain two splits 'train' and 'eval'.
      output_dict: Output dict from key to a list of artifacts, including:
        - sampled_examples: sampled examples, only for the given
          splits as specified in splits. May also include copies of the
          other non-sampled spits, as specified by null_classes.
      exec_properties: A dict of execution properties, including:
        - name: Optional unique name. Necessary if multiple components are
          declared in the same pipeline.
        - label: The name of the column containing class names to sample by.
        - splits: A list containing splits to sample.
        - copy_others: Determines whether we copy over the splits that aren't
          sampled, or just exclude them from the output artifact. Defaults
          to True.
        - shards: The number of files that each sampled split should
          contain. Default 0 is Beam's tfrecordio function's default.
        - null_classes: A list determining which classes that we should
          not sample. Defaults to None.
        - sampling_strategy: An enum of type SamplingStrategy, determining if
          the executor should over or undersample.
    Returns:
      None
    """

    self._log_startup(input_dict, output_dict, exec_properties)

    label = exec_properties[spec.SAMPLER_LABEL_KEY]
    sampling_strategy = exec_properties[spec.SAMPLER_SAMPLE_KEY]
    splits = json_utils.loads(exec_properties[spec.SAMPLER_SPLIT_KEY])
    copy_others = exec_properties[spec.SAMPLER_COPY_KEY]
    shards = exec_properties[spec.SAMPLER_SHARDS_KEY]
    null_classes = json_utils.loads(exec_properties[spec.SAMPLER_CLASSES_KEY])

    input_artifact = artifact_utils.get_single_instance(
        input_dict[spec.SAMPLER_INPUT_KEY])
    output_artifact = artifact_utils.get_single_instance(
        output_dict[spec.SAMPLER_OUTPUT_KEY])

    if sampling_strategy not in (spec.SamplingStrategy.UNDERSAMPLE,
                                 spec.SamplingStrategy.OVERSAMPLE):
      raise ValueError("Invalid sampling strategy!")

    if not splits:
      raise ValueError("No splits are marked for sampling!")

    for split in splits:
      if split not in input_artifact.split_names:
        raise ValueError(
            f"Invalid split name {split} is not in input artifact!")

    if shards < 0:
      raise ValueError("Shards value must be non-negative!")

    if copy_others:
      output_artifact.split_names = input_artifact.split_names
    else:
      output_artifact.split_names = artifact_utils.encode_split_names(splits)

    # Fetch the input uri for each split
    split_data = {}
    for split in artifact_utils.decode_split_names(input_artifact.split_names):
      uri = artifact_utils.get_split_uri([input_artifact], split)
      split_data[split] = uri

    for split, uri in split_data.items():
      if split in splits:  # Undersampling split
        output_dir = artifact_utils.get_split_uri([output_artifact], split)
        split_dir = os.path.join(output_dir, f"Split-{split}")
        with self._CreatePipeline(split_dir) as p:
          data = read_tfexamples(p, uri, label)
          merged = sample_examples(data, null_classes, sampling_strategy)
          write_tfexamples(merged, shards, split_dir)
      elif copy_others:  # Copy the other split if copy_others is True
        input_dir = uri
        output_dir = artifact_utils.get_split_uri([output_artifact], split)
        for filename in fileio.listdir(input_dir):
          input_uri = os.path.join(input_dir, filename)
          output_uri = os.path.join(output_dir, filename)
          io_utils.copy_file(src=input_uri, dst=output_uri, overwrite=True)

def _generate_elements(example, label):
  """Function that fetches the class label from a tf.Example and returns one
  item in a K-V PCollection with the key as the label and the value as the
  string-parsed tf.Example.

  Args:
    example: a tf.Example in serialized format, taken directly from a
      TFRecordDataset.
    label: string containing the name of the categorical variable that we are
      extracting from the example.
  Returns:
    Tuple with two items. First item is a class label; second item is the input
      tf.Example, deserialized and parsed from string format.
  """

  class_label = None
  parsed = tf.train.Example.FromString(example.numpy())
  if parsed.features.feature[label].int64_list.value:
    val = parsed.features.feature[label].int64_list.value
    if len(val) > 0:
      class_label = val[0]
  else:
    val = parsed.features.feature[label].bytes_list.value
    if len(val) > 0:
      class_label = val[0].decode()
  return (class_label, parsed)

def sample_data(class_key, val, key_counts_dict=None, goal_count=0):
  """Function called in a Beam pipeline that performs sampling using Python's
  random over each row of data.

  Args:
    class_key: Class key for the current row.
    val: TFExample value for current row.
    key_counts_dict: Dictionary containing element count for each class key.
    goal_count: How many elements should each class have at the end of
      execution.

  Returns:
    Iterator of TFExample
  """

  class_count = key_counts_dict[class_key]
  percentage = goal_count / class_count

  # Deterministic number of times value should be returned (used when oversampling)
  count = int(percentage)
  data = [val] * count

  # Randomly return value based on percentage of class should be returned overall
  rand = random.random()
  if rand < (percentage % 1):
    data.append(val)

  # Uncomment below to debug behaviour when testing
  # print(f"Returning {data} from {percentage} ({goal_count} / {class_count}) -> {rand} < {percentage % 1}")
  for item in data:
    yield item

def filter_null(item, keep_null=False, null_vals=None):
  """Function that returns or doesn't return the inputted item if its first
  value is either a False value or is in the inputted null_vals list.

  This function first determines if the item's first value is equivalent to
  False using bool(), with one exception; 0 is considered as "True". If the
  first value is in null_vals, the first value is automatically considered a
  "null value" and is therefore considered to be False. If the value is False,
  then None is returned; if the value is True, then the original item is
  returned. The keep_null value reverses this, so True values return None,
  and False values return the item.

  Args:
    item: Tuple whose first value determines whether it is returned or not.
      Should always be a two-value tuple, with the first value being the class
      value and the second value being all examples that belong to that class.
    keep_null: Determines whether we keep False/"null" values or True/not
      "null" values.
    null_vals: List containing values that should be considered as False/"null".
  Returns:
    None or the inputted item, depending on if the item is False/in null_vals,
      and then depending on the value of keep_null.
  """
  if item[0] == 0:
    keep = True
  else:
    keep = bool(item[0])

  if null_vals and str(item[0]) in null_vals and keep:
    keep = False
  keep ^= keep_null

  return item if keep else None

def read_tfexamples(p, uri, label):
  """Function that reads tf.Examples from tfRecord files and converts them
  to a K-V PCollection usable by Beam."""

  dataset = tf.data.TFRecordDataset(tf.data.Dataset.list_files(f'{uri}/*'),
                                    compression_type="GZIP")

  # Take the input TFRecordDataset and extract the class label that we want.
  # Output format is a K-V PCollection: {class_label: TFRecord in string format}
  data = (p
          | "DatasetToPCollection" >> beam.Create(dataset)
          | "MapToLabel" >> beam.Map(_generate_elements, label))
  return data

def sample_examples(data, null_classes, sampling_strategy):
  """Function that performs the sampling given a label-mapped dataset."""

  # Finds the minimum frequency of all classes in the input label.
  # Output is a singleton PCollection with the minimum # of examples.
  def find_minimum(elements):
    return min(elements or [0])

  def find_maximum(elements):
    return max(elements or [0])

  if sampling_strategy == spec.SamplingStrategy.UNDERSAMPLE:
    sample_fn = find_minimum
  elif sampling_strategy == spec.SamplingStrategy.OVERSAMPLE:
    sample_fn = find_maximum
  else:
    raise ValueError("Invalid value for sampling_strategy variable!")
  # Dictionary containing count of elements per class
  key_counts = (data | "CountPerKey" >> beam.combiners.Count.PerKey())

  # Calculate how many elements should be in each class in the end
  goal_count = (key_counts
                | "FilterNullCount" >>
                beam.Filter(lambda x: filter_null(x, null_vals=null_classes))
                | "Values" >> beam.Values()
                | "GetSample" >> beam.CombineGlobally(sample_fn))

  # Actually performs the sampling functionality.
  # Output is a PCollection of TFExample
  res = (data
         | "FilterNull" >>
         beam.Filter(lambda x: filter_null(x, null_vals=null_classes))
         | "Sample" >> beam.FlatMapTuple(
             sample_data,
             goal_count=beam.pvalue.AsSingleton(goal_count),
             key_counts_dict=beam.pvalue.AsDict(key_counts)))

  # Take out all the null values from the beginning and put them back in the pipeline
  null = (data
          | "ExtractNull" >> beam.Filter(
              lambda x: filter_null(x, keep_null=True, null_vals=null_classes))
          | "NullValues" >> beam.Values())
  return (res, null) | "Merge PCollections" >> beam.Flatten()

def write_tfexamples(examples, shards, output_dir):
  # Write the final set of TFRecords to the output artifact's files.
  _ = (examples
       | "Serialize" >> beam.Map(lambda x: x.SerializeToString())
       | "WriteToTFRecord" >> beam.io.tfrecordio.WriteToTFRecord(
           output_dir,
           file_name_suffix=".gz",
           num_shards=shards,
           compression_type=beam.io.filesystem.CompressionTypes.GZIP,
       ))


# reqDataInsertion.py
# Created by Nate O, modified by Mickey G for Project flextape use
# To Import Req data to mongodb, within the flextape pipeline

import pandas as pd
import pymongo
from pymongo import MongoClient
from tqdm import tqdm
import sys
import time
import os

print("reqDataInsertion.py taking over...")

if not sys.argv[1] and not sys.argv[2]:
    raise ValueError('Arguments Needed: Date String (mmddyyyy-hhmmss), Write Location: One of (dev, prod, both)')
date = sys.argv[1]
writelocation = ['dev', 'prod'] if sys.argv[2] == 'both' else [sys.argv[2]]

serverlocation = os.environ['RUBIXLOCATION']

if serverlocation == 'local':
    filepathprefix = "/home/rubix/Desktop/Project-Ducttape/data/"
elif serverlocation == 'ohio':
    filepathprefix = "/home/ubuntu/Projects/flextape/"
else:
    raise EnvironmentError('Environment Variable "RUBIXLOCATION" seems not to be set.')

filepath = filepathprefix + "req-6/" + date + "/PLIT_REQ_6-" + date + ".xlsx"

print("--- Reading " + filepath + " ---")

insertionItems = pd.read_excel(filepath, skiprows = 1)
insertionItems['Due'] = insertionItems['Due'].fillna(pd.Timestamp('2001-01-01 00:00:00'))

client = MongoClient()
insertionItems.columns = [c.replace(' ', '_') for c in insertionItems.columns]
insertionItems.columns = [c.replace('/', '_') for c in insertionItems.columns]

print("--- Pushing Data to mongo ---")

print(list(insertionItems))

print("Removing Entries to be Updated...")

print("Adding Data...")

na_values = {"Address_2": ""}
insertionItems = insertionItems.fillna(value = na_values)

for location in writelocation:
    uniqueReqIDs = dict((reqno, False) for reqno in insertionItems['Req_ID'].unique().tolist())
    uniqueItemArr = insertionItems['Item'].unique().tolist()
    dbname = 'rubix-' + serverlocation + '-' + location
    print('Using database ' + dbname)
    db = client[dbname]
    for row in tqdm(insertionItems.itertuples()):
        if not uniqueReqIDs[row.Req_ID]:
            db.REQ_DATA.update({'REQ_No': row.Req_ID}, {
                '$set': {
                    'REQ_No': row.Req_ID,
                    'Account': row.Account,
                    'Business_Unit': row.Unit,
                    'Buyer': row.Buyer,
                    'Currency' : row.Currency,
                    'Department' : {
                        "Number": row.Dept_Loc
                    },
                    "Fund": row.Fund,
                    "Origin": row.Origin,
                    "REQ_Date": row.Req_Date,
                    "Requester": row.Requester,
                    "Ship_To": {
                        "Description": row.Descr,
                        "Address_1": row.Address_1,
                        "Address_2": row.Address_2,
                        "City": row.City,
                        "State": row.St,
                        "Zip_Code": row.Postal,
                        "Country": row.Cntry
                    },
                    "Status": row.Status,
                    "Approved_By" : row.By,
                    "Approved_On": row.Date,
                    "Vendor": {
                        "Number": row.Vendor,
                        "Name": row.Name
                    },
                    "lines": []
            }
            }, upsert = True)
            uniqueReqIDs[row.Req_ID] = True

    for row in tqdm(insertionItems.itertuples()):
        db.REQ_DATA.update({"REQ_No": row.Req_ID}, {
            '$addToSet': {
                "lines": {
                        "Line_No": row.Line,
                        "Unit_Price": row.Base_Price,
                        "Line_Total": row.Amount,
                        "Schedule_No": row.Sched_Num,
                        "Quantity": row.Req_Qty,
                        "Due_Date": row.Due,
                        "More_Info": row.More_Info,
                        "UOM": row.UOM,
                        "Item": row.Item
                    }
            }})
        # db.REQ_DATA.find({"lines": {$elemMatch: {"Item": "02545903"}}})

    db.LAST_UPDATED.update({'dbname': "REQ_DATA"}, {'$set': {'last_updated_time': time.time()}})

    print("Req Update Done!")


from ops import launch

def count_participants(sessionsCount, sessionCapacity, teams):
    people = sum(teams)

    memo = [None] * (sessionCapacity + 1)
    participants = 0
    sessionsDone = 0

    waitingForSession = 0
    while True:
        if memo[waitingForSession] is not None and sessionsDone - memo[waitingForSession][1] <= sessionsCount - sessionsDone:
            cycleAddedPeople = participants - memo[waitingForSession][0]
            cycleDoneSessions = sessionsDone - memo[waitingForSession][1]
            repeats = (sessionsCount - sessionsDone) // cycleDoneSessions

            participants += cycleAddedPeople * repeats
            sessionsDone += cycleDoneSessions * repeats

            if sessionsDone == sessionsCount:
                return participants
        else:
            participantsBefore = participants
            waitingForSessionBefore = waitingForSession
            sessionsDoneBefore = sessionsDone

            for team in teams:
                if waitingForSession + team > sessionCapacity:
                    sessionsDone += 1
                    participants += waitingForSession
                    waitingForSession = 0
                    if sessionsDone == sessionsCount:
                        return participants
                waitingForSession += team
            if sessionsDoneBefore == sessionsDone or waitingForSession == sessionCapacity:
                sessionsDone += 1
                participants += waitingForSession
                waitingForSession = 0
                if sessionsDone == sessionsCount:
                    return participants
            memo[waitingForSessionBefore] = (participantsBefore, sessionsDoneBefore)

def task(input, output):
    R, K, N = [int(x) for x in input.readline().split(' ')]
    assert N > 0 and R > 0 and K > 0
    teams = [int(x) for x in input.readline().split(' ')]
    assert len(teams) == N
    output.write(str(count_participants(R, K, teams)) + '\n')
    print('Done')

launch(task, 'G-lehky.txt')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: karanrochlani
"""

from PIL import Image
import os
import numpy as np

set1=[]
number=[]

for i in range(0,43):
    s=str(i)
    cur_path=os.path.join('/Users/karanrochlani/Desktop/Traffic-Sign-Classification/untitled folder 2/traffic sign recognition/Train',s)
    images = os.listdir(cur_path)
    for a in images:
            try:
                path=os.path.join(cur_path,a)
                image = Image.open(path)
                image = image.resize((30,30))
                image = np.array(image)
                set1.append(image)
                number.append(1)
            except:
                print("Error")

set1 = np.array(set1)
number = np.array(number)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(set1, number, test_size=0.2, random_state=42)

from keras.utils import to_categorical
y_train = to_categorical(y_train, 43)
y_test = to_categorical(y_test, 43)

from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPool2D
from keras.layers import Flatten
from keras.layers import Dense

classifier = Sequential()
classifier.add(Conv2D(filters=32, kernel_size=(5,5), activation='relu', input_shape=X_train.shape[1:]))
classifier.add(Conv2D(filters=32, kernel_size=(5,5), activation='relu'))
classifier.add(MaxPool2D(pool_size=(2, 2)))

classifier.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
classifier.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
classifier.add(MaxPool2D(pool_size=(2, 2)))

classifier.add(Flatten())

classifier.add(Dense(256, activation='relu'))

classifier.add(Dense(43, activation='sigmoid'))

classifier.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

classifier.fit(X_train, y_train, batch_size=32, epochs=2, validation_data=(X_test, y_test))


#!/usr/bin/env python

from setuptools import setup, find_packages

version = '0.1dev'

with open('README.md') as f:
    readme = f.read()

with open('LICENSE') as f:
    license = f.read()

with open('requirements.txt') as f:
    required = f.read().splitlines()

setup(
    name='carlosmatos',
    version=version,
    description='Parser for simple Graph structure',
    long_description=readme,
    keywords=['test'],
    author='Philipp Ehmele',
    author_email='philipp_ehm@protonmail.com',
    license=license,
    scripts=['scripts/carlosmatos'],
    install_requires=required,
    packages=find_packages(exclude='docs'),
    include_package_data=True
)