<|start_filename|>src/networks/model-net/core/model-net-sched.c<|end_filename|>
/*
 * Copyright (C) 2014 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

/* A scheduler interface for use by model-net. */

#include <assert.h>
#include <ross.h>

#include "codes/model-net-sched.h"
#include "codes/model-net-lp.h"
#include "codes/model-net-sched-impl.h"
#include "codes/quicklist.h"

#define X(a,b,c) b,
char * sched_names [] = {
    SCHEDULER_TYPES
};
#undef X

/// general scheduler functions

void model_net_sched_init(
        const model_net_sched_cfg_params * params,
        int is_recv_queue,
        struct model_net_method *method,
        model_net_sched *sched){
    if (params->type >= MAX_SCHEDS){
        fprintf(stderr, "unknown scheduler type");
        abort();
    }
    else{
        sched->impl = sched_interfaces[params->type];
    }
    sched->type = params->type;
    sched->impl->init(method, params, is_recv_queue, &sched->dat);
}

int model_net_sched_next(
        tw_stime *poffset,
        model_net_sched *sched,
        void *rc_event_save,
        model_net_sched_rc *sched_rc,
        tw_lp *lp){
    return sched->impl->next(poffset, sched->dat, rc_event_save, sched_rc, lp);
}

void model_net_sched_next_rc(
        model_net_sched *sched,
        const void *rc_event_save,
        const model_net_sched_rc *sched_rc,
        tw_lp *lp) {
    sched->impl->next_rc(sched->dat, rc_event_save, sched_rc, lp);
}

void model_net_sched_add(
        const model_net_request *req,
        const mn_sched_params * sched_params,
        int remote_event_size,
        void * remote_event,
        int local_event_size,
        void * local_event,
        model_net_sched *sched,
        model_net_sched_rc *sched_rc,
        tw_lp *lp){
    sched->impl->add(req, sched_params, remote_event_size, remote_event,
            local_event_size, local_event, sched->dat, sched_rc, lp);
}

void model_net_sched_add_rc(
        model_net_sched *sched,
        const model_net_sched_rc *sched_rc,
        tw_lp *lp){
    sched->impl->add_rc(sched->dat, sched_rc, lp);
}

void model_net_sched_set_default_params(mn_sched_params *sched_params){
    sched_params->prio = -1;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */


<|start_filename|>tests/mapping_test.c<|end_filename|>
/*
 * Copyright (C) 2014 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#include <ross.h>
#include "codes/configuration.h"
#include "codes/codes_mapping.h"
#include "codes/codes.h"

/* define three types of lps for mapping test */

typedef struct state_s{
    int id_global, id_by_group, id_by_anno, id_by_group_anno;
    char group_name[MAX_NAME_LENGTH], lp_name[MAX_NAME_LENGTH],
         anno[MAX_NAME_LENGTH];
} state;

static void init(state *ns, tw_lp *lp){
    int dummy;
    // only need the names
    codes_mapping_get_lp_info(lp->gid, ns->group_name, &dummy,
            ns->lp_name, &dummy, ns->anno, &dummy, &dummy);
    const char * anno = codes_mapping_get_annotation_by_lpid(lp->gid);
    // annotation check
    if (    (anno == NULL && ns->anno[0] != '\0') ||
            (anno != NULL && strcmp(anno, ns->anno) != 0)){
        fprintf(stderr, "LP %llu: annotations don't match: "
                "_get_lp_info:\"%s\", _get_anno_by_lpid:\"%s\"\n",
                LLU(lp->gid), ns->anno, anno);
    }

    ns->id_global = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
    ns->id_by_group = codes_mapping_get_lp_relative_id(lp->gid, 1, 0);
    ns->id_by_anno = codes_mapping_get_lp_relative_id(lp->gid, 0, 1);
    ns->id_by_group_anno = codes_mapping_get_lp_relative_id(lp->gid, 1, 1);

    // relative ID check - looking up the gid works correctly
    tw_lpid id_from_global_rel =
        codes_mapping_get_lpid_from_relative(ns->id_global, NULL,
                ns->lp_name, NULL, 0);
    tw_lpid id_from_group_rel =
        codes_mapping_get_lpid_from_relative(ns->id_by_group, ns->group_name,
                ns->lp_name, NULL, 0);
    tw_lpid id_from_anno_rel =
        codes_mapping_get_lpid_from_relative(ns->id_by_anno, NULL,
                ns->lp_name, anno, 1);
    tw_lpid id_from_group_anno_rel =
        codes_mapping_get_lpid_from_relative(ns->id_by_group_anno, ns->group_name,
                ns->lp_name, anno, 1);
    if (lp->gid != id_from_global_rel){
        fprintf(stderr, "LP %llu (%s): "
                "global relative id %d doesn't match (got %llu)\n",
                LLU(lp->gid), ns->lp_name, ns->id_global, LLU(id_from_global_rel));
    }
    if (lp->gid != id_from_group_rel){
        fprintf(stderr, "LP %llu (%s): "
                "group %s relative id %d doesn't match (got %llu)\n",
                LLU(lp->gid), ns->lp_name, ns->group_name, ns->id_by_group,
                LLU(id_from_group_rel));
    }
    if (lp->gid != id_from_anno_rel){
        fprintf(stderr, "LP %llu (%s): "
                "anno \"%s\" relative id %d doesn't match (got %llu)\n",
                LLU(lp->gid), ns->lp_name, ns->anno, ns->id_by_group,
                LLU(id_from_anno_rel));
    }
    if (lp->gid != id_from_group_anno_rel){
        fprintf(stderr, "LP %llu (%s): "
                "group %s anno \"%s\" relative id %d doesn't match (got %llu)\n",
                LLU(lp->gid), ns->lp_name, ns->group_name, ns->anno, ns->id_by_group,
                LLU(id_from_group_anno_rel));
    }


    // output-based check - print out IDs, compare against expected
    char tmp[512];
    if (ns->anno[0]=='\0')
        tmp[0] = '\0';
    else
        sprintf(tmp, "@%s", ns->anno);
    printf("TEST2 %2llu %2d %2d %2d %2d %s%s\n", LLU(lp->gid), ns->id_global,
            ns->id_by_group, ns->id_by_anno, ns->id_by_group_anno, ns->lp_name,
            tmp);
}

tw_lptype a_lp = {
    (init_f) init,
    (pre_run_f) NULL,
    (event_f) NULL,
    (revent_f) NULL,
    (commit_f) NULL,
    (final_f)  NULL,
    (map_f) codes_mapping,
    sizeof(state),
};
tw_lptype b_lp = {
    (init_f) init,
    (pre_run_f) NULL,
    (event_f) NULL,
    (revent_f) NULL,
    (commit_f) NULL,
    (final_f) NULL,
    (map_f) codes_mapping,
    sizeof(state),
};
tw_lptype c_lp = {
    (init_f) init,
    (pre_run_f) NULL,
    (event_f) NULL,
    (revent_f) NULL,
    (commit_f) NULL,
    (final_f) NULL,
    (map_f) codes_mapping,
    sizeof(state),
};

#define TEST_TO_NM(_expected_nm, _input, _fn_type) \
    do { \
        char const * _anm = \
                codes_mapping_get_ ##_fn_type ##_name_by_cid(_input); \
        char const * _enm = \
                (_expected_nm) == NULL ? "" : (_expected_nm); \
        if ((_expected_nm) == NULL && _anm != NULL) { \
            fprintf(stderr, "%s:%d: " \
                    " cid->name check error (expected NULL, got %s)\n", \
                    __FILE__, __LINE__, _anm);\
            return 1; \
        } else if ((_expected_nm) != NULL && strcmp(_enm, _anm) != 0) { \
            fprintf(stderr, "%s:%d: " \
                    " cid->name check error (expected %s, got %s)\n", \
                    __FILE__, __LINE__, \
                    (_expected_nm) == NULL ? "null" : (_expected_nm), _anm);\
            return 1; \
        } \
    } while (0);

#define TEST_TO_CID(_expected_cid, _input, _name_type, _fn_type) \
    do { \
        int _ecid = (_expected_cid); \
        int _acid = \
            codes_mapping_get_ ##_fn_type ##_cid_by_ ##_name_type (_input); \
        if (_ecid != _acid) { \
            fprintf(stderr, "%s:%d: " #_name_type \
                    " name->cid check error (expected %d, got %d)\n", \
                    __FILE__, __LINE__, _ecid, _acid);\
            return 1; \
        } \
    } while (0);

#define TEST_TO_CID_NM(_expected_cid, _input, _fn_type) \
    TEST_TO_CID(_expected_cid, _input, name, _fn_type)
#define TEST_TO_CID_LP(_expected_cid, _input, _fn_type) \
    TEST_TO_CID(_expected_cid, _input, lpid, _fn_type)

static int test_cids()
{
    TEST_TO_CID_NM(0, "GRP1", group)
    TEST_TO_CID_NM(1, "GRP2", group)
    TEST_TO_NM("GRP1", 0, group)
    TEST_TO_NM("GRP2", 1, group)

    TEST_TO_CID_NM(0, "a", lp)
    TEST_TO_CID_NM(1, "b", lp)
    TEST_TO_CID_NM(2, "c", lp)
    TEST_TO_NM("a", 0, lp)
    TEST_TO_NM("b", 1, lp)
    TEST_TO_NM("c", 2, lp)

    TEST_TO_CID_NM(0, "foo", anno)
    TEST_TO_CID_NM(1, "bar", anno)
    TEST_TO_CID_NM(2, NULL, anno)
    TEST_TO_CID_NM(2, "", anno)
    TEST_TO_NM("foo", 0, anno)
    TEST_TO_NM("bar", 1, anno)
    TEST_TO_NM(NULL, 2, anno)

    TEST_TO_CID_LP(0, 0, group)
    TEST_TO_CID_LP(0, 1, group)
    TEST_TO_CID_LP(0, 9, group)
    TEST_TO_CID_LP(1, 10, group)
    TEST_TO_CID_LP(1, 11, group)
    TEST_TO_CID_LP(1, 24, group)

    TEST_TO_CID_LP(0, 0, lp)
    TEST_TO_CID_LP(1, 1, lp)
    TEST_TO_CID_LP(2, 9, lp)
    TEST_TO_CID_LP(2, 10, lp)
    TEST_TO_CID_LP(1, 11, lp)
    TEST_TO_CID_LP(0, 24, lp)

    TEST_TO_CID_LP(2, 0, anno)
    TEST_TO_CID_LP(2, 1, anno)
    TEST_TO_CID_LP(2, 9, anno)
    TEST_TO_CID_LP(0, 8, anno)
    TEST_TO_CID_LP(1, 10, anno)
    TEST_TO_CID_LP(0, 11, anno)
    TEST_TO_CID_LP(0, 24, anno)

    return 0;
}

static char conf_file_name[128] = {'\0'};
static const tw_optdef app_opt [] =
{
	TWOPT_GROUP("codes-mapping test case" ),
    TWOPT_CHAR("codes-config", conf_file_name, "name of codes configuration file"),
	TWOPT_END()
};

int main(int argc, char *argv[])
{
    tw_opt_add(app_opt);
    tw_init(&argc, &argv);

    if (!conf_file_name[0]){
        fprintf(stderr, "Expected \"codes-config\" option, please see --help.\n");
        MPI_Finalize();
        return 1;
    }
    if (configuration_load(conf_file_name, MPI_COMM_WORLD, &config)){
        fprintf(stderr, "Error loading config file %s.\n", conf_file_name);
        MPI_Finalize();
        return 1;
    }

    lp_type_register("a", &a_lp);
    lp_type_register("b", &b_lp);
    lp_type_register("c", &c_lp);

    codes_mapping_setup();

    printf("# test 2 format:\n"
           "# <lp> rel id <global <group-wise> <anno.-wise> <both> <lp name>\n");

    const char * lps[]    = {"a", "b", "c"};
    const char * groups[] = {NULL, "GRP1", "GRP2"};
    const char * annos[]  = {NULL, "foo", "bar"};
    char lpnm[128];
    for (int l = 0; l < 3; l++){
        for (int g = 0; g < 3; g++){
            for (int a = 0; a < 3; a++){
                if (annos[a]==NULL)
                    sprintf(lpnm, "%s", lps[l]);
                else
                    sprintf(lpnm, "%s@%s", lps[l], annos[a]);
                printf("TEST1 %2d %6s %s\n",
                    codes_mapping_get_lp_count(groups[g], 0, lps[l],
                        annos[a], 0),
                    groups[g], lpnm);
            }
            printf("TEST1 %2d %6s %s ignore annos\n",
                codes_mapping_get_lp_count(groups[g], 0, lps[l], NULL, 2),
                groups[g], lps[l]);
        }
    }

    tw_run();

    int rc = test_cids();

    tw_end();

    return rc;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */


<|start_filename|>src/networks/model-net/slimfly.c<|end_filename|>
/*
 * Copyright (C) 2013 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

// Local router ID: 0 --- total_router-1
// Router LP ID
// Terminal LP ID

#include <ross.h>

#include "codes/jenkins-hash.h"
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#include "codes/model-net-lp.h"
#include "codes/net/slimfly.h"
#include "sys/file.h"
#include "codes/quickhash.h"
#include "codes/rc-stack.h"

#define CREDIT_SIZE 8
#define MEAN_PROCESS 1.0

/* collective specific parameters */
#define SLIMFLY_HASH_TABLE_SIZE 65536

// debugging parameters
#define TRACK -9
//#define TRACK 100001
#define TRACK_MSG 0
#define TRACK_OUTPUT 0
#define DEBUG 0
#define DEBUG_ROUTING 0
#define LOAD_FROM_FILE 0
#define SLIMFLY_CONNECTIONS 1
#define MSG_TIMES 0       //Collects msg send times and outputs lp-io-dir

#define LP_CONFIG_NM_TERM (model_net_lp_config_names[SLIMFLY])
#define LP_METHOD_NM_TERM (model_net_method_names[SLIMFLY])
#define LP_CONFIG_NM_ROUT (model_net_lp_config_names[SLIMFLY_ROUTER])
#define LP_METHOD_NM_ROUT (model_net_method_names[SLIMFLY_ROUTER])

/* Begin Visualization Piece */
#define TERMINAL_SENDS_RECVS_LOG 0
#define ROUTER_SENDS_RECVS_LOG 0
#define TERMINAL_OCCUPANCY_LOG 0
#define ROUTER_OCCUPANCY_LOG 0
#define N_COLLECT_POINTS 100

/*unsigned long terminal_sends[TEMP_NUM_TERMINALS][N_COLLECT_POINTS];
  unsigned long terminal_recvs[TEMP_NUM_TERMINALS][N_COLLECT_POINTS];
  unsigned long router_sends[TEMP_NUM_ROUTERS][N_COLLECT_POINTS];
  unsigned long router_recvs[TEMP_NUM_ROUTERS][N_COLLECT_POINTS];
  int vc_occupancy_storage_router[TEMP_NUM_ROUTERS][TEMP_RADIX][TEMP_NUM_VC][N_COLLECT_POINTS];
  int vc_occupancy_storage_terminal[TEMP_NUM_TERMINALS][TEMP_NUM_VC][N_COLLECT_POINTS];
  */
FILE * slimfly_terminal_sends_recvs_log = NULL;
FILE * slimfly_router_sends_recvs_log = NULL;
FILE * slimfly_router_occupancy_log=NULL;
FILE * slimfly_terminal_occupancy_log=NULL;
FILE * slimfly_results_log=NULL;
int TEMP_RADIX;
int TEMP_NUM_VC;
int slim_total_routers_noah;
int slim_total_terminals_noah;
/* End Visualization Piece */

/*Begin Misc*/
FILE * MMS_input_file=NULL;
int csf_ratio = 1;						//Constant selected to balance the ratio between minimal and indirect routes
int num_indirect_routes = 4;			//Number of indirect (Valiant) routes to use in Adaptive routing methods
float adaptive_threshold = 0.1;

int *X;
int *X_prime;
int X_size;
/*End Misc*/

static double maxd(double a, double b) { return a < b ? b : a; }

/* minimal and non-minimal packet counts for adaptive routing*/
static int minimal_count=0, nonmin_count=0;

long slimfly_packet_gen = 0, slimfly_packet_fin = 0;

typedef struct slimfly_param slimfly_param;
/* annotation-specific parameters (unannotated entry occurs at the
 * last index) */
static uint64_t                  num_params = 0;
static slimfly_param         * all_params = NULL;
static const config_anno_map_t * anno_map   = NULL;

/* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH];
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;

/* router magic number */
int slim_router_magic_num = 0;

/* terminal magic number */
int slim_terminal_magic_num = 0;

typedef struct slim_terminal_message_list slim_terminal_message_list;
struct slim_terminal_message_list {
    slim_terminal_message msg;
    char* event_data;
    slim_terminal_message_list *next;
    slim_terminal_message_list *prev;
};

void slim_init_terminal_message_list(slim_terminal_message_list *this,
        slim_terminal_message *inmsg) {
    this->msg = *inmsg;
    this->event_data = NULL;
    this->next = NULL;
    this->prev = NULL;
}

void slim_delete_terminal_message_list(slim_terminal_message_list *this) {
    if(this->event_data != NULL) free(this->event_data);
    free(this);
}

struct slimfly_param
{
    int sf_type;
    int num_rails;
    int ports_per_nic;
    int num_injection_queues;
    int rail_select;        /*Selection of rail routing method*/
    // configuration parameters
    int num_routers; 		/*NUM_ROUTER Number of routers in a group*/
    double local_bandwidth;	/*LOCAL_BANDWIDTH bandwidth of the router-router channels within a group */
    double global_bandwidth;/*GLOBAL_BANDWIDTH bandwidth of the inter-group router connections */
    double cn_bandwidth;	/*NODE_BANDWIDTH bandwidth of the compute node channels connected to routers */
    int num_vcs; 			/*NUM_VC number of virtual channels */
    int local_vc_size; 		/*LOCAL_VC_SIZE buffer size of the router-router channels */
    int global_vc_size; 	/*GLOBAL_VC_SIZE buffer size of the global channels */
    int cn_vc_size; 		/*TERMINAL_VC_SIZE buffer size of the compute node channels */
    int chunk_size; 		/*CHUNK_SIZE full-sized packets are broken into smaller chunks.*/
    // derived parameters
    int num_cn; 			/*NUM_TERMINALS*/
    int num_groups; 		/*No relation to slim fly*/
    int radix;				/*RADIX*/
    int slim_total_routers;
    int slim_total_terminals;
    int num_global_channels;
    double cn_delay;
    double local_delay;
    double global_delay;
    double credit_delay;
    //slimfly added
    double router_delay;	/*Router processing delay moving packet from input port to output port*/
    double link_delay;		/*Network link latency. Currently encorporated into the arrival time*/
    int num_local_channels;
};

struct sfly_hash_key
{
    uint64_t message_id;
    tw_lpid sender_id;
};

struct sfly_qhash_entry
{
    struct sfly_hash_key key;
    char * remote_event_data;
    int num_chunks;
    int remote_event_size;
    struct qhash_head hash_link;
};

/* handles terminal and router events like packet generate/send/receive/buffer */
typedef enum event_t event_t;
typedef struct terminal_state terminal_state;
typedef struct router_state router_state;

/* slimfly compute node data structure */
struct terminal_state
{
    uint64_t packet_counter;

    int packet_gen;
    int packet_fin;

#if MSG_TIMES
    int * msg_send_times;
    int * msg_rail_select;
    char output_buf2[4096];
#endif

    int router_id;
    tw_lpid * router_lp;
    int terminal_id;

    // Each terminal will have an input and output channel with the router
    int* vc_occupancy; // NUM_VC
    int num_vcs;
    tw_stime* terminal_available_time;
    slim_terminal_message_list **terminal_msgs;
    slim_terminal_message_list **terminal_msgs_tail;
    int *in_send_loop;
    // Terminal generate, sends and arrival T_SEND, T_ARRIVAL, T_GENERATE
    // Router-Router Intra-group sends and receives RR_LSEND, RR_LARRIVE
    // Router-Router Inter-group sends and receives RR_GSEND, RR_GARRIVE
    struct mn_stats slimfly_stats_array[CATEGORY_MAX];

    struct rc_stack * st;
    int *issueIdle;
    int *terminal_length;

    const char * anno;
    const slimfly_param *params;

    struct qhash_table *rank_tbl;
    uint64_t rank_tbl_pop;

    tw_stime   total_time;
    uint64_t total_msg_size;
    double total_hops;
    long finished_msgs;
    long finished_chunks;
    long finished_packets;

    tw_stime *last_buf_full;
    tw_stime *busy_time;

    char output_buf[512];
};

/* terminal event type (1-4) */
enum event_t
{
    T_GENERATE=1,
    T_ARRIVE,
    T_SEND,
    T_BUFFER,
    R_SEND,
    R_ARRIVE,
    R_BUFFER
};
/* status of a virtual channel can be idle, active, allocated or wait for credit */
enum vc_status
{
    VC_IDLE,
    VC_ACTIVE,
    VC_ALLOC,
    VC_CREDIT
};

/* whether the last hop of a packet was global, local or a terminal */
enum last_hop
{
    GLOBAL,
    LOCAL,
    TERMINAL
};

/* three forms of routing algorithms available, adaptive routing is not
 * accurate and fully functional in the current version as the formulas
 * for detecting load on global channels are not very accurate */
enum ROUTING_ALGO
{
    MINIMAL = 0,
    NON_MINIMAL,
    ADAPTIVE
};

enum RAIL_SELECTION_ALGO
{
    RAIL_CONGESTION=1,  // Selects rail with minimal injection congestion
    RAIL_PATH,          // Selects the rail that provides minimal path congestion is tie breaker
    RAIL_DEDICATED,      // Selects a specific rail
    RAIL_RAND
};

struct router_state
{
    int router_id;
    int group_id;

    int* global_channel;
    int* local_channel;

    tw_stime* next_output_available_time;
    slim_terminal_message_list ***pending_msgs;
    slim_terminal_message_list ***pending_msgs_tail;
    slim_terminal_message_list ***queued_msgs;
    slim_terminal_message_list ***queued_msgs_tail;
    int *in_send_loop;
    struct rc_stack * st;

    tw_stime** last_buf_full;
    tw_stime* busy_time;
    tw_stime* busy_time_sample;

    char output_buf[4096];
    char output_buf2[4096];

    int** vc_occupancy;
    int64_t* link_traffic;	//Aren't used

    const char * anno;
    const slimfly_param *params;

    int* prev_hist_num;	//Aren't used
    int* cur_hist_num;	//Aren't used
};

/* ROSS Instrumentation Support */
struct slimfly_cn_sample
{
   tw_lpid terminal_id;
   tw_stime end_time;
   int vc_occupancy; // will sum occupancy for all vc
};

struct slimfly_router_sample
{
   tw_lpid router_id;
   int* vc_occupancy; // sum for all vc for each port
   tw_stime end_time;
};

void slimfly_event_collect(slim_terminal_message *m, tw_lp *lp, char *buffer, int *collect_flag);
void slimfly_model_stat_collect(terminal_state *s, tw_lp *lp, char *buffer);
static void ross_slimfly_sample_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_cn_sample *sample);
static void ross_slimfly_sample_rc_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_cn_sample *sample);
static void ross_slimfly_rsample_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_router_sample *sample);
static void ross_slimfly_rsample_rc_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_router_sample *sample);
int get_path_length_from_terminal(int src, int dest, const slimfly_param *p);
void get_router_connections(int src_router_id, int num_global_channels, int num_local_channels,
        int total_routers, int* local_channels, int* global_channels, int sf_type, const slimfly_param * p);

st_model_types slimfly_model_types[] = {
    {(ev_trace_f) slimfly_event_collect,
     sizeof(int),
     (model_stat_f) slimfly_model_stat_collect,
     0, // update this when changing slimfly_model_stat_collect
     (sample_event_f) ross_slimfly_sample_fn,
     (sample_revent_f) ross_slimfly_sample_rc_fn,
     sizeof(struct slimfly_cn_sample) } , 
    {(ev_trace_f) slimfly_event_collect,
     sizeof(int),
     (model_stat_f) slimfly_model_stat_collect,
     0, // update this when changing slimfly_model_stat_collect
     (sample_event_f) ross_slimfly_rsample_fn,
     (sample_revent_f) ross_slimfly_rsample_rc_fn,
     0 } , //updated in slim_router_setup() since it's based on the radix 
    {NULL, 0, NULL, 0, NULL, NULL, 0}
};
/* End of ROSS model stats collection */

static short routing = MINIMAL;

static tw_stime         slimfly_total_time = 0;
static tw_stime         slimfly_max_latency = 0;

static long long       total_hops = 0;
static long long       N_finished_packets = 0;
static long long       total_msg_sz = 0;
static long long       N_finished_msgs = 0;
static long long       N_finished_chunks = 0;

static int slimfly_rank_hash_compare(
        void *key, struct qhash_head *link)
{
    struct sfly_hash_key *message_key = (struct sfly_hash_key *)key;
    struct sfly_qhash_entry *tmp = NULL;

    tmp = qhash_entry(link, struct sfly_qhash_entry, hash_link);

    if (tmp->key.message_id == message_key->message_id
            && tmp->key.sender_id == message_key->sender_id)
        return 1;

    return 0;
}
static int slimfly_hash_func(void *k, int table_size)
{
    struct sfly_hash_key *tmp = (struct sfly_hash_key *)k;
    uint64_t key = (~tmp->message_id) + (tmp->message_id << 18);
    key = key * 21;
    key = ~key ^ (tmp->sender_id >> 4);
    key = key * tmp->sender_id;
    return (int)(key & (table_size - 1));
}

/* convert GiB/s and bytes to ns */
static tw_stime bytes_to_ns(uint64_t bytes, double GB_p_s)
{
    tw_stime time;

    /* bytes to GB */
    time = ((double)bytes)/(1024.0*1024.0*1024.0);
    /* MB to s */
    time = time / GB_p_s;
    /* s to ns */
    time = time * 1000.0 * 1000.0 * 1000.0;

    return(time);
}

/* returns the slimfly message size */
static int slimfly_get_msg_sz(void)
{
    return sizeof(slim_terminal_message);
}

static void free_tmp(void * ptr)
{
    struct sfly_qhash_entry * sfly = ptr;
    if(sfly->remote_event_data)
        free(sfly->remote_event_data);
    if(sfly)
        free(sfly);
}

static void append_to_terminal_message_list(
        slim_terminal_message_list ** thisq,
        slim_terminal_message_list ** thistail,
        int index,
        slim_terminal_message_list *msg) {
    if(thisq[index] == NULL) {
        thisq[index] = msg;
    } else {
        thistail[index]->next = msg;
        msg->prev = thistail[index];
    }
    thistail[index] = msg;
}

static void prepend_to_terminal_message_list(
        slim_terminal_message_list ** thisq,
        slim_terminal_message_list ** thistail,
        int index,
        slim_terminal_message_list *msg) {
    if(thisq[index] == NULL) {
        thistail[index] = msg;
    } else {
        thisq[index]->prev = msg;
        msg->next = thisq[index];
    }
    thisq[index] = msg;
}

static slim_terminal_message_list* return_head(
        slim_terminal_message_list ** thisq,
        slim_terminal_message_list ** thistail,
        int index) {
    slim_terminal_message_list *head = thisq[index];
    if(head != NULL) {
        thisq[index] = head->next;
        if(head->next != NULL) {
            head->next->prev = NULL;
            head->next = NULL;
        } else {
            thistail[index] = NULL;
        }
    }
    return head;
}

static slim_terminal_message_list* return_tail(
        slim_terminal_message_list ** thisq,
        slim_terminal_message_list ** thistail,
        int index) {
    slim_terminal_message_list *tail = thistail[index];
    assert(tail);
    if(tail->prev != NULL) {
        tail->prev->next = NULL;
        thistail[index] = tail->prev;
        tail->prev = NULL;
    } else {
        thistail[index] = NULL;
        thisq[index] = NULL;
    }
    return tail;
}

tw_stime* buff_time_storage_create(terminal_state *s)
{
    tw_stime* storage = (tw_stime*)malloc(s->params->ports_per_nic * sizeof(tw_stime));
    return storage;
}

void buff_time_storage_delete(void * ptr)
{
    if (ptr)
        free(ptr);
}

static void slimfly_read_config(const char * anno, slimfly_param *params){
    uint32_t h1 = 0, h2 = 0;
    bj_hashlittle2(LP_METHOD_NM_ROUT, strlen(LP_METHOD_NM_ROUT), &h1, &h2);
    slim_router_magic_num = h1 + h2;
    bj_hashlittle2(LP_METHOD_NM_TERM, strlen(LP_METHOD_NM_TERM), &h1, &h2);
    slim_terminal_magic_num = h1 + h2;
    // shorthand
    slimfly_param *p = params;

    p->sf_type = 0;
    configuration_get_value_int(&config, "PARAMS", "sf_type", anno, &p->sf_type);
    if(!g_tw_mynode) printf("SF type is %d\n", p->sf_type);

    p->num_rails = 1;
    configuration_get_value_int(&config, "PARAMS", "num_rails", anno, &p->num_rails);
    if(!g_tw_mynode) printf("SF num rails is %d\n", p->num_rails);

    p->num_injection_queues = 1;
    configuration_get_value_int(&config, "PARAMS", "num_injection_queues", anno, &p->num_injection_queues);
    if(!p->num_injection_queues)
        p->num_injection_queues = 1;

    p->ports_per_nic = 1;
    if(p->num_rails > 1)
        p->ports_per_nic = p->num_rails; //TODO: should ports_per_nic == num rails?

    configuration_get_value_int(&config, "PARAMS", "num_routers", anno,
            &p->num_routers);
    if(p->num_routers <= 0) {
        p->num_routers = 4;
        fprintf(stderr, "Number of dimensions not specified, setting to %d\n",
                p->num_routers);
    }

    configuration_get_value_int(&config, "PARAMS", "num_vcs", anno, &p->num_vcs);
    if(!p->num_vcs) {
        p->num_vcs = 4;
        fprintf(stderr, "Virtual channel size not specified, setting to %d\n", p->num_vcs);
    }

    configuration_get_value_int(&config, "PARAMS", "local_vc_size", anno, &p->local_vc_size);
    if(!p->local_vc_size) {
        p->local_vc_size = 1024;
        fprintf(stderr, "Buffer size of local channels not specified, setting to %d\n", p->local_vc_size);
    }

    configuration_get_value_int(&config, "PARAMS", "global_vc_size", anno, &p->global_vc_size);
    if(!p->global_vc_size) {
        p->global_vc_size = 2048;
        fprintf(stderr, "Buffer size of global channels not specified, setting to %d\n", p->global_vc_size);
    }

    configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno, &p->cn_vc_size);
    if(!p->cn_vc_size) {
        p->cn_vc_size = 1024;
        fprintf(stderr, "Buffer size of compute node channels not specified, setting to %d\n", p->cn_vc_size);
    }

    configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size);
    if(!p->chunk_size) {
        p->chunk_size = 512;
        fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size);
    }

    configuration_get_value_double(&config, "PARAMS", "local_bandwidth", anno, &p->local_bandwidth);
    if(!p->local_bandwidth) {
        p->local_bandwidth = 5.25;
        fprintf(stderr, "Bandwidth of local channels not specified, setting to %lf\n", p->local_bandwidth);
    }

    configuration_get_value_double(&config, "PARAMS", "global_bandwidth", anno, &p->global_bandwidth);
    if(!p->global_bandwidth) {
        p->global_bandwidth = 4.7;
        fprintf(stderr, "Bandwidth of global channels not specified, setting to %lf\n", p->global_bandwidth);
    }

    configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno, &p->cn_bandwidth);
    if(!p->cn_bandwidth) {
        p->cn_bandwidth = 5.25;
        fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", p->cn_bandwidth);
    }

    configuration_get_value_int(&config, "PARAMS", "local_channels", anno, &p->num_local_channels);
    if(!p->num_local_channels) {
        p->num_local_channels = 2;
        fprintf(stderr, "Number of Local channels not specified, setting to %d\n", p->num_local_channels);
    }

    configuration_get_value_int(&config, "PARAMS", "global_channels", anno, &p->num_global_channels);
    if(!p->num_global_channels) {
        p->num_global_channels = 2;
        fprintf(stderr, "Number of Global channels not specified, setting to %d\n", p->num_global_channels);
    }

    configuration_get_value_int(&config, "PARAMS", "num_terminals", anno, &p->num_cn);
    if(!p->num_cn) {
        p->num_cn = 2;
        fprintf(stderr, "Number of terminals not specified, setting to %d\n", p->num_cn);
    }

    p->router_delay = -1;
    configuration_get_value_double(&config, "PARAMS", "router_delay", anno, &p->router_delay);
    if(p->router_delay < 0) {
        p->router_delay = 0;
        fprintf(stderr, "Router delay not specified, setting to %lf\n", p->router_delay);
    }

    char       **values;
    size_t       length;
    int ret = configuration_get_multivalue(&config, "PARAMS", "generator_set_X", anno, &values, &length);
    if (ret != 1)
        tw_error(TW_LOC, "unable to read PARAMS:generator_set_X\n");
    if (length < 2)
        fprintf(stderr, "generator set X less than 2 elements\n");

    X = (int*)calloc(length, sizeof(int));
    for (size_t i = 0; i < length; i++)
    {
        X[i] = atoi(values[i]);
    }
    free(values);

    ret = configuration_get_multivalue(&config, "PARAMS", "generator_set_X_prime", anno, &values, &length);
    if (ret != 1)
        tw_error(TW_LOC, "unable to read PARAMS:generator_set_X_prime\n");
    if (length < 2)
        fprintf(stderr, "generator set  X_prime less than 2 elements\n");

    X_size = length;
    X_prime = (int*)calloc(length, sizeof(int));
    for (size_t i = 0; i < length; i++)
    {
        X_prime[i] = atoi(values[i]);
    }
    free(values);

    char routing_str[MAX_NAME_LENGTH];
    configuration_get_value(&config, "PARAMS", "routing", anno, routing_str,
            MAX_NAME_LENGTH);
    if(strcmp(routing_str, "minimal") == 0)
        routing = MINIMAL;
    else if(strcmp(routing_str, "nonminimal")==0 ||
            strcmp(routing_str,"non-minimal")==0)
        routing = NON_MINIMAL;
    else if (strcmp(routing_str, "adaptive") == 0)
        routing = ADAPTIVE;
    else
    {
        fprintf(stderr,
                "No routing protocol specified, setting to minimal routing\n");
        routing = -1;
    }

    char rail_select_str[MAX_NAME_LENGTH];
    configuration_get_value(&config, "PARAMS", "rail_select", anno, rail_select_str,
            MAX_NAME_LENGTH);
    if(strcmp(rail_select_str, "dedicated") == 0)
        p->rail_select = RAIL_DEDICATED;
    else if(strcmp(rail_select_str, "congestion")==0)
        p->rail_select = RAIL_CONGESTION;
    else if(strcmp(rail_select_str, "path")==0)
        p->rail_select = RAIL_PATH;
    else if(strcmp(rail_select_str, "rand")==0)
        p->rail_select = RAIL_RAND;
    else {
        p->rail_select = RAIL_DEDICATED;
    }
    if (p->sf_type == 0)
        p->rail_select = RAIL_DEDICATED;
    if(!g_tw_mynode) printf("SF rail selection is %d\n", p->rail_select);

    TEMP_RADIX = p->num_local_channels + p->num_global_channels + p->num_cn;
    TEMP_NUM_VC = p->num_vcs;

    // set the derived parameters
    p->num_groups = p->num_routers * 2;
    p->radix = (p->num_cn + p->num_global_channels + p->num_local_channels);
    p->slim_total_routers = p->num_groups * p->num_routers;
    p->slim_total_terminals = p->slim_total_routers * p->num_cn;
    slim_total_routers_noah = p->num_groups * p->num_routers;
    slim_total_terminals_noah = p->slim_total_routers * p->num_cn;
    int rank;
    MPI_Comm_rank(MPI_COMM_CODES, &rank);
    if(!rank) {
        printf("\n Total nodes %d total routers %d total groups %d num_terminals %d num_routers %d radix %d local_channels %d global_channels %d \n",
                p->num_cn * p->slim_total_routers, p->slim_total_routers, p->num_groups, p->num_cn, p->num_routers,
                p->radix, p->num_local_channels, p->num_global_channels);
    }

    p->cn_delay = bytes_to_ns(p->chunk_size, p->cn_bandwidth*p->num_rails);
    p->local_delay = bytes_to_ns(p->chunk_size, p->local_bandwidth);
    p->global_delay = bytes_to_ns(p->chunk_size, p->global_bandwidth);
    p->credit_delay = bytes_to_ns(8.0, p->local_bandwidth); //assume 8 bytes packet

}

static void slimfly_configure(){
    anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM_TERM);
    assert(anno_map);
    num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
    all_params = malloc(num_params * sizeof(*all_params));

    for (uint64_t i = 0; i < (uint64_t)anno_map->num_annos; i++){
        const char * anno = anno_map->annotations[i].ptr;
        slimfly_read_config(anno, &all_params[i]);
    }
    if (anno_map->has_unanno_lp > 0){
        slimfly_read_config(NULL, &all_params[anno_map->num_annos]);
    }
}

/* report slimfly statistics like average and maximum packet latency, average number of hops traversed */
static void slimfly_report_stats()
{
    // printf("slim report stats\n");

    long long avg_hops, total_finished_packets, total_finished_chunks;
    long long total_finished_msgs, final_msg_sz;
    tw_stime avg_time, max_time;
    int total_minimal_packets, total_nonmin_packets;
    long total_gen, total_fin;

    MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &slimfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &slimfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);

    MPI_Reduce( &slimfly_packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce( &slimfly_packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);

    MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
    MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);

    /* print statistics */
    if(!g_tw_mynode)
    {
        printf(" Total packets routed minimally: %d. Total packets routed non-minimally: %d\n",total_minimal_packets, total_nonmin_packets);
        printf(" Average number of hops traversed %f average chunk latency %lf us maximum chunk latency %lf us avg message size %lf bytes finished messages %lld finished chunks %lld \n",
                (float)avg_hops/total_finished_chunks, avg_time/(total_finished_chunks*1000), max_time/1000, (float)final_msg_sz/total_finished_msgs, total_finished_msgs, total_finished_chunks);
        printf(" Total packets generated %ld finished %ld \n", total_gen, total_fin);
    }

#if ROUTER_OCCUPANCY_LOG
    if(tw_ismaster())
    {
        printf("Printing Realtime Router Occupancy Log Files\n");
        for(k=0; k<slim_total_routers_noah; k++)
        {
            sprintf( log, "vc-occupancy/routers/slimfly_router_occupancy_log.%d.txt", k );
            slimfly_router_occupancy_log=fopen(log, "w+");
            if(slimfly_router_occupancy_log == NULL)
                tw_error(TW_LOC, "\n Failed to open slimfly router occupancy log file \n");

            for( t=0; t<N_COLLECT_POINTS; t++ )
            {
                if(t == 0)
                {
                    fprintf(slimfly_router_occupancy_log, "%d, ",t*100/N_COLLECT_POINTS);
                }
                else
                {
                    fprintf(slimfly_router_occupancy_log, "\n%d, ",t*100/N_COLLECT_POINTS);
                }
                for( j=0; j<TEMP_RADIX; j++)
                {
                    for(i=0; i<TEMP_NUM_VC; i++)
                    {
                        if( k==0 && t==0)
                            printf("k:%d, j:%d, i:%d, t:%d\n",k,j,i,t);
                        fprintf(slimfly_router_occupancy_log, "%d, ", vc_occupancy_storage_router[k][j][i][t]);
                    }
                }
            }
        }
    }
    fclose(slimfly_router_occupancy_log);
#endif

#if TERMINAL_OCCUPANCY_LOG
    if(tw_ismaster())
    {
        printf("Printing Realtime Terminal Occupancy Log Files\n");
        for(k=0; k<slim_total_terminals_noah; k++)
        {
            sprintf( log, "vc-occupancy/terminals/slimfly_terminal_occupancy_log.%d.txt", k );
            slimfly_terminal_occupancy_log=fopen(log, "w+");
            if(slimfly_terminal_occupancy_log == NULL)
            {
                printf("Failed to open slimfly terminal occupancy log file vc-occupancy/terminals/slimfly_terminal_occupancy_log.%d.txt\n",k);
                tw_error(TW_LOC, "\n Failed to open slimfly terminal occupancy log file \n");
            }

            for( i=0; i<N_COLLECT_POINTS; i++ )
            {
                if(i == 0)
                {
                    fprintf(slimfly_terminal_occupancy_log, "%d, ",i*100/N_COLLECT_POINTS);
                }
                else
                {
                    fprintf(slimfly_terminal_occupancy_log, "\n%d, ",i*100/N_COLLECT_POINTS);
                }
                for( j=0; j<1; j++)
                {
                    fprintf(slimfly_terminal_occupancy_log, "%d, ", vc_occupancy_storage_terminal[k][j][i]);
                }
            }
        }
    }
    fclose(slimfly_terminal_occupancy_log);
#endif

#if TERMINAL_SENDS_RECVS_LOG
    if(tw_ismaster())
    {
        printf("Printing Realtime Terminal sends & recvs Log Files\n");
        sprintf( log, "slimfly_terminal_sends_recvs_log.txt");
        slimfly_terminal_sends_recvs_log=fopen(log, "w+");
        if(slimfly_terminal_sends_recvs_log == NULL)
            tw_error(TW_LOC, "\n Failed to open slimfly terminal sends & recvs log file \n");
        for( i=0; i<N_COLLECT_POINTS; i++ )
        {
            fprintf(slimfly_terminal_sends_recvs_log, "%d",i*100/N_COLLECT_POINTS);
            for(j=0; j<slim_total_terminals_noah; j++)
            {
                fprintf(slimfly_terminal_sends_recvs_log, ", %lu", terminal_sends[j][i]);
                fprintf(slimfly_terminal_sends_recvs_log, ", %lu", terminal_recvs[j][i]);
            }
            fprintf(slimfly_terminal_sends_recvs_log, "\n");
        }
    }
    fclose(slimfly_terminal_sends_recvs_log);
#endif

#if ROUTER_SENDS_RECVS_LOG
    if(tw_ismaster())
    {
        printf("Printing Realtime Router ROUTER sends & recvs Log Files\n");
        sprintf( log, "slimfly_router_sends_recvs_log.txt");
        slimfly_router_sends_recvs_log=fopen(log, "w+");
        if(slimfly_router_sends_recvs_log == NULL)
            tw_error(TW_LOC, "\n Failed to open slimfly router sends & recvs log file \n");
        for( i=0; i<N_COLLECT_POINTS; i++ )
        {
            fprintf(slimfly_router_sends_recvs_log, "%d",i*100/N_COLLECT_POINTS);
            for(j=0; j<slim_total_routers_noah; j++)
            {
                fprintf(slimfly_router_sends_recvs_log, ", %lu", router_sends[j][i]);
                fprintf(slimfly_router_sends_recvs_log, ", %lu", router_recvs[j][i]);
            }
            fprintf(slimfly_router_sends_recvs_log, "\n");
        }
    }
    fclose(slimfly_router_sends_recvs_log);
#endif

    return;
}

//This method returns the relative router ID of the router attached to the Terminal term_id on rail rail_id.
int slim_get_associated_router_id_from_terminal(int term_gid, int rail_id, int num_rails)
{
    int term_rel_id = codes_mapping_get_lp_relative_id(term_gid,0,0);

    int num_term_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM, NULL, 0); // number of terminals per rep
    int num_routers = codes_mapping_get_lp_count(lp_group_name, 0 ,LP_CONFIG_NM_ROUT, NULL, 0);

    int default_router_id = term_rel_id / num_term_lps; // the router id that is assigned to the given terminal by default - relative.

    int num_routers_per_rail = num_routers / num_rails;

    tw_lpid router_lpgid;
    if (rail_id%2 == 0) { //even rails have normal mapping
        codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, NULL, 1, default_router_id, rail_id, &router_lpgid);
    }
    else { //odd rails have reverse mapping
        int rep_id = num_routers_per_rail - default_router_id - 1;
        codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, NULL, 1, rep_id, rail_id, &router_lpgid);
    }

    int router_id = codes_mapping_get_lp_relative_id(router_lpgid, 0, 0); //get the router ID from the lpid

    return router_id;
}

/* initialize a slimfly compute node terminal */
void slim_terminal_init( terminal_state * s,
        tw_lp * lp )
{
    s->packet_gen = 0;
    s->packet_fin = 0;

    char anno[MAX_NAME_LENGTH];

    // Assign the global router ID
    // TODO: be annotation-aware
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
    if (anno[0] == '\0'){
        s->anno = NULL;
        s->params = &all_params[num_params-1];
    }
    else{
        s->anno = strdup(anno);
        int id = configuration_get_annotation_index(anno, anno_map);
        s->params = &all_params[id];
    }

    int num_term_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
            s->anno, 0); //keep in mind that this is the number of terminals per rep!

    int num_routers = codes_mapping_get_lp_count(lp_group_name, 0 ,"modelnet_slimfly_router",
            s->anno, 0);

    int num_routers_per_rail = num_routers / s->params->ports_per_nic;

#if MSG_TIMES
    s->msg_send_times = (int*)calloc(200,sizeof(int));
    s->msg_rail_select = (int*)calloc(200,sizeof(int));
#endif

    s->terminal_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);  
    s->router_id=(int)s->terminal_id / (num_term_lps); //this is the id for the default rail 0
    s->router_lp=(tw_lpid*)calloc(s->params->ports_per_nic, sizeof(tw_lpid));
    
    // printf("Terminal: %d",s->terminal_id);

    if (s->params->sf_type == 1) { //fitfly
        //Assign Routers on each rail:
        for (int i = 0; i < s->params->ports_per_nic; i++)
        {
            if (i%2 == 0) { //even rails have normal mapping
                codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, NULL, 1, s->router_id, i, &s->router_lp[i]);
                // printf("  - Rail %d = %d",i, codes_mapping_get_lp_relative_id(s->router_lp[i],0,0));
            }
            else { //odd rails have reverse mapping
                int rep_id = num_routers_per_rail - s->router_id - 1;
                codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, NULL, 1, rep_id, i, &s->router_lp[i]);
                // printf("  - Rail %d = %d",i, codes_mapping_get_lp_relative_id(s->router_lp[i],0,0));
            }
        }
        // printf("\n");
    }
    else { //default slimfly
        codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", NULL, 1,
            s->router_id, 0, &s->router_lp[0]);
    }

#if SLIMFLY_CONNECTIONS
    int written = 0;
    written += sprintf(s->output_buf + written, "%d, %d, ", s->terminal_id, s->params->slim_total_terminals + s->router_id);
#endif

// TODO: Reintroduce fitfly multi rail connection output
//     //Assign router from second rail
//     if(s->params->sf_type == 1){
//         codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", NULL, 1,
//             (num_routers-1) - s->router_id - (num_routers/2), 1, &s->router_lp[1]);
// #if SLIMFLY_CONNECTIONS
//         written += sprintf(s->output_buf + written, "%d, %d, ", s->terminal_id, s->params->slim_total_terminals + (num_routers-1) - s->router_id);
// #endif
//     }

#if SLIMFLY_CONNECTIONS
    lp_io_write(lp->gid, "slimfly-config-terminal-connections", written, s->output_buf);
#endif

    if(s->terminal_id == 1723)
        printf("terminal_id:%d, router_id:%d, router_lp[0]:%d, router_lp[1]:%d\n",s->terminal_id, s->router_id, (int)s->router_lp[0], (int)s->router_lp[1]);

    s->terminal_available_time = (tw_stime*)calloc(s->params->ports_per_nic, sizeof(tw_stime));
    s->busy_time = (tw_stime*)calloc(s->params->ports_per_nic, sizeof(tw_stime));
    s->issueIdle = (int*)calloc(s->params->ports_per_nic, sizeof(int));
    s->in_send_loop = (int*)calloc(s->params->ports_per_nic, sizeof(int));
    s->terminal_length = (int*)calloc(s->params->ports_per_nic, sizeof(int));
    s->vc_occupancy = (int*)calloc(s->params->ports_per_nic, sizeof(int));
    s->last_buf_full = (tw_stime*)calloc(s->params->ports_per_nic, sizeof(tw_stime));
    s->terminal_msgs = (slim_terminal_message_list**)calloc(s->params->ports_per_nic, sizeof(slim_terminal_message_list*));
    s->terminal_msgs_tail = (slim_terminal_message_list**)calloc(s->params->ports_per_nic, sizeof(slim_terminal_message_list*));

    for(int i = 0; i < s->params->ports_per_nic; i++) {
        s->terminal_available_time[i] = 0.0;
        s->vc_occupancy[i] = 0;
        s->last_buf_full[i] = 0;
        s->terminal_msgs[i] = NULL;
        s->terminal_msgs_tail[i] = NULL;
        s->terminal_length[i] = 0;
        s->in_send_loop[i] = 0;
        s->issueIdle[i] = 0;
        s->busy_time[i] = 0;
    }
    s->packet_counter = 0;
    s->finished_msgs = 0;
    s->finished_chunks = 0;
    s->finished_packets = 0;
    s->total_time = 0.0;
    s->total_msg_size = 0;

    rc_stack_create(&s->st);

    s->rank_tbl = qhash_init(slimfly_rank_hash_compare, slimfly_hash_func, SLIMFLY_HASH_TABLE_SIZE);

    if(!s->rank_tbl)
        tw_error(TW_LOC, "\n Hash table not initialized! ");

    return;
}


/* sets up the router virtual channels, global channels,
 * local channels, compute node channels */
void slim_router_setup(router_state * r, tw_lp * lp)
{
    char anno[MAX_NAME_LENGTH];
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);

    if (anno[0] == '\0'){
        r->anno = NULL;
        r->params = &all_params[num_params-1];
    } else{
        r->anno = strdup(anno);
        int id = configuration_get_annotation_index(anno, anno_map);
        r->params = &all_params[id];
    }

    // shorthand
    const slimfly_param *p = r->params;

    r->router_id=mapping_rep_id + mapping_offset * r->params->slim_total_routers;
    r->group_id=r->router_id/p->num_routers;

    r->global_channel = (int*)calloc(p->num_global_channels, sizeof(int));
    r->local_channel = (int*)calloc(p->num_local_channels, sizeof(int));
    r->next_output_available_time = (tw_stime*)calloc(p->radix, sizeof(tw_stime));
    r->link_traffic = (int64_t*)calloc(p->radix, sizeof(int64_t));
    r->cur_hist_num = (int*)calloc(p->radix, sizeof(int));
    r->prev_hist_num = (int*)calloc(p->radix, sizeof(int));

    r->vc_occupancy = (int**)calloc(p->radix, sizeof(int*));
    r->in_send_loop = (int*)calloc(p->radix, sizeof(int));
    r->pending_msgs =
        (slim_terminal_message_list***)calloc(p->radix, sizeof(slim_terminal_message_list**));
    r->pending_msgs_tail =
        (slim_terminal_message_list***)calloc(p->radix, sizeof(slim_terminal_message_list**));
    r->queued_msgs =
        (slim_terminal_message_list***)calloc(p->radix, sizeof(slim_terminal_message_list**));
    r->queued_msgs_tail =
        (slim_terminal_message_list***)calloc(p->radix, sizeof(slim_terminal_message_list**));

   r->last_buf_full = (tw_stime**)calloc(p->radix, sizeof(tw_stime*));
    r->busy_time = (tw_stime*)calloc(p->radix, sizeof(tw_stime));

    // ROSS Instrumentation
    if (g_st_use_analysis_lps  && g_st_model_stats)
       lp->model_types->sample_struct_sz = sizeof(struct slimfly_router_sample) + sizeof(int) * p->radix;

    rc_stack_create(&r->st);

    for(int i=0; i < p->radix; i++)
    {
        // Set credit & router occupancy
        r->next_output_available_time[i]=0;
        r->link_traffic[i]=0;
        r->cur_hist_num[i] = 0;
        r->prev_hist_num[i] = 0;

        r->busy_time[i] = 0.0;

        r->in_send_loop[i] = 0;
        r->vc_occupancy[i] = (int*)calloc(p->num_vcs, sizeof(int));
        r->pending_msgs[i] = (slim_terminal_message_list**)calloc(p->num_vcs, 
                sizeof(slim_terminal_message_list*));
        r->last_buf_full[i] = (tw_stime*)calloc(p->num_vcs, sizeof(tw_stime));
        r->pending_msgs_tail[i] = (slim_terminal_message_list**)calloc(p->num_vcs,
                sizeof(slim_terminal_message_list*));
        r->queued_msgs[i] = (slim_terminal_message_list**)calloc(p->num_vcs,
                sizeof(slim_terminal_message_list*));
        r->queued_msgs_tail[i] = (slim_terminal_message_list**)calloc(p->num_vcs,
                sizeof(slim_terminal_message_list*));
        for(int j = 0; j < p->num_vcs; j++) {
            r->last_buf_full[i][j] = 0.0;
            r->vc_occupancy[i][j] = 0;
            r->pending_msgs[i][j] = NULL;
            r->pending_msgs_tail[i][j] = NULL;
            r->queued_msgs[i][j] = NULL;
            r->queued_msgs_tail[i][j] = NULL;
        }
    }

#if LOAD_FROM_FILE
    //Load input MMS router and node layout/connection graph from file
    char log[500];
    sprintf( log, "simulation-input-files/MMS.%d/MMS.%d.%d.bsconf", p->num_global_channels+p->num_local_channels, p->num_global_channels+p->num_local_channels, p->num_cn);
    MMS_input_file = fopen( log, "r");
    if( MMS_input_file == NULL )
        tw_error( TW_LOC, "Failed to Open Slim_fly input MMS layout file: %s The reason *may* have been %s\n",log,strerror(errno));

    int i,j;
    rewind(MMS_input_file);
    char one_word[16];
    int one_number = 0;
    int temp;
    //Skip over preceeding lines
    for(i=0;i<r->router_id;i++)
    {
        for(j=0;j<=p->num_local_channels+p->num_global_channels+p->num_cn;j++)
        {
            temp = fscanf(MMS_input_file,"%s",one_word); // got one word from the file
            temp = fscanf(MMS_input_file,"%d",&one_number);
        }
    }
    //Skip over self in MMS file
    temp = fscanf(MMS_input_file,"%s",one_word); // got one word from the file /
    temp = fscanf(MMS_input_file,"%d",&one_number);
    //Assign Local router connections
    for(i=0;i<p->num_local_channels;i++)
    {
        temp = fscanf(MMS_input_file,"%s",one_word); // got one word from the file /
        temp = fscanf(MMS_input_file,"%d",&one_number);
        r->local_channel[i] = one_number;
    }
    //Set global router connections according to input MMS file
    for(i=0;i<p->num_global_channels;i++)
    {
        temp = fscanf(MMS_input_file,"%s",one_word); // got one word from the file
        temp = fscanf(MMS_input_file,"%d",&one_number);
        r->global_channel[i] = one_number;
    }
    fclose(MMS_input_file);
#else
    get_router_connections(r->router_id, p->num_global_channels, p->num_local_channels,
        p->slim_total_routers, r->local_channel, r->global_channel, p->sf_type, p);
#endif

#if SLIMFLY_CONNECTIONS
    int written = 0;
    int i;

    for(i=0;i<r->params->num_local_channels;i++){
        written += sprintf(r->output_buf + written, "%d, %d, ", r->params->slim_total_terminals + r->router_id, r->params->slim_total_terminals + r->local_channel[i]);
    }
    for(i=0;i<r->params->num_global_channels;i++){
        written += sprintf(r->output_buf + written, "%d, %d, ", r->params->slim_total_terminals + r->router_id, r->params->slim_total_terminals + r->global_channel[i]);
    }
    lp_io_write(lp->gid, "slimfly-config-router-connections", written, r->output_buf);
#endif

    return;
}


/* slimfly packet event , generates a slimfly packet on the compute node */
static tw_stime slimfly_packet_event(
        model_net_request const * req,
        uint64_t message_offset,
        uint64_t packet_size,
        tw_stime offset,
        mn_sched_params const * sched_params,
        void const * remote_event,
        void const * self_event,
        tw_lp *sender,
        int is_last_pckt)
{
    // printf("slim packet event\n");

    (void)message_offset;
    (void)sched_params;

    tw_event * e_new;
    tw_stime xfer_to_nic_time;
    slim_terminal_message * msg;
    char* tmp_ptr;
#if DEBUG
    printf("\x1B[34m(%lf) packet_event() msg_id:%lu rail_id:%d\x1b[0m\n",tw_now(sender), req->msg_id, req->queue_offset);
#endif
    xfer_to_nic_time = codes_local_latency(sender);
    //e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender);
    //msg = tw_event_data(e_new);
    //printf("%llu packet_event() xfer to nic time: %llu, offset: %llu\n",LLU(tw_now(sender)),LLU(xfer_to_nic_time),LLU(offset));
    e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset,
            sender, SLIMFLY, (void**)&msg, (void**)&tmp_ptr);
    strcpy(msg->category, req->category);
    msg->final_dest_gid = req->final_dest_lp;
    msg->total_size = req->msg_size;
    msg->sender_lp=req->src_lp;
    msg->sender_mn_lp = sender->gid;
    msg->packet_size = packet_size;
    msg->travel_start_time = tw_now(sender);
    msg->remote_event_size_bytes = 0;
    msg->local_event_size_bytes = 0;
    msg->type = T_GENERATE;
    msg->dest_terminal_id = req->dest_mn_lp;
    msg->rail_id = req->queue_offset;
    msg->message_id = req->msg_id;
    msg->is_pull = req->is_pull;
    msg->pull_size = req->pull_size;
    msg->magic = slim_terminal_magic_num;
    msg->msg_start_time = req->msg_start_time;

    if(is_last_pckt) /* Its the last packet so pass in remote and local event information*/
    {
        if(req->remote_event_size > 0)
        {
            msg->remote_event_size_bytes = req->remote_event_size;
            memcpy(tmp_ptr, remote_event, req->remote_event_size);
            tmp_ptr += req->remote_event_size;
        }
        if(req->self_event_size > 0)
        {
            msg->local_event_size_bytes = req->self_event_size;
            memcpy(tmp_ptr, self_event, req->self_event_size);
            tmp_ptr += req->self_event_size;
        }
    }
    //printf("\n slimfly remote event %d local event %d last packet %d %lf ", msg->remote_event_size_bytes, msg->local_event_size_bytes, is_last_pckt, xfer_to_nic_time);
    tw_event_send(e_new);
    return xfer_to_nic_time;
}

/* slimfly packet event reverse handler */
static void slimfly_packet_event_rc(tw_lp *sender)
{
    // printf("slimfly packet event rc\n");
    codes_local_latency_reverse(sender);
    return;
}

/*When a packet is sent from the current router and a buffer slot becomes available, a credit is sent back to schedule another packet event*/
void slim_router_credit_send(router_state * s, slim_terminal_message * msg, tw_lp * lp, int sq)
{
    // printf("slim router credit send\n");

    tw_event * buf_e;
    tw_stime ts;
    slim_terminal_message * buf_msg;

    int dest = 0,  type = R_BUFFER;
    int is_terminal = 0;

    const slimfly_param *p = s->params;

    // Notify sender terminal about available buffer space
    if(msg->last_hop == TERMINAL) {
        dest = msg->src_terminal_id;
        type = T_BUFFER;
        is_terminal = 1;
    } else if(msg->last_hop == GLOBAL) {
        dest = msg->intm_lp_id;
    } else if(msg->last_hop == LOCAL) {
        dest = msg->intm_lp_id;
    } else
        printf("\n Invalid message type");

#if TRACK_OUTPUT
    if( msg->packet_ID == TRACK )
    {
        printf("sending credit to gid:%d\n",dest);
    }
#endif

    ts = g_tw_lookahead + p->credit_delay +  tw_rand_unif(lp->rng);
    msg->rng_calls++;

    if (is_terminal) {
        buf_e = model_net_method_event_new(dest, ts, lp, SLIMFLY,
                (void**)&buf_msg, NULL);
        buf_msg->magic = slim_terminal_magic_num;
    } else {
        buf_e = model_net_method_event_new(dest, ts, lp, SLIMFLY_ROUTER, (void**)&buf_msg, NULL);
        buf_msg->magic = slim_router_magic_num;
    }

    if(sq == -1) {
        buf_msg->vc_index = msg->vc_index;
        buf_msg->output_chan = msg->output_chan;
    } else {
        buf_msg->vc_index = msg->saved_vc;
        buf_msg->output_chan = msg->saved_channel;
    }

    buf_msg->type = type;

    tw_event_send(buf_e);
    return;
}

void slim_packet_generate_rc(terminal_state * s, tw_bf * bf, slim_terminal_message * msg, tw_lp * lp)
{
    // printf("slimfly packet gen rc\n");

    s->packet_gen--;
    slimfly_packet_gen--;

    for(int i = 0; i < msg->rng_calls; i++)
    {
        tw_rand_reverse_unif(lp->rng);
    }

    // tw_rand_reverse_unif(lp->rng); //rng_calls

    // if (bf->c1)
        // tw_rand_reverse_unif(lp->rng); //rng_calls

    int num_chunks = msg->packet_size/s->params->chunk_size;
    if(msg->packet_size % s->params->chunk_size)
        num_chunks++;

    if(!num_chunks)
        num_chunks = 1;

    int i;
    for(i = 0; i < num_chunks; i++)
    {
        slim_delete_terminal_message_list(return_tail(s->terminal_msgs,
                    s->terminal_msgs_tail, msg->saved_vc));
        s->terminal_length[msg->saved_vc] -= s->params->chunk_size;
    }
    if(bf->c5)
    {
        codes_local_latency_reverse(lp);
        s->in_send_loop[msg->saved_vc] = 0;
    }

    tw_stime* bts = rc_stack_pop(s->st);
    for(int j = 0; j < s->params->num_injection_queues; j++) 
    {
        s->last_buf_full[j] = bts[j];
        if (s->terminal_length[j] >= s->params->cn_vc_size)
            s->issueIdle[j] = 0;
    }
    buff_time_storage_delete(bts);

    struct mn_stats* stat;
    stat = model_net_find_stats(msg->category, s->slimfly_stats_array);
    stat->send_count--;
    stat->send_bytes -= msg->packet_size;
    stat->send_time -= (1/s->params->cn_bandwidth) * msg->packet_size;
}

/* generates packet at the current slimfly compute node */
void slim_packet_generate(terminal_state * s, tw_bf * bf, slim_terminal_message * msg, tw_lp * lp)
{
    // printf("slim packet gen\n");

    slimfly_packet_gen++;
    s->packet_gen++;

    tw_stime ts, nic_ts;

    assert(lp->gid != msg->dest_terminal_id);
    const slimfly_param *p = s->params;

    uint64_t i;
    int total_event_size;
    uint64_t num_chunks = msg->packet_size / p->chunk_size;
    double cn_delay = s->params->cn_delay;

    if (msg->packet_size < (unsigned)s->params->chunk_size)
        num_chunks++;

    if (msg->packet_size < (unsigned)s->params->chunk_size)
        cn_delay = bytes_to_ns(msg->packet_size % s->params->chunk_size, s->params->cn_bandwidth);

    nic_ts = g_tw_lookahead + (num_chunks * cn_delay) + tw_rand_unif(lp->rng);
    msg->rng_calls++;

    msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter;
    msg->my_N_hop = 0;
    msg->my_l_hop = 0;
    msg->my_g_hop = 0;
    msg->intm_group_id = -1;
    msg->intm_router_id = -1;

    int target_queue = msg->rail_id;
    int path_tie = 0;


    // This for some reason is innaccurate - use old one below but it is hardcoded for 2 rails
    // if (s->params->rail_select == RAIL_PATH) {
    //     // set starting rail to first rail (rail 0)
    //     target_queue = 0;

    //     int path_lengths[s->params->ports_per_nic];
    //     int min_len = 999999;
    //     for(int rail_id = 0; rail_id < s->params->ports_per_nic; rail_id++)
    //     {
    //         //get the router relative ID connected to the destination terminal on this rail
    //         int dest_router_rel_id = slim_get_associated_router_id_from_terminal(msg->dest_terminal_id, rail_id, s->params->ports_per_nic);

    //         //get the router relative ID connected to this terminal on this rail
    //         int src_router_rel_id = codes_mapping_get_lp_relative_id(s->router_lp[rail_id],0,0);
    //         path_lengths[rail_id] = get_path_length_from_terminal(src_router_rel_id, dest_router_rel_id, p);

    //         if (path_lengths[rail_id] < min_len) {
    //             min_len = path_lengths[rail_id];
    //             target_queue = rail_id;
    //             path_tie = 0; //reset as we found a new min
    //         }
    //         else if (path_lengths[rail_id] == min_len) {
    //             path_tie = 1; //this was the same as the current min, tie.
    //         }
    //     }
    // }
    // else if (s->params->rail_select == RAIL_RAND) {
    //     target_queue = tw_rand_integer(lp->rng, 0,p->num_rails-1);
    //     msg->rng_calls++;
    // }

    if(s->params->rail_select == RAIL_PATH){
        if (s->params->num_rails > 2)
            tw_error(TW_LOC,"This rail path selection method doesn't support greater than 2 rails. see notes in slim_packet_generate()\n");
        // Set starting rail to the first rail (rail 0)
        target_queue = 0;
        // Get information on destination terminal from its LP ID
        codes_mapping_get_lp_info(msg->dest_terminal_id, lp_group_name,
                &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id,
                &mapping_offset);
        // Get number of terminal LPs per repetition so we can calculate local/relative router ID for the destination terminal
        int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
                s->anno, 0);
        // Compute relative id of router in the first rail that is connected to the destination terminal
        int rail_one_dest_router_rel_id = (mapping_offset + (mapping_rep_id * num_lps)) / s->params->num_cn;
        // Get path length from rail 1 source router to rail 1 dest router
        int path_length_rail1 = get_path_length_from_terminal(s->router_id, rail_one_dest_router_rel_id, p);
        // Compute relative id of router in the second rail connected to destination compute terminal
        int rail_two_dest_router_rel_id = s->params->slim_total_routers - 1 - rail_one_dest_router_rel_id;
        // Get path length from rail 2 src router to rail 2 dest router
        int path_length_rail2 = get_path_length_from_terminal(s->params->slim_total_routers - 1 - s->router_id, rail_two_dest_router_rel_id, p);
        // Compare rail path lengths
        if( path_length_rail2 < path_length_rail1 ){
            target_queue = 1;
        }else if( path_length_rail2 == path_length_rail1 ){
            // Set path tie so we can break the tie with the RAIL_CONGESTION method below
            path_tie = 1;
        }   
    }

    if(s->params->rail_select == RAIL_CONGESTION || path_tie) {
        bf->c1 = 1;
        target_queue = tw_rand_integer(lp->rng,0,p->ports_per_nic-1);
        msg->rng_calls++;
        int curr_buffer = s->vc_occupancy[target_queue];
        for(int j = 0; j < s->params->ports_per_nic; j++) {
            int next = (msg->rail_id + j) % s->params->ports_per_nic;
            if(s->vc_occupancy[next] < curr_buffer) {
                curr_buffer =  s->vc_occupancy[next];
                target_queue = next;
            }
        }
    }

    msg->saved_vc = target_queue;

    for(i = 0; i < num_chunks; i++)
    {
        slim_terminal_message_list *cur_chunk = (slim_terminal_message_list*)calloc(1,
                sizeof(slim_terminal_message_list));
        slim_init_terminal_message_list(cur_chunk, msg);

        if(msg->remote_event_size_bytes + msg->local_event_size_bytes > 0)
        {
            cur_chunk->event_data = (char*)calloc(1,
                    msg->remote_event_size_bytes + msg->local_event_size_bytes);
        }

        void * m_data_src = model_net_method_get_edata(SLIMFLY, msg);
        if (msg->remote_event_size_bytes)
        {
            memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes);
        }
        if (msg->local_event_size_bytes)
        {
            m_data_src = (char*)m_data_src + msg->remote_event_size_bytes;
            memcpy((char*)cur_chunk->event_data + msg->remote_event_size_bytes,
                    m_data_src, msg->local_event_size_bytes);
        }

        cur_chunk->msg.chunk_id = i;
        append_to_terminal_message_list(s->terminal_msgs, s->terminal_msgs_tail,
                target_queue, cur_chunk);
        s->terminal_length[target_queue] += s->params->chunk_size;
    }

    tw_stime *bts = buff_time_storage_create(s); //mallocs space to push onto the rc stack -- free'd in rc
    
    //TODO: Inspect this and verify that we should be looking at each port always
    for(int j=0; j<s->params->num_injection_queues; j++){
        bts[j] = s->last_buf_full[j];
        if(s->terminal_length[j] < s->params->cn_vc_size)
        {
#if DEBUG
            printf("(%lf) packet_generate() idle_event2 time offset:%lf\n",tw_now(lp),nic_ts);
#endif   
            double dither = .0001 * tw_rand_unif(lp->rng);
            msg->rng_calls++;
            model_net_method_idle_event2(nic_ts+dither, 0, j, lp);
        }
        else
        {
            bf->c11 = 1;
            s->issueIdle[j] = 1;
            if(s->last_buf_full[j] == 0.0)
            {
                bf->c8 = 1;
                /* TODO: Assumes a single vc from terminal to router */
                s->last_buf_full[j] = tw_now(lp);;
            }
        }
    }

    rc_stack_push(lp, bts, buff_time_storage_delete, s->st);

    if(s->in_send_loop[target_queue] == 0)
    {
        bf->c5 = 1;
        ts = codes_local_latency(lp);
        slim_terminal_message *m;
        tw_event* e = model_net_method_event_new(lp->gid, ts, lp, SLIMFLY,
                (void**)&m, NULL);
        m->type = T_SEND;
        m->vc_index = target_queue;
        m->magic = slim_terminal_magic_num;
        s->in_send_loop[target_queue] = 1;
        tw_event_send(e);
    }

    total_event_size = model_net_get_msg_sz(SLIMFLY) +
        msg->remote_event_size_bytes + msg->local_event_size_bytes;
    mn_stats* stat;
    stat = model_net_find_stats(msg->category, s->slimfly_stats_array);
    stat->send_count++;
    stat->send_bytes += msg->packet_size;
    stat->send_time += (1/p->cn_bandwidth) * msg->packet_size;
    if(stat->max_event_size < total_event_size)
        stat->max_event_size = total_event_size;

    return;
}

void slim_packet_send_rc(terminal_state * s, tw_bf * bf, slim_terminal_message * msg, tw_lp * lp)
{
    // printf("slimfly packet send rc\n");
    for(int i = 0; i < msg->rng_calls; i++)
    {
        tw_rand_reverse_unif(lp->rng);
    }

    if (bf->c10) {
        s->last_buf_full[msg->vc_index] = 0.0;
    }

    if(bf->c1) {
        s->in_send_loop[msg->vc_index] = msg->saved_send_loop;
        return;
    }

    // tw_rand_reverse_unif(lp->rng);
    s->terminal_available_time[msg->vc_index] = msg->saved_available_time;
    if(bf->c2) {
        codes_local_latency_reverse(lp);
    }

    s->terminal_length[msg->vc_index] += s->params->chunk_size;
    s->packet_counter--;
    s->vc_occupancy[msg->vc_index] -= s->params->chunk_size;

    // if(bf->c3) {
    //     tw_rand_reverse_unif(lp->rng);
    // }
    if(bf->c4) {
        s->in_send_loop[msg->vc_index] = 1;
    }
    if(bf->c5)
    {
        s->issueIdle[msg->vc_index] = 1;
        // tw_rand_reverse_unif(lp->rng);
        if(bf->c6)
        {
            tw_stime *bts = rc_stack_pop(s->st);
            s->busy_time[msg->vc_index] = msg->saved_total_time;
            s->last_buf_full[msg->vc_index] = bts[msg->vc_index];
            buff_time_storage_delete(bts);
        }
    }

    slim_terminal_message_list* cur_entry = rc_stack_pop(s->st);

    prepend_to_terminal_message_list(s->terminal_msgs,
            s->terminal_msgs_tail, msg->vc_index, cur_entry);

    return;
}

/* sends the packet from the current slimfly compute node to the attached router */
void slim_packet_send(terminal_state * s, tw_bf * bf, slim_terminal_message * msg,
        tw_lp * lp)
{
    // printf("slim pacekt send\n");
    tw_stime ts;
    tw_event *e;
    slim_terminal_message *m;


    slim_terminal_message_list* cur_entry = s->terminal_msgs[msg->vc_index];
    if(s->vc_occupancy[msg->vc_index] + s->params->chunk_size > s->params->cn_vc_size)
    {
        if(s->last_buf_full[msg->vc_index] == 0.0)
        {
            bf->c10 = 1;
            s->last_buf_full[msg->vc_index] = tw_now(lp);
        }
    }

    if(s->vc_occupancy[msg->vc_index] + s->params->chunk_size > s->params->cn_vc_size
            || cur_entry == NULL)
    {
        bf->c1 = 1;
        msg->saved_send_loop = s->in_send_loop[msg->vc_index];
        s->in_send_loop[msg->vc_index] = 0;
        return;
    }

    uint64_t num_chunks = cur_entry->msg.packet_size/s->params->chunk_size;
    if(cur_entry->msg.packet_size % s->params->chunk_size)
        num_chunks++;

    if(!num_chunks)
        num_chunks = 1;

    tw_stime delay = s->params->cn_delay;
    if((cur_entry->msg.packet_size % s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1))
        delay = bytes_to_ns(cur_entry->msg.packet_size % s->params->chunk_size, s->params->cn_bandwidth); 

    msg->saved_available_time = s->terminal_available_time[msg->vc_index];
    ts = g_tw_lookahead + delay + tw_rand_unif(lp->rng);
    msg->rng_calls++;
    s->terminal_available_time[msg->vc_index] = maxd(s->terminal_available_time[msg->vc_index], tw_now(lp));
    s->terminal_available_time[msg->vc_index] += ts;

    ts = s->terminal_available_time[msg->vc_index] - tw_now(lp);
    
    void *remote_event;
    e = model_net_method_event_new(s->router_lp[msg->vc_index], ts, lp, SLIMFLY_ROUTER, (void**)&m, &remote_event);
    memcpy(m, &cur_entry->msg, sizeof(slim_terminal_message));
    if (m->remote_event_size_bytes)
    {   
        memcpy(remote_event, cur_entry->event_data, m->remote_event_size_bytes);
        // memcpy(model_net_method_get_edata(SLIMFLY, m), cur_entry->event_data,
        //         m->remote_event_size_bytes);
    }

    //if((int)lp->gid == 49)
    //    printf("%llu \x1B[34m-->send time offset:%llu Packet sending from terminal %d sending to router %d on rail %d \x1b[0m\n", LLU(ts), LLU(tw_now(lp)),(int)lp->gid, (int)s->router_lp[msg->vc_index], msg->vc_index);

    //if(msg->packet_ID == TRACK)
#if DEBUG
        printf("%llu \x1B[34m-->Packet sending from terminal %d sending to router LP:%d relID:%d to arrive at %llu\x1b[0m\n", LLU(tw_now(lp)),(int)lp->gid, (int)s->router_lp[msg->vc_index], s->router_id, LLU(ts+tw_now(lp)));
#endif

    m->origin_router_id = s->router_lp[msg->vc_index];
    m->type = R_ARRIVE;
    m->src_terminal_id = lp->gid;
    m->rail_id = msg->vc_index;
    m->vc_index = msg->vc_index;
    m->last_hop = TERMINAL;
    m->intm_group_id = -1;
    m->intm_router_id = -1;
    m->magic = slim_router_magic_num;
    m->path_type = -1;
    m->local_event_size_bytes = 0;
    m->local_id = s->terminal_id;
    tw_event_send(e);

    //printf("s->router_lp[%d]:%llu, s->router_lp[1]:%llu, s->router_id:%d, router_id:%llu, packet_id:%llu\n",msg->vc_index,LLU(s->router_lp[msg->vc_index]),LLU(s->router_lp[1]),s->router_id,LLU(router_id),LLU(m->packet_ID));
#if DEBUG
    if( m->packet_ID == TRACK)
    {
        printf( "(%lf) [Terminal lp:%d, rel:%d] packet_ID %lld message_ID %d is sending ",tw_now(lp), (int)lp->gid, s->terminal_id, m->packet_ID, (int)m->message_id);
        printf("to router lpID:%d. ", (int)s->router_lp[msg->vc_index]);
        printf("Travel start time is %f. ", m->travel_start_time);
        printf("My hop now is %d\n",m->my_N_hop);
    }
#endif

#if TERMINAL_SENDS_RECVS_LOG || TERMINAL_OCCUPANCY_LOG
    int index = floor(N_COLLECT_POINTS*(tw_now(lp)/g_tw_ts_end));
#endif

#if TERMINAL_SENDS_RECVS_LOG
    terminal_sends[s->terminal_id][index]++;
#endif

    if(cur_entry->msg.chunk_id == num_chunks - 1 && (cur_entry->msg.local_event_size_bytes > 0))
    {
        bf->c2 = 1;
        tw_stime local_ts = codes_local_latency(lp);
        tw_event *e_new = tw_event_new(cur_entry->msg.sender_lp, local_ts, lp);
        slim_terminal_message* m_new = tw_event_data(e_new);
        void *local_event = (char*)cur_entry->event_data +
            cur_entry->msg.remote_event_size_bytes;
        memcpy(m_new, local_event, cur_entry->msg.local_event_size_bytes);
        tw_event_send(e_new);
    }
    s->packet_counter++;
    s->vc_occupancy[msg->vc_index] += s->params->chunk_size;
#if TERMINAL_OCCUPANCY_LOG
    vc_occupancy_storage_terminal[s->terminal_id][0][index] = s->vc_occupancy[0]/s->params->chunk_size;
#endif
    cur_entry = return_head(s->terminal_msgs, s->terminal_msgs_tail, msg->vc_index);
    rc_stack_push(lp, cur_entry, (void*)slim_delete_terminal_message_list, s->st);
    s->terminal_length[msg->vc_index] -= s->params->chunk_size;

#if MSG_TIMES
    s->msg_send_times[s->packet_counter-1] = (int)tw_now(lp);
    s->msg_rail_select[s->packet_counter-1] = msg->vc_index;
#endif

    cur_entry = s->terminal_msgs[msg->vc_index];

    if(cur_entry != NULL && s->vc_occupancy[msg->vc_index] + s->params->chunk_size <= s->params->cn_vc_size)
    {
        bf->c3 = 1;
        slim_terminal_message *m_new;
        ts += tw_rand_unif(lp->rng);
        msg->rng_calls++;
        tw_event* e_new = model_net_method_event_new(lp->gid, ts, lp, SLIMFLY,
                (void**)&m_new, NULL);
        m_new->type = T_SEND;
        m_new->vc_index = msg->vc_index;
        m_new->magic = slim_terminal_magic_num;
        tw_event_send(e_new);
    }
    else
    {
        bf->c4 = 1;
        s->in_send_loop[msg->vc_index] = 0;
    }
    if(s->issueIdle[msg->vc_index])
    {
        bf->c5 = 1;
        s->issueIdle[msg->vc_index] = 0;
        ts += tw_rand_unif(lp->rng);
        msg->rng_calls++;
        //printf("packet_send() idle_event2\n");
        model_net_method_idle_event2(ts, 0, msg->vc_index, lp);

        if(s->last_buf_full[msg->vc_index] > 0.0)
        {
            bf->c6 = 1;
            msg->saved_total_time = s->busy_time[msg->vc_index];
            
            tw_stime* bts = buff_time_storage_create(s);
            bts[msg->vc_index] = s->last_buf_full[msg->vc_index];
            rc_stack_push(lp, bts, buff_time_storage_delete, s->st);

            s->busy_time[msg->vc_index] += (tw_now(lp) - s->last_buf_full[msg->vc_index]);
            s->last_buf_full[msg->vc_index] = 0.0;
        }
    }
    return;
}

void slim_packet_arrive_rc(terminal_state * s, tw_bf * bf, slim_terminal_message * msg, tw_lp * lp)
{
    // printf("slimfly packet arrive rc\n");
    for(int i = 0; i < msg->rng_calls; i++)
    {
        tw_rand_reverse_unif(lp->rng);
    }

    if(bf->c31)
    {
	    s->packet_fin--;
	    slimfly_packet_fin--;
    }

    // tw_rand_reverse_unif(lp->rng);
    if(msg->path_type == MINIMAL)
        minimal_count--;
    if(msg->path_type == NON_MINIMAL)
        nonmin_count--;

    N_finished_chunks--;
    s->finished_chunks--;

    total_hops -= msg->my_N_hop;
    s->total_hops -= msg->my_N_hop;
    slimfly_total_time -= (tw_now(lp) - msg->travel_start_time);
    s->total_time = msg->saved_avg_time;

    struct qhash_head * hash_link = NULL;
    struct sfly_qhash_entry * tmp = NULL;

    struct sfly_hash_key key;
    key.message_id = msg->message_id;
    key.sender_id = msg->sender_lp;

    hash_link = qhash_search(s->rank_tbl, &key);
    tmp = qhash_entry(hash_link, struct sfly_qhash_entry, hash_link);

    mn_stats* stat;
    stat = model_net_find_stats(msg->category, s->slimfly_stats_array);
    stat->recv_time -= (tw_now(lp) - msg->travel_start_time);

    if(bf->c1)
    {
        stat->recv_count--;
        stat->recv_bytes -= msg->packet_size;
        N_finished_packets--;
        s->finished_packets--;
    }
    if(bf->c3)
        slimfly_max_latency = msg->saved_available_time;

    if(bf->c7)
    {
        // if(bf->c8) 
            // tw_rand_reverse_unif(lp->rng);

        s->finished_msgs--;
        total_msg_sz -= msg->total_size;
        N_finished_msgs--;
        s->total_msg_size -= msg->total_size;

        struct sfly_qhash_entry * d_entry_pop = rc_stack_pop(s->st);
        qhash_add(s->rank_tbl, &key, &(d_entry_pop->hash_link));
        s->rank_tbl_pop++;

        hash_link = &(d_entry_pop->hash_link);
        tmp = d_entry_pop;


        if(bf->c4)
            model_net_event_rc2(lp, &msg->event_rc);
    }

    assert(tmp);
    tmp->num_chunks--;
    if(bf->c5)
    {
      qhash_del(hash_link);
      free_tmp(tmp);	
      s->rank_tbl_pop--;
    }

    return;
}
void slim_send_remote_event(terminal_state * s, slim_terminal_message * msg, tw_lp * lp, tw_bf * bf, char * event_data, int remote_event_size)
{
    // printf("slim send remote event\n");

    (void)s;
    void * tmp_ptr = model_net_method_get_edata(SLIMFLY, msg);
    tw_stime ts = g_tw_lookahead + tw_rand_unif(lp->rng);
    msg->rng_calls++;

    if (msg->is_pull){
        bf->c4 = 1;
        struct codes_mctx mc_dst =
            codes_mctx_set_global_direct(msg->sender_mn_lp);
        struct codes_mctx mc_src =
            codes_mctx_set_global_direct(lp->gid);
        int net_id = model_net_get_id(LP_METHOD_NM_TERM);

        model_net_set_msg_param(MN_MSG_PARAM_START_TIME, MN_MSG_PARAM_START_TIME_VAL, &(msg->msg_start_time));

        msg->event_rc = model_net_event_mctx(net_id, &mc_src, &mc_dst, msg->category,
                msg->sender_lp, msg->pull_size, ts,
                remote_event_size, tmp_ptr, 0, NULL, lp);
    }
    else{
        tw_event * e = tw_event_new(msg->final_dest_gid, ts, lp);
        void * m_remote = tw_event_data(e);
        memcpy(m_remote, event_data, remote_event_size);
        tw_event_send(e);
    }
    return;
}
/* packet arrives at the destination terminal */
void slim_packet_arrive(terminal_state * s, tw_bf * bf, slim_terminal_message * msg,
        tw_lp * lp) {
    // printf("slim packet arrive\n");

    // NIC aggregation - should this be a separate function?
    // Trigger an event on receiving server

    tw_stime ts = g_tw_lookahead + s->params->credit_delay + tw_rand_unif(lp->rng);
    msg->rng_calls++;
    
    // no method_event here - message going to router
    tw_event * buf_e;
    slim_terminal_message * buf_msg;
    buf_e = model_net_method_event_new(s->router_lp[msg->rail_id], ts, lp, SLIMFLY_ROUTER, (void**)&buf_msg, NULL);
    buf_msg->magic = slim_router_magic_num;
    buf_msg->vc_index = msg->vc_index;
    buf_msg->output_chan = msg->output_chan;
    buf_msg->type = R_BUFFER;
    tw_event_send(buf_e);

    bf->c1 = 0;
    bf->c3 = 0;
    bf->c4 = 0;
    bf->c7 = 0;

    N_finished_chunks++;
    s->finished_chunks++;

    /* WE do not allow self messages through slimfly */
    assert(lp->gid != msg->src_terminal_id);

    uint64_t num_chunks = msg->packet_size / s->params->chunk_size;
    uint64_t total_chunks = msg->total_size / s->params->chunk_size;

    if(msg->total_size % s->params->chunk_size)
        total_chunks++;

    if(!total_chunks)
        total_chunks = 1;

    if (msg->packet_size % s->params->chunk_size)
        num_chunks++;

    if(!num_chunks)
        num_chunks = 1;

    if(msg->chunk_id == num_chunks - 1)
    {
        bf->c31 = 1;
        s->packet_fin++;
        slimfly_packet_fin++;
    }

    if(msg->path_type == MINIMAL)
        minimal_count++;

    if(msg->path_type == NON_MINIMAL)
        nonmin_count++;
    
    if(msg->path_type != MINIMAL && msg->path_type != NON_MINIMAL)
        printf("\n Wrong message path type %d ", msg->path_type);

    msg->saved_avg_time = s->total_time;
    s->total_time += (tw_now(lp) - msg->travel_start_time);
    slimfly_total_time += tw_now( lp ) - msg->travel_start_time;
    total_hops += msg->my_N_hop;
    s->total_hops += msg->my_N_hop;

    mn_stats* stat = model_net_find_stats(msg->category, s->slimfly_stats_array);
    stat->recv_time += (tw_now(lp) - msg->travel_start_time);

#if DEBUG
    if( msg->packet_ID == TRACK /*
                                   && msg->message_id == TRACK_MSG*/)
    {
        printf( "(%lf) [Terminal %d] packet %lld has arrived. ", tw_now(lp), (int)lp->gid, msg->packet_ID);

        printf("travel start time is %f. ", msg->travel_start_time);

        printf("My hop now is %d\n\n",msg->my_N_hop);
    }
#endif

    /* Now retreieve the number of chunks completed from the hash and update
     * them */
    void *m_data_src = model_net_method_get_edata(SLIMFLY, msg);
    struct qhash_head *hash_link = NULL;
    struct sfly_qhash_entry * tmp = NULL;
    struct sfly_hash_key key;
    key.message_id = msg->message_id;
    key.sender_id = msg->sender_lp;

    hash_link = qhash_search(s->rank_tbl, &key);

    if(hash_link)
        tmp = qhash_entry(hash_link, struct sfly_qhash_entry, hash_link);

    /* If an entry does not exist then create one */
    if(!hash_link)
    {
        bf->c5 = 1;
        struct sfly_qhash_entry * d_entry = calloc(1, sizeof(struct sfly_qhash_entry));
        d_entry->num_chunks = 0;
        d_entry->key = key;
        d_entry->remote_event_data = NULL;
        d_entry->remote_event_size = 0;
        qhash_add(s->rank_tbl, &key, &(d_entry->hash_link));
        s->rank_tbl_pop++;

        hash_link = &(d_entry->hash_link);
        // tmp = d_entry;
    }

    if(hash_link)
        tmp = qhash_entry(hash_link, struct sfly_qhash_entry, hash_link);

    assert(tmp);
    tmp->num_chunks++;

    /* if its the last chunk of the packet then handle the remote event data */
    if(msg->chunk_id == num_chunks - 1)
    {
        bf->c1 = 1;
        stat->recv_count++;
        stat->recv_bytes += msg->packet_size;

        N_finished_packets++;
        s->finished_packets++;
    }
    if(msg->remote_event_size_bytes > 0 && !tmp->remote_event_data)
    {
        /* Retreive the remote event entry */
        tmp->remote_event_data = (void*)calloc(1, msg->remote_event_size_bytes);
        assert(tmp->remote_event_data);
        tmp->remote_event_size = msg->remote_event_size_bytes;
        memcpy(tmp->remote_event_data, m_data_src, msg->remote_event_size_bytes);
    }
    if (slimfly_max_latency < tw_now( lp ) - msg->travel_start_time) {
        bf->c3 = 1;
        msg->saved_available_time = slimfly_max_latency;
        slimfly_max_latency = tw_now( lp ) - msg->travel_start_time;
    }
    /* If all chunks of a message have arrived then send a remote event to the
     * callee*/
    if((uint64_t)tmp->num_chunks >= total_chunks)
    {
        bf->c7 = 1;

        N_finished_msgs++;
        total_msg_sz += msg->total_size;
        s->total_msg_size += msg->total_size;
        s->finished_msgs++;

        if(tmp->remote_event_data && tmp->remote_event_size > 0) {
            bf->c8 = 1;
            slim_send_remote_event(s, msg, lp, bf, tmp->remote_event_data, tmp->remote_event_size);
        }
        /* Remove the hash entry */
        qhash_del(hash_link);
        rc_stack_push(lp, tmp, free_tmp, s->st);
        s->rank_tbl_pop--;
    }
#if TERMINAL_SENDS_RECVS_LOG
    int index = floor(N_COLLECT_POINTS*(tw_now(lp)/g_tw_ts_end));
    terminal_recvs[s->terminal_id][index]++;
#endif
    return;
}

void slim_terminal_buf_update_rc(terminal_state * s,
        tw_bf * bf,
        slim_terminal_message * msg,
        tw_lp * lp)
{
    // printf("slimfly term buf update rc\n");

    (void)msg;
    s->vc_occupancy[msg->vc_index] += s->params->chunk_size;
    codes_local_latency_reverse(lp);
    if(bf->c1) {
        s->in_send_loop[msg->vc_index] = 0;
    }
    return;
}
/* update the compute node-router channel buffer */
void slim_terminal_buf_update(terminal_state * s,
        tw_bf * bf,
        slim_terminal_message * msg,
        tw_lp * lp)
{
    // printf("slim terminal buf update\n");
    (void)msg;
    bf->c1 = 0;

    tw_stime ts = codes_local_latency(lp);
    s->vc_occupancy[msg->vc_index] -= s->params->chunk_size;

#if TERMINAL_OCCUPANCY_LOG
    int index = floor(N_COLLECT_POINTS*(tw_now(lp)/g_tw_ts_end));
    vc_occupancy_storage_terminal[s->terminal_id][0][index] = s->vc_occupancy[0]/s->params->chunk_size;
#endif

    if(s->in_send_loop[msg->vc_index] == 0 && s->terminal_msgs[msg->vc_index] != NULL) {
        slim_terminal_message *m;
        bf->c1 = 1;
        tw_event* e = model_net_method_event_new(lp->gid, ts, lp, SLIMFLY, (void**)&m, NULL);
        m->type = T_SEND;
        m->vc_index = msg->vc_index;
        m->magic = slim_terminal_magic_num;
        s->in_send_loop[msg->vc_index] = 1;
        tw_event_send(e);
    }

    return;
}

void slim_terminal_event( terminal_state * s,
        tw_bf * bf,
        slim_terminal_message * msg,
        tw_lp * lp )
{
    // printf("slim terminal event\n");

    *(int *)bf = (int)0;
    assert(msg->magic == slim_terminal_magic_num);

    rc_stack_gc(lp, s->st);
    switch(msg->type)
    {
        case T_GENERATE:
            slim_packet_generate(s,bf,msg,lp);
            break;

        case T_ARRIVE:
            slim_packet_arrive(s,bf,msg,lp);
            break;

        case T_SEND:
            slim_packet_send(s,bf,msg,lp);
            break;

        case T_BUFFER:
            slim_terminal_buf_update(s, bf, msg, lp);
            break;

        default:
            printf("\n LP %d Terminal message type not supported %d ", (int)lp->gid, msg->type);
            tw_error(TW_LOC, "Msg type not supported");
    }
}

void slimfly_terminal_final( terminal_state * s,
        tw_lp * lp )
{
    // printf("slim terminal final\n");

    model_net_print_stats(lp->gid, s->slimfly_stats_array);

    int written = 0;
    if(!s->terminal_id)
        written = sprintf(s->output_buf, "# Format <LP id> <Terminal ID> <Total Data Size> <Total Packet Latency> <# Flits/Packets finished> <Packets Generated> <Avg hops> <Busy Time>\n");

    tw_stime final_terminal_busy_time = 0;
    for(int i=0; i<s->params->num_injection_queues;i++){
        final_terminal_busy_time += s->busy_time[i];
    }

    written += sprintf(s->output_buf + written, "%llu %u %llu %lf %ld %d %lf %lf\n",
            LLU(lp->gid), s->terminal_id, LLU(s->total_msg_size), s->total_time,
            s->finished_packets, s->packet_gen, (double)s->total_hops/s->finished_chunks,
            final_terminal_busy_time);

    lp_io_write(lp->gid, "slimfly-msg-stats", written, s->output_buf);

 //   if(s->terminal_msgs[0] != NULL)
 //     printf("[%llu] leftover terminal messages \n", LLU(lp->gid));

#if MSG_TIMES
    int written2 = 0;
    if(!s->terminal_id)
        written2 = sprintf(s->output_buf2, " Format <Terminal ID> <Send Time> <Rail>\n");
    for(int i=0; i<200; i++)
        written2 += sprintf(s->output_buf2 + written2, "%u %d %d\n", s->terminal_id, s->msg_send_times[i], s->msg_rail_select[i]);
    lp_io_write(lp->gid, "slimfly-msg-times",written2, s->output_buf2);
#endif

    qhash_finalize(s->rank_tbl);
    rc_stack_destroy(s->st);
    free(s->vc_occupancy);
    free(s->terminal_msgs);
    free(s->terminal_msgs_tail);
}

void slimfly_router_final(router_state * s,
        tw_lp * lp)
{
    // printf("slim router final\n");

    (void)lp;

    free(s->global_channel);
    int i, j;
    for(i = 0; i < s->params->radix; i++) {
        for(j = 0; j < s->params->num_vcs; j++) {
            if(s->queued_msgs[i][j] != NULL) {
              printf("[%llu] leftover queued messages %d %d %d\n", LLU(lp->gid), i, j,
                     s->vc_occupancy[i][j]);
            }
            if(s->pending_msgs[i][j] != NULL) {
             printf("[%llu] lefover pending messages %d %d\n", LLU(lp->gid), i, j);
           }
        }
    }
    rc_stack_destroy(s->st);
    int written = 0;
    if(s->router_id == 0)
    {
        /* write metadata file */
        /*        char meta_fname[64];
                  sprintf(meta_fname, "slimfly-msg-stats.meta");

                  FILE * fp = fopen(meta_fname, "w");
                  fprintf(fp, "# Format <LP ID> <Group ID> <Router ID> <Busy time per router port(s)>");
                  fprintf(fp, "# Router ports in the order: %d local channels, %d global channels", 
                  p->num_routers, p->num_global_channels);
                  fclose(fp);
                  */        written = sprintf(s->output_buf,"# Format <LP ID> <Group ID> <Router ID> <Busy time per router port(s)>");
    }
    written += sprintf(s->output_buf + written, "\n %llu %d %d", 
            LLU(lp->gid),
            s->group_id,
            s->router_id);
    for(int d = 0; d < s->params->radix; d++) 
        written += sprintf(s->output_buf + written, " %lf", s->busy_time[d]);

    lp_io_write(lp->gid, "slimfly-router-stats", written, s->output_buf);

    written = 0;
    if(!s->router_id)
    {
        written = sprintf(s->output_buf2, "# Format <LP ID> <Group ID> <Router ID> <Link traffic per router port(s)>");
        written += sprintf(s->output_buf2 + written, "# Router ports in the order: %d local channels, %d global channels",
                s->params->num_local_channels, s->params->num_global_channels);
    }
    written += sprintf(s->output_buf2 + written, "\n %llu %d %d",
            LLU(lp->gid),
            s->group_id,
            s->router_id);

    for(int d = 0; d < s->params->radix; d++) 
        written += sprintf(s->output_buf2 + written, " %lld", LLD(s->link_traffic[d]));

    assert(written < 4096);
    lp_io_write(lp->gid, "slimfly-router-traffic", written, s->output_buf2);
}


/** Get the local and global router connections for the given source router
 *  @param[in] src_router_id            The local/relative ID for the source router
 *  @param[in] num_global_channels      Number of global connections
 *  @param[in] num_local_channels       Number of local connections
 *  @param[in] total_routers            The total number of routers in one rail
 *  @param[in] *local_channels          Integer array pointer for storing the local connections
 *  @param[in] *global_channels         Integer array pointer for storing the global connections
 */
void get_router_connections(int src_router_id, int num_global_channels, int num_local_channels,
        int total_routers, int* local_channels, int* global_channels, int sf_type, const slimfly_param *p){
    //Compute MMS router layout/connection graph
    int rid_s = src_router_id;	// ID for source router
    int num_rails = 1;
    int routers_per_rail = total_routers;
    int rail_id = 0;
    if(sf_type == 1) {
        num_rails = p->num_rails;
        routers_per_rail = total_routers; //dumb name
        rail_id = (src_router_id/routers_per_rail);
        //Convert source ID for second rail of Fit Fly to be in reverse order of the first rail - that was commented out
        if (rid_s >= total_routers){
            rid_s = rid_s % routers_per_rail;
            //rid_s = (total_routers - 1) - rid_s;
        }
    }

    int rid_d;						// ID for dest. router
    int s_s,s_d;					// subgraph location for source and destination routers
    int i_s,i_d;					// x or m coordinates for source and destination routers
    int j_s,j_d; 					// y or c coordinates for source and destination routers
    int k;
    int local_idx = 0;
    int global_idx = 0;
    int generator_size = X_size;

    for(rid_d=0;rid_d<total_routers;rid_d++)
    {
        // Decompose source and destination Router IDs into 3D subgraph coordinates (subgraph,i,j)
        if(rid_d >= total_routers/2)
        {
            s_d = 1;
            i_d = (rid_d - total_routers/2) /  num_global_channels;
            j_d = (rid_d - total_routers/2) %  num_global_channels;
        }
        else
        {
            s_d = 0;
            i_d = rid_d /  num_global_channels;
            j_d = rid_d %  num_global_channels;
        }
        if(rid_s >= total_routers/2)
        {
            s_s = 1;
            i_s = (rid_s - total_routers/2) /  num_global_channels;
            j_s = (rid_s - total_routers/2) %  num_global_channels;
        }
        else
        {
            s_s = 0;
            i_s = rid_s /  num_global_channels;
            j_s = rid_s %  num_global_channels;
        }
        // Check for subgraph 0 local connections
        if(s_s==0 && s_d==0)
        {
            if(i_s==i_d)							// equation (2) y-y' is in X'
            {
                for(k=0;k<generator_size;k++)
                {
                    if(abs(j_s-j_d)==X[k])
                    {
                        if(src_router_id >= total_routers) //greater than a single rail?
                            local_channels[local_idx++] = rid_d + routers_per_rail*rail_id;
                        else
                            local_channels[local_idx++] = rid_d;
                    }
                }
            }
        }
        // Check if global connections
        if(s_s==0 && s_d==1)
        {
            if(j_s == (i_d*i_s + j_d) % num_global_channels)							// equation (3) y=mx+c
            {
                if(src_router_id >= total_routers)
                    global_channels[global_idx++] = rid_d + routers_per_rail*rail_id;
                else
                    global_channels[global_idx++] = rid_d;
            }
        }
    }

    // Loop over second subgraph source routers
    for(rid_d=total_routers-1;rid_d>=0;rid_d--)
    {
        // Decompose source and destination Router IDs into 3D subgraph coordinates (subgraph,i,j)
        if(rid_d >= total_routers/2)
        {
            s_d = 1;
            i_d = (rid_d -  total_routers/2) /  num_global_channels;
            j_d = (rid_d -  total_routers/2) %  num_global_channels;
        }
        else
        {
            s_d = 0;
            i_d = rid_d /  num_global_channels;
            j_d = rid_d %  num_global_channels;
        }
        if(rid_s >= total_routers/2)
        {
            s_s = 1;
            i_s = (rid_s -  total_routers/2) /  num_global_channels;
            j_s = (rid_s -  total_routers/2) %  num_global_channels;
        }
        else
        {
            s_s = 0;
            i_s = rid_s /  num_global_channels;
            j_s = rid_s %  num_global_channels;
        }
        // Check for subgraph 1 local connections
        if(s_s==1 && s_d==1)
        {
            if(i_s==i_d)							// equation (2) c-c' is in X'
            {
                for(k=0;k<generator_size;k++)
                {
                    if(abs(j_s-j_d)==X_prime[k])
                    {
                        if(src_router_id >= total_routers)
                            local_channels[local_idx++] = rid_d + routers_per_rail*rail_id;
                        else
                            local_channels[local_idx++] = rid_d;
                    }
                }
            }
        }
        // Check if global connections
        if(s_s==1 && s_d==0)
        {
            if(j_d == (i_s*i_d + j_s) %  num_global_channels)							// equation (3) y=mx+c
            {
                if(src_router_id >= total_routers)
                    global_channels[global_idx++] = rid_d + routers_per_rail*rail_id;
                else
                    global_channels[global_idx++] = rid_d;
            }
        }
    }

    assert(local_idx == num_local_channels);
    assert(global_idx == num_global_channels);
}

/** Get the length (number of hops) in the route/path from a source terminal to dest router
 *  @param[in] dest         Local/relative ID of the destination router
 *  @param[in] src          Local/relative ID of the source terminal
 *  @param[out] num_hops 	number of hops in the minimal path/route
 */
int get_path_length_from_terminal(int src, int dest, const slimfly_param *p)
{
    int *local_channel = (int*) calloc(p->num_local_channels,sizeof(int)); //NM TODO: DYNAMIC ALLOCS ARE TIME EXPENSIVE - this function is called every packet generate?!
    int *global_channel = (int*) calloc(p->num_global_channels,sizeof(int));
    get_router_connections(src, p->num_global_channels, p->num_local_channels,
            p->slim_total_routers, local_channel, global_channel, p->sf_type,p);
    int i, num_hops=2;
    for(i=0;i<p->num_global_channels;i++)
    {
        if(global_channel[i] == dest)
        {
            num_hops = 1;
        }
    }
    for(i=0;i<p->num_local_channels;i++)
    {
        if(local_channel[i] == dest)
        {
            num_hops = 1;
        }
    }

    free(local_channel);
    free(global_channel);

    return num_hops;
}

/** Get the length (number of hops) in the route/path starting with a local src router to ending dest router
 *  @param[in] 	dest 		The relative ID for the destination router
 *  @param[in] 	src   		The state for the current local router
 *  @param[out] num_hops 	number of hops in the minimal path/route
 */
int get_path_length_local(router_state * src, int dest)
{
    int i, num_hops=2;
    for(i=0;i<src->params->num_global_channels;i++)
    {
        if(src->global_channel[i] == dest)
        {
            num_hops = 1;
        }
    }
    for(i=0;i<src->params->num_local_channels;i++)
    {
        if(src->local_channel[i] == dest)
        {
            num_hops = 1;
        }
    }
    return num_hops;
}

void get3DCoordinates(int router_id, int *s, int *i, int *j, router_state * r)
{
    // Convert second rail relative ID to it's corresponding 1st rail relative ID
    router_id = router_id % r->params->slim_total_routers;
    // Decompose source and destination Router IDs into 3D subgraph coordinates (subgraph,i,j)
    if(router_id >= r->params->slim_total_routers/2)
    {
        *s = 1;
        *i = (router_id - r->params->slim_total_routers/2) / r->params->num_global_channels;
        *j = (router_id - r->params->slim_total_routers/2) % r->params->num_global_channels;
    }
    else
    {
        *s = 0;
        *i = router_id / r->params->num_global_channels;
        *j = router_id % r->params->num_global_channels;
    }
}

/** Get the length (number of hops) in the route/path starting with a global src router to ending dest router
 *  @param[in] 	dest 		The relative ID for the destination router
 *  @param[in] 	src   		The state for the global router
 *  @param[out] num_hops 	number of hops in the minimal path/route
 */
int get_path_length_global(int src, int dest, router_state * r)
{
    int j,num_hops=2;
    int s_s,s_d,i_s,i_d,j_s,j_d;
    // Get corresponding graph coordinates for source and destination routers
    get3DCoordinates(src,&s_s,&i_s,&j_s,r);
    get3DCoordinates(dest,&s_d,&i_d,&j_d,r);

    if(s_s == s_d)
    {
        if(s_s == 0)
        {
            //Case 1a local connection same subgroup 0
            for(j=0;j<X_size;j++)
            {
                if(abs(j_s-j_d) == X[j])
                {
                    num_hops = 1;
                    break;
                }
            }
        }
        else
        {
            //Case 1b local connection same subgroup 1
            for(j=0;j<X_size;j++)
            {
                if(abs(j_s-j_d) == X_prime[j])
                {
                    num_hops = 1;
                    break;
                }
            }
        }
    }
    else
    {
        if(s_s == 0)
        {
            //Case 1c global connection s_s=0, s_d=1
            if(j_s == (i_d*i_s+j_d) % r->params->num_routers)
            {
                num_hops = 1;
            }
        }
        else
        {
            //Case 1d global connection s_s=1, s_d=0
            if(j_d == (i_s*i_d+j_s) % r->params->num_routers)
            {
                num_hops = 1;
            }
        }
    }
    return num_hops;
}

/** Get the next router along the minimal path to the destination using the 3 paper connection equations
 *  Cases:
 *     1. Rs is directly connected to Rd.
 *       a. Subgraph 0 local connection
 *       b. Subgraph 1 local connection
 *       c. Subgraph 0 global connection to subgraph 1
 *       d. Subgraph 1 global connection to subgraph 0
 *     2. Rs is indirectly connected to Rd in separate subgraphs. (global and local path)
 *       a. Rs local to Rm and global to Rd. Rs and Rm in subgraph 0. Rd in subgraph 1.
 *       b. Rs global to Rm and local to Rd. Rs in subgraph 0. Rd and Rm in subgraph 1.
 *       c. Rs local to Rm and global to Rd. Rs and Rm in subgraph 1. Rd in subgraph 0.
 *       d. Rs global to Rm and local to Rd. Rs in subgraph 1. Rd and Rs in subgraph 0.
 *     3. Rs and Rd are in same group connected through Rm also in same group. (2 local paths)
 *       a. All in subgraph 0.
 *       b. All in subgraph 1.
 *     4. Rs and Rd in same subgraph globally connected to intermediate router in other subgraph (2 global paths)
 *       a. Rs and Rd in subgraph 0. Rm in subgraph 1.
 *       b. Rs and Rd in subgraph 1. Rm in subgraph 0.
 *  @param[in] rid The ID for the destination router
 *  @param[in] r   The state for the current router
 *  @param[out] router_id The ID for a router in the destination router group
 */
//////////////////////////////////////// Get router in the group which has a global channel to group id gid /////////////////////////////////
tw_lpid getMinimalRouterFromEquations(slim_terminal_message * msg, int rid, router_state * r)
{
    int s_s,s_d,i_s,i_d,j_s,j_d;
    int i,j;
    int match = 0;
    tw_lpid router_id = 0;
    // Get corresponding graph coordinates for source and destination routers
    get3DCoordinates((int)r->router_id,&s_s,&i_s,&j_s,r);
    get3DCoordinates(rid,&s_d,&i_d,&j_d,r);
#if DEBUG_ROUTING
    if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
    {
        printf("source:%d, s_s:%d, i_s:%d, j_s:%d\n",(int)r->router_id,s_s, i_s, j_s);
        printf("destination:%d s_d:%d, i_d:%d, j_d:%d\n",rid,s_d, i_d, j_d);
    }
#endif

    // Check for case 1
    for(i=0;i<r->params->num_local_channels;i++)
    {
#if DEBUG_ROUTING
        if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
        {
            printf("comparing local:%d with dest:%d\n",r->local_channel[i],rid);
        }
#endif
        if(r->local_channel[i] == rid)
        {
            router_id = rid;
            match = 1; 				// Case 1a and 1b
#if DEBUG_ROUTING
            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
            {
                printf("match:%d case:1a/1b\n",(int)router_id);
            }
#endif
            break;
        }
    }
    if(match == 0)
    {
        for(i=0;i<r->params->num_global_channels;i++)
        {
#if DEBUG_ROUTING
            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
            {
                printf("comparing global:%d with dest:%d\n",r->global_channel[i],rid);
            }
#endif
            if(r->global_channel[i] == rid)
            {
                router_id = rid;
                match = 1;			// Case 1c
#if DEBUG_ROUTING
                if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                {
                    printf("match case:1c\n");
                }
#endif
                break;
            }
        }
    }
    if(match == 0)
    {
        //Loop over all global and local connections
        for(i=0;i<r->params->num_global_channels+r->params->num_local_channels;i++)
        {
            int s_m,i_m,j_m;
            if(i<r->params->num_global_channels)
            {
                get3DCoordinates(r->global_channel[i],&s_m,&i_m,&j_m,r);
            }
            else
            {
                get3DCoordinates(r->local_channel[i-r->params->num_global_channels],&s_m,&i_m,&j_m,r);
            }
#if DEBUG_ROUTING
            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
            {
                printf("possible case 2,3,4\n");
                if(i<r->params->num_global_channels)
                {
                    printf("global_id:%d, s_m:%d, i_m:%d, j_m:%d\n",(int)r->global_channel[i%GLOBAL_CHANNELS],s_m, i_m, j_m);
                    printf("case 4b,2a result comparing global%d and dest%d: %d=%d\n",r->global_channel[i%GLOBAL_CHANNELS],rid,j_m,(i_d*i_m+j_d) % NUM_ROUTER);
                    printf("case 4a,2c result comparing global%d and dest%d: %d=%d\n",r->global_channel[i%GLOBAL_CHANNELS],rid,j_d,(i_m*i_d+j_m) % NUM_ROUTER);
                }
                else
                {
                    printf("local_id:%d, s_m:%d, i_m:%d, j_m:%d\n",(int)r->local_channel[i%GLOBAL_CHANNELS],s_m, i_m, j_m);
                    printf("case 4b,2a result comparing local%d and dest%d: %d=%d\n",r->local_channel[i%GLOBAL_CHANNELS],rid,j_m,(i_d*i_m+j_d) % NUM_ROUTER);
                    printf("case 4a,2c result comparing local%d and dest%d: %d=%d\n",r->local_channel[i%GLOBAL_CHANNELS],rid,j_d,(i_m*i_d+j_m) % NUM_ROUTER);
                }
            }
#endif
            if(s_s == s_d)
            {
                if(s_s == s_m)
                {
                    if(s_s == 0)
                    {
                        if(i_m == i_d && i_m == i_s)
                        {
#if DEBUG_ROUTING
                            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                            {
                                printf("possible case:3a\n");
                            }
#endif
                            for(j=0;j<X_size;j++)
                            {
                                if(abs(j_m-j_d) == X[j])
                                {
                                    if(i<r->params->num_global_channels)
                                    {
                                        router_id = r->global_channel[i];
                                    }
                                    else
                                    {
                                        router_id = r->local_channel[i-r->params->num_global_channels];
                                    }
                                    match = 1;			// Case 3a
#if DEBUG_ROUTING
                                    if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                                    {
                                        printf("match case:3a\n");
                                    }
#endif
                                    break;
                                }
                            }
                        }
                    }
                    else
                    {
                        if(i_m == i_d && i_m == i_s)
                        {
#if DEBUG_ROUTING
                            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                            {
                                printf("possible case:3b\n");
                            }
#endif
                            for(j=0;j<X_size;j++)
                            {
                                if(abs(j_m-j_d) == X_prime[j])
                                {
                                    if(i<r->params->num_global_channels)
                                    {
                                        router_id = r->global_channel[i];
                                    }
                                    else
                                    {
                                        router_id = r->local_channel[i-r->params->num_global_channels];
                                    }
                                    match = 1;			// Case 3b
#if DEBUG_ROUTING
                                    if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                                    {
                                        printf("match case:3b\n");
                                    }
#endif
                                }
                            }
                        }
                    }
                }
                else
                {
                    if(s_s == 0)
                    {
#if DEBUG_ROUTING
                        if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                        {
                            printf("possible case:4a\n");
                        }
#endif
                        if(j_d == (i_m*i_d+j_m) % r->params->num_routers)
                        {
                            if(i<r->params->num_global_channels)
                            {
                                router_id = r->global_channel[i];
                            }
                            else
                            {
                                router_id = r->local_channel[i-r->params->num_global_channels];
                            }
                            match = 1;				// Case 4a
#if DEBUG_ROUTING
                            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                            {
                                printf("match case:4a\n");
                            }
#endif
                            break;
                        }

                    }
                    else
                    {
#if DEBUG_ROUTING
                        if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                        {
                            printf("possible case:4b\n");
                        }
#endif
                        if(j_m == (i_d*i_m+j_d) % r->params->num_routers)
                        {
                            if(i<r->params->num_global_channels)
                            {
                                router_id = r->global_channel[i];
                            }
                            else
                            {
                                router_id = r->local_channel[i-r->params->num_global_channels];
                            }
                            match = 1;				// Case 4b
#if DEBUG_ROUTING
                            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                            {
                                printf("match case:4b\n");
                            }
#endif
                            break;
                        }
                    }
                }
            }
            else
            {
                if(s_s == s_m)
                {
                    if(s_s == 0)
                    {
                        if(i_m == i_s)
                        {
#if DEBUG_ROUTING
                            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                            {
                                printf("possible case:2a\n");
                            }
#endif
                            if(j_m == (i_d*i_m+j_d) % r->params->num_routers)
                            {
                                if(i<r->params->num_global_channels)
                                {
                                    router_id = r->global_channel[i];
                                }
                                else
                                {
                                    router_id = r->local_channel[i-r->params->num_global_channels];
                                }
                                match = 1;				// Case 2a
#if DEBUG_ROUTING
                                if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                                {
                                    printf("match case:2a\n");
                                }
#endif
                                break;
                            }
                        }
                    }
                    else
                    {
                        if(i_m == i_s)
                        {
#if DEBUG_ROUTING
                            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                            {
                                printf("possible case:2c\n");
                            }
#endif
                            if(j_d == (i_m*i_d+j_m) % r->params->num_routers)
                            {
                                if(i<r->params->num_global_channels)
                                {
                                    router_id = r->global_channel[i];
                                }
                                else
                                {
                                    router_id = r->local_channel[i-r->params->num_global_channels];
                                }
                                match = 1;				// Case 2c
#if DEBUG_ROUTING
                                if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                                {
                                    printf("match case:2c\n");
                                }
#endif
                                break;
                            }
                        }
                    }
                }
                else
                {
                    if(s_s == 0)
                    {
                        if(i_m == i_d)
                        {
#if DEBUG_ROUTING
                            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                            {
                                printf("possible case:2b\n");
                            }
#endif
                            for(j=0;j<X_size;j++)
                            {
                                if(abs(j_d-j_m) == X_prime[j])
                                {
                                    if(i<r->params->num_global_channels)
                                    {
                                        router_id = r->global_channel[i];
                                    }
                                    else
                                    {
                                        router_id = r->local_channel[i-r->params->num_global_channels];
                                    }
                                    match = 1;			// Case 2b
#if DEBUG_ROUTING
                                    if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                                    {
                                        printf("match case:2b\n");
                                    }
#endif
                                }
                            }
                        }
                    }
                    else
                    {
                        if(i_m == i_d)
                        {
#if DEBUG_ROUTING
                            if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                            {
                                printf("possible case:2d\n");
                            }
#endif
                            for(j=0;j<X_size;j++)
                            {
                                if(abs(j_m-j_d) == X[j])
                                {
                                    if(i<r->params->num_global_channels)
                                    {
                                        router_id = r->global_channel[i];
                                    }
                                    else
                                    {
                                        router_id = r->local_channel[i-r->params->num_global_channels];
                                    }
                                    match = 1;			// Case 2d
#if DEBUG_ROUTING
                                    if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
                                    {
                                        printf("match case:2d\n");
                                    }
#endif
                                    break;
                                }
                            }
                        }
                    }
                }
            }
            if(match==1)
                break;
        }
    }
#if DEBUG_ROUTING
    if( msg->packet_ID == TRACK && msg->chunk_id == num_chunks-1)
    {
        if(match == 1)
        {
            printf("source:%d destination:%d match:%d\n",(int)r->router_id,rid,(int)router_id);
        }
        else
        {
            printf("source:%d destination:%d no match so defaulting to router:%d\n",(int)r->router_id,rid,(int)router_id);
        }
    }
#endif
    if(match == 0 && (int)r->router_id != 0)
    {
        printf("packet_ID:%d source:%d destination:%d no match so defaulting to router:%d\n",(int)msg->packet_ID,(int)r->router_id,rid,(int)router_id);
    }
    return router_id;
}

/* get the next stop for the current packet
 * determines if it is a router within a group, a router in another group
 * or the destination terminal */
tw_lpid slim_get_next_stop(router_state * s,
        tw_bf * bf,
        slim_terminal_message * msg,
        tw_lp * lp,
        int path,
        int dest_router_rel_id,
        int intm_id)
{
    // printf("slim get next stop\n");
    (void)lp;
    (void)msg;
    (void)bf;

    int next_stop_rel_id = -1;
    tw_lpid next_stop_lp_id = -1;

    int local_router_rel_id = s->router_id;

    int rail_id = msg->rail_id;
    int actual_total_num_routers = codes_mapping_get_lp_count(lp_group_name, 0 ,"modelnet_slimfly_router", s->anno, 0);
    int rpr = actual_total_num_routers / s->params->num_rails; //number routers per rail

    /* If the packet has arrived at the destination router */
    if(dest_router_rel_id == local_router_rel_id)
    {
        next_stop_lp_id = msg->dest_terminal_id;
#if TRACK_OUTPUT
        if( msg->packet_ID == TRACK )
        {
            printf( "\x1b[32m-->Finishing Routing at current router:%d (gid:%d) to dest_terminal(globalID):%d through randm intm router:%d next stop:%d (globalid:%d)\x1b[0m\n",s->router_id,(int)lp->gid,(int)msg->dest_terminal_id,msg->intm_router_id,next_stop_rel_id,(int)next_stop_lp_id);
        }
#endif
        return next_stop_lp_id;
    }
    /* Assign inter-mediate destination for non-minimal routing (selecting a random router) */
    if(msg->last_hop == TERMINAL && path == NON_MINIMAL)
    {
        msg->intm_router_id = intm_id;
        next_stop_rel_id=getMinimalRouterFromEquations(msg, msg->intm_router_id, s);
        if(next_stop_rel_id > s->params->slim_total_routers-1)
            codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", NULL, 1, next_stop_rel_id - (rpr * rail_id), rail_id, &next_stop_lp_id);
        else
            codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", s->anno, 0, next_stop_rel_id, 0, &next_stop_lp_id);
#if TRACK_OUTPUT
        //if( msg->packet_ID == TRACK )
        {
            printf( "\x1b[32m-->Starting First leg NonMinimal from current router:%d (gid:%d) to dest_router:%d through randm intm router:%d next stop:%d (globalid:%d)\x1b[0m\n",s->router_id,(int)lp->gid,dest_router_rel_id,msg->intm_router_id,next_stop_rel_id,(int)next_stop_lp_id);
        }
#endif
        return next_stop_lp_id;
    }
    /* If the packet has arrived at the inter-mediate router for non-minimal routing. Reset the intm router now. */
    if(path == NON_MINIMAL && msg->intm_router_id == s->router_id)
    {
#if TRACK_OUTPUT
        //if( msg->packet_ID == TRACK )
        {
            printf( "\x1b[32m-->Arrived at NonMinimal intm_router %d s->router_id:%d local_router_rel_id:%d (gid:%d) dest_router:%d next stop:%d (globalid:%d)\x1b[0m\n",msg->intm_router_id,s->router_id,local_router_rel_id,(int)lp->gid,dest_router_rel_id,next_stop_rel_id,(int)next_stop_lp_id);
        }
#endif
        msg->intm_router_id = -1;//no inter-mediate router
    }
    /* If intermediate router is set, route minimally to intermediate router*/
    if(path == NON_MINIMAL && msg->intm_router_id >= 0 && (msg->intm_router_id != local_router_rel_id))
    {
        next_stop_rel_id=getMinimalRouterFromEquations(msg, msg->intm_router_id, s);
        if(next_stop_rel_id > s->params->slim_total_routers-1)
            codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", NULL, 1, next_stop_rel_id - (rpr * rail_id), rail_id, &next_stop_lp_id);
        else
            codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", s->anno, 0, next_stop_rel_id, 0, &next_stop_lp_id);
#if TRACK_OUTPUT
        //if( msg->packet_ID == TRACK )
        {
            printf( "\x1b[32m-->Continuing First leg Non-Minimal Routing from current router:%d (gid:%d) to dest_router:%d through randm intm router:%d next stop:%d (globalid:%d)\x1b[0m\n",s->router_id,(int)lp->gid,dest_router_rel_id,msg->intm_router_id,next_stop_rel_id,(int)next_stop_lp_id);
        }
#endif
        return next_stop_lp_id;
    }
    /* No intermediate router set, then route to destination*/
    if(path == NON_MINIMAL && msg->intm_router_id < 0)
    {
        next_stop_rel_id=getMinimalRouterFromEquations(msg, dest_router_rel_id, s);
        if(next_stop_rel_id > s->params->slim_total_routers-1)
            codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", NULL, 1, next_stop_rel_id - (rpr * rail_id), rail_id, &next_stop_lp_id);
        else
            codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", s->anno, 0, next_stop_rel_id, 0, &next_stop_lp_id);
#if TRACK_OUTPUT
        //if( msg->packet_ID == TRACK )
        {
            printf( "\x1b[32m-->Second leg Non-Minimal Routing from current router:%d (gid:%d) to dest_router:%d through randm intm router:%d next stop:%d (globalid:%d)\x1b[0m\n",s->router_id,(int)lp->gid,dest_router_rel_id,msg->intm_router_id,next_stop_rel_id,(int)next_stop_lp_id);
        }
#endif
        return next_stop_lp_id;
    }
    if(path == MINIMAL)
    {
        next_stop_rel_id=getMinimalRouterFromEquations(msg, dest_router_rel_id, s);
        //printf("lp->gid:%llu, packet_id:%llu, s->router_id:%d, dest_router_rel_id:%d, next_stop_rel_id:%d\n",LLU(lp->gid), LLU(msg->packet_ID), s->router_id, dest_router_rel_id, next_stop_rel_id);
    }

    if(next_stop_rel_id > s->params->slim_total_routers-1)
        codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", NULL, 1, next_stop_rel_id - (rpr * rail_id), rail_id, &next_stop_lp_id);
    else
        codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", s->anno, 0, next_stop_rel_id, 0, &next_stop_lp_id);
    //printf("next_stop_rel_id:%d, next_stop_lp_id:%llu\n",next_stop_rel_id,LLU(next_stop_lp_id));

#if TRACK_OUTPUT
    //if( msg->packet_ID == TRACK && msg->last_hop == TERMINAL)
    {
        printf( "\x1b[32m-->Starting Minimal Routing from current router:%d (gid:%d) to dest_router:%d next stop:%d (globalid:%d)\x1b[0m\n",s->router_id,(int)lp->gid,dest_router_rel_id,next_stop_rel_id,(int)next_stop_lp_id);
    }
    //if( msg->packet_ID == TRACK && msg->last_hop != TERMINAL)
    {
        printf( "\x1b[32m-->Continuing Minimal Routing from current router:%d (gid:%d) to dest_router:%d \x1b[0m\n",s->router_id,(int)lp->gid,dest_router_rel_id);
    }
#endif

    return next_stop_lp_id;
}
/* gets the output port corresponding to the next stop of the message */
int slim_get_output_port( router_state * s,
        slim_terminal_message * msg,
        tw_lp * lp,
        tw_lpid next_stop_lp_id )
{
    // printf("slim get output port\n");

    (void)lp;

    int output_port = -1, terminal_id;
    codes_mapping_get_lp_info(msg->dest_terminal_id, lp_group_name,
            &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id,
            &mapping_offset);
    int num_lps = codes_mapping_get_lp_count(lp_group_name,1,LP_CONFIG_NM_TERM,s->anno,0);
    terminal_id = (mapping_rep_id * num_lps) + mapping_offset;

    if(next_stop_lp_id == msg->dest_terminal_id)
    {
        output_port = s->params->num_local_channels + s->params->num_global_channels +
            ( terminal_id % s->params->num_cn);
    }
    else
    {
        codes_mapping_get_lp_info(next_stop_lp_id, lp_group_name, &mapping_grp_id,
                NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
        int next_stop_rel_id = mapping_rep_id + mapping_offset * s->params->slim_total_routers;

        for(int i=0; i<s->params->num_global_channels; i++)
        {
            if(s->global_channel[i] == next_stop_rel_id)
            {
                output_port = s->params->num_local_channels + i;
                break;
            }
            else if(i<s->params->num_local_channels && s->local_channel[i] == next_stop_rel_id)
            {
                output_port = i;
                break;
            }
        }
        if(output_port == -1)
        {
            printf("\n output port not found %d. Local LPID:%d local_rel_id:%d next stop lp_id %d next_stop rel_id %d \n", output_port, (int)lp->gid, s->router_id, (int)next_stop_lp_id, next_stop_rel_id);
            exit(0);
        }
    }

    return output_port;
}


/* UGAL (first condition is from booksim), output port equality check comes from Dally slimfly'09*/
static int do_adaptive_routing( router_state * s,
        slim_terminal_message * msg,
        tw_lp * lp,
        int dest_router_rel_id,
        int * intm_id)
{
    int i;
    int *nonmin_out_port = (int *)malloc(num_indirect_routes * sizeof(int));
    int *num_nonmin_hops = (int *)malloc(num_indirect_routes * sizeof(int));
    int *nonmin_port_count = (int *)malloc(num_indirect_routes * sizeof(int));
    int next_stop;
    int minimal_out_port = -1;
    float *cost_nonminimal = (float *)(int *)malloc(num_indirect_routes * sizeof(float));
    float cost_minimal;
    tw_lpid *nonmin_next_stop_lp_id = (tw_lpid *)malloc(num_indirect_routes * sizeof(tw_lpid));
    tw_lpid minimal_next_stop_lp_id;

    int rail_id = msg->rail_id;
    int actual_total_num_routers = codes_mapping_get_lp_count(lp_group_name, 0 ,"modelnet_slimfly_router", s->anno, 0);
    int rpr = actual_total_num_routers / s->params->num_rails; //number routers per rail

    //Compute the next stop on the minimal path and get port number
    int minimal_next_stop_rel_id = getMinimalRouterFromEquations(msg, dest_router_rel_id, s);
    if(minimal_next_stop_rel_id > s->params->slim_total_routers-1)
        codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", NULL, 1, minimal_next_stop_rel_id - (rpr * rail_id), rail_id, &minimal_next_stop_lp_id);
    else
        codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", s->anno, 0, minimal_next_stop_rel_id, 0, &minimal_next_stop_lp_id);
    minimal_out_port = slim_get_output_port(s, msg, lp, minimal_next_stop_lp_id);

    //Compute the next stop on the non-minimal paths and get port number
    int nonmin_next_stop_rel_id;
    for(i=0;i<num_indirect_routes;i++)
    {
        nonmin_next_stop_rel_id = getMinimalRouterFromEquations(msg, intm_id[i], s);
        //codes_mapping_get_lp_id(lp_group_name, "slimfly_router", s->anno, 0, nonmin_next_stop_rel_id, 0, &nonmin_next_stop_lp_id[i]);
        if(nonmin_next_stop_rel_id > s->params->slim_total_routers-1)
            codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", NULL, 1, nonmin_next_stop_rel_id - (rpr * rail_id), rail_id, &nonmin_next_stop_lp_id[i]);
        else
            codes_mapping_get_lp_id(lp_group_name, "modelnet_slimfly_router", s->anno, 0, nonmin_next_stop_rel_id, 0, &nonmin_next_stop_lp_id[i]);
        nonmin_out_port[i] = slim_get_output_port(s, msg, lp, nonmin_next_stop_lp_id[i]);
    }

    //Only computed on first hop so vc will always be 0 for both min and nonmin
    int nomin_vc = 0;
    int min_vc = 0;

    //Calculate number of hops in random indirect routes above
    for(i=0;i<num_indirect_routes;i++)
    {
        //Compute # of hops from src to intm router
        num_nonmin_hops[i] = get_path_length_local(s, intm_id[i]);
        //Compute # of hops from intm to dest router
        num_nonmin_hops[i] += get_path_length_from_terminal(intm_id[i], dest_router_rel_id, s->params);
    }
    //Calculate number of hops in minimal path
    int num_min_hops = get_path_length_local(s, dest_router_rel_id);

    //Determine port occupancy for all minimal and nonminimal paths
    int min_port_count = s->vc_occupancy[minimal_out_port][min_vc];
    for(i=0;i<num_indirect_routes;i++)
    {
        nonmin_port_count[i] = s->vc_occupancy[nonmin_out_port[i]][nomin_vc];
    }

    //Calculate cost of all paths/routes
    cost_minimal = min_port_count;
    for(i=0;i<num_indirect_routes;i++)
    {
        cost_nonminimal[i] = ((float)num_nonmin_hops[i]/(float)num_min_hops) * (float)nonmin_port_count[i] * csf_ratio;
#if DEBUG
        //if( msg->packet_ID == TRACK)
        {
            printf( "\x1b[31mnonmin_next_stop_lp_id[%d]:%d num_hops_nonmin[%d]:%d, output_port_nonmin[%d]:%d port_occupancy_non_min[%d]:%d cost_indirect[%d]:%f\n      min_next_stop:%d       num_hops_min:%d,       output_port_min:%d        port_occupancy_min:%d     cost_minimal:%f\x1b[0m\n",
                i,(int)nonmin_next_stop_lp_id[i],i,num_nonmin_hops[i],i,nonmin_out_port[i],i,nonmin_port_count[i],i,cost_nonminimal[i],(int)minimal_next_stop_lp_id,num_min_hops,minimal_out_port,min_port_count,cost_minimal);
        }
#endif
    }

    //If minCost=="Minimal", route minimally; Else route Valiantly according to corresponding path
    msg->path_type = MINIMAL;
    next_stop = minimal_next_stop_lp_id;
    float min_cost = cost_minimal;
    msg->intm_router_id = -1;		//So far no intermediate router
    for(i=0;i<num_indirect_routes;i++)
    {

        if(cost_nonminimal[i] < min_cost)
        {
            min_cost = (int)cost_nonminimal[i];
            msg->path_type = NON_MINIMAL;
            next_stop = nonmin_next_stop_lp_id[i];
            msg->intm_router_id = intm_id[i];
        }
    }

    if(msg->path_type == MINIMAL)
    {
        //if(msg->packet_ID == TRACK && msg->message_id == TRACK_MSG)
#if DEBUG
            printf("\n (%lf) [Router %d] Packet %d routing minimally \n", tw_now(lp), (int)lp->gid, (int)msg->packet_ID);
#endif
    }
    else
    {
        //if(msg->packet_ID == TRACK && msg->message_id == TRACK_MSG)
#if DEBUG
            printf("\n (%lf) [Router %d] Packet %d routing non-minimally \n", tw_now(lp), (int)lp->gid, (int)msg->packet_ID);
#endif
    }
    return next_stop;
}

void slim_router_packet_receive_rc(router_state * s,
        tw_bf * bf,
        slim_terminal_message * msg,
        tw_lp * lp)
{
    // printf("slimfly router packet receive rc\n");

    for(int i = 0; i < msg->rng_calls; i++)
    {
        tw_rand_reverse_unif(lp->rng);
    }

    int output_port = msg->saved_vc;
    int output_chan = msg->saved_channel;

    // if(bf->c1)
        // tw_rand_reverse_unif(lp->rng);

    // if(bf->c5){
    //     for(int i=0;i<num_indirect_routes;i++){
    //         tw_rand_reverse_unif(lp->rng);
    //     }
    // }

    if(bf->c2) {
        // tw_rand_reverse_unif(lp->rng);
        slim_terminal_message_list * tail = return_tail(s->pending_msgs[output_port],
                    s->pending_msgs_tail[output_port], output_chan);
        slim_delete_terminal_message_list(tail);
        s->vc_occupancy[output_port][output_chan] -= s->params->chunk_size;
        if(bf->c3) {
            codes_local_latency_reverse(lp);
            s->in_send_loop[output_port] = 0;
        }
    }
    if(bf->c4) {
          if(bf->c22)
          {
        s->last_buf_full[output_port][output_chan] = msg->saved_busy_time;
          }
        slim_delete_terminal_message_list(return_tail(s->queued_msgs[output_port],
                    s->queued_msgs_tail[output_port], output_chan));
    }
}

/* Packet arrives at the router and a credit is sent back to the sending terminal/router */
    void
slim_router_packet_receive( router_state * s,
        tw_bf * bf,
        slim_terminal_message * msg,
        tw_lp * lp )
{
    // printf("slim router packet receive\n");

    bf->c1 = 0;
    bf->c2 = 0;
    bf->c3 = 0;
    bf->c4 = 0;
    bf->c5 = 0;
    bf->c6 = 0;
    bf->c7 = 0;
    bf->c8 = 0;

    tw_stime ts;
    int i;
    int next_stop = -1, output_port = -1, output_chan = -1;
    int dest_router_rel_id;

    int rail_id = msg->rail_id;

    codes_mapping_get_lp_info(msg->dest_terminal_id, lp_group_name,
            &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id,
            &mapping_offset);
    int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
            s->anno, 0);
    // Compute relative id of router in the first rail that is connected to the destination node
    int default_rail_dest_router_rel_id = (mapping_offset + (mapping_rep_id * num_lps)) /
            s->params->num_cn;
    if(s->router_id > s->params->slim_total_routers -1)
        // Compute relative id of router in the second rail connected to destination compute node
        if (rail_id % 2 == 0)
            dest_router_rel_id = (rail_id)*s->params->slim_total_routers + default_rail_dest_router_rel_id;
        else
            dest_router_rel_id = (rail_id+1) * s->params->slim_total_routers - 1 - default_rail_dest_router_rel_id;
    else
        dest_router_rel_id = default_rail_dest_router_rel_id;

    int intm_id = -1;
    int *intm_router;		//Array version of intm_id for use in Adaptive routing
    int local_grp_id = (s->router_id % s->params->slim_total_routers) / s->params->num_routers;
    
    slim_terminal_message_list * cur_chunk = (slim_terminal_message_list *)calloc(1, 
            sizeof(slim_terminal_message_list));
    slim_init_terminal_message_list(cur_chunk, msg);


#if TRACK_OUTPUT
    if( msg->packet_ID == TRACK )
    {
        printf( "\x1b[32m-->arrived at router relID:%d lpID:%d. Final destination terminal lp_id:%d router rel_id:%d\x1b[0m\n",s->router_id, (int)lp->gid, 
                (int)msg->dest_terminal_id, dest_router_rel_id);
    }
#endif

    if(routing == NON_MINIMAL)
    {
        bf->c1 = 1;
        intm_id = tw_rand_integer(lp->rng, 0, s->params->slim_total_routers - 1);
        msg->rng_calls++;

        if((int)(intm_id/s->params->num_routers) == local_grp_id)
        {
            //        intm_id = (local_grp_id + 2) % s->params->num_groups;
            intm_id = (intm_id +  s->params->num_routers) % s->params->slim_total_routers;
        }

        // If in the second rail, adjust the relative router id of the intermediate router accordingly to be in the second rail
        if(s->router_id >= s->params->slim_total_routers)
            intm_id += s->params->slim_total_routers*rail_id;
    }
    if(msg->last_hop == TERMINAL && routing == ADAPTIVE)
    {
        if(dest_router_rel_id == s->router_id){
            cur_chunk->msg.path_type = MINIMAL; /*defaults to the routing algorithm if we don't have adaptive routing here*/
            next_stop = slim_get_next_stop(s, bf, &(cur_chunk->msg), lp, cur_chunk->msg.path_type, dest_router_rel_id, intm_id);
        }else{
            intm_router = (int *)malloc(num_indirect_routes * sizeof(int)); 	//indirect == nonMinimal == valiant
            //Generate n_I many indirect routes through intermediate random routers
            bf->c5 = 1;
            for(i=0;i<num_indirect_routes;i++)
            {
                intm_router[i] = tw_rand_integer(lp->rng, 0, s->params->slim_total_routers-1);
                msg->rng_calls++;
                if(dest_router_rel_id >= s->params->slim_total_routers)
                    intm_router[i] += s->params->slim_total_routers*rail_id;

                if(intm_router[i] == dest_router_rel_id)	//Check if same as dest_router
                {
                    intm_router[i] = (intm_router[i]+1) % (s->params->slim_total_routers-1);
                    if(dest_router_rel_id >= s->params->slim_total_routers)
                        intm_router[i] += s->params->slim_total_routers*rail_id;
                }
                if(intm_router[i] == (int)lp->gid)			//Check if same as source router
                {
                    intm_router[i] = (intm_router[i]+1) % (s->params->slim_total_routers-1);
                    if(dest_router_rel_id >= s->params->slim_total_routers)
                        intm_router[i] += s->params->slim_total_routers*rail_id;
                }
            }
            next_stop = do_adaptive_routing(s, &(cur_chunk->msg), lp, dest_router_rel_id, intm_router);
            free(intm_router);
        }
    }
    else
    {
        if(routing == MINIMAL || routing == NON_MINIMAL)
            cur_chunk->msg.path_type = routing; /*defaults to the routing algorithm if we don't have adaptive routing here*/
        next_stop = slim_get_next_stop(s, bf, &(cur_chunk->msg), lp, cur_chunk->msg.path_type, dest_router_rel_id, intm_id);
    }
    assert(cur_chunk->msg.path_type == MINIMAL || cur_chunk->msg.path_type == NON_MINIMAL);
    if(msg->remote_event_size_bytes > 0)
    {
        void *m_data_src = model_net_method_get_edata(SLIMFLY_ROUTER, msg);
        cur_chunk->event_data = (char*)calloc(1,msg->remote_event_size_bytes);
        memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes);
    }

    output_port = slim_get_output_port(s, &(cur_chunk->msg), lp, next_stop);
    assert(output_port >= 0);
    output_chan = 0;
    int max_vc_size = s->params->cn_vc_size;

#if TRACK_OUTPUT
    if( msg->packet_ID == TRACK )
    {
        printf( "\x1b[32m-->next stop:%d \x1b[0m\n",next_stop);
    }
#endif

    cur_chunk->msg.vc_index = output_port;
    cur_chunk->msg.next_stop = next_stop;

    if(output_port < s->params->num_local_channels)
    {
        max_vc_size = s->params->local_vc_size;
        output_chan = msg->my_N_hop;
//        if(msg->my_N_hop >3)
//            output_chan = 3;
    }
    else if(output_port < (s->params->num_local_channels + s->params->num_global_channels))
    {
        max_vc_size = s->params->global_vc_size;
        output_chan = msg->my_N_hop;
//        if(msg->my_N_hop >3)
//            output_chan = 3;
    }

#if TRACK_OUTPUT
    if( msg->packet_ID == TRACK )
//    if(output_chan > 2)
    {
        codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
                &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
        int local_router_id = (mapping_offset * s->params->slim_total_routers + mapping_rep_id);
        printf("packet_id:%d, current router:%d dest_router:%d next stop lp_id:%d output port:%d output chan:%d\n",(int)msg->packet_ID, local_router_id,dest_router_rel_id,next_stop,output_port,output_chan);
    }
#endif

    cur_chunk->msg.output_chan = output_chan;
    cur_chunk->msg.my_N_hop++;

    if(msg->path_type == MINIMAL)
        assert(output_chan < 3);	//Minimal routing needs 2 vcs
    else
        assert(output_chan < 5);	//Non-minimal needs 4 vcs

    assert(output_port < s->params->radix);

    if(s->vc_occupancy[output_port][output_chan] + s->params->chunk_size <= max_vc_size)
    {
#if TRACK_OUTPUT
        if( msg->packet_ID == TRACK )
        {
            printf("vc occupancy available\n");
        }
#endif
        bf->c2 = 1;
        slim_router_credit_send(s, msg, lp, -1);
        append_to_terminal_message_list( s->pending_msgs[output_port], s->pending_msgs_tail[output_port], output_chan, cur_chunk);
        s->vc_occupancy[output_port][output_chan] += s->params->chunk_size;

#if ROUTER_OCCUPANCY_LOG
        int index = floor(N_COLLECT_POINTS*(tw_now(lp)/g_tw_ts_end));
        vc_occupancy_storage_router[s->router_id][output_port][output_chan][index] = s->vc_occupancy[output_port][output_chan]/s->params->chunk_size;
#endif

        if(s->in_send_loop[output_port] == 0) {
            bf->c3 = 1;
            slim_terminal_message *m;
            ts = codes_local_latency(lp);
            tw_event *e = model_net_method_event_new(lp->gid, ts, lp, SLIMFLY_ROUTER, (void**)&m, NULL);
            m->type = R_SEND;
            m->magic = slim_router_magic_num;
            m->vc_index = output_port;

            tw_event_send(e);
            s->in_send_loop[output_port] = 1;
        }
    }
    else
    {
#if TRACK_OUTPUT
        //if( msg->packet_ID == TRACK )
        {
            printf("vc occupancy not available\n");
        }
#endif
        bf->c4 = 1;
        cur_chunk->msg.saved_vc = msg->vc_index;
        cur_chunk->msg.saved_channel = msg->output_chan;
        append_to_terminal_message_list( s->queued_msgs[output_port], s->queued_msgs_tail[output_port], output_chan, cur_chunk);
        if(s->pending_msgs[output_port][output_chan] == NULL && s->last_buf_full[output_port][output_chan] == 0.0)
          {
            bf->c22 = 1;
            msg->saved_busy_time = s->last_buf_full[output_port][output_chan];
            s->last_buf_full[output_port][output_chan] = tw_now(lp);
          }
    }

    msg->saved_vc = output_port;
    msg->saved_channel = output_chan;

#if ROUTER_SENDS_RECVS_LOG
    router_recvs[s->router_id][index]++;
#endif

    return;
}

void slim_router_packet_send_rc(router_state * s,
        tw_bf * bf,
        slim_terminal_message * msg, tw_lp * lp)
{
    // printf("slimfly router packet send rc\n");
    for(int i = 0; i < msg->rng_calls; i++)
    {
        tw_rand_reverse_unif(lp->rng);
    }

    int output_port = msg->saved_vc;
    int output_chan = msg->saved_channel;
    if(bf->c1) {
        s->in_send_loop[output_port] = 1;
        return;
    }

    // tw_rand_reverse_unif(lp->rng);
    slim_terminal_message_list * cur_entry = rc_stack_pop(s->st);
    assert(cur_entry);

    if(bf->c11)
    {
        s->link_traffic[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size;
    }
    if(bf->c12)
    {
        s->link_traffic[output_port] -= s->params->chunk_size;
    }
    s->next_output_available_time[output_port] = msg->saved_available_time;

    prepend_to_terminal_message_list(s->pending_msgs[output_port],
            s->pending_msgs_tail[output_port], output_chan, cur_entry);

    // if(bf->c3) {
    //     tw_rand_reverse_unif(lp->rng);
    // }

    if(bf->c4) {
        s->in_send_loop[output_port] = 1;
    }

}
/* routes the current packet to the next stop */
    void
slim_router_packet_send( router_state * s,
        tw_bf * bf,
        slim_terminal_message * msg, tw_lp * lp)
{
    // printf("slim router packet send\n");

    tw_stime ts;
    tw_event *e;
    slim_terminal_message *m;
    int output_port = msg->vc_index;
    int output_chan = 3;

    slim_terminal_message_list *cur_entry = s->pending_msgs[output_port][3];
    if(cur_entry == NULL)
    {
        cur_entry = s->pending_msgs[output_port][2];
        output_chan = 2;
        if(cur_entry == NULL)
        {
            cur_entry = s->pending_msgs[output_port][1];
            output_chan = 1;
            if(cur_entry == NULL)
            {
                cur_entry = s->pending_msgs[output_port][0];
                output_chan = 0;
            }
        }
    }
    msg->saved_vc = output_port;
    msg->saved_channel = output_chan;

    if(cur_entry == NULL)
    {
        bf->c1 = 1;
        s->in_send_loop[output_port] = 0;
        //printf("[%d] Router skipping send at begin %d \n", lp->gid, output_port);
        return;
    }

    int to_terminal = 1, global = 0;
    double delay = s->params->cn_delay;
    double bandwidth = s->params->cn_bandwidth;

    if(output_port < s->params->num_local_channels)
    {
        to_terminal = 0;
        delay = s->params->local_delay;
        bandwidth = s->params->local_bandwidth;
    }
    else if(output_port < s->params->num_local_channels + s->params->num_global_channels)
    {
        to_terminal = 0;
        global = 1;
        delay = s->params->global_delay;
        bandwidth = s->params->global_bandwidth;
    }

    uint64_t num_chunks = cur_entry->msg.packet_size / s->params->chunk_size;
    if(msg->packet_size % s->params->chunk_size)
        num_chunks++;
    if(!num_chunks)
        num_chunks = 1;

    double bytetime = delay;

    if((cur_entry->msg.packet_size % s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1))
        bytetime = bytes_to_ns(cur_entry->msg.packet_size % s->params->chunk_size, bandwidth); 

    ts = g_tw_lookahead + tw_rand_unif(lp->rng) + bytetime + s->params->router_delay;
    msg->rng_calls++;

    msg->saved_available_time = s->next_output_available_time[output_port];
    s->next_output_available_time[output_port] =
        maxd(s->next_output_available_time[output_port], tw_now(lp));
    s->next_output_available_time[output_port] += ts;

    ts = s->next_output_available_time[output_port] - tw_now(lp);
    // dest can be a router or a terminal, so we must check
    void * m_data;
    if (to_terminal)
    {
        assert(cur_entry->msg.next_stop == cur_entry->msg.dest_terminal_id);
        e = model_net_method_event_new(cur_entry->msg.next_stop,
                s->next_output_available_time[output_port] - tw_now(lp), lp,
                SLIMFLY, (void**)&m, &m_data);
    }
    else
    {
        e = model_net_method_event_new(cur_entry->msg.next_stop, s->next_output_available_time[output_port] - tw_now(lp), lp,
            SLIMFLY_ROUTER, (void**)&m, &m_data);
    }
    memcpy(m, &cur_entry->msg, sizeof(slim_terminal_message));
    if (m->remote_event_size_bytes){
        memcpy(m_data, cur_entry->event_data, m->remote_event_size_bytes);
    }

    if(global)
        m->last_hop = GLOBAL;
    else
        m->last_hop = LOCAL;

    m->local_id = s->router_id;
    m->intm_lp_id = lp->gid;
    m->magic = slim_router_magic_num;

    if((cur_entry->msg.packet_size % s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1)) {
        bf->c11 = 1;
        s->link_traffic[output_port] +=  (cur_entry->msg.packet_size %
            s->params->chunk_size); 
    } else {
        bf->c12 = 1;
        s->link_traffic[output_port] += s->params->chunk_size;
    }

    /* Determine the event type. If the packet has arrived at the final
     * destination router then it should arrive at the destination terminal
     * next.*/
    if(to_terminal)
    {
        m->type = T_ARRIVE;
        m->magic = slim_terminal_magic_num;
    }
    else
    {
        /* The packet has to be sent to another router */
        m->magic = slim_router_magic_num;
        m->type = R_ARRIVE;
    }
    tw_event_send(e);

    cur_entry = return_head(s->pending_msgs[output_port],
            s->pending_msgs_tail[output_port], output_chan);
    rc_stack_push(lp, cur_entry, (void*)slim_delete_terminal_message_list, s->st);

    cur_entry = s->pending_msgs[output_port][3];

    s->next_output_available_time[output_port] -= s->params->router_delay;
    ts -= s->params->router_delay;

    if(cur_entry == NULL) cur_entry = s->pending_msgs[output_port][2];
    if(cur_entry == NULL) cur_entry = s->pending_msgs[output_port][1];
    if(cur_entry == NULL) cur_entry = s->pending_msgs[output_port][0];
    if(cur_entry != NULL)
    {
        bf->c3 = 1;
        slim_terminal_message *m_new;
        ts = ts + g_tw_lookahead * tw_rand_unif(lp->rng);
        msg->rng_calls++;
        tw_event *e_new = model_net_method_event_new(lp->gid, ts, lp, SLIMFLY_ROUTER, (void**)&m_new, NULL);
        m_new->type = R_SEND;
        m_new->magic = slim_router_magic_num;
        m_new->vc_index = output_port;
        tw_event_send(e_new);
    }
    else
    {
        bf->c4 = 1;
        s->in_send_loop[output_port] = 0;
    }

#if TRACK_OUTPUT
    if( m->packet_ID == TRACK )
    {
        printf("router packet sending \n");
    }
#endif

#if ROUTER_SENDS_RECVS_LOG
    int index = floor(N_COLLECT_POINTS*(tw_now(lp)/g_tw_ts_end));
    router_sends[s->router_id][index]++;
#endif

    return;
}

void slim_router_buf_update_rc(router_state * s,
        tw_bf * bf,
        slim_terminal_message * msg,
        tw_lp * lp)
{
    // printf("slimfly router buf update rc\n");
    for(int i = 0; i < msg->rng_calls; i++)
    {
        tw_rand_reverse_unif(lp->rng);
    }

    int indx = msg->vc_index;
    int output_chan = msg->output_chan;
    // printf("index: %d  output chan: %d\n",indx, output_chan);
    s->vc_occupancy[indx][output_chan] += s->params->chunk_size;
    if(bf->c3)
    {
        s->busy_time[indx] = msg->saved_rcv_time;
        s->last_buf_full[indx][output_chan] = msg->saved_busy_time;
    }
    if(bf->c1) {
        slim_terminal_message_list* head = return_tail(s->pending_msgs[indx],
                s->pending_msgs_tail[indx], output_chan);
        // tw_rand_reverse_unif(lp->rng);
        prepend_to_terminal_message_list(s->queued_msgs[indx],
                s->queued_msgs_tail[indx], output_chan, head);
        s->vc_occupancy[indx][output_chan] -= s->params->chunk_size;
    }
    if(bf->c2) {
        codes_local_latency_reverse(lp);
        s->in_send_loop[indx] = 0;
    }
}
/* Update the buffer space associated with this router LP */
void slim_router_buf_update(router_state * s, tw_bf * bf, slim_terminal_message * msg, tw_lp * lp)
{
    // printf("slim router buf update\n");

    int indx = msg->vc_index;
    int output_chan = msg->output_chan;
    s->vc_occupancy[indx][output_chan] -= s->params->chunk_size;

    if(s->last_buf_full[indx][output_chan] > 0.0)
    {
        bf->c3 = 1;
        msg->saved_rcv_time = s->busy_time[indx];
        msg->saved_busy_time = s->last_buf_full[indx][output_chan];
        s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
        s->last_buf_full[indx][output_chan] = 0.0;
    }

#if ROUTER_OCCUPANCY_LOG
    int index = floor(N_COLLECT_POINTS*(tw_now(lp)/g_tw_ts_end));
    vc_occupancy_storage_router[s->router_id][indx][output_chan][index] = s->vc_occupancy[indx][output_chan]/s->params->chunk_size;
#endif
    if(s->queued_msgs[indx][output_chan] != NULL) {
        bf->c1 = 1;
        slim_terminal_message_list *head = return_head(s->queued_msgs[indx],
                s->queued_msgs_tail[indx], output_chan);
        slim_router_credit_send(s, &head->msg, lp, 1);
        append_to_terminal_message_list(s->pending_msgs[indx],
                s->pending_msgs_tail[indx], output_chan, head);
        s->vc_occupancy[indx][output_chan] += s->params->chunk_size;
#if ROUTER_OCCUPANCY_LOG
        vc_occupancy_storage_router[s->router_id][indx][output_chan][index] = s->vc_occupancy[indx][output_chan]/s->params->chunk_size;
#endif
    }
    if(s->in_send_loop[indx] == 0 && s->pending_msgs[indx][output_chan] != NULL) {
        bf->c2 = 1;
        slim_terminal_message *m;
        tw_stime ts = codes_local_latency(lp);
        tw_event *e = model_net_method_event_new(lp->gid, ts, lp, SLIMFLY_ROUTER, (void**)&m, NULL);
        m->type = R_SEND;
        m->vc_index = indx;
        m->magic = slim_router_magic_num;
        s->in_send_loop[indx] = 1;
        tw_event_send(e);
    }

    return;
}

void slim_router_event(router_state * s, tw_bf * bf, slim_terminal_message * msg,
        tw_lp * lp) {
    assert(msg->magic == slim_router_magic_num);
    rc_stack_gc(lp, s->st);
    switch(msg->type)
    {
        case R_SEND: // Router has sent a packet to an intra-group router (local channel)
            slim_router_packet_send(s, bf, msg, lp);
            break;

        case R_ARRIVE: // Router has received a packet from an intra-group router (local channel)
            slim_router_packet_receive(s, bf, msg, lp);
            break;

        case R_BUFFER:
            slim_router_buf_update(s, bf, msg, lp);
            break;

        default:
            printf("\n (%lf) [Router %d] Router Message type not supported %d dest "
                    "terminal id %d packet ID %d ", tw_now(lp), (int)lp->gid, msg->type,
                    (int)msg->dest_terminal_id, (int)msg->packet_ID);
            tw_error(TW_LOC, "Msg type not supported");
            break;
    }
}

/* Reverse computation handler for a terminal event */
void slim_terminal_rc_event_handler(terminal_state * s, tw_bf * bf,
        slim_terminal_message * msg, tw_lp * lp) {
    switch(msg->type)
    {
        case T_GENERATE:
            slim_packet_generate_rc(s, bf, msg, lp);
            break;

        case T_SEND:
            slim_packet_send_rc(s, bf, msg, lp);
            break;

        case T_ARRIVE:
            slim_packet_arrive_rc(s, bf, msg, lp);
            break;

        case T_BUFFER:
            slim_terminal_buf_update_rc(s, bf, msg, lp);
            break;
    }
}

/* Reverse computation handler for a router event */
void slim_router_rc_event_handler(router_state * s, tw_bf * bf,
        slim_terminal_message * msg, tw_lp * lp) {
    switch(msg->type) {
        case R_SEND:
            slim_router_packet_send_rc(s, bf, msg, lp);
            break;
        case R_ARRIVE:
            slim_router_packet_receive_rc(s, bf, msg, lp);
            break;

        case R_BUFFER:
            slim_router_buf_update_rc(s, bf, msg, lp);
            break;
    }
}

/* slimfly compute node and router LP types */
tw_lptype slimfly_lps[] =
{
    // Terminal handling functions
    {
        (init_f)slim_terminal_init,
        (pre_run_f) NULL,
        (event_f) slim_terminal_event,
        (revent_f) slim_terminal_rc_event_handler,
        (commit_f) NULL,
        (final_f) slimfly_terminal_final,
        (map_f) codes_mapping,
        sizeof(terminal_state)
    },
    {
        (init_f) slim_router_setup,
        (pre_run_f) NULL,
        (event_f) slim_router_event,
        (revent_f) slim_router_rc_event_handler,
        (commit_f) NULL,
        (final_f) slimfly_router_final,
        (map_f) codes_mapping,
        sizeof(router_state),
    },
    {NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0},
};



/* For ROSS Instrumentation */
void slimfly_event_collect(slim_terminal_message *m, tw_lp *lp, char *buffer, int *collect_flag)
{
    (void)lp;
    (void)collect_flag;
    int type = (int) m->type;
    memcpy(buffer, &type, sizeof(type));
}

void slimfly_model_stat_collect(terminal_state *s, tw_lp *lp, char *buffer)
{
    (void)lp;
    (void)s;
    (void)buffer;

    return;
}

static void ross_slimfly_sample_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_cn_sample *sample)
{
    (void)bf;

    sample->terminal_id = s->terminal_id;
    sample->end_time = tw_now(lp);
    sample->vc_occupancy = 0;

    // sum vc_occupancy
    int i;
    for (i = 0; i < s->num_vcs; i++)
        sample->vc_occupancy += s->vc_occupancy[i];

    return;
}

static void ross_slimfly_sample_rc_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_cn_sample *sample)
{
    (void)lp;
    (void)bf;
    (void)s;
    (void)sample;
    
    return;
}

static void ross_slimfly_rsample_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_router_sample *sample)
{
    (void)bf;

    const slimfly_param * p = s->params; 
    int i, j, total;

    sample->router_id = s->router_id;
    sample->end_time = tw_now(lp);
    sample->vc_occupancy = (int*)((&sample->end_time) + 1);

    // sum vc occupancy for each port
    for(i = 0; i < p->radix; i++)
    {
        total = 0;
        for (j = 0; j < p->num_vcs; j++)
            total += s->vc_occupancy[i][j];
        sample->vc_occupancy[i] = total;
    }

    return;
}

static void ross_slimfly_rsample_rc_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct slimfly_router_sample *sample)
{
    (void)lp;
    (void)bf;
    (void)s;
    (void)sample;
    
    return;
}

static const st_model_types *slimfly_get_cn_model_types()
{
    return(&slimfly_model_types[0]);
}

static const st_model_types *slimfly_get_router_model_types()
{
    return(&slimfly_model_types[1]);
}

static void slimfly_register_model_types(st_model_types *base_type)
{
    st_model_type_register(LP_CONFIG_NM_TERM, base_type);
}

static void slimfly_router_register_model_types(st_model_types *base_type)
{
    st_model_type_register(LP_CONFIG_NM_ROUT, base_type);
}
/***  END of ROSS Instrumentation support */


/* returns the slimfly lp type for lp registration */
static const tw_lptype* slimfly_cn_get_lp_type(void)
{
    return(&slimfly_lps[0]);
}

static const tw_lptype* slimfly_router_get_lp_type(void)
{
    return(&slimfly_lps[1]);
}

static void slimfly_register(tw_lptype *base_type) {
    lp_type_register(LP_CONFIG_NM_TERM, base_type);
}

static void slimfly_router_register(tw_lptype *base_type) {
    lp_type_register(LP_CONFIG_NM_ROUT, base_type);
}

/*** END of ROSS event tracing additions */

/* data structure for slimfly statistics */
struct model_net_method slimfly_method = 
{
    0,
    slimfly_configure,
    slimfly_register,
    slimfly_packet_event,
    slimfly_packet_event_rc,
    NULL,
    NULL,
    slimfly_cn_get_lp_type,
    slimfly_get_msg_sz,
    slimfly_report_stats,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    slimfly_register_model_types,
    slimfly_get_cn_model_types,
};

struct model_net_method slimfly_router_method =
{
    0,
    NULL,
    slimfly_router_register,
    NULL,
    NULL,
    NULL,
    NULL,
    slimfly_router_get_lp_type,
    slimfly_get_msg_sz,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    NULL,
    slimfly_router_register_model_types,
    slimfly_get_router_model_types,
};


// struct model_net_method slimfly_method =
// {
//     .mn_configure = slimfly_configure,
//     .mn_register = slimfly_register,
//     .model_net_method_packet_event = slimfly_packet_event,
//     .model_net_method_packet_event_rc = slimfly_packet_event_rc,
//     .model_net_method_recv_msg_event = NULL,
//     .model_net_method_recv_msg_event_rc = NULL,
//     .mn_get_lp_type = slimfly_get_cn_lp_type,
//     .mn_get_msg_sz = slimfly_get_msg_sz,
//     .mn_report_stats = slimfly_report_stats,
//     .mn_model_stat_register = slimfly_register_model_types,
//     .mn_get_model_stat_types = slimfly_get_cn_model_types
// };



<|start_filename|>src/networks/model-net/dragonfly.c<|end_filename|>
/*
 * Copyright (C) 2013 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

// Local router ID: 0 --- total_router-1
// Router LP ID 
// Terminal LP ID

#include <ross.h>

#define DEBUG_LP 892
#include "codes/jenkins-hash.h"
#include "codes/codes_mapping.h"
#include "codes/codes.h"
#include "codes/model-net.h"
#include "codes/model-net-method.h"
#include "codes/model-net-lp.h"
#include "codes/net/dragonfly.h"
#include "sys/file.h"
#include "codes/quickhash.h"
#include "codes/rc-stack.h"

#ifdef ENABLE_CORTEX
#include <cortex/cortex.h>
#include <cortex/topology.h>
#endif

#define MEAN_PROCESS 1.0

/* collective specific parameters */
#define TREE_DEGREE 4
#define LEVEL_DELAY 1000
#define DRAGONFLY_COLLECTIVE_DEBUG 0
#define NUM_COLLECTIVES  1
#define COLLECTIVE_COMPUTATION_DELAY 5700
#define DRAGONFLY_FAN_OUT_DELAY 20.0
#define WINDOW_LENGTH 0
#define DFLY_HASH_TABLE_SIZE 4999

// debugging parameters
#define TRACK -1
#define TRACK_PKT -1
#define TRACK_MSG -1
#define PRINT_ROUTER_TABLE 1
#define DEBUG 0
#define USE_DIRECT_SCHEME 1
#define MAX_STATS 65536

#define LP_CONFIG_NM_TERM (model_net_lp_config_names[DRAGONFLY])
#define LP_METHOD_NM_TERM (model_net_method_names[DRAGONFLY])
#define LP_CONFIG_NM_ROUT (model_net_lp_config_names[DRAGONFLY_ROUTER])
#define LP_METHOD_NM_ROUT (model_net_method_names[DRAGONFLY_ROUTER])

#ifdef ENABLE_CORTEX
/* This structure is defined at the end of the file */
extern cortex_topology dragonfly_cortex_topology;
#endif

int debug_slot_count = 0;
long term_ecount, router_ecount, term_rev_ecount, router_rev_ecount;
long packet_gen = 0, packet_fin = 0;

static double maxd(double a, double b) { return a < b ? b : a; }

/* minimal and non-minimal packet counts for adaptive routing*/
static int minimal_count=0, nonmin_count=0;
static int num_routers_per_mgrp = 0;

typedef struct dragonfly_param dragonfly_param;
/* annotation-specific parameters (unannotated entry occurs at the 
 * last index) */
static uint64_t                  num_params = 0;
static dragonfly_param         * all_params = NULL;
static const config_anno_map_t * anno_map   = NULL;

/* global variables for codes mapping */
static char lp_group_name[MAX_NAME_LENGTH];
static int mapping_grp_id, mapping_type_id, mapping_rep_id, mapping_offset;

/* router magic number */
int router_magic_num = 0;

/* terminal magic number */
int terminal_magic_num = 0;

FILE * dragonfly_log = NULL;

int sample_bytes_written = 0;
int sample_rtr_bytes_written = 0;

char dfly_cn_sample_file[MAX_NAME_LENGTH];
char dfly_rtr_sample_file[MAX_NAME_LENGTH];

typedef struct terminal_message_list terminal_message_list;
struct terminal_message_list {
    terminal_message msg;
    char* event_data;
    terminal_message_list *next;
    terminal_message_list *prev;
};

static void init_terminal_message_list(terminal_message_list *this, 
    terminal_message *inmsg) {
    this->msg = *inmsg;
    this->event_data = NULL;
    this->next = NULL;
    this->prev = NULL;
}

static void delete_terminal_message_list(terminal_message_list *this) {
    if(this->event_data != NULL) free(this->event_data);
    free(this);
}

struct dragonfly_param
{
    // configuration parameters
    int num_routers; /*Number of routers in a group*/
    double local_bandwidth;/* bandwidth of the router-router channels within a group */
    double global_bandwidth;/* bandwidth of the inter-group router connections */
    double cn_bandwidth;/* bandwidth of the compute node channels connected to routers */
    int num_vcs; /* number of virtual channels */
    int local_vc_size; /* buffer size of the router-router channels */
    int global_vc_size; /* buffer size of the global channels */
    int cn_vc_size; /* buffer size of the compute node channels */
    int chunk_size; /* full-sized packets are broken into smaller chunks.*/
    // derived parameters
    int num_cn;
    int num_groups;
    int num_real_groups;
    int radix;
    int total_routers;
    int total_terminals;
    int num_global_channels;
    double cn_delay;
    double local_delay;
    double global_delay;
    int credit_size;
    double local_credit_delay;
    double global_credit_delay;
    double cn_credit_delay;
    double router_delay;
};

struct dfly_hash_key
{
    uint64_t message_id;
    tw_lpid sender_id;
};

struct dfly_router_sample
{
    tw_lpid router_id;
    tw_stime* busy_time;
    int64_t* link_traffic_sample;
    tw_stime end_time;
    long fwd_events;
    long rev_events;
};

struct dfly_cn_sample
{
   tw_lpid terminal_id;
   long fin_chunks_sample;
   long data_size_sample;
   double fin_hops_sample;
   tw_stime fin_chunks_time;
   tw_stime busy_time_sample;
   tw_stime end_time;
   long fwd_events;
   long rev_events;
};

struct dfly_qhash_entry
{
   struct dfly_hash_key key;
   char * remote_event_data;
   uint64_t num_chunks;
   int remote_event_size;
   struct qhash_head hash_link;
};

/* handles terminal and router events like packet generate/send/receive/buffer */
typedef enum event_t event_t;
typedef struct terminal_state terminal_state;
typedef struct router_state router_state;

/* dragonfly compute node data structure */
struct terminal_state
{
   uint64_t packet_counter;

   int packet_gen;
   int packet_fin;

   // Dragonfly specific parameters
   unsigned int router_id;
   unsigned int terminal_id;

   // Each terminal will have an input and output channel with the router
   int* vc_occupancy; // NUM_VC
   int num_vcs;
   tw_stime terminal_available_time;
   terminal_message_list **terminal_msgs;
   terminal_message_list **terminal_msgs_tail;
   int in_send_loop;
// Terminal generate, sends and arrival T_SEND, T_ARRIVAL, T_GENERATE
// Router-Router Intra-group sends and receives RR_LSEND, RR_LARRIVE
// Router-Router Inter-group sends and receives RR_GSEND, RR_GARRIVE
   struct mn_stats dragonfly_stats_array[CATEGORY_MAX];
  /* collective init time */
  tw_stime collective_init_time;

  /* node ID in the tree */ 
   tw_lpid node_id;

   /* messages sent & received in collectives may get interchanged several times so we have to save the 
     origin server information in the node's state */
   tw_lpid origin_svr; 
  
  /* parent node ID of the current node */
   tw_lpid parent_node_id;
   /* array of children to be allocated in terminal_init*/
   tw_lpid* children;

   /* children of a node can be less than or equal to the tree degree */
   int num_children;

   short is_root;
   short is_leaf;

   struct rc_stack * st;
   int issueIdle;
   int terminal_length;

   /* to maintain a count of child nodes that have fanned in at the parent during the collective
      fan-in phase*/
   int num_fan_nodes;

   const char * anno;
   dragonfly_param *params;

   struct qhash_table *rank_tbl;
   uint64_t rank_tbl_pop;

   tw_stime   total_time;
   uint64_t total_msg_size;
   double total_hops;
   long finished_msgs;
   long finished_chunks;
   long finished_packets;

   tw_stime * last_buf_full;
   tw_stime busy_time;
   char output_buf[4096];
   /* For LP suspend functionality */
   int error_ct;

   /* For sampling */
   long fin_chunks_sample;
   long data_size_sample;
   double fin_hops_sample;
   tw_stime fin_chunks_time;
   tw_stime busy_time_sample;

   char sample_buf[4096];
   struct dfly_cn_sample * sample_stat;
   int op_arr_size;
   int max_arr_size;
   
   /* for logging forward and reverse events */
   long fwd_events;
   long rev_events;

   /* following used for ROSS model-level stats collection */
   long fin_chunks_ross_sample;
   long data_size_ross_sample;
   long fin_hops_ross_sample;
   tw_stime fin_chunks_time_ross_sample;
   tw_stime busy_time_ross_sample;
   struct dfly_cn_sample ross_sample;
};

/* terminal event type (1-4) */
enum event_t
{
  T_GENERATE=1,
  T_ARRIVE,
  T_SEND,
  T_BUFFER,
  R_SEND,
  R_ARRIVE,
  R_BUFFER,
  D_COLLECTIVE_INIT,
  D_COLLECTIVE_FAN_IN,
  D_COLLECTIVE_FAN_OUT
};
/* status of a virtual channel can be idle, active, allocated or wait for credit */
enum vc_status
{
   VC_IDLE,
   VC_ACTIVE,
   VC_ALLOC,
   VC_CREDIT
};

/* whether the last hop of a packet was global, local or a terminal */
enum last_hop
{
   GLOBAL,
   LOCAL,
   TERMINAL
};

/* three forms of routing algorithms available, adaptive routing is not
 * accurate and fully functional in the current version as the formulas
 * for detecting load on global channels are not very accurate */
enum ROUTING_ALGO
{
    MINIMAL = 0,
    NON_MINIMAL,
    ADAPTIVE,
    PROG_ADAPTIVE
};

struct router_state
{
   unsigned int router_id;
   int group_id;
   int op_arr_size;
   int max_arr_size;

   int* global_channel; 
   
   tw_stime* next_output_available_time;
   tw_stime* cur_hist_start_time;
   tw_stime** last_buf_full;

   tw_stime* busy_time;
   tw_stime* busy_time_sample;

   terminal_message_list ***pending_msgs;
   terminal_message_list ***pending_msgs_tail;
   terminal_message_list ***queued_msgs;
   terminal_message_list ***queued_msgs_tail;
   int *in_send_loop;
   int *queued_count;
   struct rc_stack * st;
   
   int** vc_occupancy;
   int64_t* link_traffic;
   int64_t * link_traffic_sample;

   const char * anno;
   dragonfly_param *params;

   int* prev_hist_num;
   int* cur_hist_num;
   
   char output_buf[4096];
   char output_buf2[4096];

   struct dfly_router_sample * rsamples;
   
   long fwd_events;
   long rev_events;

   /* following used for ROSS model-level stats collection */
   tw_stime* busy_time_ross_sample;
   int64_t * link_traffic_ross_sample;
   struct dfly_router_sample ross_rsample;
};

/* ROSS Instrumentation Support */
void dragonfly_event_collect(terminal_message *m, tw_lp *lp, char *buffer, int *collect_flag);
void dragonfly_model_stat_collect(terminal_state *s, tw_lp *lp, char *buffer);
void dfly_router_model_stat_collect(router_state *s, tw_lp *lp, char *buffer);
static void ross_dragonfly_rsample_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct dfly_router_sample *sample);
static void ross_dragonfly_rsample_rc_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct dfly_router_sample *sample);
static void ross_dragonfly_sample_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct dfly_cn_sample *sample);
static void ross_dragonfly_sample_rc_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct dfly_cn_sample *sample);

st_model_types dragonfly_model_types[] = {
    {(ev_trace_f) dragonfly_event_collect,
     sizeof(int),
     (model_stat_f) dragonfly_model_stat_collect,
     sizeof(tw_lpid) + sizeof(long) * 2 + sizeof(double) + sizeof(tw_stime) *2,
     (sample_event_f) ross_dragonfly_sample_fn,
     (sample_revent_f) ross_dragonfly_sample_rc_fn,
     sizeof(struct dfly_cn_sample) } , 
    {(ev_trace_f) dragonfly_event_collect,
     sizeof(int),
     (model_stat_f) dfly_router_model_stat_collect,
     0, //updated in router_setup() since it's based on the radix
     (sample_event_f) ross_dragonfly_rsample_fn,
     (sample_revent_f) ross_dragonfly_rsample_rc_fn,
     0 } , //updated in router_setup() since it's based on the radix    
    {NULL, 0, NULL, 0, NULL, NULL, 0}
};
/* End of ROSS model stats collection */

static short routing = MINIMAL;

static tw_stime         dragonfly_total_time = 0;
static tw_stime         dragonfly_max_latency = 0;
static tw_stime         max_collective = 0;


static long long       total_hops = 0;
static long long       N_finished_packets = 0;
static long long       total_msg_sz = 0;
static long long       N_finished_msgs = 0;
static long long       N_finished_chunks = 0;

static int dragonfly_rank_hash_compare(
        void *key, struct qhash_head *link)
{
    struct dfly_hash_key *message_key = (struct dfly_hash_key *)key;
    struct dfly_qhash_entry *tmp = NULL;

    tmp = qhash_entry(link, struct dfly_qhash_entry, hash_link);
    
    if (tmp->key.message_id == message_key->message_id
            && tmp->key.sender_id == message_key->sender_id)
        return 1;

    return 0;
}
static int dragonfly_hash_func(void *k, int table_size)
{
    struct dfly_hash_key *tmp = (struct dfly_hash_key *)k;
    //uint32_t pc = 0, pb = 0;	
    //bj_hashlittle2(tmp, sizeof(*tmp), &pc, &pb);
    uint64_t key = (~tmp->message_id) + (tmp->message_id << 18);
    key = key * 21;
    key = ~key ^ (tmp->sender_id >> 4);
    key = key * tmp->sender_id; 
    return (int)(key & (table_size - 1));
    //return (int)(pc % (table_size - 1));
}

/* convert GiB/s and bytes to ns */
static tw_stime bytes_to_ns(uint64_t bytes, double GB_p_s)
{
    tw_stime time;

    /* bytes to GB */
    time = ((double)bytes)/(1024.0*1024.0*1024.0);
    /* GiB to s */
    time = time / GB_p_s;
    /* s to ns */
    time = time * 1000.0 * 1000.0 * 1000.0;

    return(time);
}

/* returns the dragonfly message size */
static int dragonfly_get_msg_sz(void)
{
	   return sizeof(terminal_message);
}

static void free_tmp(void * ptr)
{
    struct dfly_qhash_entry * dfly = ptr; 
    
    if(dfly->remote_event_data)
        free(dfly->remote_event_data);
   
    if(dfly)
        free(dfly);
}
static void append_to_terminal_message_list(  
        terminal_message_list ** thisq,
        terminal_message_list ** thistail,
        int index, 
        terminal_message_list *msg) {
    if(thisq[index] == NULL) {
        thisq[index] = msg;
    } else {
        thistail[index]->next = msg;
        msg->prev = thistail[index];
    } 
    thistail[index] = msg;
}

static void prepend_to_terminal_message_list(  
        terminal_message_list ** thisq,
        terminal_message_list ** thistail,
        int index, 
        terminal_message_list *msg) {
    if(thisq[index] == NULL) {
        thistail[index] = msg;
    } else {
        thisq[index]->prev = msg;
        msg->next = thisq[index];
    } 
    thisq[index] = msg;
}

static terminal_message_list* return_head(
        terminal_message_list ** thisq,
        terminal_message_list ** thistail,
        int index) {
    terminal_message_list *head = thisq[index];
    if(head != NULL) {
        thisq[index] = head->next;
        if(head->next != NULL) {
            head->next->prev = NULL;
            head->next = NULL;
        } else {
            thistail[index] = NULL;
        }
    }
    return head;
}

static terminal_message_list* return_tail(
        terminal_message_list ** thisq,
        terminal_message_list ** thistail,
        int index) {
    terminal_message_list *tail = thistail[index];
    assert(tail);
    if(tail->prev != NULL) {
        tail->prev->next = NULL;
        thistail[index] = tail->prev;
        tail->prev = NULL;
    } else {
        thistail[index] = NULL;
        thisq[index] = NULL;
    }
    return tail;
}

static void dragonfly_read_config(const char * anno, dragonfly_param *params){
    uint32_t h1 = 0, h2 = 0; 
    bj_hashlittle2(LP_METHOD_NM_TERM, strlen(LP_METHOD_NM_TERM), &h1, &h2);
    terminal_magic_num = h1 + h2;
    
    bj_hashlittle2(LP_METHOD_NM_ROUT, strlen(LP_METHOD_NM_ROUT), &h1, &h2);
    router_magic_num = h1 + h2;
    // shorthand
    dragonfly_param *p = params;

    int rc = configuration_get_value_int(&config, "PARAMS", "num_routers", anno,
            &p->num_routers);
    if(rc) {
        p->num_routers = 4;
        fprintf(stderr, "Number of dimensions not specified, setting to %d\n",
                p->num_routers);
    }

    p->num_vcs = 3;

    rc = configuration_get_value_int(&config, "PARAMS", "local_vc_size", anno, &p->local_vc_size);
    if(rc) {
        p->local_vc_size = 1024;
        fprintf(stderr, "Buffer size of local channels not specified, setting to %d\n", p->local_vc_size);
    }

    rc = configuration_get_value_int(&config, "PARAMS", "global_vc_size", anno, &p->global_vc_size);
    if(rc) {
        p->global_vc_size = 2048;
        fprintf(stderr, "Buffer size of global channels not specified, setting to %d\n", p->global_vc_size);
    }

    rc = configuration_get_value_int(&config, "PARAMS", "cn_vc_size", anno, &p->cn_vc_size);
    if(rc) {
        p->cn_vc_size = 1024;
        fprintf(stderr, "Buffer size of compute node channels not specified, setting to %d\n", p->cn_vc_size);
    }

    rc = configuration_get_value_int(&config, "PARAMS", "chunk_size", anno, &p->chunk_size);
    if(rc) {
        p->chunk_size = 512;
        fprintf(stderr, "Chunk size for packets is specified, setting to %d\n", p->chunk_size);
    }

    rc = configuration_get_value_double(&config, "PARAMS", "local_bandwidth", anno, &p->local_bandwidth);
    if(rc) {
        p->local_bandwidth = 5.25;
        fprintf(stderr, "Bandwidth of local channels not specified, setting to %lf\n", p->local_bandwidth);
    }

    rc = configuration_get_value_double(&config, "PARAMS", "global_bandwidth", anno, &p->global_bandwidth);
    if(rc) {
        p->global_bandwidth = 4.7;
        fprintf(stderr, "Bandwidth of global channels not specified, setting to %lf\n", p->global_bandwidth);
    }

    rc = configuration_get_value_double(&config, "PARAMS", "cn_bandwidth", anno, &p->cn_bandwidth);
    if(rc) {
        p->cn_bandwidth = 5.25;
        fprintf(stderr, "Bandwidth of compute node channels not specified, setting to %lf\n", p->cn_bandwidth);
    }

    p->router_delay = 50;
    configuration_get_value_double(&config, "PARAMS", "router_delay", anno,
            &p->router_delay);

    configuration_get_value(&config, "PARAMS", "cn_sample_file", anno, dfly_cn_sample_file,
            MAX_NAME_LENGTH);
    configuration_get_value(&config, "PARAMS", "rt_sample_file", anno, dfly_rtr_sample_file,
            MAX_NAME_LENGTH);
    
    char routing_str[MAX_NAME_LENGTH];
    configuration_get_value(&config, "PARAMS", "routing", anno, routing_str,
            MAX_NAME_LENGTH);
    if(strcmp(routing_str, "minimal") == 0)
        routing = MINIMAL;
    else if(strcmp(routing_str, "nonminimal")==0 || 
            strcmp(routing_str,"non-minimal")==0)
        routing = NON_MINIMAL;
    else if (strcmp(routing_str, "adaptive") == 0)
        routing = ADAPTIVE;
    else if (strcmp(routing_str, "prog-adaptive") == 0)
	routing = PROG_ADAPTIVE;
    else
    {
        fprintf(stderr, 
                "No routing protocol specified, setting to minimal routing\n");
        routing = -1;
    }

    // set the derived parameters
    p->num_cn = p->num_routers/2;
    p->num_global_channels = p->num_routers/2;
    p->num_groups = p->num_routers * p->num_cn + 1;
    p->radix = (p->num_routers + p->num_global_channels + p->num_cn);
    p->total_routers = p->num_groups * p->num_routers;
    p->total_terminals = p->total_routers * p->num_cn;
    int rank;
    MPI_Comm_rank(MPI_COMM_CODES, &rank);
    if(!rank) {
        printf("\n Total nodes %d routers %d groups %d radix %d \n",
                p->num_cn * p->total_routers, p->total_routers, p->num_groups,
                p->radix);
    }
    
    p->cn_delay = bytes_to_ns(p->chunk_size, p->cn_bandwidth);
    p->local_delay = bytes_to_ns(p->chunk_size, p->local_bandwidth);
    p->global_delay = bytes_to_ns(p->chunk_size, p->global_bandwidth);

    //CREDIT DELAY CONFIGURATION LOGIC ------------
    int myRank;
    MPI_Comm_rank(MPI_COMM_CODES, &myRank);

    rc = configuration_get_value_int(&config, "PARAMS", "credit_size", anno, &p->credit_size);
    if (rc) {
        p->credit_size = 8;
        if(!myRank)
            fprintf(stderr, "credit_size not specified, using default: %d\n", p->credit_size);
    }

    double general_credit_delay;
    int credit_delay_unset = configuration_get_value_double(&config, "PARAMS", "credit_delay", anno, &general_credit_delay); 
    int local_credit_delay_unset = configuration_get_value_double(&config, "PARAMS", "local_credit_delay", anno, &p->local_credit_delay);
    int global_credit_delay_unset = configuration_get_value_double(&config, "PARAMS", "global_credit_delay", anno, &p->global_credit_delay);
    int cn_credit_delay_unset = configuration_get_value_double(&config, "PARAMS", "cn_credit_delay", anno, &p->cn_credit_delay);

    int auto_credit_delay_flag;
    rc = configuration_get_value_int(&config, "PARAMS", "auto_credit_delay", anno, &auto_credit_delay_flag);
    if (rc) {
        auto_credit_delay_flag = 0;
    }
    else {
        if(!myRank && auto_credit_delay_flag)
            fprintf(stderr, "auto_credit_delay flag enabled. All credit delays will be calculated based on their respective bandwidths\n");
    }

    //If the user specifies a general "credit_delay" AND any of the more specific credit delays, throw an error to make sure they correct their configuration
    if (!credit_delay_unset && !(local_credit_delay_unset || global_credit_delay_unset || cn_credit_delay_unset))
        tw_error(TW_LOC, "\nCannot set both a general credit delay and specific (local/global/cn) credit delays. Check configuration file.");
    
    //If the user specifies ANY credit delays general or otherwise AND has the auto credit delay flag enabled, throw an error to make sure they correct the conflicting configuration
    if ((!credit_delay_unset || !local_credit_delay_unset || !global_credit_delay_unset || !cn_credit_delay_unset) && auto_credit_delay_flag)
        tw_error(TW_LOC, "\nCannot set both a credit delay (general or specific) and also enable auto credit delay calculation. Check Configuration file.");

    //If the user doesn't specify either general or specific credit delays - calculate credit delay based on local bandwidth.
    //This is old legacy behavior that is left in to make sure that the credit delay configurations of old aren't semantically different
    //Other possible way to program this would be to make each credit delay be set based on their respective bandwidths but this semantically
    //changes the behavior of old configuration files. (although it would be more accurate)
    if (credit_delay_unset && local_credit_delay_unset && global_credit_delay_unset && cn_credit_delay_unset && !auto_credit_delay_flag) {
        p->local_credit_delay = bytes_to_ns(p->credit_size, p->local_bandwidth);
        p->global_credit_delay = p->local_credit_delay;
        p->cn_credit_delay = p->local_credit_delay;
        if(!myRank)
            fprintf(stderr, "no credit_delay specified - all credit delays set to %.2f\n",p->local_credit_delay);
    }
    //If the user doesn't specify a general credit delay but leaves any of the specific credit delay values unset, then we need to set those (the above conditional handles if none of them had been set)
    else if (credit_delay_unset) {
        if (local_credit_delay_unset) {
            p->local_credit_delay = bytes_to_ns(p->credit_size, p->local_bandwidth);
            if(!myRank && !auto_credit_delay_flag) //if the auto credit delay flag is true then we've already printed what we're going to do
                fprintf(stderr, "local_credit_delay not specified, using calculation based on local bandwidth: %.2f\n", p->local_credit_delay);
        }
        if (global_credit_delay_unset) {
            p->global_credit_delay = bytes_to_ns(p->credit_size, p->global_bandwidth);
            if(!myRank && !auto_credit_delay_flag)
                fprintf(stderr, "global_credit_delay not specified, using calculation based on global bandwidth: %.2f\n", p->global_credit_delay);   
        }
        if (cn_credit_delay_unset) {
            p->cn_credit_delay = bytes_to_ns(p->credit_size, p->cn_bandwidth);
            if(!myRank && !auto_credit_delay_flag)
                fprintf(stderr, "cn_credit_delay not specified, using calculation based on cn bandwidth: %.2f\n", p->cn_credit_delay);
        }
    }
    //If the user specifies a general credit delay (but didn't specify any specific credit delays) then we set all specific credit delays to the general
    else if (!credit_delay_unset) {
        p->local_credit_delay = general_credit_delay;
        p->global_credit_delay = general_credit_delay;
        p->cn_credit_delay = general_credit_delay;
        
        if(!myRank)
            fprintf(stderr, "general credit_delay specified - all credit delays set to %.2f\n",general_credit_delay);
    }
    //END CREDIT DELAY CONFIGURATION LOGIC ----------------

}

static void dragonfly_configure(){
    anno_map = codes_mapping_get_lp_anno_map(LP_CONFIG_NM_TERM);
    assert(anno_map);
    num_params = anno_map->num_annos + (anno_map->has_unanno_lp > 0);
    all_params = malloc(num_params * sizeof(*all_params));

    for (int i = 0; i < anno_map->num_annos; i++){
        const char * anno = anno_map->annotations[i].ptr;
        dragonfly_read_config(anno, &all_params[i]);
    }
    if (anno_map->has_unanno_lp > 0){
        dragonfly_read_config(NULL, &all_params[anno_map->num_annos]);
    }
#ifdef ENABLE_CORTEX
	model_net_topology = dragonfly_cortex_topology;
#endif
}

/* report dragonfly statistics like average and maximum packet latency, average number of hops traversed */
static void dragonfly_report_stats()
{
   long long avg_hops, total_finished_packets, total_finished_chunks;
   long long total_finished_msgs, final_msg_sz;
   tw_stime avg_time, max_time;
   int total_minimal_packets, total_nonmin_packets;
   long total_gen, total_fin;

   MPI_Reduce( &total_hops, &avg_hops, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &N_finished_packets, &total_finished_packets, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &N_finished_msgs, &total_finished_msgs, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &N_finished_chunks, &total_finished_chunks, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &total_msg_sz, &final_msg_sz, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &dragonfly_total_time, &avg_time, 1,MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &dragonfly_max_latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_CODES);
   
   MPI_Reduce( &packet_gen, &total_gen, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   MPI_Reduce( &packet_fin, &total_fin, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_CODES);
   if(routing == ADAPTIVE || routing == PROG_ADAPTIVE)
    {
	MPI_Reduce(&minimal_count, &total_minimal_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
 	MPI_Reduce(&nonmin_count, &total_nonmin_packets, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_CODES);
    }

   /* print statistics */
   if(!g_tw_mynode)
   {	
      printf(" Average number of hops traversed %f average chunk latency %lf us maximum chunk latency %lf us avg message size %lf bytes finished messages %lld finished chunks %lld \n", 
              (float)avg_hops/total_finished_chunks, avg_time/(total_finished_chunks*1000), max_time/1000, (float)final_msg_sz/total_finished_msgs, total_finished_msgs, total_finished_chunks);
     if(routing == ADAPTIVE || routing == PROG_ADAPTIVE)
              printf("\n ADAPTIVE ROUTING STATS: %d chunks routed minimally %d chunks routed non-minimally completed packets %lld \n", 
                      total_minimal_packets, total_nonmin_packets, total_finished_chunks);
 
      printf("\n Total packets generated %ld finished %ld \n", total_gen, total_fin);
   }
   return;
}

static void dragonfly_collective_init(terminal_state * s,
           		   tw_lp * lp)
{
    // TODO: be annotation-aware
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
    int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
            NULL, 1);
    int num_reps = codes_mapping_get_group_reps(lp_group_name);
    s->node_id = (mapping_rep_id * num_lps) + mapping_offset;

    int i;
   /* handle collective operations by forming a tree of all the LPs */
   /* special condition for root of the tree */
   if( s->node_id == 0)
    {
        s->parent_node_id = -1;
        s->is_root = 1;
   }
   else
   {
       s->parent_node_id = (s->node_id - ((s->node_id - 1) % TREE_DEGREE)) / TREE_DEGREE;
       s->is_root = 0;
   }
   s->children = (tw_lpid*)malloc(TREE_DEGREE * sizeof(tw_lpid));

   /* set the isleaf to zero by default */
   s->is_leaf = 1;
   s->num_children = 0;

   /* calculate the children of the current node. If its a leaf, no need to set children,
      only set isleaf and break the loop*/

   for( i = 0; i < TREE_DEGREE; i++ )
    {
        tw_lpid next_child = (TREE_DEGREE * s->node_id) + i + 1;
        if(next_child < ((tw_lpid)num_lps * (tw_lpid)num_reps))
        {
            s->num_children++;
            s->is_leaf = 0;
            s->children[i] = next_child;
        }
        else
           s->children[i] = -1;
    }

#if DRAGONFLY_COLLECTIVE_DEBUG == 1
   printf("\n LP %ld parent node id ", s->node_id);

   for( i = 0; i < TREE_DEGREE; i++ )
        printf(" child node ID %ld ", s->children[i]);
   printf("\n");

   if(s->is_leaf)
        printf("\n LP %ld is leaf ", s->node_id);
#endif
}

/* initialize a dragonfly compute node terminal */
static void 
terminal_init( terminal_state * s, 
	       tw_lp * lp )
{
    s->packet_gen = 0;
    s->packet_fin = 0;

    
    int i;
    char anno[MAX_NAME_LENGTH];

    // Assign the global router ID
    // TODO: be annotation-aware
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);
    if (anno[0] == '\0'){
        s->anno = NULL;
        s->params = &all_params[num_params-1];
    }
    else{
        s->anno = strdup(anno);
        int id = configuration_get_annotation_index(anno, anno_map);
        s->params = &all_params[id];
    }

   //int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
   //        s->anno, 0);

   s->terminal_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);  
   
   s->router_id=(int)s->terminal_id / s->params->num_cn;
   s->terminal_available_time = 0.0;
   s->packet_counter = 0;
   
   s->finished_msgs = 0;
   s->finished_chunks = 0;
   s->finished_packets = 0;
   s->total_time = 0.0;
   s->total_msg_size = 0;
   s->num_vcs = 1;

   s->last_buf_full = (tw_stime*)malloc(s->num_vcs * sizeof(tw_stime));
   s->busy_time = 0.0;

   s->fwd_events = 0;
   s->rev_events = 0;

   rc_stack_create(&s->st);
   s->vc_occupancy = (int*)malloc(s->num_vcs * sizeof(int));

   for( i = 0; i < s->num_vcs; i++ )
    {
      s->last_buf_full[i] = 0.0;
      s->vc_occupancy[i]=0;
    }


   s->rank_tbl = NULL;
   s->terminal_msgs = 
       (terminal_message_list**)malloc(1*sizeof(terminal_message_list*));
   s->terminal_msgs_tail = 
       (terminal_message_list**)malloc(1*sizeof(terminal_message_list*));
   s->terminal_msgs[0] = NULL;
   s->terminal_msgs_tail[0] = NULL;
   s->terminal_length = 0;
   s->in_send_loop = 0;
   s->issueIdle = 0;

   /* set up for ROSS stats sampling */
   s->fin_chunks_ross_sample = 0;
   s->data_size_ross_sample = 0;
   s->fin_hops_ross_sample = 0;
   s->fin_chunks_time_ross_sample = 0.0;
   s->busy_time_ross_sample = 0.0;

   dragonfly_collective_init(s, lp);
   return;
}

/* sets up the router virtual channels, global channels, 
 * local channels, compute node channels */
static void router_setup(router_state * r, tw_lp * lp)
{
    
    char anno[MAX_NAME_LENGTH];
    codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
            &mapping_type_id, anno, &mapping_rep_id, &mapping_offset);

    if (anno[0] == '\0'){
        r->anno = NULL;
        r->params = &all_params[num_params-1];
    } else{
        r->anno = strdup(anno);
        int id = configuration_get_annotation_index(anno, anno_map);
        r->params = &all_params[id];
    }

    dragonfly_param *p = r->params;
    p->num_real_groups = codes_mapping_get_lp_count(lp_group_name, 0, LP_CONFIG_NM_ROUT, NULL, 1);
    assert(p->num_real_groups > 0);
    if(p->num_real_groups % p->num_routers)
    {
        tw_error(TW_LOC, "\n Config error: num_routers specified %d "
                "does not divide num_router per group %d  ",
                p->num_real_groups , p->num_routers);
    }
    p->num_real_groups = p->num_real_groups/p->num_routers;
    
    num_routers_per_mgrp = codes_mapping_get_lp_count (lp_group_name, 1, LP_METHOD_NM_ROUT,
            NULL, 0);
    /*int num_grp_reps = codes_mapping_get_group_reps(lp_group_name);
    if(p->total_routers != num_grp_reps * num_routers_per_mgrp)
        tw_error(TW_LOC, "\n Config error: num_routers specified %d total routers computed in the network %d "
                "does not match with repetitions * dragonfly_router %d  ",
                p->num_routers, p->total_routers, num_grp_reps * num_routers_per_mgrp);
    */
   r->router_id=mapping_rep_id + mapping_offset;
   r->group_id=r->router_id/p->num_routers;

   r->fwd_events = 0;
   r->rev_events = 0;
   r->ross_rsample.fwd_events = 0;
   r->ross_rsample.rev_events = 0;

   r->global_channel = (int*)malloc(p->num_global_channels * sizeof(int));
   r->next_output_available_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
   r->cur_hist_start_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
   r->link_traffic = (int64_t*)malloc(p->radix * sizeof(int64_t));
   r->link_traffic_sample = (int64_t*)malloc(p->radix * sizeof(int64_t));
   r->cur_hist_num = (int*)malloc(p->radix * sizeof(int));
   r->prev_hist_num = (int*)malloc(p->radix * sizeof(int));
   
   r->vc_occupancy = (int**)malloc(p->radix * sizeof(int*));
   r->in_send_loop = (int*)malloc(p->radix * sizeof(int));
   r->pending_msgs = 
    (terminal_message_list***)malloc(p->radix * sizeof(terminal_message_list**));
   r->pending_msgs_tail = 
    (terminal_message_list***)malloc(p->radix * sizeof(terminal_message_list**));
   r->queued_msgs = 
    (terminal_message_list***)malloc(p->radix * sizeof(terminal_message_list**));
   r->queued_msgs_tail = 
    (terminal_message_list***)malloc(p->radix * sizeof(terminal_message_list**));
   r->queued_count = (int*)malloc(p->radix * sizeof(int));
   r->last_buf_full = (tw_stime**)malloc(p->radix * sizeof(tw_stime*));
   r->busy_time = (tw_stime*)malloc(p->radix * sizeof(tw_stime));
   r->busy_time_sample = (tw_stime*)malloc(p->radix * sizeof(tw_stime));

   /* set up for ROSS stats sampling */
   r->link_traffic_ross_sample = (int64_t*)calloc(p->radix, sizeof(int64_t));
   r->busy_time_ross_sample = (tw_stime*)calloc(p->radix, sizeof(tw_stime));
   if (g_st_model_stats)
       lp->model_types->mstat_sz = sizeof(tw_lpid) + (sizeof(int64_t) + sizeof(tw_stime)) * p->radix;
   if (g_st_use_analysis_lps && g_st_model_stats)
       lp->model_types->sample_struct_sz = sizeof(struct dfly_router_sample) + (sizeof(tw_stime) + sizeof(int64_t)) * p->radix;
   r->ross_rsample.busy_time = (tw_stime*)calloc(p->radix, sizeof(tw_stime));
   r->ross_rsample.link_traffic_sample = (int64_t*)calloc(p->radix, sizeof(int64_t));

   rc_stack_create(&r->st);
   for(int i=0; i < p->radix; i++)
    {
       // Set credit & router occupancy
    r->busy_time[i] = 0.0;
    r->busy_time_sample[i] = 0.0;
	r->next_output_available_time[i]=0;
	r->cur_hist_start_time[i] = 0;
    r->link_traffic[i]=0;
    r->link_traffic_sample[i] = 0;
	r->cur_hist_num[i] = 0;
	r->prev_hist_num[i] = 0;
    r->queued_count[i] = 0;    
    r->in_send_loop[i] = 0;
    r->vc_occupancy[i] = (int*)malloc(p->num_vcs * sizeof(int));
    r->last_buf_full[i] = (tw_stime*)malloc(p->num_vcs * sizeof(tw_stime));
    r->pending_msgs[i] = (terminal_message_list**)malloc(p->num_vcs * 
        sizeof(terminal_message_list*));
    r->pending_msgs_tail[i] = (terminal_message_list**)malloc(p->num_vcs * 
        sizeof(terminal_message_list*));
    r->queued_msgs[i] = (terminal_message_list**)malloc(p->num_vcs * 
        sizeof(terminal_message_list*));
    r->queued_msgs_tail[i] = (terminal_message_list**)malloc(p->num_vcs * 
        sizeof(terminal_message_list*));
        for(int j = 0; j < p->num_vcs; j++) {
            r->last_buf_full[i][j] = 0.0;
            r->vc_occupancy[i][j] = 0;
            r->pending_msgs[i][j] = NULL;
            r->pending_msgs_tail[i][j] = NULL;
            r->queued_msgs[i][j] = NULL;
            r->queued_msgs_tail[i][j] = NULL;
        }
    }

#if DEBUG == 1
//   printf("\n LP ID %d VC occupancy radix %d Router %d is connected to ", lp->gid, p->radix, r->router_id);
#endif 
   //round the number of global channels to the nearest even number
#if USE_DIRECT_SCHEME
       int first = r->router_id % p->num_routers;
       for(int i=0; i < p->num_global_channels; i++)
        {
            int target_grp = first;
            if(target_grp == r->group_id) {
                target_grp = p->num_groups - 1;
            }
            int my_pos = r->group_id % p->num_routers;
            if(r->group_id == p->num_groups - 1) {
                my_pos = target_grp % p->num_routers;
            }
            r->global_channel[i] = target_grp * p->num_routers + my_pos;
            first += p->num_routers;
        }
#else
   int router_offset = (r->router_id % p->num_routers) * 
    (p->num_global_channels / 2) + 1;
   for(int i=0; i < p->num_global_channels; i++)
    {
      if(i % 2 != 0)
          {
             r->global_channel[i]=(r->router_id + (router_offset * p->num_routers))%p->total_routers;
             router_offset++;
          }
          else
           {
             r->global_channel[i]=r->router_id - ((router_offset) * p->num_routers);
           }
        if(r->global_channel[i]<0)
         {
           r->global_channel[i]=p->total_routers+r->global_channel[i]; 
	 }
#if DEBUG == 1
    printf("\n channel %d ", r->global_channel[i]);
#endif 
    }
#endif

#if DEBUG == 1
   printf("\n");
#endif
   return;
}	


/* dragonfly packet event , generates a dragonfly packet on the compute node */
static tw_stime dragonfly_packet_event(
        model_net_request const * req,
        uint64_t message_offset,
        uint64_t packet_size,
        tw_stime offset,
        mn_sched_params const * sched_params,
        void const * remote_event,
        void const * self_event,
        tw_lp *sender,
        int is_last_pckt)
{
    (void)message_offset;
    (void)sched_params;
    tw_event * e_new;
    tw_stime xfer_to_nic_time;
    terminal_message * msg;
    char* tmp_ptr;

    xfer_to_nic_time = codes_local_latency(sender); 
    //e_new = tw_event_new(sender->gid, xfer_to_nic_time+offset, sender);
    //msg = tw_event_data(e_new);
    e_new = model_net_method_event_new(sender->gid, xfer_to_nic_time+offset,
            sender, DRAGONFLY, (void**)&msg, (void**)&tmp_ptr);
    strcpy(msg->category, req->category);
    msg->final_dest_gid = req->final_dest_lp;
    msg->total_size = req->msg_size;
    msg->sender_lp=req->src_lp;
    msg->sender_mn_lp = sender->gid;
    msg->packet_size = packet_size;
    msg->travel_start_time = tw_now(sender);
    msg->remote_event_size_bytes = 0;
    msg->local_event_size_bytes = 0;
    msg->type = T_GENERATE;
    msg->dest_terminal_id = req->dest_mn_lp;
    msg->message_id = req->msg_id;
    msg->is_pull = req->is_pull;
    msg->pull_size = req->pull_size;
    msg->magic = terminal_magic_num; 
    msg->msg_start_time = req->msg_start_time;

    if(is_last_pckt) /* Its the last packet so pass in remote and local event information*/
      {
	if(req->remote_event_size > 0)
	 {
		msg->remote_event_size_bytes = req->remote_event_size;
		memcpy(tmp_ptr, remote_event, req->remote_event_size);
		tmp_ptr += req->remote_event_size;
	}
	if(req->self_event_size > 0)
	{
		msg->local_event_size_bytes = req->self_event_size;
		memcpy(tmp_ptr, self_event, req->self_event_size);
		tmp_ptr += req->self_event_size;
	}
     }
	   //printf("\n dragonfly remote event %d local event %d last packet %d %lf ", msg->remote_event_size_bytes, msg->local_event_size_bytes, is_last_pckt, xfer_to_nic_time);
    tw_event_send(e_new);
    return xfer_to_nic_time;
}

/* dragonfly packet event reverse handler */
static void dragonfly_packet_event_rc(tw_lp *sender)
{
	  codes_local_latency_reverse(sender);
	    return;
}

/* given two group IDs, find the router of the src_gid that connects to the dest_gid*/
tw_lpid getRouterFromGroupID(int dest_gid, 
		    int src_gid,
		    int num_routers,
            int total_groups)
{
#if USE_DIRECT_SCHEME
  int dest = dest_gid;
  if(dest == total_groups - 1) {
      dest = src_gid;
  }
  return src_gid * num_routers + (dest % num_routers);
#else
  int group_begin = src_gid * num_routers;
  int group_end = (src_gid * num_routers) + num_routers-1;
  int offset = (dest_gid * num_routers - group_begin) / num_routers;
  
  if((dest_gid * num_routers) < group_begin)
    offset = (group_begin - dest_gid * num_routers) / num_routers; // take absolute value
  
  int half_channel = num_routers / 4;
  int index = (offset - 1)/(half_channel * num_routers);
  
  offset=(offset - 1) % (half_channel * num_routers);

  // If the destination router is in the same group
  tw_lpid router_id;

  if(index % 2 != 0)
    router_id = group_end - (offset / half_channel); // start from the end
  else
    router_id = group_begin + (offset / half_channel);

  return router_id;
#endif
}	

/*When a packet is sent from the current router and a buffer slot becomes available, a credit is sent back to schedule another packet event*/
static void router_credit_send(router_state * s, terminal_message * msg, 
  tw_lp * lp, int sq) {
  tw_event * buf_e;
  tw_stime ts;
  terminal_message * buf_msg;

  int dest = 0,  type = R_BUFFER;
  int is_terminal = 0;
  double credit_delay;

  const dragonfly_param *p = s->params;
 
  // Notify sender terminal about available buffer space
  if(msg->last_hop == TERMINAL) {
    dest = msg->src_terminal_id;
    type = T_BUFFER;
    is_terminal = 1;
    credit_delay = p->cn_credit_delay;
  } 
  else if(msg->last_hop == GLOBAL) {
    dest = msg->intm_lp_id;
    credit_delay = p->global_credit_delay;
  }
   else if(msg->last_hop == LOCAL) {
    dest = msg->intm_lp_id;
    credit_delay = p->local_credit_delay;
  }
   else
    printf("\n Invalid message type");

  ts = g_tw_lookahead + credit_delay +  tw_rand_unif(lp->rng);
	
  if (is_terminal) {
    buf_e = model_net_method_event_new(dest, ts, lp, DRAGONFLY, 
      (void**)&buf_msg, NULL);
    buf_msg->magic = terminal_magic_num;
  } else {
    buf_e = model_net_method_event_new(dest, ts, lp, DRAGONFLY_ROUTER,
            (void**)&buf_msg, NULL);
    buf_msg->magic = router_magic_num;
  }
 
  if(sq == -1) {
    buf_msg->vc_index = msg->vc_index;
    buf_msg->output_chan = msg->output_chan;
  } else {
    buf_msg->vc_index = msg->saved_vc;
    buf_msg->output_chan = msg->saved_channel;
  }
  
  buf_msg->type = type;

  tw_event_send(buf_e);
  return;
}

static void packet_generate_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp)
{
   s->packet_gen--;
   packet_gen--;
   
   tw_rand_reverse_unif(lp->rng);

   int num_chunks = msg->packet_size/s->params->chunk_size;
   if(msg->packet_size % s->params->chunk_size)
       num_chunks++;

   if(!num_chunks)
       num_chunks = 1;

   int i;
   for(i = 0; i < num_chunks; i++) {
        delete_terminal_message_list(return_tail(s->terminal_msgs, 
          s->terminal_msgs_tail, 0));
        s->terminal_length -= s->params->chunk_size;
   }
    if(bf->c5) {
        codes_local_latency_reverse(lp);
        s->in_send_loop = 0;
    }
      if(bf->c11) {
        s->issueIdle = 0;
        if(bf->c8)
        {
            s->last_buf_full[0] = msg->saved_busy_time;
        }
      }
     struct mn_stats* stat;
     stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
     stat->send_count--;
     stat->send_bytes -= msg->packet_size;
     stat->send_time -= (1/s->params->cn_bandwidth) * msg->packet_size;
}

/* generates packet at the current dragonfly compute node */
static void packet_generate(terminal_state * s, tw_bf * bf, terminal_message * msg, 
  tw_lp * lp) {
  packet_gen++;
  s->packet_gen++;

  tw_stime ts, nic_ts;

  assert(lp->gid != msg->dest_terminal_id);
  const dragonfly_param *p = s->params;

  int total_event_size;
  uint64_t num_chunks = msg->packet_size / p->chunk_size;
  if (msg->packet_size % s->params->chunk_size) 
      num_chunks++;

  if(!num_chunks)
    num_chunks = 1;

  nic_ts = g_tw_lookahead + (num_chunks * s->params->cn_delay) + tw_rand_unif(lp->rng);
  
  msg->packet_ID = lp->gid + g_tw_nlp * s->packet_counter;
  msg->my_N_hop = 0;
  msg->my_l_hop = 0;
  msg->my_g_hop = 0;
  msg->intm_group_id = -1;

  //if(msg->dest_terminal_id == TRACK)
  if(msg->packet_ID == LLU(TRACK_PKT))
    printf("\n Packet %llu generated at terminal %d dest %llu size %llu num chunks %llu ", 
            msg->packet_ID, s->terminal_id, LLU(msg->dest_terminal_id),
            LLU(msg->packet_size), LLU(num_chunks));

  for(uint64_t i = 0; i < num_chunks; i++)
  {
    terminal_message_list *cur_chunk = (terminal_message_list*)malloc(
      sizeof(terminal_message_list));
    msg->origin_router_id = s->router_id;
    init_terminal_message_list(cur_chunk, msg);
  

    if(msg->remote_event_size_bytes + msg->local_event_size_bytes > 0) {
      cur_chunk->event_data = (char*)malloc(
          msg->remote_event_size_bytes + msg->local_event_size_bytes);
    }
    
    void * m_data_src = model_net_method_get_edata(DRAGONFLY, msg);
    if (msg->remote_event_size_bytes){
      memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes);
    }
    if (msg->local_event_size_bytes){ 
      m_data_src = (char*)m_data_src + msg->remote_event_size_bytes;
      memcpy((char*)cur_chunk->event_data + msg->remote_event_size_bytes, 
          m_data_src, msg->local_event_size_bytes);
    }

    cur_chunk->msg.chunk_id = i;
    cur_chunk->msg.origin_router_id = s->router_id;
    append_to_terminal_message_list(s->terminal_msgs, s->terminal_msgs_tail,
      0, cur_chunk);
    s->terminal_length += s->params->chunk_size;
  }

  if(s->terminal_length < 2 * s->params->cn_vc_size) {
    model_net_method_idle_event(nic_ts, 0, lp);
  } else {
    bf->c11 = 1;
    s->issueIdle = 1;
      if(s->last_buf_full[0] == 0.0)
      {
        bf->c8 = 1;
        msg->saved_busy_time = s->last_buf_full[0];
        /* TODO: Assumes a single vc from terminal to router */
        s->last_buf_full[0] = tw_now(lp);
      }
  }
  
  if(s->in_send_loop == 0) {
    bf->c5 = 1;
    ts = codes_local_latency(lp);
    terminal_message *m;
    tw_event* e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY, 
      (void**)&m, NULL);
    m->type = T_SEND;
    m->magic = terminal_magic_num;
    s->in_send_loop = 1;
    tw_event_send(e);
  }

  total_event_size = model_net_get_msg_sz(DRAGONFLY) + 
      msg->remote_event_size_bytes + msg->local_event_size_bytes;
  mn_stats* stat;
  stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
  stat->send_count++;
  stat->send_bytes += msg->packet_size;
  stat->send_time += (1/p->cn_bandwidth) * msg->packet_size;
  if(stat->max_event_size < total_event_size)
	  stat->max_event_size = total_event_size;

  return;
}

static void packet_send_rc(terminal_state * s, tw_bf * bf, terminal_message * msg,
        tw_lp * lp)
{
      if(bf->c1) {
        s->in_send_loop = 1;
        if(bf->c10)
            s->last_buf_full[0] = msg->saved_busy_time;
        return;
      }
      
      tw_rand_reverse_unif(lp->rng);
      s->terminal_available_time = msg->saved_available_time;
      if(bf->c2) {
        codes_local_latency_reverse(lp);
      }
     
      s->terminal_length += s->params->chunk_size;
      s->packet_counter--;
      s->vc_occupancy[0] -= s->params->chunk_size;

      terminal_message_list* cur_entry = rc_stack_pop(s->st);

      prepend_to_terminal_message_list(s->terminal_msgs, 
              s->terminal_msgs_tail, 0, cur_entry);
      if(bf->c3) {
        tw_rand_reverse_unif(lp->rng);
      }
      if(bf->c4) {
        s->in_send_loop = 1;
      }
      if(bf->c5)
      {
          tw_rand_reverse_unif(lp->rng);
          s->issueIdle = 1;
          if(bf->c6)
          {
            s->busy_time = msg->saved_total_time;
            s->last_buf_full[0] = msg->saved_busy_time;
            s->busy_time_sample = msg->saved_sample_time;
            s->ross_sample.busy_time_sample = msg->saved_sample_time;
            s->busy_time_ross_sample = msg->saved_busy_time_ross;
          }
      }
      return;
}
/* sends the packet from the current dragonfly compute node to the attached router */
static void packet_send(terminal_state * s, tw_bf * bf, terminal_message * msg, 
  tw_lp * lp) {
  
  tw_stime ts;
  tw_event *e;
  terminal_message *m;
  tw_lpid router_id;

  terminal_message_list* cur_entry = s->terminal_msgs[0];

  if(s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size)
  {
      if(s->last_buf_full[0] == 0.0)
      {
        bf->c10 = 1;
        msg->saved_busy_time = s->last_buf_full[0];
        s->last_buf_full[0] = tw_now(lp);
      }
  }
  if(s->vc_occupancy[0] + s->params->chunk_size > s->params->cn_vc_size 
      || cur_entry == NULL) {
    bf->c1 = 1;
    s->in_send_loop = 0;
    return;
  }

  uint64_t num_chunks = cur_entry->msg.packet_size/s->params->chunk_size;
  if(cur_entry->msg.packet_size % s->params->chunk_size)
    num_chunks++;

  if(!num_chunks)
      num_chunks = 1;
  
  tw_stime delay = s->params->cn_delay;
  if((cur_entry->msg.packet_size % s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1))
       delay = bytes_to_ns(cur_entry->msg.packet_size % s->params->chunk_size, s->params->cn_bandwidth); 

  msg->saved_available_time = s->terminal_available_time;
  ts = g_tw_lookahead + delay + tw_rand_unif(lp->rng);
  s->terminal_available_time = maxd(s->terminal_available_time, tw_now(lp));
  s->terminal_available_time += ts;

  ts = s->terminal_available_time - tw_now(lp);
  //TODO: be annotation-aware
  codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id, NULL,
      &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
  codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, NULL, 1,
      s->router_id, 0, &router_id);
  // we are sending an event to the router, so no method_event here
  void * remote_event;
  e = model_net_method_event_new(router_id, ts, lp,
          DRAGONFLY_ROUTER, (void**)&m, &remote_event);
  memcpy(m, &cur_entry->msg, sizeof(terminal_message));
  if (m->remote_event_size_bytes){
    memcpy(remote_event, cur_entry->event_data, m->remote_event_size_bytes);
  }

  m->type = R_ARRIVE;
  m->src_terminal_id = lp->gid;
  m->vc_index = 0;
  m->last_hop = TERMINAL;
  m->intm_group_id = -1;
  m->magic = router_magic_num;
  m->path_type = -1;
  m->local_event_size_bytes = 0;
  tw_event_send(e);


  if(cur_entry->msg.chunk_id == num_chunks - 1 && 
      (cur_entry->msg.local_event_size_bytes > 0)) {
    bf->c2 = 1;
    tw_stime local_ts = codes_local_latency(lp);
    tw_event *e_new = tw_event_new(cur_entry->msg.sender_lp, local_ts, lp);
    void * m_new = tw_event_data(e_new);
    void *local_event = (char*)cur_entry->event_data + 
      cur_entry->msg.remote_event_size_bytes;
    memcpy(m_new, local_event, cur_entry->msg.local_event_size_bytes);
    tw_event_send(e_new);
  }
  s->packet_counter++;
  s->vc_occupancy[0] += s->params->chunk_size;
  cur_entry = return_head(s->terminal_msgs, s->terminal_msgs_tail, 0); 
  rc_stack_push(lp, cur_entry, (void*)delete_terminal_message_list, s->st);
  s->terminal_length -= s->params->chunk_size;

  cur_entry = s->terminal_msgs[0];

  /* if there is another packet inline then schedule another send event */
  if(cur_entry != NULL &&
    s->vc_occupancy[0] + s->params->chunk_size <= s->params->cn_vc_size) {
    bf->c3 = 1;
    terminal_message *m_new;
    ts += tw_rand_unif(lp->rng);
    e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY, 
      (void**)&m_new, NULL);
    m_new->type = T_SEND;
    m_new->magic = terminal_magic_num;
    tw_event_send(e);
  } else {
      /* If not then the LP will wait for another credit or packet generation */
    bf->c4 = 1;
    s->in_send_loop = 0;
  }
  if(s->issueIdle) {
    bf->c5 = 1;
    s->issueIdle = 0;
    ts += tw_rand_unif(lp->rng);
    model_net_method_idle_event(ts, 0, lp);
   
    if(s->last_buf_full[0] > 0.0)
    {
        bf->c6 = 1;
        msg->saved_total_time = s->busy_time;
        msg->saved_sample_time = s->busy_time_sample;
        msg->saved_busy_time_ross = s->busy_time_ross_sample;

        s->busy_time += (tw_now(lp) - s->last_buf_full[0]);
        s->busy_time_sample += (tw_now(lp) - s->last_buf_full[0]);
        s->ross_sample.busy_time_sample += (tw_now(lp) - s->last_buf_full[0]);
        s->busy_time_ross_sample += (tw_now(lp) - s->last_buf_full[0]);
        s->last_buf_full[0] = 0.0;
    }
  }
  return;
}

static void packet_arrive_rc(terminal_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp)
{
    if(bf->c31)
    {
      s->packet_fin--;
      packet_fin--;
    }
      tw_rand_reverse_unif(lp->rng);
      if(msg->path_type == MINIMAL)
        minimal_count--;
      if(msg->path_type == NON_MINIMAL)
        nonmin_count--;

      N_finished_chunks--;
      s->finished_chunks--;
      s->fin_chunks_sample--;
      s->ross_sample.fin_chunks_sample--;
      s->fin_chunks_ross_sample--;

      total_hops -= msg->my_N_hop;
       s->total_hops -= msg->my_N_hop;
       s->fin_hops_sample -= msg->my_N_hop;
       s->ross_sample.fin_hops_sample -= msg->my_N_hop;
       s->fin_hops_ross_sample -= msg->my_N_hop;
       dragonfly_total_time  = msg->saved_total_time;
       s->fin_chunks_time = msg->saved_sample_time;
       s->ross_sample.fin_chunks_time = msg->saved_sample_time;
       s->fin_chunks_time_ross_sample = msg->saved_fin_chunks_ross;
       s->total_time = msg->saved_avg_time;
      
      struct qhash_head * hash_link = NULL;
      struct dfly_qhash_entry * tmp = NULL; 
      
      struct dfly_hash_key key;
      key.message_id = msg->message_id;
      key.sender_id = msg->sender_lp;
      
      hash_link = qhash_search(s->rank_tbl, &key);
      tmp = qhash_entry(hash_link, struct dfly_qhash_entry, hash_link);
      
      mn_stats* stat;
      stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
      stat->recv_time = msg->saved_rcv_time;

      if(bf->c1)
      {
        stat->recv_count--;
        stat->recv_bytes -= msg->packet_size;
        N_finished_packets--;
        s->finished_packets--;
      }
       if(bf->c3)
          dragonfly_max_latency = msg->saved_available_time;
       
       if(bf->c7)
        {
            //assert(!hash_link);
            if(bf->c8) 
              tw_rand_reverse_unif(lp->rng);
            N_finished_msgs--;
            s->finished_msgs--;
            total_msg_sz -= msg->total_size;
            s->total_msg_size -= msg->total_size;
            s->data_size_sample -= msg->total_size;
            s->ross_sample.data_size_sample -= msg->total_size;
            s->data_size_ross_sample -= msg->total_size;

	        struct dfly_qhash_entry * d_entry_pop = rc_stack_pop(s->st);
            qhash_add(s->rank_tbl, &key, &(d_entry_pop->hash_link));
            s->rank_tbl_pop++; 

            if(s->rank_tbl_pop >= DFLY_HASH_TABLE_SIZE)
                tw_error(TW_LOC, "\n Exceeded allocated qhash size, increase hash size in dragonfly model");

            hash_link = &(d_entry_pop->hash_link);
            tmp = d_entry_pop; 

            if(bf->c4)
                model_net_event_rc2(lp, &msg->event_rc);
        }
      
       assert(tmp);
       tmp->num_chunks--;

   if(bf->c5)
	{
	   qhash_del(hash_link);
	   free_tmp(tmp);	
       s->rank_tbl_pop--;
	}
       return;
}
static void send_remote_event(terminal_state * s, terminal_message * msg, tw_lp * lp, tw_bf * bf, char * event_data, int remote_event_size)
{
        (void)s;
        void * tmp_ptr = model_net_method_get_edata(DRAGONFLY, msg);
        //tw_stime ts = g_tw_lookahead + bytes_to_ns(msg->remote_event_size_bytes, (1/s->params->cn_bandwidth));
        tw_stime ts = g_tw_lookahead + tw_rand_unif(lp->rng);
        if (msg->is_pull){
            bf->c4 = 1;
            struct codes_mctx mc_dst =
                codes_mctx_set_global_direct(msg->sender_mn_lp);
            struct codes_mctx mc_src =
                codes_mctx_set_global_direct(lp->gid);
            int net_id = model_net_get_id(LP_METHOD_NM_TERM);

            model_net_set_msg_param(MN_MSG_PARAM_START_TIME, MN_MSG_PARAM_START_TIME_VAL, &(msg->msg_start_time));
            
            msg->event_rc = model_net_event_mctx(net_id, &mc_src, &mc_dst, msg->category,
                    msg->sender_lp, msg->pull_size, ts,
                    remote_event_size, tmp_ptr, 0, NULL, lp);
        }
        else{
            tw_event * e = tw_event_new(msg->final_dest_gid, ts, lp);
            void * m_remote = tw_event_data(e);
            memcpy(m_remote, event_data, remote_event_size);
            tw_event_send(e); 
        }
    return;
}
/* packet arrives at the destination terminal */
static void packet_arrive(terminal_state * s, tw_bf * bf, terminal_message * msg, 
  tw_lp * lp) {

    // NIC aggregation - should this be a separate function?
    // Trigger an event on receiving server

    if(!s->rank_tbl)
        s->rank_tbl = qhash_init(dragonfly_rank_hash_compare, dragonfly_hash_func, DFLY_HASH_TABLE_SIZE);
    
    struct dfly_hash_key key;
    key.message_id = msg->message_id; 
    key.sender_id = msg->sender_lp;
    
    struct qhash_head *hash_link = NULL;
    struct dfly_qhash_entry * tmp = NULL;
      
    hash_link = qhash_search(s->rank_tbl, &key);
    
    if(hash_link)
        tmp = qhash_entry(hash_link, struct dfly_qhash_entry, hash_link);

    uint64_t total_chunks = msg->total_size / s->params->chunk_size;

    if(msg->total_size % s->params->chunk_size)
          total_chunks++;

    if(!total_chunks)
          total_chunks = 1;

    /*if(tmp)
    {
        if(tmp->num_chunks >= total_chunks || tmp->num_chunks < 0)
        {
           //tw_output(lp, "\n invalid number of chunks %d for LP %ld ", tmp->num_chunks, lp->gid);
           tw_lp_suspend(lp, 0, 0);
           return;
        }
    }*/
    assert(lp->gid == msg->dest_terminal_id);

    if(msg->packet_ID == LLU(TRACK_PKT))
        printf("\n Packet %llu arrived at lp %llu hops %d", msg->packet_ID, LLU(lp->gid), msg->my_N_hop);
  
  tw_stime ts = g_tw_lookahead + s->params->cn_credit_delay + tw_rand_unif(lp->rng);

  // no method_event here - message going to router
  tw_event * buf_e;
  terminal_message * buf_msg;
  buf_e = model_net_method_event_new(msg->intm_lp_id, ts, lp,
          DRAGONFLY_ROUTER, (void**)&buf_msg, NULL);
  buf_msg->magic = router_magic_num;
  buf_msg->vc_index = msg->vc_index;
  buf_msg->output_chan = msg->output_chan;
  buf_msg->type = R_BUFFER;
  tw_event_send(buf_e);

  bf->c1 = 0;
  bf->c3 = 0;
  bf->c4 = 0;
  bf->c7 = 0;

  /* Total overall finished chunks in simulation */
  N_finished_chunks++;
  /* Finished chunks on a LP basis */
  s->finished_chunks++;
  /* Finished chunks per sample */
  s->fin_chunks_sample++;
  s->ross_sample.fin_chunks_sample++;
  s->fin_chunks_ross_sample++;

  /* WE do not allow self messages through dragonfly */
  assert(lp->gid != msg->src_terminal_id);

  uint64_t num_chunks = msg->packet_size / s->params->chunk_size;
  if (msg->packet_size % s->params->chunk_size)
    num_chunks++;

  if(!num_chunks)
     num_chunks = 1;

  if(msg->path_type == MINIMAL)
    minimal_count++;

  if(msg->path_type == NON_MINIMAL)
    nonmin_count++;

  if(msg->chunk_id == num_chunks - 1)
  {
    bf->c31 = 1;
    s->packet_fin++;
    packet_fin++;
  }
  if(msg->path_type != MINIMAL && msg->path_type != NON_MINIMAL)
    printf("\n Wrong message path type %d ", msg->path_type);

  /* save the sample time */
    msg->saved_sample_time = s->fin_chunks_time;
    s->fin_chunks_time += (tw_now(lp) - msg->travel_start_time);
    s->ross_sample.fin_chunks_time += (tw_now(lp) - msg->travel_start_time);
    msg->saved_fin_chunks_ross = s->fin_chunks_time_ross_sample;
    s->fin_chunks_time_ross_sample += (tw_now(lp) - msg->travel_start_time);
    
    /* save the total time per LP */
    msg->saved_avg_time = s->total_time;
    s->total_time += (tw_now(lp) - msg->travel_start_time); 

    msg->saved_total_time = dragonfly_total_time;
    dragonfly_total_time += tw_now( lp ) - msg->travel_start_time;
    total_hops += msg->my_N_hop;
    s->total_hops += msg->my_N_hop;
    s->fin_hops_sample += msg->my_N_hop;
    s->ross_sample.fin_hops_sample += msg->my_N_hop;
    s->fin_hops_ross_sample += msg->my_N_hop;

    mn_stats* stat = model_net_find_stats(msg->category, s->dragonfly_stats_array);
    msg->saved_rcv_time = stat->recv_time;
    stat->recv_time += (tw_now(lp) - msg->travel_start_time);

#if DEBUG == 1
 if( msg->packet_ID == TRACK 
          && msg->chunk_id == num_chunks-1
          && msg->message_id == TRACK_MSG)
  {
    printf( "(%lf) [Terminal %d] packet %lld has arrived  \n",
        tw_now(lp), (int)lp->gid, msg->packet_ID);

    printf("travel start time is %f\n",
        msg->travel_start_time);

    printf("My hop now is %d\n",msg->my_N_hop);
  }
#endif

   /* Now retreieve the number of chunks completed from the hash and update
    * them */
   void *m_data_src = model_net_method_get_edata(DRAGONFLY, msg);

   /* If an entry does not exist then create one */
   if(!tmp)
   {
        bf->c5 = 1;
       struct dfly_qhash_entry * d_entry = malloc(sizeof (struct dfly_qhash_entry));
       d_entry->num_chunks = 0;
       d_entry->key = key;
       d_entry->remote_event_data = NULL;
       d_entry->remote_event_size = 0;
       qhash_add(s->rank_tbl, &key, &(d_entry->hash_link));
       s->rank_tbl_pop++;
            
       if(s->rank_tbl_pop >= DFLY_HASH_TABLE_SIZE)
            tw_error(TW_LOC, "\n Exceeded allocated qhash size, increase hash size in dragonfly model");
      
       hash_link = &(d_entry->hash_link);
       tmp = d_entry;
   }
    
    assert(tmp);
    tmp->num_chunks++;

    if(msg->chunk_id == num_chunks - 1)
    {
        bf->c1 = 1;
        stat->recv_count++;
        stat->recv_bytes += msg->packet_size;

        N_finished_packets++;
        s->finished_packets++;
    }
    /* if its the last chunk of the packet then handle the remote event data */
    if(msg->remote_event_size_bytes > 0 && !tmp->remote_event_data)
    {
        /* Retreive the remote event entry */
         tmp->remote_event_data = (void*)malloc(msg->remote_event_size_bytes);
         assert(tmp->remote_event_data);
         tmp->remote_event_size = msg->remote_event_size_bytes; 
         memcpy(tmp->remote_event_data, m_data_src, msg->remote_event_size_bytes);
    }
        if (dragonfly_max_latency < tw_now( lp ) - msg->travel_start_time) {
          bf->c3 = 1;
          msg->saved_available_time = dragonfly_max_latency;
          dragonfly_max_latency = tw_now( lp ) - msg->travel_start_time;
        }
    /* If all chunks of a message have arrived then send a remote event to the
     * callee*/
    //assert(tmp->num_chunks <= total_chunks);

    if(tmp->num_chunks >= total_chunks)
    {
        bf->c7 = 1;

        N_finished_msgs++;
        total_msg_sz += msg->total_size;
        s->total_msg_size += msg->total_size;
        s->finished_msgs++;
        s->data_size_sample += msg->total_size;
        s->ross_sample.data_size_sample += msg->total_size;
        s->data_size_ross_sample += msg->total_size;
        
        if(tmp->remote_event_data && tmp->remote_event_size > 0) {
            bf->c8 = 1;
            send_remote_event(s, msg, lp, bf, tmp->remote_event_data, tmp->remote_event_size);
        }
        
        /* Remove the hash entry */
        qhash_del(hash_link);
        rc_stack_push(lp, tmp, free_tmp, s->st);
        s->rank_tbl_pop--;
   }
  return;
}

/* collective operation for the torus network */
static void dragonfly_collective(char const * category, int message_size, int remote_event_size, const void* remote_event, tw_lp* sender)
{
    tw_event * e_new;
    tw_stime xfer_to_nic_time;
    terminal_message * msg;
    tw_lpid local_nic_id;
    char* tmp_ptr;

    codes_mapping_get_lp_info(sender->gid, lp_group_name, &mapping_grp_id,
            NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
    codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_TERM, NULL, 1,
            mapping_rep_id, mapping_offset, &local_nic_id);

    xfer_to_nic_time = codes_local_latency(sender);
    e_new = model_net_method_event_new(local_nic_id, xfer_to_nic_time,
            sender, DRAGONFLY, (void**)&msg, (void**)&tmp_ptr);

    msg->remote_event_size_bytes = message_size;
    strcpy(msg->category, category);
    msg->sender_svr=sender->gid;
    msg->type = D_COLLECTIVE_INIT;

    tmp_ptr = (char*)msg;
    tmp_ptr += dragonfly_get_msg_sz();
    if(remote_event_size > 0)
     {
            msg->remote_event_size_bytes = remote_event_size;
            memcpy(tmp_ptr, remote_event, remote_event_size);
            tmp_ptr += remote_event_size;
     }

    tw_event_send(e_new);
    return;
}

/* reverse for collective operation of the dragonfly network */
static void dragonfly_collective_rc(int message_size, tw_lp* sender)
{
    (void)message_size;
     codes_local_latency_reverse(sender);
     return;
}

static void send_collective_remote_event(terminal_state * s,
                        terminal_message * msg,
                        tw_lp * lp)
{
    // Trigger an event on receiving server
    if(msg->remote_event_size_bytes)
     {
            tw_event* e;
            tw_stime ts;
            terminal_message * m;
            ts = (1/s->params->cn_bandwidth) * msg->remote_event_size_bytes;
            e = tw_event_new(s->origin_svr, ts, lp);
            m = tw_event_data(e);
            char* tmp_ptr = (char*)msg;
            tmp_ptr += dragonfly_get_msg_sz();
            memcpy(m, tmp_ptr, msg->remote_event_size_bytes);
            tw_event_send(e);
     }
}

static void node_collective_init(terminal_state * s,
                        terminal_message * msg,
                        tw_lp * lp)
{
        tw_event * e_new;
        tw_lpid parent_nic_id;
        tw_stime xfer_to_nic_time;
        terminal_message * msg_new;
        int num_lps;

        msg->saved_busy_time = s->collective_init_time;
        s->collective_init_time = tw_now(lp);
	s->origin_svr = msg->sender_svr;
	
        if(s->is_leaf)
        {
            //printf("\n LP %ld sending message to parent %ld ", s->node_id, s->parent_node_id);
            /* get the global LP ID of the parent node */
            // TODO: be annotation-aware
            codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id,
                    NULL, &mapping_type_id, NULL, &mapping_rep_id,
                    &mapping_offset);
            num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
                    s->anno, 0);
            codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_TERM, s->anno, 0,
                    s->parent_node_id/num_lps, (s->parent_node_id % num_lps),
                    &parent_nic_id);

           /* send a message to the parent that the LP has entered the collective operation */
            xfer_to_nic_time = g_tw_lookahead + LEVEL_DELAY;
            //e_new = tw_event_new(parent_nic_id, xfer_to_nic_time, lp);
	    void* m_data;
	    e_new = model_net_method_event_new(parent_nic_id, xfer_to_nic_time,
            	lp, DRAGONFLY, (void**)&msg_new, (void**)&m_data);
	    	
            memcpy(msg_new, msg, sizeof(terminal_message));
	    if (msg->remote_event_size_bytes){
        	memcpy(m_data, model_net_method_get_edata(DRAGONFLY, msg),
                	msg->remote_event_size_bytes);
      	    }
	    
            msg_new->type = D_COLLECTIVE_FAN_IN;
            msg_new->sender_node = s->node_id;

            tw_event_send(e_new);
        }
        return;
}

static void node_collective_fan_in(terminal_state * s,
                        tw_bf * bf,
                        terminal_message * msg,
                        tw_lp * lp)
{
        int i;
        s->num_fan_nodes++;

        codes_mapping_get_lp_info(lp->gid, lp_group_name, &mapping_grp_id,
                NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
        int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
                s->anno, 0);

        tw_event* e_new;
        terminal_message * msg_new;
        tw_stime xfer_to_nic_time;

        bf->c1 = 0;
        bf->c2 = 0;

        /* if the number of fanned in nodes have completed at the current node then signal the parent */
        if((s->num_fan_nodes == s->num_children) && !s->is_root)
        {
            bf->c1 = 1;
            msg->saved_fan_nodes = s->num_fan_nodes-1;
            s->num_fan_nodes = 0;
            tw_lpid parent_nic_id;
            xfer_to_nic_time = g_tw_lookahead + LEVEL_DELAY;

            /* get the global LP ID of the parent node */
            codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_TERM, s->anno, 0,
                    s->parent_node_id/num_lps, (s->parent_node_id % num_lps),
                    &parent_nic_id);

           /* send a message to the parent that the LP has entered the collective operation */
            //e_new = tw_event_new(parent_nic_id, xfer_to_nic_time, lp);
            //msg_new = tw_event_data(e_new);
	    void * m_data;
      	    e_new = model_net_method_event_new(parent_nic_id,
              xfer_to_nic_time,
              lp, DRAGONFLY, (void**)&msg_new, &m_data);
	    
            memcpy(msg_new, msg, sizeof(terminal_message));
            msg_new->type = D_COLLECTIVE_FAN_IN;
            msg_new->sender_node = s->node_id;

            if (msg->remote_event_size_bytes){
	        memcpy(m_data, model_net_method_get_edata(DRAGONFLY, msg),
        	        msg->remote_event_size_bytes);
      	   }
	    
            tw_event_send(e_new);
      }

      /* root node starts off with the fan-out phase */
      if(s->is_root && (s->num_fan_nodes == s->num_children))
      {
           bf->c2 = 1;
           msg->saved_fan_nodes = s->num_fan_nodes-1;
           s->num_fan_nodes = 0;
           send_collective_remote_event(s, msg, lp);

           for( i = 0; i < s->num_children; i++ )
           {
                tw_lpid child_nic_id;
                /* Do some computation and fan out immediate child nodes from the collective */
                xfer_to_nic_time = g_tw_lookahead + COLLECTIVE_COMPUTATION_DELAY + LEVEL_DELAY + tw_rand_exponential(lp->rng, (double)LEVEL_DELAY/50);

                /* get global LP ID of the child node */
                codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_TERM, NULL, 1,
                        s->children[i]/num_lps, (s->children[i] % num_lps),
                        &child_nic_id);
                //e_new = tw_event_new(child_nic_id, xfer_to_nic_time, lp);

                //msg_new = tw_event_data(e_new);
                void * m_data;
	        e_new = model_net_method_event_new(child_nic_id,
                xfer_to_nic_time,
		lp, DRAGONFLY, (void**)&msg_new, &m_data);

		memcpy(msg_new, msg, sizeof(terminal_message));
	        if (msg->remote_event_size_bytes){
	                memcpy(m_data, model_net_method_get_edata(DRAGONFLY, msg),
        	               msg->remote_event_size_bytes);
      		}
		
                msg_new->type = D_COLLECTIVE_FAN_OUT;
                msg_new->sender_node = s->node_id;

                tw_event_send(e_new);
           }
      }
}

static void node_collective_fan_out(terminal_state * s,
                        tw_bf * bf,
                        terminal_message * msg,
                        tw_lp * lp)
{
        int i;
        int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
                NULL, 1);
        bf->c1 = 0;
        bf->c2 = 0;

        send_collective_remote_event(s, msg, lp);

        if(!s->is_leaf)
        {
            bf->c1 = 1;
            tw_event* e_new;
            nodes_message * msg_new;
            tw_stime xfer_to_nic_time;

           for( i = 0; i < s->num_children; i++ )
           {
                xfer_to_nic_time = g_tw_lookahead + DRAGONFLY_FAN_OUT_DELAY + tw_rand_exponential(lp->rng, (double)DRAGONFLY_FAN_OUT_DELAY/10);

                if(s->children[i] > 0)
                {
                        tw_lpid child_nic_id;

                        /* get global LP ID of the child node */
                        codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_TERM,
                                s->anno, 0, s->children[i]/num_lps,
                                (s->children[i] % num_lps), &child_nic_id);
                        //e_new = tw_event_new(child_nic_id, xfer_to_nic_time, lp);
                        //msg_new = tw_event_data(e_new);
                        //memcpy(msg_new, msg, sizeof(nodes_message) + msg->remote_event_size_bytes);
			void* m_data;
			e_new = model_net_method_event_new(child_nic_id,
							xfer_to_nic_time,
					                lp, DRAGONFLY, (void**)&msg_new, &m_data);
		        memcpy(msg_new, msg, sizeof(nodes_message));
		        if (msg->remote_event_size_bytes){
			        memcpy(m_data, model_net_method_get_edata(DRAGONFLY, msg),
			                msg->remote_event_size_bytes);
      			}

                        //TODO: this is probably incorrect - need to fix
                        msg_new->type = (nodes_event_t) D_COLLECTIVE_FAN_OUT;
                        msg_new->sender_node = s->node_id;
                        tw_event_send(e_new);
                }
           }
         }
	//printf("\n Fan out phase completed %ld ", lp->gid);
        if(max_collective < tw_now(lp) - s->collective_init_time )
          {
              bf->c2 = 1;
              max_collective = tw_now(lp) - s->collective_init_time;
          }
}

static void ross_dragonfly_rsample_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct dfly_router_sample *sample)
{
    (void)lp;
    (void)bf;

    const dragonfly_param * p = s->params; 
    int i = 0;

    sample->router_id = s->router_id;
    sample->end_time = tw_now(lp);
    sample->fwd_events = s->ross_rsample.fwd_events;
    sample->rev_events = s->ross_rsample.rev_events;
    sample->busy_time = (tw_stime*)((&sample->rev_events) + 1);
    sample->link_traffic_sample = (int64_t*)((&sample->busy_time[0]) + p->radix);

    for(; i < p->radix; i++)
    {
        sample->busy_time[i] = s->ross_rsample.busy_time[i]; 
        sample->link_traffic_sample[i] = s->ross_rsample.link_traffic_sample[i]; 
    }

    /* clear up the current router stats */
    s->ross_rsample.fwd_events = 0;
    s->ross_rsample.rev_events = 0;

    for( i = 0; i < p->radix; i++)
    {
        s->ross_rsample.busy_time[i] = 0;
        s->ross_rsample.link_traffic_sample[i] = 0;
    }
}

static void ross_dragonfly_rsample_rc_fn(router_state * s, tw_bf * bf, tw_lp * lp, struct dfly_router_sample *sample)
{
    (void)lp;
    (void)bf;
    
    const dragonfly_param * p = s->params;
    int i =0;

    for(; i < p->radix; i++)
    {
        s->ross_rsample.busy_time[i] = sample->busy_time[i];
        s->ross_rsample.link_traffic_sample[i] = sample->link_traffic_sample[i];
    }

    s->ross_rsample.fwd_events = sample->fwd_events;
    s->ross_rsample.rev_events = sample->rev_events;
}

static void ross_dragonfly_sample_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct dfly_cn_sample *sample)
{
    (void)lp;
    (void)bf;
    
    sample->terminal_id = s->terminal_id;
    sample->fin_chunks_sample = s->ross_sample.fin_chunks_sample;
    sample->data_size_sample = s->ross_sample.data_size_sample;
    sample->fin_hops_sample = s->ross_sample.fin_hops_sample;
    sample->fin_chunks_time = s->ross_sample.fin_chunks_time;
    sample->busy_time_sample = s->ross_sample.busy_time_sample;
    sample->end_time = tw_now(lp);
    sample->fwd_events = s->ross_sample.fwd_events;
    sample->rev_events = s->ross_sample.rev_events;

    s->ross_sample.fin_chunks_sample = 0;
    s->ross_sample.data_size_sample = 0;
    s->ross_sample.fin_hops_sample = 0;
    s->ross_sample.fwd_events = 0;
    s->ross_sample.rev_events = 0;
    s->ross_sample.fin_chunks_time = 0;
    s->ross_sample.busy_time_sample = 0;
}

static void ross_dragonfly_sample_rc_fn(terminal_state * s, tw_bf * bf, tw_lp * lp, struct dfly_cn_sample *sample)
{
    (void)lp;
    (void)bf;

    s->ross_sample.busy_time_sample = sample->busy_time_sample;
    s->ross_sample.fin_chunks_time = sample->fin_chunks_time;
    s->ross_sample.fin_hops_sample = sample->fin_hops_sample;
    s->ross_sample.data_size_sample = sample->data_size_sample;
    s->ross_sample.fin_chunks_sample = sample->fin_chunks_sample;
    s->ross_sample.fwd_events = sample->fwd_events;
    s->ross_sample.rev_events = sample->rev_events;
}

static void dragonfly_rsample_init(router_state * s,
        tw_lp * lp)
{
   (void)lp;
   int i = 0;
   const dragonfly_param * p = s->params;

   assert(p->radix);

   s->max_arr_size = MAX_STATS;
   s->rsamples = malloc(MAX_STATS * sizeof(struct dfly_router_sample)); 
   for(; i < s->max_arr_size; i++)
   {
    s->rsamples[i].busy_time = malloc(sizeof(tw_stime) * p->radix); 
    s->rsamples[i].link_traffic_sample = malloc(sizeof(int64_t) * p->radix);
   }
}
static void dragonfly_rsample_rc_fn(router_state * s,
        tw_bf * bf,
        terminal_message * msg, 
        tw_lp * lp)
{
    (void)bf;
    (void)lp;
    (void)msg;

    s->op_arr_size--;
    int cur_indx = s->op_arr_size;
    struct dfly_router_sample stat = s->rsamples[cur_indx];

    const dragonfly_param * p = s->params;
    int i =0;

    for(; i < p->radix; i++)
    {
        s->busy_time_sample[i] = stat.busy_time[i];
        s->link_traffic_sample[i] = stat.link_traffic_sample[i];
    }

    for( i = 0; i < p->radix; i++)
    {
        stat.busy_time[i] = 0;
        stat.link_traffic_sample[i] = 0;
    }
    s->fwd_events = stat.fwd_events;
    s->rev_events = stat.rev_events;
}

static void dragonfly_rsample_fn(router_state * s,
        tw_bf * bf,
        terminal_message * msg, 
        tw_lp * lp)
{
  (void)bf;
  (void)lp;
  (void)msg;

  const dragonfly_param * p = s->params; 

  if(s->op_arr_size >= s->max_arr_size) 
  {
    struct dfly_router_sample * tmp = malloc((MAX_STATS + s->max_arr_size) * sizeof(struct dfly_router_sample));
    memcpy(tmp, s->rsamples, s->op_arr_size * sizeof(struct dfly_router_sample));
    free(s->rsamples);
    s->rsamples = tmp;
    s->max_arr_size += MAX_STATS;
  }

  int i = 0;
  int cur_indx = s->op_arr_size; 

  s->rsamples[cur_indx].router_id = s->router_id;
  s->rsamples[cur_indx].end_time = tw_now(lp);
  s->rsamples[cur_indx].fwd_events = s->fwd_events;
  s->rsamples[cur_indx].rev_events = s->rev_events;

  for(; i < p->radix; i++)
  {
    s->rsamples[cur_indx].busy_time[i] = s->busy_time_sample[i]; 
    s->rsamples[cur_indx].link_traffic_sample[i] = s->link_traffic_sample[i]; 
  }

  s->op_arr_size++;

  /* clear up the current router stats */
  s->fwd_events = 0;
  s->rev_events = 0;

  for( i = 0; i < p->radix; i++)
  {
    s->busy_time_sample[i] = 0;
    s->link_traffic_sample[i] = 0;
  }
}

static void dragonfly_rsample_fin(router_state * s,
        tw_lp * lp)
{
    (void)lp;
    const dragonfly_param * p = s->params;

    if(s->router_id == 0)
    {
    
        /* write metadata file */
        char meta_fname[64];
        sprintf(meta_fname, "dragonfly-router-sampling.meta");

        FILE * fp = fopen(meta_fname, "w");
        fprintf(fp, "Router sample struct format: \nrouter_id (tw_lpid) \nbusy time for each of the %d links (double) \n"
                "link traffic for each of the %d links (int64_t) \nsample end time (double) forward events per sample \nreverse events per sample ",
                p->radix, p->radix);
        fprintf(fp, "\n\nOrdering of links \n%d local (router-router same group) channels \n%d global (router-router remote group)"
                " channels \n%d terminal channels", p->radix/2, p->radix/4, p->radix/4);
        fclose(fp);
    }
    char rt_fn[MAX_NAME_LENGTH];
    if(strcmp(dfly_rtr_sample_file, "") == 0)
        sprintf(rt_fn, "dragonfly-router-sampling-%ld.bin", g_tw_mynode); 
    else
        sprintf(rt_fn, "%s-%ld.bin", dfly_rtr_sample_file, g_tw_mynode);
    
    int i = 0;

    int size_sample = sizeof(tw_lpid) + p->radix * (sizeof(int64_t) + sizeof(tw_stime)) + sizeof(tw_stime) + 2 * sizeof(long);
    FILE * fp = fopen(rt_fn, "a");
    fseek(fp, sample_rtr_bytes_written, SEEK_SET);

    for(; i < s->op_arr_size; i++)
    {
        fwrite((void*)&(s->rsamples[i].router_id), sizeof(tw_lpid), 1, fp);
        fwrite(s->rsamples[i].busy_time, sizeof(tw_stime), p->radix, fp);
        fwrite(s->rsamples[i].link_traffic_sample, sizeof(int64_t), p->radix, fp);
        fwrite((void*)&(s->rsamples[i].end_time), sizeof(tw_stime), 1, fp);
        fwrite((void*)&(s->rsamples[i].fwd_events), sizeof(long), 1, fp);
        fwrite((void*)&(s->rsamples[i].rev_events), sizeof(long), 1, fp);
    }
    sample_rtr_bytes_written += (s->op_arr_size * size_sample);
    fclose(fp);
}
static void dragonfly_sample_init(terminal_state * s,
        tw_lp * lp)
{
    (void)lp;
    s->fin_chunks_sample = 0;
    s->data_size_sample = 0;
    s->fin_hops_sample = 0;
    s->fin_chunks_time = 0;
    s->busy_time_sample = 0;

    s->op_arr_size = 0;
    s->max_arr_size = MAX_STATS;

    s->sample_stat = malloc(MAX_STATS * sizeof(struct dfly_cn_sample));
    
    /*char buf[1024];
    int written = 0;
    if(!s->terminal_id)
    {
       written = sprintf(buf, "# Format <LP ID> <Terminal ID> <Data size> <Avg packet latency> <#flits/packets finished> <Avg hops> <Busy Time>"); 
        lp_io_write(lp->gid, "dragonfly-sampling-stats", written, buf);
    }*/
}
static void dragonfly_sample_rc_fn(terminal_state * s,
        tw_bf * bf,
        terminal_message * msg, 
        tw_lp * lp)
{
    (void)lp;
    (void)bf;
    (void)msg;

    s->op_arr_size--;
    int cur_indx = s->op_arr_size;
    struct dfly_cn_sample stat = s->sample_stat[cur_indx];
    s->busy_time_sample = stat.busy_time_sample;
    s->fin_chunks_time = stat.fin_chunks_time;
    s->fin_hops_sample = stat.fin_hops_sample;
    s->data_size_sample = stat.data_size_sample;
    s->fin_chunks_sample = stat.fin_chunks_sample;
    s->fwd_events = stat.fwd_events;
    s->rev_events = stat.rev_events;

    stat.busy_time_sample = 0;
    stat.fin_chunks_time = 0;
    stat.fin_hops_sample = 0;
    stat.data_size_sample = 0;
    stat.fin_chunks_sample = 0;
    stat.end_time = 0;
    stat.terminal_id = 0;
    stat.fwd_events = 0;
    stat.rev_events = 0;
}

static void dragonfly_sample_fn(terminal_state * s,
        tw_bf * bf,
        terminal_message * msg,
        tw_lp * lp)
{
    (void)lp;
    (void)msg;
    (void)bf;
    
    //int i = 0, free_slots = 0;

    /* checkout which samples are past the GVT, write them to the file */
    /*for(i = 0; i < s->op_arr_size; i++)
    {
        if(s->sample_stat[i].end_time == 0 || s->sample_stat[i].end_time >= lp->pe->GVT)
            break;

        free_slots++;
    }
    if(free_slots > 0)
    {
        fwrite(s->sample_stat, sizeof(struct dfly_sample_stats), free_slots, fp);
        s->op_arr_size -= free_slots;
        memmove(s->sample_stat, &(s->sample_stat[free_slots]), sizeof(struct dfly_sample_stats) * s->op_arr_size);
    }*/

    if(s->op_arr_size >= s->max_arr_size)
    {
        /* In the worst case, copy array to a new memory location, its very
         * expensive operation though */
        struct dfly_cn_sample * tmp = malloc((MAX_STATS + s->max_arr_size) * sizeof(struct dfly_cn_sample));
        memcpy(tmp, s->sample_stat, s->op_arr_size * sizeof(struct dfly_cn_sample));
        free(s->sample_stat);
        s->sample_stat = tmp;
        s->max_arr_size += MAX_STATS;
    }
    
    int cur_indx = s->op_arr_size;

    s->sample_stat[cur_indx].terminal_id = s->terminal_id;
    s->sample_stat[cur_indx].fin_chunks_sample = s->fin_chunks_sample;
    s->sample_stat[cur_indx].data_size_sample = s->data_size_sample;
    s->sample_stat[cur_indx].fin_hops_sample = s->fin_hops_sample;
    s->sample_stat[cur_indx].fin_chunks_time = s->fin_chunks_time;
    s->sample_stat[cur_indx].busy_time_sample = s->busy_time_sample;
    s->sample_stat[cur_indx].end_time = tw_now(lp);
    s->sample_stat[cur_indx].fwd_events = s->fwd_events;
    s->sample_stat[cur_indx].rev_events = s->rev_events;

    s->op_arr_size++;
    s->fin_chunks_sample = 0;
    s->data_size_sample = 0;
    s->fin_hops_sample = 0;
    s->fwd_events = 0;
    s->rev_events = 0;
    s->fin_chunks_time = 0;
    s->busy_time_sample = 0;
}

static void dragonfly_sample_fin(terminal_state * s,
        tw_lp * lp)
{
    (void)lp;
 

    /* int i = 0;
     * for(; i < s->op_arr_size; i++)
    {
        printf("\n Terminal id %ld data size sample %ld fin chunks %ld end time %lf ",
                s->terminal_id, s->sample_stat[i].data_size_sample, s->sample_stat[i].fin_chunks_sample, s->sample_stat[i].end_time);
    }*/
    if(!g_tw_mynode)
    {
    
        /* write metadata file */
        char meta_fname[64];
        sprintf(meta_fname, "dragonfly-cn-sampling.meta");

        FILE * fp = fopen(meta_fname, "w");
        fprintf(fp, "Compute node sample format\nterminal_id (tw_lpid) \nfinished chunks (long)"
                "\ndata size per sample (long) \nfinished hops (double) \ntime to finish chunks (double)"
                "\nbusy time (double)\nsample end time(double) \nforward events (long) \nreverse events (long)");
        fclose(fp);
    }
    char rt_fn[MAX_NAME_LENGTH];
    if(strncmp(dfly_cn_sample_file, "", 10) == 0)
        sprintf(rt_fn, "dragonfly-cn-sampling-%ld.bin", g_tw_mynode); 
    else
        sprintf(rt_fn, "%s-%ld.bin", dfly_cn_sample_file, g_tw_mynode);

    FILE * fp = fopen(rt_fn, "a");
    fseek(fp, sample_bytes_written, SEEK_SET);
    fwrite(s->sample_stat, sizeof(struct dfly_cn_sample), s->op_arr_size, fp);
    fclose(fp);

    sample_bytes_written += (s->op_arr_size * sizeof(struct dfly_cn_sample));
}

static void terminal_buf_update_rc(terminal_state * s,
		    tw_bf * bf, 
		    terminal_message * msg, 
		    tw_lp * lp)
{
      (void)msg;
      s->vc_occupancy[0] += s->params->chunk_size;
      codes_local_latency_reverse(lp);
      if(bf->c1) {
        s->in_send_loop = 0;
      }

      return;
}
/* update the compute node-router channel buffer */
static void 
terminal_buf_update(terminal_state * s, 
		    tw_bf * bf, 
		    terminal_message * msg, 
		    tw_lp * lp)
{
  (void)msg;

  bf->c1 = 0;
  bf->c2 = 0;
  bf->c3 = 0;

  tw_stime ts = codes_local_latency(lp);
  s->vc_occupancy[0] -= s->params->chunk_size;
  

  if(s->in_send_loop == 0 && s->terminal_msgs[0] != NULL) {
    terminal_message *m;
    bf->c1 = 1;
    tw_event* e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY, 
        (void**)&m, NULL);
    m->type = T_SEND;
    m->magic = terminal_magic_num;
    s->in_send_loop = 1;
    tw_event_send(e);
  }
  return;
}

static void 
terminal_event( terminal_state * s, 
		tw_bf * bf, 
		terminal_message * msg, 
		tw_lp * lp )
{
  s->fwd_events++;
  s->ross_sample.fwd_events++;
  //*(int *)bf = (int)0;
  assert(msg->magic == terminal_magic_num);

  rc_stack_gc(lp, s->st);
  switch(msg->type)
    {
    case T_GENERATE:
       packet_generate(s,bf,msg,lp);
    break;
    
    case T_ARRIVE:
        packet_arrive(s,bf,msg,lp);
    break;
    
    case T_SEND:
      packet_send(s,bf,msg,lp);
    break;
    
    case T_BUFFER:
       terminal_buf_update(s, bf, msg, lp);
     break;
    
    case D_COLLECTIVE_INIT:
      node_collective_init(s, msg, lp);
    break;

    case D_COLLECTIVE_FAN_IN:
      node_collective_fan_in(s, bf, msg, lp);
    break;

    case D_COLLECTIVE_FAN_OUT:
      node_collective_fan_out(s, bf, msg, lp);
    break;
    
    default:
       printf("\n LP %d Terminal message type not supported %d ", (int)lp->gid, msg->type);
       tw_error(TW_LOC, "Msg type not supported");
    }
}

static void 
dragonfly_terminal_final( terminal_state * s, 
      tw_lp * lp )
{
	model_net_print_stats(lp->gid, s->dragonfly_stats_array);
  
    if(s->terminal_id == 0)
    {
        /* write metadata file */
        char meta_fname[64];
        sprintf(meta_fname, "dragonfly-msg-stats.meta");

        FILE * fp = fopen(meta_fname, "w");
        fprintf(fp, "# Format <LP id> <Terminal ID> <Total Data Size> <Aggregate packet latency> <# Flits/Packets finished> <Avg hops> <Busy Time>");

        fclose(fp);
    }
    
    int written = 0;
    if(!s->terminal_id)
        written = sprintf(s->output_buf, "# Format <LP id> <Terminal ID> <Total Data Size> <Aggregate packet latency> <# Flits/Packets finished> <Avg hops> <Busy Time>");

    written += sprintf(s->output_buf + written, "\n %llu %u %"PRId64" %lf %ld %lf %lf",
            LLU(lp->gid), s->terminal_id, s->total_msg_size, (double)s->total_time/s->finished_packets, 
            s->finished_packets, (double)s->total_hops/s->finished_chunks,
            s->busy_time);

    lp_io_write(lp->gid, "dragonfly-msg-stats", written, s->output_buf); 
    
    if(s->terminal_msgs[0] != NULL) 
      printf("[%llu] leftover terminal messages \n", LLU(lp->gid));


    //if(s->packet_gen != s->packet_fin)
    //    printf("\n generated %d finished %d ", s->packet_gen, s->packet_fin);
   
    if(s->rank_tbl)
        qhash_finalize(s->rank_tbl);
    
    rc_stack_destroy(s->st);
    free(s->vc_occupancy);
    free(s->terminal_msgs);
    free(s->terminal_msgs_tail);
    free(s->children);
}

static void dragonfly_router_final(router_state * s,
		tw_lp * lp)
{
   //rc_stack_gc(lp, s->st);
   free(s->global_channel);
    int i, j;
    for(i = 0; i < s->params->radix; i++) {
      for(j = 0; j < 3; j++) {
        if(s->queued_msgs[i][j] != NULL) {
          printf("[%llu] leftover queued messages %d %d %d\n", LLU(lp->gid), i, j,
          s->vc_occupancy[i][j]);
        }
        if(s->pending_msgs[i][j] != NULL) {
          printf("[%llu] lefover pending messages %d %d\n", LLU(lp->gid), i, j);
        }
      }
    }

    rc_stack_destroy(s->st);
    
    const dragonfly_param *p = s->params;
    int written = 0;
    if(s->router_id == 0)
    {
        /* write metadata file */
        char meta_fname[64];
        sprintf(meta_fname, "dragonfly-msg-stats.meta");

        FILE * fp = fopen(meta_fname, "w");
        fprintf(fp, "# Format <LP ID> <Group ID> <Router ID> <Busy time per router port(s)>");
        fprintf(fp, "# Router ports in the order: %d local channels, %d global channels", 
                p->num_routers, p->num_global_channels);
        fclose(fp);
    }
    written += sprintf(s->output_buf + written, "\n %llu %d %d", 
            LLU(lp->gid),
            s->router_id / p->num_routers,
            s->router_id % p->num_routers);
    for(int d = 0; d < p->num_routers + p->num_global_channels; d++) 
        written += sprintf(s->output_buf + written, " %lf", s->busy_time[d]);

    lp_io_write(lp->gid, "dragonfly-router-stats", written, s->output_buf);

    written = 0;
    if(!s->router_id)
    {
        written = sprintf(s->output_buf2, "# Format <LP ID> <Group ID> <Router ID> <Link traffic per router port(s)>");
        written += sprintf(s->output_buf2 + written, "# Router ports in the order: %d local channels, %d global channels",
            p->num_routers, p->num_global_channels);
    }
    written += sprintf(s->output_buf2 + written, "\n %llu %d %d",
        LLU(lp->gid),
        s->router_id / p->num_routers,
        s->router_id % p->num_routers);

    for(int d = 0; d < p->num_routers + p->num_global_channels; d++) 
        written += sprintf(s->output_buf2 + written, " %lld", LLD(s->link_traffic[d]));

    assert(written < 4096);
    lp_io_write(lp->gid, "dragonfly-router-traffic", written, s->output_buf2);
}

/* Get the number of hops for this particular path source and destination groups */
int get_num_hops(int local_router_id,
		 int dest_router_id,
		 int num_routers,
                 int total_groups)
{
   int local_grp_id = local_router_id / num_routers;
   int dest_group_id = dest_router_id / num_routers;
   int num_hops = 4;

   /* Already at the destination router */
   if(local_router_id == dest_router_id)
    {
	return 1; /* already at the destination, traverse one hop only*/
    }
   else if(local_grp_id == dest_group_id)
    {
		return 2; /* in the same group, each router is connected so 2 additional hops to traverse (source and dest routers). */		
    }	

     /* if the router in the source group has direct connection to the destination group */
     int src_connecting_router = getRouterFromGroupID(dest_group_id, 
        local_grp_id, num_routers, total_groups);

     if(src_connecting_router == local_router_id)		
		num_hops--;

     int dest_connecting_router = getRouterFromGroupID(local_grp_id, 
        dest_group_id, num_routers, total_groups);	

     if(dest_connecting_router == dest_router_id)	
			num_hops--;

     return num_hops;
}

/* get the next stop for the current packet
 * determines if it is a router within a group, a router in another group
 * or the destination terminal */
tw_lpid 
get_next_stop(router_state * s, 
		      terminal_message * msg, 
		      int path,
		      int dest_router_id,
              int intm_id)
{
   int dest_lp;
   tw_lpid router_dest_id;
   int dest_group_id;

   int local_router_id = s->router_id;
   int origin_grp_id = msg->origin_router_id / s->params->num_routers;

   dest_group_id = dest_router_id / s->params->num_routers;

  /* If the packet has arrived at the destination router */
   if(dest_router_id == local_router_id)
    {
        dest_lp = msg->dest_terminal_id;
        return dest_lp;
    }
   if(s->group_id == dest_group_id)
   {
       codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_router_id,
          0, &router_dest_id);
       return router_dest_id;
   }
   /* Generate inter-mediate destination for non-minimal routing (selecting a random group) */
   if(msg->last_hop == TERMINAL && path == NON_MINIMAL)
    {
    	    msg->intm_group_id = intm_id;
    }
  /******************** DECIDE THE DESTINATION GROUP ***********************/
  /* It means that the packet has arrived at the inter-mediate group for non-minimal routing. Reset the group now. */
  /* Intermediate group ID is set. Divert the packet to the intermediate group. */
  if(path == NON_MINIMAL && msg->intm_group_id >= 0 &&
          (s->group_id == origin_grp_id))
   {
      dest_group_id = msg->intm_group_id;
   }
 
/********************** DECIDE THE ROUTER IN THE DESTINATION GROUP ***************/ 
  /* It means the packet has arrived at the destination group. Now divert it to the destination router. */
      /* Packet is at the source or intermediate group. Find a router that has a path to the destination group. */
  dest_lp=getRouterFromGroupID(dest_group_id, 
    s->group_id, s->params->num_routers, 
    s->params->num_groups);

      if(dest_lp == local_router_id)
      {
#if USE_DIRECT_SCHEME
          int my_pos = s->group_id % s->params->num_routers;
          if(s->group_id == s->params->num_groups - 1) {
              my_pos = dest_group_id % s->params->num_routers;
          }
          dest_lp = dest_group_id * s->params->num_routers + my_pos;
#else
        for(int i=0; i < s->params->num_global_channels; i++)
           {
            if(s->global_channel[i] / s->params->num_routers == dest_group_id)
                dest_lp=s->global_channel[i];
          }
#endif
      }
  codes_mapping_get_lp_id(lp_group_name, LP_CONFIG_NM_ROUT, s->anno, 0, dest_lp,
          0, &router_dest_id);

  return router_dest_id;
}
/* gets the output port corresponding to the next stop of the message */
int 
get_output_port( router_state * s, 
		terminal_message * msg, 
		int next_stop )
{
  int output_port = -1, terminal_id;
  codes_mapping_get_lp_info(msg->dest_terminal_id, lp_group_name,
          &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id,
          &mapping_offset);
  int num_lps = codes_mapping_get_lp_count(lp_group_name,1,LP_CONFIG_NM_TERM,s->anno,0);
  terminal_id = (mapping_rep_id * num_lps) + mapping_offset;

  if((tw_lpid)next_stop == msg->dest_terminal_id)
   {
      output_port = s->params->num_routers + s->params->num_global_channels +
          ( terminal_id % s->params->num_cn);
    }
    else
    {
     codes_mapping_get_lp_info(next_stop, lp_group_name, &mapping_grp_id,
             NULL, &mapping_type_id, NULL, &mapping_rep_id, &mapping_offset);
     int local_router_id = mapping_rep_id + mapping_offset;
     int intm_grp_id = local_router_id / s->params->num_routers;

     if(intm_grp_id != s->group_id)
      {
#if USE_DIRECT_SCHEME
          int target_grp = intm_grp_id;
          if(target_grp == s->params->num_groups - 1) {
              target_grp = s->group_id;
          }
          output_port = s->params->num_routers + (target_grp) / 
                s->params->num_routers;
#else
        for(int i=0; i < s->params->num_global_channels; i++)
         {
           if(s->global_channel[i] == local_router_id)
             output_port = s->params->num_routers + i;
          }
#endif
      }
      else
       {
        output_port = local_router_id % s->params->num_routers;
       }
    }
    return output_port;
}


/* UGAL (first condition is from booksim), output port equality check comes from <NAME>'09*/
static int do_adaptive_routing( router_state * s,
				 terminal_message * msg,
				 int dest_router_id,
                 int intm_id) {
  int next_stop;
  int minimal_out_port = -1, nonmin_out_port = -1;
  // decide which routing to take
  // get the queue occupancy of both the minimal and non-minimal output ports 
  int minimal_next_stop=get_next_stop(s, msg, MINIMAL, dest_router_id, -1);
  minimal_out_port = get_output_port(s, msg, minimal_next_stop);
  int nonmin_next_stop = get_next_stop(s, msg, NON_MINIMAL, dest_router_id, intm_id);
  nonmin_out_port = get_output_port(s, msg, nonmin_next_stop);
  
#if 0
TODO: do we need this code? nomin_vc and min_vc not used anywhere...
  int nomin_vc = 0;
  if(nonmin_out_port < s->params->num_routers) {
    nomin_vc = msg->my_l_hop;
  } else if(nonmin_out_port < (s->params->num_routers + 
        s->params->num_global_channels)) {
    nomin_vc = msg->my_g_hop;
  }
  int min_vc = 0;
  if(minimal_out_port < s->params->num_routers) {
    min_vc = msg->my_l_hop;
  } else if(minimal_out_port < (s->params->num_routers + 
        s->params->num_global_channels)) {
    min_vc = msg->my_g_hop;
  }
#endif
  int min_port_count = s->vc_occupancy[minimal_out_port][0] + 
      s->vc_occupancy[minimal_out_port][1] + s->vc_occupancy[minimal_out_port][2]
      + s->queued_count[minimal_out_port];

  // Now get the expected number of hops to be traversed for both routes 
  //int num_min_hops = get_num_hops(s->router_id, dest_router_id, 
  //    s->params->num_routers, s->params->num_groups);

  int intm_router_id = getRouterFromGroupID(msg->intm_group_id, 
      s->router_id / s->params->num_routers, s->params->num_routers, 
      s->params->num_groups);

  int num_nonmin_hops = 6;
  
  if(msg->intm_group_id >= 0)
  {
      num_nonmin_hops = get_num_hops(s->router_id, intm_router_id, 
      s->params->num_routers, s->params->num_groups) + 
    get_num_hops(intm_router_id, dest_router_id, s->params->num_routers,
        s->params->num_groups);
  }
  assert(num_nonmin_hops <= 6);

  /* average the local queues of the router */
  unsigned int q_avg = 0;
  int i;
  for( i = 0; i < s->params->radix; i++)
  {
    if( i != minimal_out_port)
      q_avg += s->vc_occupancy[i][0] + s->vc_occupancy[i][1] +
        s->vc_occupancy[i][2];
  }
  q_avg = q_avg / (s->params->radix - 1);

  //int min_out_chan = minimal_out_port;
  //int nonmin_out_chan = nonmin_out_port;

  /* Adding history window approach, not taking the queue status at every 
   * simulation time thats why, we are maintaining the current history 
   * window number and an average of the previous history window number. */
  //int min_hist_count = s->cur_hist_num[min_out_chan] + 
  //  (s->prev_hist_num[min_out_chan]/2);
  //int nonmin_hist_count = s->cur_hist_num[nonmin_out_chan] + 
  //  (s->prev_hist_num[min_out_chan]/2);

  int nonmin_port_count = s->vc_occupancy[nonmin_out_port][0] +
      s->vc_occupancy[nonmin_out_port][1] + s->vc_occupancy[nonmin_out_port][2]
      + s->queued_count[nonmin_out_port];
  // VARIATION 1:
  // if(num_min_hops * min_port_count <= num_nonmin_hops * nonmin_port_count) {
  // VARIATION 2:
  //if(num_min_hops * min_port_count <= (num_nonmin_hops * (q_avg + 1))) {
    if(min_port_count <= nonmin_port_count) {
    msg->path_type = MINIMAL;
    next_stop = minimal_next_stop;
    msg->intm_group_id = -1;
  }
  else
  {
    msg->path_type = NON_MINIMAL;
    next_stop = nonmin_next_stop;
  }
  return next_stop;
}

static void router_packet_receive_rc(router_state * s,
        tw_bf * bf,
        terminal_message * msg,
        tw_lp * lp)
{
    router_rev_ecount++;
	router_ecount--;
      
    int output_port = msg->saved_vc;
    int output_chan = msg->saved_channel;

    tw_rand_reverse_unif(lp->rng);
      
    if(bf->c2) {
        tw_rand_reverse_unif(lp->rng);
        terminal_message_list * tail = return_tail(s->pending_msgs[output_port], s->pending_msgs_tail[output_port], output_chan);
        delete_terminal_message_list(tail);
        s->vc_occupancy[output_port][output_chan] -= s->params->chunk_size;
        if(bf->c3) {
          codes_local_latency_reverse(lp);
          s->in_send_loop[output_port] = 0;
        }
      }
      if(bf->c4) {
          if(bf->c22)
          {
            s->last_buf_full[output_port][output_chan] = msg->saved_busy_time;
          }
      delete_terminal_message_list(return_tail(s->queued_msgs[output_port], 
          s->queued_msgs_tail[output_port], output_chan));
      s->queued_count[output_port] -= s->params->chunk_size; 
      }
}

/* Packet arrives at the router and a credit is sent back to the sending terminal/router */
static void 
router_packet_receive( router_state * s, 
			tw_bf * bf, 
			terminal_message * msg, 
			tw_lp * lp )
{
  router_ecount++;

  bf->c1 = 0;
  bf->c2 = 0;
  bf->c3 = 0;
  bf->c4 = 0;
  bf->c5 = 0;

  tw_stime ts;

  int next_stop = -1, output_port = -1, output_chan = -1;

  codes_mapping_get_lp_info(msg->dest_terminal_id, lp_group_name,
      &mapping_grp_id, NULL, &mapping_type_id, NULL, &mapping_rep_id,
      &mapping_offset); 
  int num_lps = codes_mapping_get_lp_count(lp_group_name, 1, LP_CONFIG_NM_TERM,
      s->anno, 0);
  int dest_router_id = (mapping_offset + (mapping_rep_id * num_lps)) / 
    s->params->num_cn;
  int local_grp_id = s->router_id / s->params->num_routers;

  int intm_id = tw_rand_integer(lp->rng, 1, s->params->num_real_groups - 1); 
  intm_id = (local_grp_id + intm_id) % s->params->num_real_groups;

  /* progressive adaptive routing makes a check at every node/router at the 
   * source group to sense congestion. Once it does and decides on taking 
   * non-minimal path, it does not check any longer. */
  terminal_message_list * cur_chunk = (terminal_message_list*)malloc(sizeof(terminal_message_list));
 init_terminal_message_list(cur_chunk, msg);
  
  if(routing == PROG_ADAPTIVE
      && msg->path_type != NON_MINIMAL
      && (unsigned int) local_grp_id == ( msg->origin_router_id / s->params->num_routers)) {
    next_stop = do_adaptive_routing(s, &(cur_chunk->msg), dest_router_id, intm_id);	
  } else if(msg->last_hop == TERMINAL && routing == ADAPTIVE) {
    next_stop = do_adaptive_routing(s, &(cur_chunk->msg), dest_router_id, intm_id);
  } else {
    if(routing == MINIMAL || routing == NON_MINIMAL)	
      cur_chunk->msg.path_type = routing; /*defaults to the routing algorithm if we 
                                don't have adaptive routing here*/
    assert(cur_chunk->msg.path_type == MINIMAL || cur_chunk->msg.path_type == NON_MINIMAL);
    next_stop = get_next_stop(s, &(cur_chunk->msg), cur_chunk->msg.path_type, dest_router_id, intm_id);
  }
  
  if(msg->remote_event_size_bytes > 0) {
    void *m_data_src = model_net_method_get_edata(DRAGONFLY_ROUTER, msg);
    cur_chunk->event_data = (char*)malloc(msg->remote_event_size_bytes);
    memcpy(cur_chunk->event_data, m_data_src, msg->remote_event_size_bytes);
  }
  output_port = get_output_port(s, &(cur_chunk->msg), next_stop); 
  assert(output_port >= 0);
  output_chan = 0;
  int max_vc_size = s->params->cn_vc_size;

  cur_chunk->msg.vc_index = output_port;
  cur_chunk->msg.next_stop = next_stop;

  if(msg->packet_ID == LLU(TRACK_PKT))
      printf("\n Router packet %llu arrived lp id %llu final dest %llu output port %d ", msg->packet_ID, LLU(lp->gid), LLU(msg->dest_terminal_id), output_port);
  
  if(output_port < s->params->num_routers) {
    output_chan = msg->my_l_hop;
    if(msg->my_g_hop == 1) output_chan = 1;
    if(msg->my_g_hop == 2) output_chan = 2;
    max_vc_size = s->params->local_vc_size;
    cur_chunk->msg.my_l_hop++;
  } else if(output_port < (s->params->num_routers + 
        s->params->num_global_channels)) {
    output_chan = msg->my_g_hop;
    max_vc_size = s->params->global_vc_size;
    cur_chunk->msg.my_g_hop++;
  }

  cur_chunk->msg.output_chan = output_chan;
  cur_chunk->msg.my_N_hop++;

  assert(output_chan < 3);
  assert(output_port < s->params->radix);

  if(s->vc_occupancy[output_port][output_chan] + s->params->chunk_size 
      <= max_vc_size) {
    bf->c2 = 1;
    router_credit_send(s, msg, lp, -1);
    append_to_terminal_message_list( s->pending_msgs[output_port], 
      s->pending_msgs_tail[output_port], output_chan, cur_chunk);
    s->vc_occupancy[output_port][output_chan] += s->params->chunk_size;
    if(s->in_send_loop[output_port] == 0) {
      bf->c3 = 1;
      terminal_message *m;
      ts = codes_local_latency(lp); 
      tw_event *e = model_net_method_event_new(lp->gid, ts, lp,
              DRAGONFLY_ROUTER, (void**)&m, NULL);
      m->type = R_SEND;
      m->magic = router_magic_num;
      m->vc_index = output_port;
      
      tw_event_send(e);
      s->in_send_loop[output_port] = 1;
    }
  } else {
    bf->c4 = 1;
    cur_chunk->msg.saved_vc = msg->vc_index;
    cur_chunk->msg.saved_channel = msg->output_chan;
    append_to_terminal_message_list( s->queued_msgs[output_port], 
      s->queued_msgs_tail[output_port], output_chan, cur_chunk);
    s->queued_count[output_port] += s->params->chunk_size;
    if(s->pending_msgs[output_port][output_chan] == NULL && s->last_buf_full[output_port][output_chan] == 0.0)
          {
            bf->c22 = 1;
            msg->saved_busy_time = s->last_buf_full[output_port][output_chan];
            s->last_buf_full[output_port][output_chan] = tw_now(lp);
          }
  }

  msg->saved_vc = output_port;
  msg->saved_channel = output_chan;
  return;
}

static void router_packet_send_rc(router_state * s, 
		    tw_bf * bf, 
		     terminal_message * msg, tw_lp * lp)
{
    router_ecount--;
    router_rev_ecount++;
    
    int output_port = msg->saved_vc;
    int output_chan = msg->saved_channel;
    if(bf->c1) {
        s->in_send_loop[output_port] = 1;
        return;  
    }
      
    tw_rand_reverse_unif(lp->rng);
      
    terminal_message_list * cur_entry = rc_stack_pop(s->st);
    assert(cur_entry);

    if(bf->c11)
    {
        s->link_traffic[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size;
        s->link_traffic_sample[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size; 
        s->ross_rsample.link_traffic_sample[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size; 
        s->link_traffic_ross_sample[output_port] -= cur_entry->msg.packet_size % s->params->chunk_size; 
    }
    if(bf->c12)
    {
        s->link_traffic[output_port] -= s->params->chunk_size;
        s->link_traffic_sample[output_port] -= s->params->chunk_size;
        s->ross_rsample.link_traffic_sample[output_port] -= s->params->chunk_size;
        s->link_traffic_ross_sample[output_port] -= s->params->chunk_size;
    }
    s->next_output_available_time[output_port] = msg->saved_available_time;

    prepend_to_terminal_message_list(s->pending_msgs[output_port],
            s->pending_msgs_tail[output_port], output_chan, cur_entry);

    if(routing == PROG_ADAPTIVE)
	{
		if(bf->c2)
		{
		   s->cur_hist_num[output_port] = s->prev_hist_num[output_port];
		   s->prev_hist_num[output_port] = msg->saved_hist_num;
		   s->cur_hist_start_time[output_port] = msg->saved_hist_start_time;	
		}
		else
		  s->cur_hist_num[output_port]--;
 	}
    if(bf->c3) {
        tw_rand_reverse_unif(lp->rng);
      }
      
    if(bf->c4) {
        s->in_send_loop[output_port] = 1;
      }
}
/* routes the current packet to the next stop */
static void 
router_packet_send( router_state * s, 
		    tw_bf * bf, 
		     terminal_message * msg, tw_lp * lp)
{
  router_ecount++;

  tw_stime ts;
  tw_event *e;
  terminal_message *m;
  int output_port = msg->vc_index;
  int output_chan = 2;

  terminal_message_list *cur_entry = s->pending_msgs[output_port][2];
  if(cur_entry == NULL) {
    cur_entry = s->pending_msgs[output_port][1];
    output_chan = 1;
    if(cur_entry == NULL) {
      cur_entry = s->pending_msgs[output_port][0];
      output_chan = 0;
    }
  }
  msg->saved_vc = output_port;
  msg->saved_channel = output_chan;

  if(cur_entry == NULL) {
    bf->c1 = 1;
    s->in_send_loop[output_port] = 0;
    return;
  }

  int to_terminal = 1, global = 0;
  double delay = s->params->cn_delay;
  double bandwidth = s->params->cn_bandwidth;

  if(output_port < s->params->num_routers) {
    to_terminal = 0;
    delay = s->params->local_delay;
    bandwidth = s->params->local_bandwidth;
  } else if(output_port < s->params->num_routers + 
    s->params->num_global_channels) {
    to_terminal = 0;
    global = 1;
    delay = s->params->global_delay;
    bandwidth = s->params->global_bandwidth;
  }

  uint64_t num_chunks = cur_entry->msg.packet_size / s->params->chunk_size;
  if(msg->packet_size % s->params->chunk_size)
      num_chunks++;
  if(!num_chunks)
      num_chunks = 1;

  double bytetime = delay;
  
  if((cur_entry->msg.packet_size % s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1))
      bytetime = bytes_to_ns(cur_entry->msg.packet_size % s->params->chunk_size, bandwidth); 
  
  ts = g_tw_lookahead + tw_rand_unif( lp->rng) + bytetime + s->params->router_delay;

  msg->saved_available_time = s->next_output_available_time[output_port];
  s->next_output_available_time[output_port] = 
    maxd(s->next_output_available_time[output_port], tw_now(lp));
  s->next_output_available_time[output_port] += ts;

  ts = s->next_output_available_time[output_port] - tw_now(lp);
  // dest can be a router or a terminal, so we must check
  void * m_data;
  if (to_terminal) {
    assert(cur_entry->msg.next_stop == cur_entry->msg.dest_terminal_id);
    e = model_net_method_event_new(cur_entry->msg.next_stop, 
        s->next_output_available_time[output_port] - tw_now(lp), lp,
        DRAGONFLY, (void**)&m, &m_data);
  } else {
    e = model_net_method_event_new(cur_entry->msg.next_stop,
            s->next_output_available_time[output_port] - tw_now(lp), lp,
            DRAGONFLY_ROUTER, (void**)&m, &m_data);
  }
  memcpy(m, &cur_entry->msg, sizeof(terminal_message));
  if (m->remote_event_size_bytes){
    memcpy(m_data, cur_entry->event_data, m->remote_event_size_bytes);
  }

  if(global)
    m->last_hop = GLOBAL;
  else
    m->last_hop = LOCAL;

  m->intm_lp_id = lp->gid;
  m->magic = router_magic_num;

  if((cur_entry->msg.packet_size % s->params->chunk_size) && (cur_entry->msg.chunk_id == num_chunks - 1)) {
      bf->c11 = 1;
      s->link_traffic[output_port] +=  (cur_entry->msg.packet_size %
              s->params->chunk_size); 
      s->link_traffic_sample[output_port] += (cur_entry->msg.packet_size % 
               s->params->chunk_size);
      s->ross_rsample.link_traffic_sample[output_port] += (cur_entry->msg.packet_size % 
               s->params->chunk_size);
      s->link_traffic_ross_sample[output_port] += (cur_entry->msg.packet_size % 
               s->params->chunk_size);
  } else {
    bf->c12 = 1;
    s->link_traffic[output_port] += s->params->chunk_size;
    s->link_traffic_sample[output_port] += s->params->chunk_size;
    s->ross_rsample.link_traffic_sample[output_port] += s->params->chunk_size;
    s->link_traffic_ross_sample[output_port] += s->params->chunk_size;
  }

  if(routing == PROG_ADAPTIVE)
  {
      if(tw_now(lp) - s->cur_hist_start_time[output_port] >= WINDOW_LENGTH) {
        bf->c2 = 1;
        msg->saved_hist_start_time = s->cur_hist_start_time[output_port];
        s->prev_hist_num[output_port] = s->cur_hist_num[output_port];
        s->cur_hist_start_time[output_port] = tw_now(lp);
        msg->saved_hist_num = s->cur_hist_num[output_port];
        s->cur_hist_num[output_port] = 1;
      } else {
        s->cur_hist_num[output_port]++;
      }
  }
  /* Determine the event type. If the packet has arrived at the final 
   * destination router then it should arrive at the destination terminal 
   * next.*/
  if(to_terminal) {
    m->type = T_ARRIVE;
    m->magic = terminal_magic_num;
  } else {
      /* The packet has to be sent to another router */
    m->magic = router_magic_num;
    m->type = R_ARRIVE;
  }
  tw_event_send(e);
  
  cur_entry = return_head(s->pending_msgs[output_port], 
    s->pending_msgs_tail[output_port], output_chan);
  rc_stack_push(lp, cur_entry, (void*)delete_terminal_message_list, s->st);
  
  cur_entry = s->pending_msgs[output_port][2];
 
  s->next_output_available_time[output_port] -= s->params->router_delay;
  ts -= s->params->router_delay;

  if(cur_entry == NULL) cur_entry = s->pending_msgs[output_port][1];
  if(cur_entry == NULL) cur_entry = s->pending_msgs[output_port][0];
  if(cur_entry != NULL) {
    bf->c3 = 1;
    terminal_message *m_new;
    ts = ts + g_tw_lookahead * tw_rand_unif(lp->rng);
    e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_ROUTER,
            (void**)&m_new, NULL);
    m_new->type = R_SEND;
    m_new->magic = router_magic_num;
    m_new->vc_index = output_port;
    tw_event_send(e);
  } else {
    bf->c4 = 1;
    s->in_send_loop[output_port] = 0;
  }
  return;
}

static void router_buf_update_rc(router_state * s,
        tw_bf * bf,
        terminal_message * msg,
        tw_lp * lp)
{
      int indx = msg->vc_index;
      int output_chan = msg->output_chan;
      s->vc_occupancy[indx][output_chan] += s->params->chunk_size;
      if(bf->c3)
      {
        s->busy_time[indx] = msg->saved_rcv_time;
        s->busy_time_sample[indx] = msg->saved_sample_time;
        s->ross_rsample.busy_time[indx] = msg->saved_sample_time;
        s->busy_time_ross_sample[indx] = msg->saved_busy_time_ross;
        s->last_buf_full[indx][output_chan] = msg->saved_busy_time;
      }
      if(bf->c1) {
        terminal_message_list* head = return_tail(s->pending_msgs[indx],
            s->pending_msgs_tail[indx], output_chan);
        tw_rand_reverse_unif(lp->rng);
        prepend_to_terminal_message_list(s->queued_msgs[indx], 
            s->queued_msgs_tail[indx], output_chan, head);
        s->vc_occupancy[indx][output_chan] -= s->params->chunk_size;
        s->queued_count[indx] += s->params->chunk_size;
      }
      if(bf->c2) {
        codes_local_latency_reverse(lp);
        s->in_send_loop[indx] = 0;
      }
}
/* Update the buffer space associated with this router LP */
static void router_buf_update(router_state * s, tw_bf * bf, terminal_message * msg, tw_lp * lp)
{
  int indx = msg->vc_index;
  int output_chan = msg->output_chan;
  s->vc_occupancy[indx][output_chan] -= s->params->chunk_size;
  
  if(s->last_buf_full[indx][output_chan])
  {
    bf->c3 = 1;
    msg->saved_rcv_time = s->busy_time[indx];
    msg->saved_busy_time = s->last_buf_full[indx][output_chan];
    msg->saved_sample_time = s->busy_time_sample[indx];
    msg->saved_busy_time_ross = s->busy_time_ross_sample[indx];
    s->busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
    s->busy_time_sample[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
    s->ross_rsample.busy_time[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
    s->busy_time_ross_sample[indx] += (tw_now(lp) - s->last_buf_full[indx][output_chan]);
    s->last_buf_full[indx][output_chan] = 0.0;
  }
  if(s->queued_msgs[indx][output_chan] != NULL) {
    bf->c1 = 1;
    terminal_message_list *head = return_head(s->queued_msgs[indx],
        s->queued_msgs_tail[indx], output_chan);
    router_credit_send(s, &head->msg, lp, 1); 
    append_to_terminal_message_list(s->pending_msgs[indx], 
      s->pending_msgs_tail[indx], output_chan, head);
    s->vc_occupancy[indx][output_chan] += s->params->chunk_size;
    s->queued_count[indx] -= s->params->chunk_size; 
  }
  if(s->in_send_loop[indx] == 0 && s->pending_msgs[indx][output_chan] != NULL) {
    bf->c2 = 1;
    terminal_message *m;
    tw_stime ts = codes_local_latency(lp);
    tw_event *e = model_net_method_event_new(lp->gid, ts, lp, DRAGONFLY_ROUTER,
            (void**)&m, NULL);
    m->type = R_SEND;
    m->vc_index = indx;
    m->magic = router_magic_num;
    s->in_send_loop[indx] = 1;
    tw_event_send(e);
  }
  
  return;
}

static void router_event(router_state * s, tw_bf * bf, terminal_message * msg, 
    tw_lp * lp) {

  s->fwd_events++;
  s->ross_rsample.fwd_events++;
  rc_stack_gc(lp, s->st);
  
  assert(msg->magic == router_magic_num);
  switch(msg->type)
  {
    case R_SEND: // Router has sent a packet to an intra-group router (local channel)
      router_packet_send(s, bf, msg, lp);
      break;

    case R_ARRIVE: // Router has received a packet from an intra-group router (local channel)
      router_packet_receive(s, bf, msg, lp);
      break;

    case R_BUFFER:
      router_buf_update(s, bf, msg, lp);
      break;

    default:
      printf("\n (%lf) [Router %d] Router Message type not supported %d dest " 
        "terminal id %d packet ID %d ", tw_now(lp), (int)lp->gid, msg->type, 
        (int)msg->dest_terminal_id, (int)msg->packet_ID);
      tw_error(TW_LOC, "Msg type not supported");
      break;
  }	   
}

/* Reverse computation handler for a terminal event */
static void terminal_rc_event_handler(terminal_state * s, tw_bf * bf, 
    terminal_message * msg, tw_lp * lp) {

   s->rev_events++;
   s->ross_sample.rev_events++;
   switch(msg->type)
   {
    case T_GENERATE:
        packet_generate_rc(s, bf, msg, lp); 
        break;

    case T_SEND:
        packet_send_rc(s, bf, msg, lp);
        break;

    case T_ARRIVE:
        packet_arrive_rc(s, bf, msg, lp);
        break;

    case T_BUFFER:
        terminal_buf_update_rc(s, bf, msg, lp); 
        break;

    case D_COLLECTIVE_INIT:
            {
                s->collective_init_time = msg->saved_busy_time;
            }
      break;
    case D_COLLECTIVE_FAN_IN: {
        int i;
        s->num_fan_nodes--;
        if(bf->c1)
        {
          s->num_fan_nodes = msg->saved_fan_nodes;
        }
        if(bf->c2)
        {
          s->num_fan_nodes = msg->saved_fan_nodes;
          for( i = 0; i < s->num_children; i++ )
            tw_rand_reverse_unif(lp->rng);
        }
      }
      break;

    case D_COLLECTIVE_FAN_OUT: {
        int i;
        if(bf->c1)
        {
          for( i = 0; i < s->num_children; i++ )
            tw_rand_reverse_unif(lp->rng);
        }
      }	 
  }
}

/* Reverse computation handler for a router event */
static void router_rc_event_handler(router_state * s, tw_bf * bf, 
  terminal_message * msg, tw_lp * lp) {
    s->rev_events++;
    s->ross_rsample.rev_events++;

    switch(msg->type) {
    case R_SEND: 
        router_packet_send_rc(s, bf, msg, lp);
    break;
    case R_ARRIVE: 
        router_packet_receive_rc(s, bf, msg, lp);
    break;

    case R_BUFFER: 
        router_buf_update_rc(s, bf, msg, lp);
    break;
  }
}

/* dragonfly compute node and router LP types */
tw_lptype dragonfly_lps[] =
{
   // Terminal handling functions
   {
    (init_f)terminal_init,
    (pre_run_f) NULL,
    (event_f) terminal_event,
    (revent_f) terminal_rc_event_handler,
    (commit_f) NULL,
    (final_f) dragonfly_terminal_final,
    (map_f) codes_mapping,
    sizeof(terminal_state)
    },
   {
     (init_f) router_setup,
     (pre_run_f) NULL,
     (event_f) router_event,
     (revent_f) router_rc_event_handler,
     (commit_f) NULL,
     (final_f) dragonfly_router_final,
     (map_f) codes_mapping,
     sizeof(router_state),
   },
   {NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0},
};

/* For ROSS event tracing */
void dragonfly_event_collect(terminal_message *m, tw_lp *lp, char *buffer, int *collect_flag)
{
    (void)lp;
    (void)collect_flag;

    int type = (int) m->type;
    memcpy(buffer, &type, sizeof(type));
}

void dragonfly_model_stat_collect(terminal_state *s, tw_lp *lp, char *buffer)
{
    (void)lp;

    int index = 0;
    tw_lpid id = 0;
    long tmp = 0;
    tw_stime tmp2 = 0;
    
    id = s->terminal_id;
    memcpy(&buffer[index], &id, sizeof(id));
    index += sizeof(id);

    tmp = s->fin_chunks_ross_sample;
    memcpy(&buffer[index], &tmp, sizeof(tmp));
    index += sizeof(tmp);
    s->fin_chunks_ross_sample = 0;

    tmp = s->data_size_ross_sample;
    memcpy(&buffer[index], &tmp, sizeof(tmp));
    index += sizeof(tmp);
    s->data_size_ross_sample = 0;

    tmp = s->fin_hops_ross_sample;
    memcpy(&buffer[index], &tmp, sizeof(tmp));
    index += sizeof(tmp);
    s->fin_hops_ross_sample = 0;

    tmp2 = s->fin_chunks_time_ross_sample;
    memcpy(&buffer[index], &tmp2, sizeof(tmp2));
    index += sizeof(tmp2);
    s->fin_chunks_time_ross_sample = 0;

    tmp2 = s->busy_time_ross_sample;
    memcpy(&buffer[index], &tmp2, sizeof(tmp2));
    index += sizeof(tmp2);
    s->busy_time_ross_sample = 0;

    return;
}

void dfly_router_model_stat_collect(router_state *s, tw_lp *lp, char *buffer)
{
    (void)lp;

    const dragonfly_param * p = s->params; 
    int i, index = 0;

    tw_lpid id = 0;
    tw_stime tmp = 0;
    int64_t tmp2 = 0;

    id = s->router_id;
    memcpy(&buffer[index], &id, sizeof(id));
    index += sizeof(id);

    for(i = 0; i < p->radix; i++)
    {
        tmp = s->busy_time_ross_sample[i];
        memcpy(&buffer[index], &tmp, sizeof(tmp));
        index += sizeof(tmp);
        s->busy_time_ross_sample[i] = 0; 

        tmp2 = s->link_traffic_ross_sample[i];
        memcpy(&buffer[index], &tmp2, sizeof(tmp2));
        index += sizeof(tmp2);
        s->link_traffic_ross_sample[i] = 0; 
    }
    return;
}

static const st_model_types  *dragonfly_get_model_types(void)
{
    return(&dragonfly_model_types[0]);
}

static const st_model_types  *dfly_router_get_model_types(void)
{
    return(&dragonfly_model_types[1]);
}

static void dragonfly_register_model_types(st_model_types *base_type)
{
    st_model_type_register(LP_CONFIG_NM_TERM, base_type);
}

static void router_register_model_types(st_model_types *base_type)
{
    st_model_type_register(LP_CONFIG_NM_ROUT, base_type);
}
/*** END of ROSS event tracing additions */

/* returns the dragonfly lp type for lp registration */
static const tw_lptype* dragonfly_get_cn_lp_type(void)
{
	   return(&dragonfly_lps[0]);
}
static const tw_lptype* router_get_lp_type(void)
{
    return (&dragonfly_lps[1]);
}

static void dragonfly_register(tw_lptype *base_type) {
    lp_type_register(LP_CONFIG_NM_TERM, base_type);
}

static void router_register(tw_lptype *base_type) {
    lp_type_register(LP_CONFIG_NM_ROUT, base_type);
}

/* data structure for dragonfly statistics */
struct model_net_method dragonfly_method =
{
    .mn_configure = dragonfly_configure,
    .mn_register = dragonfly_register,
    .model_net_method_packet_event = dragonfly_packet_event,
    .model_net_method_packet_event_rc = dragonfly_packet_event_rc,
    .model_net_method_recv_msg_event = NULL,
    .model_net_method_recv_msg_event_rc = NULL,
    .mn_get_lp_type = dragonfly_get_cn_lp_type,
    .mn_get_msg_sz = dragonfly_get_msg_sz,
    .mn_report_stats = dragonfly_report_stats,
    .mn_collective_call = dragonfly_collective,
    .mn_collective_call_rc = dragonfly_collective_rc,   
    .mn_sample_fn = (void*)dragonfly_sample_fn,    
    .mn_sample_rc_fn = (void*)dragonfly_sample_rc_fn,
    .mn_sample_init_fn = (void*)dragonfly_sample_init,
    .mn_sample_fini_fn = (void*)dragonfly_sample_fin,
    .mn_model_stat_register = dragonfly_register_model_types,
    .mn_get_model_stat_types = dragonfly_get_model_types,
};

struct model_net_method dragonfly_router_method =
{
    .mn_configure = NULL, // handled by dragonfly_configure
    .mn_register  = router_register,
    .model_net_method_packet_event = NULL,
    .model_net_method_packet_event_rc = NULL,
    .model_net_method_recv_msg_event = NULL,
    .model_net_method_recv_msg_event_rc = NULL,
    .mn_get_lp_type = router_get_lp_type,
    .mn_get_msg_sz = dragonfly_get_msg_sz,
    .mn_report_stats = NULL, // not yet supported
    .mn_collective_call = NULL,
    .mn_collective_call_rc = NULL,
    .mn_sample_fn = (void*)dragonfly_rsample_fn,
    .mn_sample_rc_fn = (void*)dragonfly_rsample_rc_fn,
    .mn_sample_init_fn = (void*)dragonfly_rsample_init,
    .mn_sample_fini_fn = (void*)dragonfly_rsample_fin,
    .mn_model_stat_register = router_register_model_types,
    .mn_get_model_stat_types = dfly_router_get_model_types,
};

#ifdef ENABLE_CORTEX

static int dragonfly_get_number_of_compute_nodes(void* topo) {
		// TODO
        (void)topo;
		return -1;
}

static int dragonfly_get_number_of_routers(void* topo) {
		// TODO
        (void)topo;
		return -1;
}

static double dragonfly_get_router_link_bandwidth(void* topo, router_id_t r1, router_id_t r2) {
        // TODO
        (void)topo;
        (void)r2;
        (void)r1;
        return -1.0;
}

static double dragonfly_get_compute_node_bandwidth(void* topo, cn_id_t node) {
        // TODO
        (void)topo;
        (void)node;
        return -1.0;
}

static int dragonfly_get_router_neighbor_count(void* topo, router_id_t r) {
        // TODO
        (void)topo;
        (void)r;
        return 0;
}

static void dragonfly_get_router_neighbor_list(void* topo, router_id_t r, router_id_t* neighbors) {
        // TODO
        (void)topo;
        (void)r;
        (void)neighbors;
}

static int dragonfly_get_router_location(void* topo, router_id_t r, int32_t* location, int size) {
        // TODO
        (void)topo;
        (void)r;
        (void)location;
        (void)size;
        return 0;
}

static int dragonfly_get_compute_node_location(void* topo, cn_id_t node, int32_t* location, int size) {
        // TODO
        (void)topo;
        (void)node;
        (void)location;
        (void)size;
        return 0;
}

static router_id_t dragonfly_get_router_from_compute_node(void* topo, cn_id_t node) {
        // TODO
        (void)topo;
        (void)node;
        return -1;
}

static int dragonfly_get_router_compute_node_count(void* topo, router_id_t r) {
        // TODO
        (void)topo;
        (void)r;
        return 0;
}

static void dragonfly_get_router_compute_node_list(void* topo, router_id_t r, cn_id_t* nodes) {
        // TODO
        (void)topo;
        (void)r;
        (void)nodes;
}

cortex_topology dragonfly_cortex_topology = {
        .internal = NULL,
		.get_number_of_routers			= dragonfly_get_number_of_routers,
		.get_number_of_compute_nodes	= dragonfly_get_number_of_compute_nodes,
        .get_router_link_bandwidth      = dragonfly_get_router_link_bandwidth,
        .get_compute_node_bandwidth     = dragonfly_get_compute_node_bandwidth,
        .get_router_neighbor_count      = dragonfly_get_router_neighbor_count,
        .get_router_neighbor_list       = dragonfly_get_router_neighbor_list,
        .get_router_location            = dragonfly_get_router_location,
        .get_compute_node_location      = dragonfly_get_compute_node_location,
        .get_router_from_compute_node   = dragonfly_get_router_from_compute_node,
        .get_router_compute_node_count  = dragonfly_get_router_compute_node_count,
        .get_router_compute_node_list   = dragonfly_get_router_compute_node_list,
};

#endif


<|start_filename|>src/workload/codes-workload.c<|end_filename|>
/*
 * Copyright (C) 2013 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#include <assert.h>

#include <ross.h>
#include <codes/codes-workload.h>
#include <codes/codes.h>

/* list of available methods.  These are statically compiled for now, but we
 * could make generators optional via autoconf tests etc. if needed
 */
/* added by pj: differ POSIX and MPI IO in darshan 3.00*/
#define	DARSHAN_POSIX_IO 	1
#define	DARSHAN_MPI_IO		0

extern struct codes_workload_method test_workload_method;
extern struct codes_workload_method iolang_workload_method;
#ifdef USE_DUMPI
extern struct codes_workload_method dumpi_trace_workload_method;
#endif

#ifdef USE_DARSHAN
#if DARSHAN_POSIX_IO
extern struct codes_workload_method darshan_posix_io_workload_method;
#elif DARSHAN_MPI_IO
extern struct codes_workload_method darshan_mpi_io_workload_method;
#endif
#endif

#ifdef USE_RECORDER
extern struct codes_workload_method recorder_io_workload_method;
#endif
#ifdef USE_ONLINE
extern struct codes_workload_method online_comm_workload_method;
#endif
extern struct codes_workload_method checkpoint_workload_method;
extern struct codes_workload_method iomock_workload_method;

static struct codes_workload_method const * method_array_default[] =
{
    &test_workload_method,
    &iolang_workload_method,
#ifdef USE_DUMPI
    &dumpi_trace_workload_method,
#endif

#ifdef USE_DARSHAN
/* added by pj: posix and mpi io */
#if	DARSHAN_POSIX_IO
    &darshan_posix_io_workload_method,
#elif DARNSHAN_MPI_IO
	/* TODO: MPI_IO */
	&darshan_mpi_io_workload_method,
#endif

#endif
#ifdef USE_ONLINE
    &online_comm_workload_method,
#endif
#ifdef USE_RECORDER
    &recorder_io_workload_method,
#endif
    &checkpoint_workload_method,
    &iomock_workload_method,
    NULL
};

// once initialized, adding a workload generator is an error
static int is_workloads_init = 0;
static int num_user_methods = 0;
static struct codes_workload_method const ** method_array = NULL;

/* This shim layer is responsible for queueing up reversed operations and
 * re-issuing them so that the underlying workload generator method doesn't
 * have to worry about reverse events.
 *
 * NOTE: we could make this faster with a smarter data structure.  For now
 * we just have a linked list of rank_queue structs, one per rank that has
 * opened the workload.  We then have a linked list off of each of those
 * to hold a lifo queue of operations that have been reversed for that rank.
 */

/* holds an operation that has been reversed */
struct rc_op
{
    struct codes_workload_op op;
    struct rc_op* next;
};

/* tracks lifo queue of reversed operations for a given rank */
struct rank_queue
{
    int app;
    int rank;
    struct rc_op *lifo;
    struct rank_queue *next;
};

static struct rank_queue *ranks = NULL;

// only call this once
static void init_workload_methods(void)
{
    if (is_workloads_init)
        return;
    if (method_array == NULL)
        method_array = method_array_default;
    else {
        // note - includes null char
        int num_default_methods =
            (sizeof(method_array_default) / sizeof(method_array_default[0]));
        printf("\n Num default methods %d ", num_default_methods);
        method_array = realloc(method_array,
                (num_default_methods + num_user_methods + 1) *
                sizeof(*method_array));
        memcpy(method_array+num_user_methods, method_array_default,
                num_default_methods * sizeof(*method_array_default));
    }
    is_workloads_init = 1;
}

codes_workload_config_return codes_workload_read_config(
        ConfigHandle * handle,
        char const * section_name,
        char const * annotation,
        int num_ranks)
{
    init_workload_methods();

    char type[MAX_NAME_LENGTH_WKLD];
    codes_workload_config_return r;
    r.type = NULL;
    r.params = NULL;

    int rc = configuration_get_value(handle, section_name, "workload_type",
            annotation, type, MAX_NAME_LENGTH_WKLD);
    if (rc <= 0)
        return r;

    for (int i = 0; method_array[i] != NULL; i++){
        struct codes_workload_method const * m = method_array[i];
        if (strcmp(m->method_name, type) == 0) {
            r.type = m->method_name;
            if (m->codes_workload_read_config == NULL)
                r.params = NULL;
            else
                r.params = m->codes_workload_read_config(handle, section_name,
                        annotation, num_ranks);
        }
    }

    return r;
}

void codes_workload_free_config_return(codes_workload_config_return *c)
{
    free(c->params);
    c->type = NULL;
    c->params = NULL;
}

int codes_workload_load(
        const char* type,
        const char* params,
        int app_id,
        int rank)
{
    init_workload_methods();

    int i;
    int ret;
    struct rank_queue *tmp;

    for(i=0; method_array[i] != NULL; i++)
    {
        if(strcmp(method_array[i]->method_name, type) == 0)
        {
            /* load appropriate workload generator */
            ret = method_array[i]->codes_workload_load(params, app_id, rank);
            if(ret < 0)
            {
                return(-1);
            }

            /* are we tracking information for this rank yet? */
            tmp = ranks;
            while(tmp)
            {
                if(tmp->rank == rank && tmp->app == app_id)
                    break;
                tmp = tmp->next;
            }
            if(tmp == NULL)
            {
                tmp = (struct rank_queue*)malloc(sizeof(*tmp));
                assert(tmp);
                tmp->app  = app_id;
                tmp->rank = rank;
                tmp->lifo = NULL;
                tmp->next = ranks;
                ranks = tmp;
            }

            return(i);
        }
    }

    fprintf(stderr, "Error: failed to find workload generator %s\n", type);
    return(-1);
}

void codes_workload_get_next(
        int wkld_id,
        int app_id,
        int rank,
        struct codes_workload_op *op)
{
    struct rank_queue *tmp;
    struct rc_op *tmp_op;

    /* first look to see if we have a reversed operation that we can
     * re-issue
     */
    tmp = ranks;
    while(tmp)
    {
        if(tmp->rank == rank && tmp->app == app_id)
            break;
        tmp = tmp->next;
    }
    if(tmp==NULL)
        printf("tmp is NULL, rank=%d, app_id = %d", rank, app_id);
    assert(tmp);
    if(tmp->lifo)
    {
        tmp_op = tmp->lifo;
        tmp->lifo = tmp_op->next;

        *op = tmp_op->op;
        free(tmp_op);
        return;
    }

    method_array[wkld_id]->codes_workload_get_next(app_id, rank, op);

    assert(op->op_type);
    return;
}

void codes_workload_get_next_rc(
        int wkld_id,
        int app_id,
        int rank,
        const struct codes_workload_op *op)
{
    (void)wkld_id; // currently unused
    struct rank_queue *tmp;
    struct rc_op *tmp_op;

    tmp = ranks;
    while(tmp)
    {
        if(tmp->rank == rank && tmp->app == app_id)
            break;
        tmp = tmp->next;
    }
    assert(tmp);

    tmp_op = (struct rc_op*)malloc(sizeof(struct rc_op));
    assert(tmp_op);
    tmp_op->op = *op;
    tmp_op->next = tmp->lifo;
    tmp->lifo = tmp_op;

    return;
}

void codes_workload_get_next_rc2(
                int wkld_id,
                int app_id,
                int rank)
{
    assert(method_array[wkld_id]->codes_workload_get_next_rc2);
    method_array[wkld_id]->codes_workload_get_next_rc2(app_id, rank);
}

/* Finalize the workload */
int codes_workload_finalize(
        const char* type,
        const char* params,
        int app_id, 
        int rank)
{
    int i;

    for(i=0; method_array[i] != NULL; i++)
    {
        if(strcmp(method_array[i]->method_name, type) == 0)
        {
                return method_array[i]->codes_workload_finalize(
                        params, app_id, rank);
        }
    }

    fprintf(stderr, "Error: failed to find workload generator %s\n", type);
    return(-1);
}
int codes_workload_get_time(const char *type, const char *params, int app_id,
		int rank, double *read_time, double *write_time, int64_t *read_bytes, int64_t *written_bytes)
{
	int i;
	init_workload_methods();

	//printf("entering rank count, method_array = %p \n", method_array);
	for(i=0; method_array[i] != NULL; i++)
	{
		//printf("%p\n", method_array[i]);
		//printf(" geting time:: method_array[%d]->method_name = %s, type = %s\n", i, method_array[i]->method_name, type);
		if(strcmp(method_array[i]->method_name, type) == 0)
		{
			if (method_array[i]->codes_workload_get_time != NULL)
				return method_array[i]->codes_workload_get_time(
						params, app_id, rank, read_time, write_time, read_bytes, written_bytes);
			else
				return -1;
		}
	}
	return 0;
}

int codes_workload_get_rank_cnt(
        const char* type,
        const char* params,
        int app_id)
{
    int i;
    init_workload_methods();

    //printf("entering rank count, method_array = %p \n", method_array);
    for(i=0; method_array[i] != NULL; i++)
    {
    	//printf("%p\n", method_array[i]);
    	//printf("method_array[%d]->method_name = %s, type = %s\n", i, method_array[i]->method_name, type);
        if(strcmp(method_array[i]->method_name, type) == 0)
        {
            if (method_array[i]->codes_workload_get_rank_cnt != NULL)
                return method_array[i]->codes_workload_get_rank_cnt(
                        params, app_id);
            else
                return -1;
        }
    }

    fprintf(stderr, "Error: failed to find workload generator %s\n", type);
    return(-1);
}

void codes_workload_print_op(
        FILE *f,
        struct codes_workload_op *op,
        int app_id,
        int rank)
{
    char *name;

    switch(op->op_type){
        case CODES_WK_END:
            fprintf(f, "op: app:%d rank:%d type:end\n", app_id, rank);
            break;
        case CODES_WK_DELAY:
            fprintf(f, "op: app:%d rank:%d type:delay seconds:%lf\n",
                    app_id, rank, op->u.delay.seconds);
            break;
        case CODES_WK_BARRIER:
            fprintf(f, "op: app:%d rank:%d type:barrier count:%d root:%d\n",
                    app_id, rank, op->u.barrier.count, op->u.barrier.root);
            break;
        case CODES_WK_OPEN:
        case CODES_WK_MPI_OPEN:
        case CODES_WK_MPI_COLL_OPEN:
            if(op->op_type == CODES_WK_OPEN) name = "open";
            if(op->op_type == CODES_WK_MPI_OPEN) name = "mpi_open";
            if(op->op_type == CODES_WK_MPI_COLL_OPEN) name = "mpi_coll_open";
            fprintf(f, "op: app:%d rank:%d type:%s file_id:%llu flag:%d\n",
                    app_id, rank, name, LLU(op->u.open.file_id), op->u.open.create_flag);
            break;
        case CODES_WK_CLOSE:
        case CODES_WK_MPI_CLOSE:
            if(op->op_type == CODES_WK_CLOSE) name = "close";
            if(op->op_type == CODES_WK_MPI_CLOSE) name = "mpi_close";
            fprintf(f, "op: app:%d rank:%d type:%s file_id:%llu\n",
                    app_id, rank, name, LLU(op->u.close.file_id));
            break;
        case CODES_WK_WRITE:
        case CODES_WK_MPI_WRITE:
        case CODES_WK_MPI_COLL_WRITE:
            if(op->op_type == CODES_WK_WRITE) name = "write";
            if(op->op_type == CODES_WK_MPI_WRITE) name = "mpi_write";
            if(op->op_type == CODES_WK_MPI_COLL_WRITE) name = "mpi_coll_write";
            fprintf(f, "op: app:%d rank:%d type:%s "
                       "file_id:%llu off:%llu size:%llu\n",
                    app_id, rank, name, LLU(op->u.write.file_id), LLU(op->u.write.offset),
                    LLU(op->u.write.size));
            break;
        case CODES_WK_READ:
        case CODES_WK_MPI_READ:
        case CODES_WK_MPI_COLL_READ:
            if(op->op_type == CODES_WK_READ) name = "read";
            if(op->op_type == CODES_WK_MPI_READ) name = "mpi_read";
            if(op->op_type == CODES_WK_MPI_COLL_READ) name = "mpi_coll_read";
            fprintf(f, "op: app:%d rank:%d type:%s "
                       "file_id:%llu off:%llu size:%llu\n",
                    app_id, rank, name, LLU(op->u.read.file_id), LLU(op->u.read.offset),
                    LLU(op->u.read.size));
            break;
        case CODES_WK_SEND:
            fprintf(f, "op: app:%d rank:%d type:send "
                    "src:%d dst:%d bytes:%"PRIu64" type:%d count:%d tag:%d "
                    "start:%.5e end:%.5e\n",
                    app_id, rank,
                    op->u.send.source_rank, op->u.send.dest_rank,
                    op->u.send.num_bytes, op->u.send.data_type,
                    op->u.send.count, op->u.send.tag,
                    op->start_time, op->end_time);
            break;
        case CODES_WK_RECV:
            fprintf(f, "op: app:%d rank:%d type:recv "
                    "src:%d dst:%d bytes:%"PRIu64" type:%d count:%d tag:%d "
                    "start:%.5e end:%.5e\n",
                    app_id, rank,
                    op->u.recv.source_rank, op->u.recv.dest_rank,
                    op->u.recv.num_bytes, op->u.recv.data_type,
                    op->u.recv.count, op->u.recv.tag,
                    op->start_time, op->end_time);
            break;
        case CODES_WK_ISEND:
            fprintf(f, "op: app:%d rank:%d type:isend "
                    "src:%d dst:%d req_id:%"PRIu32" bytes:%"PRIu64" type:%d count:%d tag:%d "
                    "start:%.5e end:%.5e\n",
                    app_id, rank,
                    op->u.send.source_rank, op->u.send.dest_rank,
                    op->u.send.req_id,
                    op->u.send.num_bytes, op->u.send.data_type,
                    op->u.send.count, op->u.send.tag,
                    op->start_time, op->end_time);
            break;
        case CODES_WK_IRECV:
            fprintf(f, "op: app:%d rank:%d type:irecv "
                    "src:%d dst:%d req_id:%"PRIu32" bytes:%"PRIu64" type:%d count:%d tag:%d "
                    "start:%.5e end:%.5e\n",
                    app_id, rank,
                    op->u.recv.source_rank, op->u.recv.dest_rank,
                    op->u.recv.req_id,
                    op->u.recv.num_bytes, op->u.recv.data_type,
                    op->u.recv.count, op->u.recv.tag,
                    op->start_time, op->end_time);
            break;
       case CODES_WK_REQ_FREE:
            fprintf(f, "op: app:%d rank:%d type:req free "
                    " req:%d ",
                    app_id, rank,
                    op->u.free.req_id);
            break;
#define PRINT_COL(_type_str) \
            fprintf(f, "op: app:%d rank:%d type:%s" \
                    " bytes:%d, start:%.5e, end:%.5e\n", app_id, rank, \
                    _type_str, op->u.collective.num_bytes, op->start_time, \
                    op->end_time)
        case CODES_WK_BCAST:
            PRINT_COL("bcast");
            break;
        case CODES_WK_ALLGATHER:
            PRINT_COL("allgather");
            break;
        case CODES_WK_ALLGATHERV:
            PRINT_COL("allgatherv");
            break;
        case CODES_WK_ALLTOALL:
            PRINT_COL("alltoall");
            break;
        case CODES_WK_ALLTOALLV:
            PRINT_COL("alltoallv");
            break;
        case CODES_WK_REDUCE:
            PRINT_COL("reduce");
            break;
        case CODES_WK_ALLREDUCE:
            PRINT_COL("allreduce");
            break;
        case CODES_WK_COL:
            PRINT_COL("collective");
            break;
#undef PRINT_COL
#define PRINT_WAIT(_type_str, _ct) \
            fprintf(f, "op: app:%d rank:%d type:%s" \
                    "num reqs:%d, start:%.5e, end:%.5e\n", \
                    app_id, rank, _type_str, _ct, op->start_time, op->end_time)
        case CODES_WK_WAITALL:
            PRINT_WAIT("waitall", op->u.waits.count);
            break;
        case CODES_WK_WAIT:
            PRINT_WAIT("wait", 1);
            break;
        case CODES_WK_WAITSOME:
            PRINT_WAIT("waitsome", op->u.waits.count);
            break;
        case CODES_WK_WAITANY:
            PRINT_WAIT("waitany", op->u.waits.count);
            break;
        case CODES_WK_IGNORE:
            break;
        default:
            fprintf(stderr,
                    "%s:%d: codes_workload_print_op: unrecognized workload type "
                    "(op code %d)\n", __FILE__, __LINE__, op->op_type);
    }
}

void codes_workload_add_method(struct codes_workload_method const * method)
{
    static int method_array_cap = 10;
    if (is_workloads_init)
        tw_error(TW_LOC,
                "adding a workload method after initialization is forbidden");
    else if (method_array == NULL){
        method_array = malloc(method_array_cap * sizeof(*method_array));
        assert(method_array);
    }

    if (num_user_methods == method_array_cap) {
        method_array_cap *= 2;
        method_array = realloc(method_array,
                method_array_cap * sizeof(*method_array));
        assert(method_array);
    }

    method_array[num_user_methods++] = method;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 *  indent-tabs-mode: nil
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */


<|start_filename|>src/workload/codes-workload-dump.c<|end_filename|>
/*
 * Copyright (C) 2014 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

#include <assert.h>
#include <getopt.h>
#include <stdio.h>
#include <codes/codes-workload.h>
#include <codes/codes.h>
#include <inttypes.h>

static char type[128] = {'\0'};
static darshan_params d_params = {"", 0}; 
static iolang_params i_params = {0, 0, "", ""};
static recorder_params r_params = {"", 0};
static dumpi_trace_params du_params = {"", 0, 0};
static online_comm_params oc_params = {"", "", 0};
static checkpoint_wrkld_params c_params = {0, 0, 0, 0, 0};
static iomock_params im_params = {0, 0, 1, 0, 0, 0};
static int n = -1;
static int start_rank = 0;

static struct option long_opts[] = 
{
    {"type", required_argument, NULL, 't'},
    {"num-ranks", required_argument, NULL, 'n'},
    {"start-rank", required_argument, NULL, 'r'},
    {"d-log", required_argument, NULL, 'l'},
    {"i-meta", required_argument, NULL, 'm'},
    {"i-use-relpath", no_argument, NULL, 'p'},
    {"r-trace-dir", required_argument, NULL, 'd'},
    {"r-nprocs", required_argument, NULL, 'x'},
    {"dumpi-log", required_argument, NULL, 'w'},
    {"workload-name", required_argument, NULL, 'b'},
    {"chkpoint-size", required_argument, NULL, 'S'},
    {"chkpoint-bw", required_argument, NULL, 'B'},
    {"chkpoint-iters", required_argument, NULL, 'i'},
    {"chkpoint-mtti", required_argument, NULL, 'M'},
    {"iomock-request-type", required_argument, NULL, 'Q'},
    {"iomock-num-requests", required_argument, NULL, 'N'},
    {"iomock-request-size", required_argument, NULL, 'z'},
    {"iomock-file-id", required_argument, NULL, 'f'},
    {"iomock-use-uniq-file-ids", no_argument, NULL, 'u'},
    {NULL, 0, NULL, 0}
};

void usage(){
    fputs(
            "Usage: codes-workload-dump --type TYPE --num-ranks N [OPTION...]\n"
            "--type: type of workload (\"darshan_io_workload\", \"iolang_workload\", dumpi-trace-workload\" etc.)\n"
            "--num-ranks: number of ranks to process (if not set, it is set by the workload)\n"
            "-s: print final workload stats\n"
            "DARSHAN OPTIONS (darshan_io_workload)\n"
            "--d-log: darshan log file\n"
            "IOLANG OPTIONS (iolang_workload)\n"
            "--i-meta: i/o language kernel meta file path\n"
            "--i-use-relpath: use i/o kernel path relative meta file path\n"
            "RECORDER OPTIONS (recorder_io_workload)\n"
            "--r-trace-dir: directory containing recorder trace files\n"
            "--r-nprocs: number of ranks in original recorder workload\n"
            "DUMPI TRACE OPTIONS (dumpi-trace-workload) \n"
            "--dumpi-log: dumpi log file \n"
            "ONLINE COMM OPTIONS (online_comm_workload) \n"
            "--workload-name : name of the workload (lammps or nekbone) \n"
            "CHECKPOINT OPTIONS (checkpoint_io_workload)\n"
            "--chkpoint-size: size of aggregate checkpoint to write\n"
            "--chkpoint-bw: checkpointing bandwidth\n"
            "--chkpoint-iters: iteration count for checkpoint workload\n"
            "--chkpoint-mtti: mean time to interrupt\n"
            "MOCK IO OPTIONS (iomock_workload)\n"
            "--iomock-request-type: whether to write or read\n"
            "--iomock-num-requests: number of writes/reads\n"
            "--iomock-request-size: size of each request\n"
            "--iomock-file-id: file id to use for requests\n"
            "--iomock-use-uniq-file-ids: whether to offset file ids by rank\n",
            stderr
            );
}

int main(int argc, char *argv[])
{
#ifdef USE_ONLINE
    ABT_init(argc, argv);
#endif
    int print_stats = 0;
    double total_delay = 0.0;
    int64_t num_barriers = 0;
    int64_t num_opens = 0;
    int64_t num_closes = 0;
    int64_t num_reads = 0;
    int64_t read_size = 0;
    int64_t num_writes = 0;
    int64_t write_size = 0;
    int64_t num_sends = 0;
    int64_t num_frees = 0;
    int64_t send_size = 0;
    int64_t num_recvs = 0;
    int64_t recv_size = 0;
    int64_t num_isends = 0;
    int64_t isend_size = 0;
    int64_t num_irecvs = 0;
    int64_t irecv_size = 0;
    int64_t num_bcasts = 0;
    int64_t bcast_size = 0;
    int64_t num_allgathers = 0;
    int64_t allgather_size = 0;
    int64_t num_allgathervs = 0;
    int64_t allgatherv_size = 0;
    int64_t num_alltoalls = 0;
    int64_t alltoall_size = 0;
    int64_t num_alltoallvs = 0;
    int64_t alltoallv_size = 0;
    int64_t num_reduces = 0;
    int64_t reduce_size = 0;
    int64_t num_allreduces = 0;
    int64_t allreduce_size = 0;
    int64_t num_collectives = 0;
    int64_t collective_size = 0;
    int64_t num_waitalls = 0;
    int64_t num_waits = 0;
    int64_t num_waitsomes = 0;
    int64_t num_waitanys = 0;
    int64_t num_testalls = 0;

    char ch;
    while ((ch = getopt_long(argc, argv, "t:n:l:b:a:m:sp:wr:S:B:R:M:Q:N:z:f:u",
                    long_opts, NULL)) != -1){
        switch (ch){
            case 't':
                strcpy(type, optarg);
                break;
            case 'n':
                n = atoi(optarg);
                assert(n>0);
                break;
            case 'l':
                strcpy(d_params.log_file_path, optarg);
                break;
            case 'b':
                strcpy(oc_params.workload_name, optarg);
                break;
            case 'm':
                strcpy(i_params.io_kernel_meta_path, optarg);
                break;
            case 'p':
                i_params.use_relpath = 1;
                break;
            case 'd':
                strcpy(r_params.trace_dir_path, optarg);
                break;
            case 'x':
                r_params.nprocs = atol(optarg);
                break;
            case 'w':
                strcpy(du_params.file_name, optarg);
                break;
            case 's':
                print_stats = 1;
                break;
            case 'r':
                start_rank = atoi(optarg);
                assert(n>0);
                break;
            case 'S':
                c_params.checkpoint_sz = atof(optarg);
                break;
            case 'B':
                c_params.checkpoint_wr_bw = atof(optarg);
                break;
            case 'i':
                c_params.total_checkpoints = atoi(optarg);
                break;
            case 'M':
                c_params.mtti = atof(optarg);
                break;
            case 'Q':
                im_params.is_write = (strcmp("write", optarg) == 0);
                break;
            case 'N':
                im_params.num_requests = atoi(optarg);
                break;
            case 'z':
                im_params.request_size = atoi(optarg);
                break;
            case 'f':
                im_params.file_id = (uint64_t) atoll(optarg);
                break;
            case 'u':
                im_params.use_uniq_file_ids = 1;
                break;
        }
    }

    if (type[0] == '\0'){
        fprintf(stderr, "Expected \"--type\" argument\n");
        usage();
        return 1;
    }

    int i;
    char *wparams;
    if (strcmp(type, "iomock_workload") == 0) {
        // TODO: more involved input checking
        wparams = (char*) &im_params;
    }
    else if (strcmp(type, "darshan_io_workload") == 0){
        if (d_params.log_file_path[0] == '\0'){
            fprintf(stderr, "Expected \"--d-log\" argument for darshan workload\n");
            usage();
            return 1;
        }
        else{
            wparams = (char*)&d_params;
        }
    }
    else if(strcmp(type, "online_comm_workload") == 0){
        if (n == -1){
            fprintf(stderr,
                    "Expected \"--num-ranks\" argument for online workload\n");
            usage();
            return 1;
        }
        else{
            oc_params.nprocs = n;
        }
        wparams = (char *)&oc_params;
    }
    else if (strcmp(type, "iolang_workload") == 0){
        if (n == -1){
            fprintf(stderr,
                    "Expected \"--num-ranks\" argument for iolang workload\n");
            usage();
            return 1;
        }
        else{
            i_params.num_cns = n;
        }
        if (i_params.io_kernel_meta_path[0] == '\0'){
            fprintf(stderr,
                    "Expected \"--i-meta\" argument for iolang workload\n");
            usage();
            return 1;
        }

        wparams = (char *)&i_params;
    }
    else if (strcmp(type, "recorder_io_workload") == 0){
        if (r_params.trace_dir_path[0] == '\0'){
            fprintf(stderr, "Expected \"--r-trace-dir\" argument for recorder workload\n");
            usage();
            return 1;
        }
        if (r_params.nprocs == 0){
            fprintf(stderr, "Expected \"--r-nprocs\" argument for recorder workload\n");
            usage();
            return 1;
        }
        else{
            wparams = (char *)&r_params;
        }
    }
   else if(strcmp(type, "dumpi-trace-workload") == 0)
	{
	if(n == -1){
            fprintf(stderr,
                    "Expected \"--num-ranks\" argument for dumpi workload\n");
            usage();
            return 1;		
	}
	else{
	    du_params.num_net_traces = n;	
	}

	if(du_params.file_name[0] == '\0' ){
            fprintf(stderr, "Expected \"--dumpi-log\" argument for dumpi workload\n");
            usage();
            return 1;
	}
	else
	{
	  wparams = (char*)&du_params;
	}
	}
    else if(strcmp(type, "checkpoint_io_workload") == 0)
    {
        if(c_params.checkpoint_sz == 0 || c_params.checkpoint_wr_bw == 0 ||
           c_params.total_checkpoints == 0 || c_params.mtti == 0)
        {
            fprintf(stderr, "All checkpoint workload arguments are required\n");
            usage();
            return 1;
        }
        else
        {
            c_params.nprocs = n;
            wparams = (char *)&c_params;
        }
    }
    else {
        fprintf(stderr, "Invalid type argument\n");
        usage();
        return 1;
    }

    /* if num_ranks not set, pull it from the workload */
    if (n == -1){
    	//printf("Getting rank count\n");
        n = codes_workload_get_rank_cnt(type, wparams, 0);
        if (n == -1) {
            fprintf(stderr,
                    "Unable to get rank count from workload. "
                    "Specify option --num-ranks\n");
            return 1;
        }
        printf("rank count = %d\n", n);
    }

    for (i = start_rank ; i < start_rank+n; i++){
        struct codes_workload_op op;
        //printf("loading %s, %d\n", type, i);
        int id = codes_workload_load(type, wparams, 0, i);
        double total_read_time = 0.0, total_write_time = 0.0;
        int64_t total_read_bytes = 0, total_written_bytes = 0;
        codes_workload_get_time(type, wparams, 0, i, &total_read_time, &total_write_time, &total_read_bytes, &total_written_bytes);
        printf("total_read_time = %f, total_write_time = %f\n", total_read_time, total_write_time);
        assert(id != -1);
        do {
            codes_workload_get_next(id, 0, i, &op);
//            codes_workload_print_op(stdout, &op, 0, i);

            switch(op.op_type)
            {
                case CODES_WK_DELAY:
                    total_delay += op.u.delay.seconds;
                    break;
                case CODES_WK_BARRIER:
                    num_barriers++;
                    break;
                case CODES_WK_OPEN:
                case CODES_WK_MPI_OPEN:
                case CODES_WK_MPI_COLL_OPEN:
                    num_opens++;
                    break;
                case CODES_WK_CLOSE:
                case CODES_WK_MPI_CLOSE:
                    num_closes++;
                    break;
                case CODES_WK_WRITE:
                case CODES_WK_MPI_WRITE:
                case CODES_WK_MPI_COLL_WRITE:
                    num_writes++;
                    write_size += op.u.write.size;
                    break;
                case CODES_WK_READ:
                case CODES_WK_MPI_READ:
                case CODES_WK_MPI_COLL_READ:
                    num_reads++;
                    read_size += op.u.write.size;
                    break;
                case CODES_WK_SEND:
                    num_sends++;
                    send_size += op.u.send.num_bytes;
                    break;
                case CODES_WK_REQ_FREE:
                    num_frees++;
                    break;

                case CODES_WK_RECV:
                    num_recvs++;
                    recv_size += op.u.recv.num_bytes;
                    break;
                case CODES_WK_ISEND:
                    num_isends++;
                    isend_size += op.u.send.num_bytes;
                    break;
                case CODES_WK_IRECV:
                    num_irecvs++;
                    irecv_size += op.u.recv.num_bytes;
                    break;
                /* NOTE: all collectives are currently represented as the
                 * generic "collective" type */
                case CODES_WK_BCAST:
                    num_bcasts++;
                    bcast_size += op.u.collective.num_bytes;
                    break;
                case CODES_WK_ALLGATHER:
                    num_allgathers++;
                    allgather_size += op.u.collective.num_bytes;
                    break;
                case CODES_WK_ALLGATHERV:
                    num_allgathervs++;
                    allgatherv_size += op.u.collective.num_bytes;
                    break;
                case CODES_WK_ALLTOALL:
                    num_alltoalls++;
                    alltoall_size += op.u.collective.num_bytes;
                    break;
                case CODES_WK_ALLTOALLV:
                    num_alltoallvs++;
                    alltoallv_size += op.u.collective.num_bytes;
                    break;
                case CODES_WK_REDUCE:
                    num_reduces++;
                    reduce_size += op.u.collective.num_bytes;
                    break;
                case CODES_WK_ALLREDUCE:
                    num_allreduces++;
                    allreduce_size += op.u.collective.num_bytes;
                    break;
                case CODES_WK_COL:
                    num_collectives++;
                    collective_size += op.u.collective.num_bytes;
                    break;
                case CODES_WK_WAITALL:
                    {
                        if(i == 0)
                        {
							int j;
							printf("\n rank %d wait_all: ", i);
							for(j = 0; j < op.u.waits.count; j++)
								printf(" %d ", op.u.waits.req_ids[j]);
							num_waitalls++;
                        }
                    }
                    break;
                case CODES_WK_WAIT:
                    num_waits++;
                    break;
                case CODES_WK_WAITSOME:
                    num_waitsomes++;
                    break;
                case CODES_WK_WAITANY:
                    num_waitanys++;
                    break;
                case CODES_WK_TESTALL:
                    num_testalls++;
                    break;
                case CODES_WK_END:
                    break;
                case CODES_WK_IGNORE:
                    break;
                default:
                    fprintf(stderr,
                            "WARNING: unknown workload op type (code %d)\n",
                            op.op_type);
            }
        } while (op.op_type != CODES_WK_END);

    if(strcmp(type, "online_comm_workload") == 0)
    {
        codes_workload_finalize(type, wparams, 0, i);
    }
    }

    if (print_stats)
    {
        fprintf(stderr, "\n* * * * * FINAL STATS * * * * * *\n");
        fprintf(stderr, "NUM_OPENS:       %"PRId64"\n", num_opens);
        fprintf(stderr, "NUM_CLOSES:      %"PRId64"\n", num_closes);
        fprintf(stderr, "NUM_BARRIERS:    %"PRId64"\n", num_barriers);
        fprintf(stderr, "TOTAL_DELAY:     %.4lf\n", total_delay);
        fprintf(stderr, "NUM_READS:       %"PRId64"\n", num_reads);
        fprintf(stderr, "READ_SIZE:       %"PRId64"\n", read_size);
        fprintf(stderr, "NUM_WRITES:      %"PRId64"\n", num_writes);
        fprintf(stderr, "WRITE_SIZE:      %"PRId64"\n", write_size);
        fprintf(stderr, "NUM_SENDS:       %"PRId64"\n", num_sends);
        fprintf(stderr, "NUM_FREES:       %"PRId64"\n", num_frees);
        fprintf(stderr, "SEND_SIZE:       %"PRId64"\n", send_size);
        fprintf(stderr, "NUM_RECVS:       %"PRId64"\n", num_recvs);
        fprintf(stderr, "RECV_SIZE:       %"PRId64"\n", recv_size);
        fprintf(stderr, "NUM_ISENDS:      %"PRId64"\n", num_isends);
        fprintf(stderr, "ISEND_SIZE:      %"PRId64"\n", isend_size);
        fprintf(stderr, "NUM_IRECVS:      %"PRId64"\n", num_irecvs);
        fprintf(stderr, "IRECV_SIZE:      %"PRId64"\n", irecv_size);
        fprintf(stderr, "NUM_BCASTS:      %"PRId64"\n", num_bcasts);
        fprintf(stderr, "BCAST_SIZE:      %"PRId64"\n", bcast_size);
        fprintf(stderr, "NUM_ALLGATHERS:  %"PRId64"\n", num_allgathers);
        fprintf(stderr, "ALLGATHER_SIZE:  %"PRId64"\n", allgather_size);
        fprintf(stderr, "NUM_ALLGATHERVS: %"PRId64"\n", num_allgathervs);
        fprintf(stderr, "ALLGATHERV_SIZE: %"PRId64"\n", allgatherv_size);
        fprintf(stderr, "NUM_ALLTOALLS:   %"PRId64"\n", num_alltoalls);
        fprintf(stderr, "ALLTOALL_SIZE:   %"PRId64"\n", alltoall_size);
        fprintf(stderr, "NUM_ALLTOALLVS:  %"PRId64"\n", num_alltoallvs);
        fprintf(stderr, "ALLTOALLV_SIZE:  %"PRId64"\n", alltoallv_size);
        fprintf(stderr, "NUM_REDUCES:     %"PRId64"\n", num_reduces);
        fprintf(stderr, "REDUCE_SIZE:     %"PRId64"\n", reduce_size);
        fprintf(stderr, "NUM_ALLREDUCE:   %"PRId64"\n", num_allreduces);
        fprintf(stderr, "ALLREDUCE_SIZE:  %"PRId64"\n", allreduce_size);
        fprintf(stderr, "NUM_COLLECTIVE:  %"PRId64"\n", num_collectives);
        fprintf(stderr, "COLLECTIVE_SIZE: %"PRId64"\n", collective_size);
        fprintf(stderr, "NUM_WAITALLS:    %"PRId64"\n", num_waitalls);
        fprintf(stderr, "NUM_WAITS:       %"PRId64"\n", num_waits);
        fprintf(stderr, "NUM_WAITSOMES:   %"PRId64"\n", num_waitsomes);
        fprintf(stderr, "NUM_WAITANYS:    %"PRId64"\n", num_waitanys);
        fprintf(stderr, "NUM_TESTALLS:    %"PRId64"\n", num_testalls);
    }

#ifdef USE_ONLINE
    ABT_finalize();
#endif
    return 0;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */


<|start_filename|>doc/example/tutorial-synthetic-ping-pong.c<|end_filename|>
/*
 * Copyright (C) 2019 <NAME>
 * See LICENSE notice in top-level directory
 */

#include "codes/model-net.h"
#include "codes/lp-io.h"
#include "codes/codes.h"
#include "codes/codes_mapping.h"
#include "codes/configuration.h"
#include "codes/lp-type-lookup.h"


static int net_id = 0;
static int PAYLOAD_SZ = 4096;
static unsigned long long num_nodes = 0;

static char lp_io_dir[256] = {'\0'};
static lp_io_handle io_handle;
static unsigned int lp_io_use_suffix = 0;
static int do_lp_io = 0;

static int num_msgs = 20;

typedef struct svr_msg svr_msg;
typedef struct svr_state svr_state;

/* global variables for codes mapping */
static char group_name[MAX_NAME_LENGTH];
static char lp_type_name[MAX_NAME_LENGTH];
static int group_index, lp_type_index, rep_id, offset;

/* type of events */
enum svr_event
{
    KICKOFF = 1,
    PING,          
    PONG        
};

struct svr_msg
{
    enum svr_event svr_event_type; //KICKOFF, PING, or PONG
    int sender_id; //ID of the sender workload LP to know who to send a PONG message back to
    int payload_value; //Some value that we will encode as an example
    model_net_event_return event_rc; //helper to encode data relating to CODES rng usage
};

struct svr_state
{
    int svr_id;           /* the ID of this server */
    int ping_msg_sent_count;   /* PING messages sent */
    int ping_msg_recvd_count;  /* PING messages received */
    int pong_msg_sent_count;   /* PONG messages sent */
    int pong_msg_recvd_count; /* PONG messages received */
    tw_stime start_ts;    /* time that this LP started sending requests */
    tw_stime end_ts;      /* time that this LP ended sending requests */
    int payload_sum;      /* the running sum of all payloads received */
};

/* declaration of functions */
static void svr_init(svr_state * s, tw_lp * lp);
static void svr_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp);
static void svr_rev_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp);
static void svr_finalize(svr_state * s, tw_lp * lp);
static tw_stime ns_to_s(tw_stime ns);
static tw_stime s_to_ns(tw_stime s);

/* ROSS lptype function callback mapping */
tw_lptype svr_lp = {
    (init_f) svr_init,
    (pre_run_f) NULL,
    (event_f) svr_event,
    (revent_f) svr_rev_event,
    (commit_f) NULL,
    (final_f)  svr_finalize,
    (map_f) codes_mapping,
    sizeof(svr_state),
};

const tw_optdef app_opt [] =
{
        TWOPT_GROUP("Model net synthetic traffic " ),
    	TWOPT_UINT("num_messages", num_msgs, "Number of PING messages to be generated per terminal "),
    	TWOPT_UINT("payload_sz",PAYLOAD_SZ, "size of the message being sent "),
        TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"),
        TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"),
        TWOPT_END()
};

const tw_lptype* svr_get_lp_type()
{
    return(&svr_lp);
}

static void svr_add_lp_type()
{
  lp_type_register("nw-lp", svr_get_lp_type());
}

static void svr_init(svr_state * s, tw_lp * lp)
{
    //Initialize State
    s->ping_msg_sent_count = 0;
    s->ping_msg_recvd_count = 0;
    s->pong_msg_sent_count = 0;
    s->pong_msg_recvd_count = 0;
    s->start_ts = 0.0;
    s->end_ts = 0.0;
    s->svr_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0); /* turns the LP Global ID into the server ID */
    s->payload_sum = 0;

    //Now we create and send a self KICKOFF message - this is a PDES coordination event and thus doesn't need to be injected into the connected network
    //so we won't use model_net_event(), that's reserved for stuff we want to send across the network

    /* Set a time from now when this message is to be received by the recipient (self in this cae.) add some tiny random noise to help avoid event ties (different events with same timestamp) */
    //the lookahead value is a value required for conservative mode execution to work, it prevents scheduling a new event within the lookahead window
    tw_stime kickoff_time = g_tw_lookahead + (tw_rand_unif(lp->rng) * .0001);

    tw_event *e;
    svr_msg *m;
    e = tw_event_new(lp->gid, kickoff_time, lp); //ROSS method to create a new event
    m = tw_event_data(e); //Gives you a pointer to the data encoded within event e
    m->svr_event_type = KICKOFF; //Set the event type so we can know how to classify the event when received
    tw_event_send(e); //ROSS method to send off the event e with the encoded data in m
}

static void handle_kickoff_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp)
{
    s->start_ts = tw_now(lp); //the time when we're starting this LP's work is NOW

    svr_msg * ping_msg = malloc(sizeof(svr_msg)); //allocate memory for new message
    
    tw_lpid local_dest = -1; //ID of a sever, relative to only servers
    tw_lpid global_dest = -1; //ID of a server LP relative to ALL LPs

    //We want to make sure we're not accidentally picking ourselves
    local_dest = tw_rand_integer(lp->rng, 1, num_nodes - 2);
    local_dest = (s->svr_id + local_dest) % num_nodes;
    //local_dest is now a number [0,num_nodes) but is assuredly not s->svr_id
    assert(local_dest >= 0);
    assert(local_dest < num_nodes);
    assert(local_dest != s->svr_id);

    ping_msg->sender_id = s->svr_id; //encode our server ID into the new ping message
    ping_msg->svr_event_type = PING; //set it to type PING
    ping_msg->payload_value = tw_rand_integer(lp->rng, 1, 10); //encode a random payload value to it from [1,10]
    
    codes_mapping_get_lp_info(lp->gid, group_name, &group_index, lp_type_name, &lp_type_index, NULL, &rep_id, &offset); //gets information from CODES necessary to get the global LP ID of a server
    global_dest = codes_mapping_get_lpid_from_relative(local_dest, group_name, lp_type_name, NULL, 0);
    s->ping_msg_sent_count++;
    m->event_rc = model_net_event(net_id, "test", global_dest, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)ping_msg, 0, NULL, lp);
}

static void handle_kickoff_rev_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp)
{
    tw_rand_reverse_unif(lp->rng); //reverse the rng call for getting a local_dest
    tw_rand_reverse_unif(lp->rng); //reverse the rng call for creating a payload value;

    s->ping_msg_sent_count--; //undo the increment of the ping_msg_sent_count in the server state
    model_net_event_rc2(lp, &m->event_rc); //undo any model_net_event calls encoded into this message
}

static void handle_ping_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp)
{
    s->ping_msg_recvd_count++; //increment the counter for ping messages received

    int original_sender = m->sender_id; //this is the server we need to send a PONG message back to
    s->payload_sum += m->payload_value; //increment our running sum of payload values received

    svr_msg * pong_msg = malloc(sizeof(svr_msg)); //allocate memory for new message
    pong_msg->sender_id = s->svr_id;
    pong_msg->svr_event_type = PONG;
    // only ping messages contain a payload value - not every value in a message struct must be utilized by all messages!

    codes_mapping_get_lp_info(lp->gid, group_name, &group_index, lp_type_name, &lp_type_index, NULL, &rep_id, &offset); //gets information from CODES necessary to get the global LP ID of a server
    tw_lpid global_dest = codes_mapping_get_lpid_from_relative(original_sender, group_name, lp_type_name, NULL, 0);
    s->pong_msg_sent_count++;
    m->event_rc = model_net_event(net_id, "test", global_dest, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)pong_msg, 0, NULL, lp);
}

static void handle_ping_rev_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp)
{
    s->ping_msg_recvd_count--; //undo the increment of the counter for ping messages received
    s->payload_sum -= m->payload_value; //undo the increment of the payload sum

    model_net_event_rc2(lp, &m->event_rc); //undo any model_net_event calls encoded into this message
}

static void handle_pong_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp)
{
    s->pong_msg_recvd_count++; //increment the counter for ping messages received

    if(s->ping_msg_sent_count >= num_msgs) //if we've sent enough ping messages, then we stop and don't send any more
    {
        b->c1 = 1; //flag that we didn't really do anything in this event so that if this event gets reversed, we don't over-aggressively revert state or RNGs
        return;
    }

    //Now we need to send another ping message back to the sender of the pong
    int pong_sender = m->sender_id; //this is the sender of the PONG message that we want to send another PING message to
    
    svr_msg * ping_msg = malloc(sizeof(svr_msg)); //allocate memory for new message
    ping_msg->sender_id = s->svr_id; //encode our server ID into the new ping message
    ping_msg->svr_event_type = PING; //set it to type PING
    ping_msg->payload_value = tw_rand_integer(lp->rng, 1, 10); //encode a random payload value to it
    
    codes_mapping_get_lp_info(lp->gid, group_name, &group_index, lp_type_name, &lp_type_index, NULL, &rep_id, &offset); //gets information from CODES necessary to get the global LP ID of a server
    tw_lpid global_dest = codes_mapping_get_lpid_from_relative(pong_sender, group_name, lp_type_name, NULL, 0);
    s->ping_msg_sent_count++;
    m->event_rc = model_net_event(net_id, "test", global_dest, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)ping_msg, 0, NULL, lp);
}

static void handle_pong_rev_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp)
{
    s->pong_msg_recvd_count--; //undo the increment of the counter for ping messages received

    if (b->c1) //if we flipped the c1 flag in the forward event
        return; //then we don't need to undo any rngs or state change

    tw_rand_reverse_unif(lp->rng); //undo the rng for the new payload value
    s->ping_msg_sent_count--;
    model_net_event_rc2(lp, &m->event_rc); //undo any model_net_event calls encoded into this message
}

static void svr_finalize(svr_state * s, tw_lp * lp)
{
    s->end_ts = tw_now(lp);

    int total_msgs_sent = s->ping_msg_sent_count + s->pong_msg_sent_count;
    int total_msg_size_sent = PAYLOAD_SZ * total_msgs_sent;
    tw_stime time_in_seconds_sent = ns_to_s(s->end_ts - s->start_ts);

    printf("Sever LPID:%llu svr_id:%d sent %d bytes in %f seconds, PINGs Sent: %d; PONGs Received: %d; PINGs Received: %d; PONGs Sent %d; Payload Sum: %d\n", (unsigned long long)lp->gid, s->svr_id, total_msg_size_sent, 
        time_in_seconds_sent, s->ping_msg_sent_count, s->pong_msg_recvd_count, s->ping_msg_recvd_count, s->pong_msg_sent_count, s->payload_sum);
}

static void svr_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp)
{
    switch (m->svr_event_type)
    {
        case KICKOFF:
            handle_kickoff_event(s, b, m, lp);
            break;
        case PING:
            handle_ping_event(s, b, m, lp);
            break;
        case PONG:
            handle_pong_event(s, b, m, lp);
            break;
        default:
            tw_error(TW_LOC, "\n Invalid message type %d ", m->svr_event_type);
            break;
    }
}

static void svr_rev_event(svr_state * s, tw_bf * b, svr_msg * m, tw_lp * lp)
{
    switch (m->svr_event_type)
    {
        case KICKOFF:
            handle_kickoff_rev_event(s, b, m, lp);
            break;
        case PING:
            handle_ping_rev_event(s, b, m, lp);
            break;
        case PONG:
            handle_pong_rev_event(s, b, m, lp);
            break;
        default:
            tw_error(TW_LOC, "\n Invalid message type %d ", m->svr_event_type);
            break;
    }
}

/* convert ns to seconds */
static tw_stime ns_to_s(tw_stime ns)
{
    return(ns / (1000.0 * 1000.0 * 1000.0));
}
static tw_stime s_to_ns(tw_stime s)
{
    return(s*1000.0*1000.0*1000.0);
}

int main(int argc, char **argv)
{
    int nprocs;
    int rank;
    int num_nets;
    int *net_ids;

    tw_opt_add(app_opt);
    tw_init(&argc, &argv);

    codes_comm_update();

    if(argc < 2)
    {
            printf("\n Usage: mpirun <args> --sync=1/2/3 -- <config_file.conf> ");
            MPI_Finalize();
            return 0;
    }

    MPI_Comm_rank(MPI_COMM_CODES, &rank);
    MPI_Comm_size(MPI_COMM_CODES, &nprocs);

    configuration_load(argv[2], MPI_COMM_CODES, &config);

    model_net_register();
    svr_add_lp_type();

    codes_mapping_setup();

    net_ids = model_net_configure(&num_nets);
    net_id = *net_ids;
    free(net_ids);

    /* 1 day of simulation time is drastically huge but it will ensure 
       that the simulation doesn't try to end before all packets are delivered */
    g_tw_ts_end = s_to_ns(24 * 60 * 60);

    num_nodes = codes_mapping_get_lp_count("MODELNET_GRP", 0, "nw-lp", NULL, 1);  //get the number of nodes so we can use this value during the simulation
    assert(num_nodes);

    if(lp_io_dir[0])
    {
        do_lp_io = 1;
        int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
        int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
        assert(ret == 0 || !"lp_io_prepare failure");
    }
    tw_run();
    if (do_lp_io){
        int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
        assert(ret == 0 || !"lp_io_flush failure");
    }
    model_net_report_stats(net_id);

    tw_end();
    return 0;
}

<|start_filename|>src/network-workloads/archived/model-net-synthetic-dfly-plus.c<|end_filename|>
/*
 * Copyright (C) 2015 University of Chicago.
 * See COPYRIGHT notice in top-level directory.
 *
 */

/*
    This workload generator has been modified to support multiple synthetic workloads at once
    the SHA for the version just prior to this change is: 7be9d65e14f2c5c132bed80b88960f9001458e19
*/

#include "codes/model-net.h"
#include "codes/lp-io.h"
#include "codes/codes.h"
#include "codes/codes_mapping.h"
#include "codes/configuration.h"
#include "codes/lp-type-lookup.h"
#include "codes/codes-jobmap.h"


static int net_id = 0;
static int traffic = 1;
static int PAYLOAD_SZ = 2048;
static double arrival_time = 1000.0;

/* whether to pull instead of push */
static int num_servers_per_rep = 0;
static int num_routers_per_grp = 0;
static int num_nodes_per_grp = 0;
static int num_nodes_per_cn = 0;
static int num_groups = 0;
static unsigned long long num_nodes = 0;

static char lp_io_dir[256] = {'\0'};
static lp_io_handle io_handle;
static unsigned int lp_io_use_suffix = 0;
static int do_lp_io = 0;
static int num_msgs = 20;
static tw_stime sampling_interval = 800000;
static tw_stime sampling_end_time = 1600000;

typedef struct svr_msg svr_msg;
typedef struct svr_state svr_state;

/* global variables for codes mapping */
static char group_name[MAX_NAME_LENGTH];
static char lp_type_name[MAX_NAME_LENGTH];
static int group_index, lp_type_index, rep_id, offset;

/* for multi synthetic workloads */ 
#define max_app 5               // maximum number of workload, increase if more is required
#define max_msg 1000000         // maximum number of message can be recorded for latency

typedef tw_stime twoDarray[max_app][max_msg];

struct codes_jobmap_ctx *jobmap_ctx;
struct codes_jobmap_params_list jobmap_p;

static int multi_syn_job = 0;
static int num_apps=0;
static int num_syn_clients=0;

static char workloads_file[8192];              
static char alloc_file[8192];
static char app_job_name[max_app][8192]; //should all be synthetic
static int app_job_nodes[max_app];
static int app_traffic[max_app];
static double app_arrival_time[max_app]; 
static int app_payload_sz[max_app];
static int app_num_msgs[max_app];
static int app_msg_chunk_ratio[max_app];


/* type of events */
enum svr_event
{
    KICKOFF,	   /* kickoff event */
    REMOTE,        /* remote event */
    LOCAL      /* local event */
};

/* type of synthetic traffic */
enum TRAFFIC
{
	UNIFORM = 1, /* sends message to a randomly selected node */
    RAND_PERM = 2, 
	NEAREST_GROUP = 3, /* sends message to the node connected to the neighboring router */
	NEAREST_NEIGHBOR = 4, /* sends message to the next node (potentially connected to the same router) */
    RANDOM_OTHER_GROUP = 5,

    //for multi-workload with job placement config
    UNIFORM_1 = 6,
    BROADCAST = 7, //Rank 0 sends msg to all other ranks
    TORNADO=9, //each MPI rank communicate with fixed offset away from it
    ALL_TO_ALL=10,
    STENCIL_3D=11,
    ND_BROADCAST = 12, /*Non delayed BC, difference from 7 is that the first broadcasting is immediately issued at time 0*/

    //backgroud noise between three contigous groups: group 0,1,2, bi-directional
    BACKGROUND_NOISE3 = 15,
    ND_BACKGROUND_NOISE3 = 18,
};

struct svr_state
{
    int msg_sent_count;   /* requests sent */
    int msg_recvd_count;  /* requests recvd */
    int local_recvd_count; /* number of local messages received */
    tw_stime start_ts;    /* time that we started sending requests */
    tw_stime end_ts;      /* time that we ended sending requests */
    int dest_id;

    //for multi workload and data accouting
    tw_lpid svr_id;
    int group_id;
    int app_id;
    int local_rank; //rank relative to job
    twoDarray *comm_time;   // comm_time[app_id][msg_id]
    int *num_msg_recvd;     // num_msg_recvd[app_id]
    char output_buf0[512];
    char output_buf1[512];
};

struct svr_msg
{
    enum svr_event svr_event_type;
    tw_lpid src;          /* source of this request or ack */
    int incremented_flag; /* helper for reverse computation */
    model_net_event_return event_rc;

    int app_id;
    tw_lpid dest;
    tw_stime issued_time;
    tw_stime arrive_time;
};

static void svr_init(
    svr_state * ns,
    tw_lp * lp);
static void svr_event(
    svr_state * ns,
    tw_bf * b,
    svr_msg * m,
    tw_lp * lp);
static void svr_rev_event(
    svr_state * ns,
    tw_bf * b,
    svr_msg * m,
    tw_lp * lp);
static void svr_finalize(
    svr_state * ns,
    tw_lp * lp);

tw_lptype svr_lp = {
    (init_f) svr_init,
    (pre_run_f) NULL,
    (event_f) svr_event,
    (revent_f) svr_rev_event,
    (commit_f) NULL,
    (final_f)  svr_finalize,
    (map_f) codes_mapping,
    sizeof(svr_state),
};

const tw_optdef app_opt [] =
{
        TWOPT_GROUP("Model net synthetic traffic " ),
    	TWOPT_UINT("traffic", traffic, "UNIFORM RANDOM=1, NEAREST NEIGHBOR=2 "),
    	TWOPT_UINT("payload_sz",PAYLOAD_SZ, "size of the message being sent "),
    	TWOPT_UINT("num_messages", num_msgs, "Number of messages to be generated per terminal "),
    	TWOPT_STIME("sampling-interval", sampling_interval, "the sampling interval "),
    	TWOPT_STIME("sampling-end-time", sampling_end_time, "sampling end time "),
	    TWOPT_STIME("arrival_time", arrival_time, "INTER-ARRIVAL TIME"),
        TWOPT_CHAR("lp-io-dir", lp_io_dir, "Where to place io output (unspecified -> no output"),
        TWOPT_UINT("lp-io-use-suffix", lp_io_use_suffix, "Whether to append uniq suffix to lp-io directory (default 0)"),

        TWOPT_CHAR("alloc_file", alloc_file, "allocation file name"),
        TWOPT_CHAR("workload_conf_file", workloads_file, "workload config file name"),

        TWOPT_END()
};

const tw_lptype* svr_get_lp_type()
{
            return(&svr_lp);
}

static void svr_add_lp_type()
{
  lp_type_register("nw-lp", svr_get_lp_type());
}

static void issue_event(
    svr_state * ns,
    tw_lp * lp)
{
    (void)ns;
    tw_event *e;
    svr_msg *m;
    tw_stime kickoff_time;

    tw_stime init;
    init = tw_now(lp);
    double noise = 1.0;
    int pattern = -1; 

    /* each server sends a dummy event to itself that will kick off the real
     * simulation
     */

    /* skew each kickoff event slightly to help avoid event ties later on */

    if(multi_syn_job){
        kickoff_time =  app_arrival_time[ns->app_id] + tw_rand_exponential(lp->rng, noise); 
    }
    else
        kickoff_time =  arrival_time + tw_rand_exponential(lp->rng, noise); 

    //kickoff_time = 1.1 * g_tw_lookahead + tw_rand_exponential(lp->rng, arrival_time);

    e = tw_event_new(lp->gid, kickoff_time, lp);
    m = tw_event_data(e);
    m->svr_event_type = KICKOFF;
    tw_event_send(e);
}

//no waiting after init svr
static void issue_event_non_delay(
    svr_state * ns,
    tw_lp * lp)
{
    (void)ns;
    tw_event *e;
    svr_msg *m;

    e = tw_event_new(lp->gid, tw_rand_unif(lp->rng)*0.0001, lp);
    m = tw_event_data(e);
    m->svr_event_type = KICKOFF;
    tw_event_send(e);
}

static void svr_init(
    svr_state * ns,
    tw_lp * lp)
{
    ns->start_ts = 0.0;
    ns->dest_id = -1;

    //message latency recording
    ns->comm_time = calloc(max_app*max_msg , sizeof(tw_stime));
    ns->num_msg_recvd = calloc(max_app, sizeof(int));

    if(!ns->comm_time || !ns->num_msg_recvd){
        printf("calloc failed for svr %lu\n", ns->svr_id);
        assert(0);
    }

    ns->svr_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);
    ns->group_id = ns->svr_id/num_nodes_per_grp;

    if(multi_syn_job)
    {
        struct codes_jobmap_id lid;
        lid = codes_jobmap_to_local_id(ns->svr_id, jobmap_ctx);

        if(lid.job == -1)
        {
            ns->app_id = -1;
            ns->local_rank = -1;
            return;
        } else {
            ns->app_id=lid.job;
            ns->local_rank=lid.rank;
        }
    } else {
        ns->app_id=0;
        ns->local_rank=ns->svr_id;
    }

    if(app_traffic[ns->app_id] == ND_BROADCAST || app_traffic[ns->app_id] == ND_BACKGROUND_NOISE3)
        issue_event_non_delay(ns, lp);    
    else
        issue_event(ns, lp);  

    return;
}

static void handle_kickoff_rev_event(
            svr_state * ns,
            tw_bf * b,
            svr_msg * m,
            tw_lp * lp)
{
    if(m->incremented_flag)
        return;

    if(b->c1)
        tw_rand_reverse_unif(lp->rng);
    
    if(b->c8)
        tw_rand_reverse_unif(lp->rng);

    if(multi_syn_job) {
         if(app_traffic[ns->app_id] == RANDOM_OTHER_GROUP) {
            tw_rand_reverse_unif(lp->rng);
            tw_rand_reverse_unif(lp->rng);
        }
    } else {
        if(traffic == RANDOM_OTHER_GROUP) {
            tw_rand_reverse_unif(lp->rng);
            tw_rand_reverse_unif(lp->rng);
        }
    }

    model_net_event_rc2(lp, &m->event_rc);
	ns->msg_sent_count--;
    tw_rand_reverse_unif(lp->rng);
}

static void handle_kickoff_event(
	    svr_state * ns,
	    tw_bf * b,
	    svr_msg * m,
	    tw_lp * lp)
{
    int pattern = -1;

    if(multi_syn_job) {
        pattern = app_traffic[ns->app_id];
        if(ns->msg_sent_count >= app_num_msgs[ns->app_id])
        {
            m->incremented_flag = 1;
            return;
        }
    } else {
        pattern = traffic; 
        if(ns->msg_sent_count >= num_msgs)
        {
            m->incremented_flag = 1;
            return;
        }
    }

    m->incremented_flag = 0;

    char anno[MAX_NAME_LENGTH];
    tw_lpid local_dest = -1, global_dest = -1;

    assert(net_id == DRAGONFLY || net_id == DRAGONFLY_PLUS || net_id == DRAGONFLY_CUSTOM); /* only supported for dragonfly model right now. */
    ns->start_ts = tw_now(lp);
    codes_mapping_get_lp_info(lp->gid, group_name, &group_index, lp_type_name, &lp_type_index, anno, &rep_id, &offset);
    int local_id = codes_mapping_get_lp_relative_id(lp->gid, 0, 0);

    //fot multi workload
    tw_lpid intm_dest_id=-1;
    int i, dest_num=0, sender_receiver_from_alloc=0;
    tw_lpid* local_dest_s;  //rank number in the alloc file. aka index in alloc file
    struct codes_jobmap_id jid;
    jid = codes_jobmap_to_local_id(ns->svr_id, jobmap_ctx);
    int num_clients = codes_jobmap_get_num_ranks(jid.job, jobmap_ctx);

    /* in case of uniform random traffic, send to a random destination. */
    if(pattern == UNIFORM)
    {
        b->c1 = 1;
        local_dest = tw_rand_integer(lp->rng, 0, num_nodes - 1);
    }
    else if(pattern == NEAREST_GROUP)
    {
        local_dest = (local_id + num_nodes_per_grp) % num_nodes;
        //printf("\n LP %ld sending to %ld num nodes %d ", local_id, local_dest, num_nodes);
    }
    else if(pattern == NEAREST_NEIGHBOR)
    {
        local_dest =  (local_id + 1) % num_nodes;
    //	 printf("\n LP %ld sending to %ld num nodes %d ", rep_id * 2 + offset, local_dest, num_nodes);
    }
   else if(pattern == RAND_PERM)
   {
       if(ns->dest_id == -1)
       {
            b->c8 = 1;
            ns->dest_id = tw_rand_integer(lp->rng, 0, num_nodes - 1); 
            local_dest = ns->dest_id;
       }
       else
       {
        local_dest = ns->dest_id; 
       }
   }
    else if(pattern == RANDOM_OTHER_GROUP)
    {
        int my_group_id = local_id / num_nodes_per_grp;

        int other_groups[num_groups-1];
        int added =0;
        for(int i = 0; i < num_groups; i++)
        {
            if(i != my_group_id){
                other_groups[added] = i;
                added++;
            }
        }
        int rand_group = other_groups[tw_rand_integer(lp->rng,0,added -1)];
        int rand_node_intra_id = tw_rand_integer(lp->rng, 0, num_nodes_per_grp-1);

        local_dest = (rand_group * num_nodes_per_grp) + rand_node_intra_id;

        int dest_group_calc = local_dest / num_nodes_per_grp;
        assert(rand_group == dest_group_calc);
        assert(rand_group != my_group_id);
        assert(dest_group_calc != my_group_id);
        
        // printf("\n LP %ld sending to %ld num nodes %d ", local_id, local_dest, num_nodes);
    }

    //synthetic pattern read from workload file
    else if(pattern == UNIFORM_1) {
        assert(multi_syn_job);
        sender_receiver_from_alloc = 1;
        b->c1 = 1;
        dest_num = 1;
        local_dest_s = calloc(dest_num, sizeof(tw_lpid));
        if(!local_dest_s)
            tw_error(TW_LOC,"Error: UNIFORM_1 calloc failed\n");

        local_dest_s[0] = tw_rand_integer(lp->rng, 0, num_clients - 1);
        //printf("UNIFORM_1: num_clients %d, local_dest %llu, ns->local_rank %llu\n", num_clients, LLU(local_dest), LLU(ns->local_rank));
            if(local_dest_s[0] == ns->local_rank){
                local_dest_s[0] = (ns->local_rank + 1) % num_clients;
            }
    }
    else if(pattern == BROADCAST || pattern == ND_BROADCAST) {
        assert(multi_syn_job);
        sender_receiver_from_alloc = 1;
        int i;
        
        if(ns->local_rank != 0)
            return;

        dest_num = num_clients-1;
        local_dest_s = calloc(dest_num, sizeof(tw_lpid));  //store rank number
        if(!local_dest_s)
            tw_error(TW_LOC,"Error: BROADCAST calloc failed\n");

        for (i = 0; i < dest_num; ++i)
            local_dest_s[i] = (ns->local_rank + i + 1) % num_clients;
    }
    else if (pattern == TORNADO) {

        int offset = 1 * num_nodes_per_grp;   //offset is number of multiple group sizes 

        assert(multi_syn_job);
        sender_receiver_from_alloc = 1;
        dest_num = 1;

        local_dest_s = calloc(dest_num, sizeof(tw_lpid));
        if(!local_dest_s)
            tw_error(TW_LOC,"Error: UNIFORM_1 calloc failed\n");

        local_dest_s[0] = (ns->local_rank + offset) % num_clients;
    }

    // 3D Stencil for 2304=12*12*16 job size
    else if(pattern == STENCIL_3D) {
        int x_size=12, y_size=12, z_size=16, x, y, z;

        assert(multi_syn_job);
        sender_receiver_from_alloc = 1;
        dest_num = 6;

        if ( (x_size*y_size*z_size) != num_clients) {
            tw_error(TW_LOC,"Error: Job size %d, not equal to cube size %d\n", num_clients, x_size*y_size*z_size);
        }

        local_dest_s = calloc(dest_num, sizeof(tw_lpid));

        int layer_size = x_size*y_size;

        z = ns->local_rank / layer_size;
        y = (ns->local_rank - z * layer_size) / x_size;
        x = (ns->local_rank - z * layer_size) - y * x_size;

        assert(x<x_size && y<y_size && z<z_size);

        /* x-axis neighbors */
        local_dest_s[0] = layer_size * z + x_size * y + (x+1) % x_size;   
        local_dest_s[1] = layer_size * z + x_size * y + (x-1+x_size) % x_size;     
        /* y-axis neighbor */
        local_dest_s[2] = layer_size * z + x_size * ((y+1) % y_size) + x;
        local_dest_s[3] = layer_size * z + x_size * ((y-1+y_size) % y_size) + x;   
        /* z-axis neighbor */
        local_dest_s[4] = layer_size * ((z + 1) % z_size) + x_size * y + x;
        local_dest_s[5] = layer_size * ((z - 1 + z_size) % z_size) + x_size * y + x; 
    }

    //background noise3
    else if (pattern == BACKGROUND_NOISE3 || pattern == ND_BACKGROUND_NOISE3 ) {
        assert(multi_syn_job);
        sender_receiver_from_alloc = 1;
        dest_num = 2;
        local_dest_s = calloc(dest_num, sizeof(tw_lpid));
        if(!local_dest_s)
            tw_error(TW_LOC,"Error: BACKGROUND3 calloc failed\n");

        switch(ns->group_id){
            case 0:
                local_dest_s[0] = (ns->local_rank + 1) % num_clients;
                local_dest_s[1] = (ns->local_rank + 2) % num_clients;
            break;

            case 1:
                local_dest_s[0] = (ns->local_rank - 1) % num_clients;
                local_dest_s[1] = (ns->local_rank + 1) % num_clients;
            break;

            case 2:
                local_dest_s[0] = (ns->local_rank - 2) % num_clients;
                local_dest_s[1] = (ns->local_rank - 1) % num_clients;
            break;

            default:
                tw_error(TW_LOC, "Node %llu in group %d should not generate BACKGROUND_NOISE3", LLU(ns->svr_id), ns->group_id);
        }
    }

    else {
        tw_error(TW_LOC, "\n Traffic pattern not recoginzed %d \n", pattern);
    }

    if(!sender_receiver_from_alloc){
        assert(local_dest < num_nodes);
        global_dest = codes_mapping_get_lpid_from_relative(local_dest, group_name, lp_type_name, NULL, 0);
        //printf("KICKOFF: intm_dest_id is %llu, global_id is %llu\n", LLU(intm_dest_id), LLU(global_dest));

        ns->msg_sent_count++;
        svr_msg * m_local = malloc(sizeof(svr_msg));
        svr_msg * m_remote = malloc(sizeof(svr_msg));

        m_local->svr_event_type = LOCAL;
        m_local->src = lp->gid;
        m_local->app_id = ns->app_id;
        m_local->dest = local_dest;
        m_local->issued_time = tw_now(lp);

        memcpy(m_remote, m_local, sizeof(svr_msg));
        m_remote->svr_event_type = REMOTE;

        if(multi_syn_job)
            m->event_rc = model_net_event(net_id, "test", global_dest, app_payload_sz[ns->app_id], 0.0, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);
        else
            m->event_rc = model_net_event(net_id, "test", global_dest, PAYLOAD_SZ, 0.0, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);

        issue_event(ns, lp);

        free(m_local);
        free(m_remote);
    }
    else {

        //newly added synthetic pattern all follows the same format: use local_dest_s, dest_num=1 if only 1 destination 
        assert(dest_num);
        assert(multi_syn_job);
        ns->msg_sent_count++;

        int count_tmp = 0;

        for (i = 0; i < dest_num; ++i)
        {

            count_tmp++;

            jid.rank = local_dest_s[i]; //rank id
            intm_dest_id = codes_jobmap_to_global_id(jid, jobmap_ctx); //node id
            assert(intm_dest_id < num_nodes);
            global_dest = codes_mapping_get_lpid_from_relative(intm_dest_id, group_name, lp_type_name, NULL, 0); //node lpid
            
            svr_msg * m_local = malloc(sizeof(svr_msg));
            svr_msg * m_remote = malloc(sizeof(svr_msg));

            m_local->svr_event_type = LOCAL;
            m_local->src = lp->gid;
            m_local->app_id = ns->app_id;
            m_local->dest = intm_dest_id;
            m_local->issued_time = tw_now(lp);

            memcpy(m_remote, m_local, sizeof(svr_msg));
            m_remote->svr_event_type = REMOTE;

            m->event_rc = model_net_event(net_id, "test", global_dest, app_payload_sz[ns->app_id], 0.0, sizeof(svr_msg), (const void*)m_remote, sizeof(svr_msg), (const void*)m_local, lp);

            //if(ns->svr_id ==0) printf("KICKOFF: msg geneerated at svr_id %llu,to intm_dest_id %llu at time %lf\n", LLU(ns->svr_id), LLU(intm_dest_id), m_local->issued_time);
            //printf("KICKOFF: msg geneerated at svr_id %llu,to intm_dest_id %llu at time %lf\n", LLU(ns->svr_id), LLU(intm_dest_id), m_local->issued_time);
            //if(ns->local_rank) printf("KICKOFF: App %d : msg geneerated at svr_id %llu,to intm_dest_id %llu at time %lf\n", ns->app_id, LLU(ns->svr_id), LLU(intm_dest_id), m_local->issued_time);

            free(m_local);
            free(m_remote);
        }
        issue_event(ns, lp);
        free(local_dest_s);
    }
    return;
}

static void handle_remote_rev_event(
            svr_state * ns,
            tw_bf * b,
            svr_msg * m,
            tw_lp * lp)
{
        (void)b;
        (void)m;
        (void)lp;
        ns->msg_recvd_count--;

        int index = -- ns->num_msg_recvd[m->app_id];

        if(app_traffic[m->app_id] < BACKGROUND_NOISE3) {
            if((*ns->comm_time)[m->app_id][index] ==  (m->arrive_time - m->issued_time) ){
                (*ns->comm_time)[m->app_id][index] = 0.0;
            } else {
                tw_error(TW_LOC, "Error:remote reverse calculatioin failed \n svr %llu found travel time %lf but record %lf at index %d\n", ns->svr_id, m->arrive_time - m->issued_time, (*ns->comm_time)[m->app_id][index], index);
            }
        }
}

static void handle_remote_event(
	    svr_state * ns,
	    tw_bf * b,
	    svr_msg * m,
	    tw_lp * lp)
{
        (void)b;
        (void)m;
        (void)lp;
	ns->msg_recvd_count++;

    if(m->dest != ns->svr_id)
        tw_error(TW_LOC, "Error: msg issued by %llu at time %lfns should arrive at %llu but reached %llu\n", (unsigned long long)m->src, m->issued_time , (unsigned long long) m->dest, (unsigned long long) ns->svr_id);

    m->arrive_time = tw_now(lp);

    int index = ns->num_msg_recvd[m->app_id]++;

    //Non-noise application
    if(app_traffic[m->app_id] < BACKGROUND_NOISE3) {
        if(index > max_msg) 
            tw_error(TW_LOC, "\n More than %d msgs received. Increase 'max_msg' value\n ", max_msg);

        (*ns->comm_time)[m->app_id][index] = m->arrive_time - m->issued_time;
    }
}

static void handle_local_rev_event(
                svr_state * ns,
                tw_bf * b,
                svr_msg * m,
                tw_lp * lp)
{
        (void)b;
        (void)m;
        (void)lp;
	ns->local_recvd_count--;
}

static void handle_local_event(
                svr_state * ns,
                tw_bf * b,
                svr_msg * m,
                tw_lp * lp)
{
        (void)b;
        (void)m;
        (void)lp;
    ns->local_recvd_count++;
}

/* convert seconds to ns */
static tw_stime s_to_ns(tw_stime ns)
{
    return(ns * (1000.0 * 1000.0 * 1000.0));
}

static void svr_finalize(
    svr_state * ns,
    tw_lp * lp)
{
    ns->end_ts = tw_now(lp);

    int i,j;
    int total_receive_verify = 0;
    int written = 0; 
    int index = 0; 
    int result;

    if (multi_syn_job){
        for (i = 0; i < num_apps; ++i)
        {

            int index = ns->num_msg_recvd[i];
            total_receive_verify += index;

            if(app_traffic[i] >= BACKGROUND_NOISE3)
                continue;

            char file[512];
            sprintf(file, "application_%d", i);
            written = 0;

            if (ns->svr_id ==0) {
                written = sprintf(ns->output_buf0, "### Application %d, Synthetic Traffic: %d, comm. time (ns)\n <msg comm. time>, <Avg. chunck comm.time>", i, app_traffic[i]);
                result=lp_io_write(lp->gid, file, written, ns->output_buf0);
                if(result!=0)
                    tw_error(TW_LOC, "\nERROR: lpio result %d, svr %llu, lpgid %llu, writting %s failed, written %d, buf %s\n", result, LLU(ns->svr_id), LLU(lp->gid), file, written, ns->output_buf0);
            }

            for (j=0; j<index; j++) {

                written = sprintf(ns->output_buf0, "\n%lf %lf",  (*ns->comm_time)[i][j], (*ns->comm_time)[i][j]/app_msg_chunk_ratio[i]);
                //printf("app[%d][%d]comm.time %lf, buf is %s\n", i, j, (*ns->comm_time)[i][j], ns->output_buf0);
                result=lp_io_write(lp->gid, file, written, ns->output_buf0);
                
                if(result!=0)
                    tw_error(TW_LOC, "\nERROR: lpio result %d, svr %llu, lpgid %llu, writting %s failed, written %d, buf %s, index %d\n", result, LLU(ns->svr_id), LLU(lp->gid), file, written, ns->output_buf0, j);
            } 
        }
    }

    written=0;
    if(ns->svr_id == 0)
        written = sprintf(ns->output_buf1, "#<LP ID> <Terminal ID> <Job ID> <Local Rank> <Total sends> <Total Recvs> <Local count> <lp running time>");

    written += sprintf(ns->output_buf1 + written, "\n %llu %llu %d %d %d %d %d %lf", 
        LLU(lp->gid), LLU(ns->svr_id), ns->app_id, ns->local_rank, ns->msg_sent_count, ns->msg_recvd_count, ns->local_recvd_count, ns->end_ts - ns->start_ts);

    result = lp_io_write(lp->gid, (char*)"mpi-synthetic-stats", written, ns->output_buf1);
    if( result != 0)
        tw_error(TW_LOC, "\nERROR: lpio result %d, svr %llu, lpgid %llu, writting %s failed, written %d, buf %s\n", result, LLU(ns->svr_id), LLU(lp->gid), (char*)"mpi-synthetic-stats", written, ns->output_buf1);
    
//    printf("server %llu recvd %d bytes in %f seconds, %f MiB/s sent_count %d recvd_count %d local_count %d \n", (unsigned long long)lp->gid, PAYLOAD_SZ*ns->msg_recvd_count, ns_to_s(ns->end_ts-ns->start_ts),
//        ((double)(PAYLOAD_SZ*ns->msg_sent_count)/(double)(1024*1024)/ns_to_s(ns->end_ts-ns->start_ts)), ns->msg_sent_count, ns->msg_recvd_count, ns->local_recvd_count);
    return;
}

static void svr_rev_event(
    svr_state * ns,
    tw_bf * b,
    svr_msg * m,
    tw_lp * lp)
{
    switch (m->svr_event_type)
    {
	case REMOTE:
		handle_remote_rev_event(ns, b, m, lp);
		break;
	case LOCAL:
		handle_local_rev_event(ns, b, m, lp);
		break;
	case KICKOFF:
		handle_kickoff_rev_event(ns, b, m, lp);
		break;
	default:
		assert(0);
		break;
    }
}

static void svr_event(
    svr_state * ns,
    tw_bf * b,
    svr_msg * m,
    tw_lp * lp)
{
   switch (m->svr_event_type)
    {
        case REMOTE:
            handle_remote_event(ns, b, m, lp);
            break;
        case LOCAL:
            handle_local_event(ns, b, m, lp);
            break;
	case KICKOFF:
	    handle_kickoff_event(ns, b, m, lp);
	    break;
        default:
            printf("\n Invalid message type %d ", m->svr_event_type);
            assert(0);
        break;
    }
}

int main(
    int argc,
    char **argv)
{
    int nprocs;
    int rank;
    int num_nets;
    int *net_ids;
    int num_routers;
    int num_router_leaf, num_router_spine;
    int chunk_size, packet_size;

    tw_opt_add(app_opt);
    tw_init(&argc, &argv);
#ifdef USE_RDAMARIS
    if(g_st_ross_rank)
    { // keep damaris ranks from running code between here up until tw_end()
#endif
    codes_comm_update();

    if(argc < 2)
    {
            printf("\n Usage: mpirun <args> --sync=2/3 mapping_file_name.conf (optional --nkp) ");
            MPI_Finalize();
            return 0;
    }

    MPI_Comm_rank(MPI_COMM_CODES, &rank);
    MPI_Comm_size(MPI_COMM_CODES, &nprocs);

    configuration_load(argv[2], MPI_COMM_CODES, &config);

    model_net_register();
    svr_add_lp_type();

    codes_mapping_setup();

    net_ids = model_net_configure(&num_nets);
    //assert(num_nets==1);
    net_id = *net_ids;
    free(net_ids);

    /* 5 days of simulation time */
    g_tw_ts_end = s_to_ns(5 * 24 * 60 * 60);
    model_net_enable_sampling(sampling_interval, sampling_end_time);

    if(!(net_id == DRAGONFLY || net_id == DRAGONFLY_PLUS || net_id == DRAGONFLY_CUSTOM))
    {
	printf("\n The test works with dragonfly model configuration only! %d %d ", DRAGONFLY_PLUS, net_id);
        MPI_Finalize();
        return 0;
    }
    num_servers_per_rep = codes_mapping_get_lp_count("MODELNET_GRP", 1, "nw-lp",
            NULL, 1);
    configuration_get_value_int(&config, "PARAMS", "num_router_leaf", NULL, &num_router_leaf);
    configuration_get_value_int(&config, "PARAMS", "num_router_spine", NULL, &num_router_spine);
    configuration_get_value_int(&config, "PARAMS", "num_routers", NULL, &num_routers);
    configuration_get_value_int(&config, "PARAMS", "num_groups", NULL, &num_groups);
    configuration_get_value_int(&config, "PARAMS", "num_cns_per_router", NULL, &num_nodes_per_cn);

    //num_routers_per_grp = num_routers;
    num_routers_per_grp = num_router_leaf + num_router_spine;

    num_nodes = num_groups * num_router_leaf * num_nodes_per_cn;
    num_nodes_per_grp = num_router_leaf * num_nodes_per_cn;

    assert(num_nodes);

    configuration_get_value_int(&config, "PARAMS", "packet_size", NULL, &packet_size);
    configuration_get_value_int(&config, "PARAMS", "chunk_size", NULL, &chunk_size);

    jobmap_ctx = NULL; 
    num_apps = 0;

    if(strlen(workloads_file) > 0 && strlen(alloc_file)) {
        multi_syn_job = 1;

        FILE *name_file = fopen(workloads_file, "r");
        if(!name_file)
            tw_error(TW_LOC, "\n Could not open file %s ", workloads_file);

        int i = 0;
        char ref = '\n';
        while(!feof(name_file))
        {
            if(i>max_app)
                tw_error(TW_LOC, "\n Support no more than %d jobs, increase 'max_app' value\n", max_app);

            ref = fscanf(name_file, "%d %s %d %lf %d %d", &app_job_nodes[i], app_job_name[i], &app_traffic[i], &app_arrival_time[i], &app_payload_sz[i], &app_num_msgs[i]);

            if(ref != EOF && strncmp(app_job_name[i], "synthetic", 9) == 0)
            {   
                num_syn_clients += app_job_nodes[i];
                num_apps++; 

                app_msg_chunk_ratio[i] = (app_payload_sz[i] + chunk_size - 1) / chunk_size;

                if(rank==0) 
                    printf("\nApp id %d, #nodes %d, traffic#: %d, arrival-time %lf, payload size %d, num msgs %d, msg chunk ratio %d\n", i, app_job_nodes[i], app_traffic[i], app_arrival_time[i], app_payload_sz[i], app_num_msgs[i], app_msg_chunk_ratio[i]);

            } else if(ref!=EOF) {
                if (strncmp(app_job_name[i], "synthetic", 9) != 0)
                   tw_error(TW_LOC, "\nOnly supprot synthetic traffic\n");

                tw_error(TW_LOC, "\nUndefined behavior in %s\n", workloads_file);
 
            }
            i++;
        }
        fclose(name_file);
        jobmap_p.alloc_file = alloc_file;
        jobmap_ctx = codes_jobmap_configure(CODES_JOBMAP_LIST, &jobmap_p);

    } else {
        multi_syn_job = 0;
        num_apps = 1;
        num_syn_clients = num_nodes;
        jobmap_ctx = codes_jobmap_configure(CODES_JOBMAP_IDENTITY, &jobmap_p);
        if(!rank)
            printf("Synthetic Job %d for whole system \n", traffic);

    }

    if(lp_io_dir[0])
    {
        do_lp_io = 1;
        int flags = lp_io_use_suffix ? LP_IO_UNIQ_SUFFIX : 0;
        int ret = lp_io_prepare(lp_io_dir, flags, &io_handle, MPI_COMM_CODES);
        assert(ret == 0 || !"lp_io_prepare failure");
    }

    assert(num_apps <= max_app);
    if(rank==0) 
        printf("\nIntotal of %d applications, begin simulation\n", num_apps);

    tw_run();
    if (do_lp_io){
        int ret = lp_io_flush(io_handle, MPI_COMM_CODES);
        assert(ret == 0 || !"lp_io_flush failure");
    }
    model_net_report_stats(net_id);

    if(multi_syn_job)
       codes_jobmap_destroy(jobmap_ctx);

#ifdef USE_RDAMARIS
    } // end if(g_st_ross_rank)
#endif
    tw_end();
    return 0;
}

/*
 * Local variables:
 *  c-indent-level: 4
 *  c-basic-offset: 4
 * End:
 *
 * vim: ft=c ts=8 sts=4 sw=4 expandtab
 */
