<repo_name>msingam/covid-chicago<file_sep>/nucluster/readme.md

# Submission workflow on the NU cluster 'Quest'

The COVID-19 project files are located in `/projects/p30781/covidproject/covid-chicago/`.

1. Navigate to the project folder: 
	`cd /projects/p30781/covidproject`

2. Prepare simulation input files or copy simulation files (optional):
  - relevant files to edit are the yaml and emodl files
  - when done a useful command is `git reset --hard HEAD` to remove all changes done on the git repository on quest ! 
  Be sure to commit edits if needed before running that. However commiting files on the main repository under `/projects/p30781/covidproject/covid-chicago/` is not recommended, as it commits to upstream and not the forked repository.
  
  
3. Submit runScenarios.py : 
	`python runScenarios.py -rl NUCLUSTER -r EMS_1 -c EMSspecific_sample_parameters.yaml -e extendedmodel.emodl  -n "testrun"`
	
4. Submit postprocessing jobs
	`cd /projects/p30781/covidproject/projects/covid_chicago/cms_sim/simulation_output/<exp_name>/sh"`
	`sbatch 2_runDataComparison.sh"`

5. Copy final files to Box
 - Recommended to zip before copying using `python cleanup_and_zip_simFiles.py --stem "<exp_name>"  --Location "NUCLUSTER" --zip_dir`
 - Via Box sync `lftp -p 990 -u  <useremail> ftps://box.com -e "mirror -R /projects/p30781/covidproject/projects/covid_chicago/cms_sim/simulation_output/<exp_name> NU-malaria-team/projects/covid_chicago/cms_sim/simulation_output/<exp_name>; exit"`
 
## Files in this folder
- cleanup.py: moves simulation folder from  `/projects/p30781/covidproject/covid-chicago/_temp/<exp_name>` to Box on quest (`/projects/p30781/covidproject/projects/covid_chicago/cms_sim/simulation_output/<exp_name>`), runs automatically after runScenarios.py
- cleanup_and_zip_simFiles.py zips and optionally deletes simulation folder in Box on quest (i.e. before transferring) and per default (!) deletes single trajectories and simulation files.

Example shell job submission files (experiment specific shell files generated in simulation folder)
- submit_cleanup.sh
- submit_combineTrajectories.sh
- submit_compareToData.sh

 
## Useful commands

Check status of submitted jobs 
`squeue -u <username>`

Cancel job
`scancel <jobid>`
`scancel -u <username> ` # all jobs for that user 

Hold and release job
`scontrol hold  <jobid>`
`scontrol release  <jobid>`

Wine error, if the wine container causes issues it may help to copy a new container like below
`mv ~/.wine ~/.wine.old99`

Reset git directory- quick way to discard all changes made and return to last PR (i.e. discard experiment specific edits in the yaml)
`git reset --hard HEAD`

Count csv files in folder 
`ls *.csv | wc -l`

Load python module
`module load python/anaconda3.6`

Load virtual environment module
``` 
module purge all
module load python/anaconda3.6
source activate /projects/p30781/anaconda3/envs/team-test-py37  
``` 
  
  <file_sep>/requirements.txt
python-dotenv>=0.12
PyYAML>=5.3
yamlordereddictloader>0.4
epyestim

matplotlib==3.1.3
more-itertools==8.2.0
numpy==1.18.1
pandas==1.0.1
seaborn==0.10.0
<file_sep>/plotters/plot_split_by_channel.py
import argparse
import os
import pandas as pd
import matplotlib.pyplot as plt
import sys
sys.path.append('../')
from load_paths import load_box_paths
import matplotlib as mpl
import matplotlib.dates as mdates
import seaborn as sns
from processing_helpers import *
import re

mpl.rcParams['pdf.fonttype'] = 42

def parse_args():
    description = "Simulation run for modeling Covid-19"
    parser = argparse.ArgumentParser(description=description)

    parser.add_argument(
        "-stem",
        "--stem",
        type=str,
        help="Name of simulation experiment"
    )
    parser.add_argument(
        "-loc",
        "--Location",
        type=str,
        help="Local or NUCLUSTER",
        default="Local"
    )
    return parser.parse_args()

def plot_on_fig(df, channels, axes, color, label) :

    for c, channel in enumerate(channels) :
        channeltitle = re.sub('_detected', '', str(channel), count=1)
        channeltitle = re.sub('_det','', str(channeltitle), count=1)

        ax = axes[c]
        mdf = df.groupby('date')[channel].agg([CI_50, CI_2pt5, CI_97pt5, CI_25, CI_75]).reset_index()
        ax.plot(mdf['date'], mdf['CI_50'], color=color, label=label)
        # ax.fill_between(mdf['date'].values, mdf['CI_2pt5'], mdf['CI_97pt5'],
        #                 color=color, linewidth=0, alpha=0.2)
        ax.fill_between(mdf['date'].values, mdf['CI_25'], mdf['CI_75'],
                        color=color, linewidth=0, alpha=0.4)
        ax.set_title(' '.join(channeltitle.split('_')), y=0.85)
        ax.xaxis.set_major_formatter(mdates.DateFormatter('%b\n%y'))

def compare_channels(channelGrp,grp="All"):
    nchannels_symp = {'channels1': ['symp_severe_cumul', 'symp_mild_cumul', 'symptomatic_severe', 'symptomatic_mild'],
                      'channels2': ['symp_severe_det_cumul', 'symp_mild_det_cumul', 'symptomatic_severe_det',
                                    'symptomatic_mild_det']}

    nchannels_infect = {'channels1': ['infected', 'presymptomatic', 'infectious_undet', 'asymp', 'asymp_cumul'],
                        'channels2': ['infected_det', 'presymptomatic_det', 'infectious_det', 'asymptomatic_det',
                                      'asymp_det_cumul']}

    nchannels_hospCrit = {
        'channels1': ['hospitalized', 'new_hospitalized', 'hosp_cumul', 'critical', 'new_critical', 'crit_cumul'],
        'channels2': ['hosp_det', 'new_detected_hospitalized', 'hosp_det_cumul', 'crit_det', 'new_detected_critical',
                      'crit_det_cumul']}

    nchannels_Vacc = {
        'channels1': ['vaccinated_cumul', 'asymp_det', 'hosp_det', 'crit_det', 'deaths_det', 'recovered_det'],
        'channels2': ['vaccinated_cumul', 'asymp_det_V', 'hosp_det_V', 'crit_det_V', 'deaths_det_V', 'recovered_det_V']}

    if channelGrp == "symp":
        nchannels = nchannels_symp
    if channelGrp == "infect":
        nchannels = nchannels_infect
    if channelGrp == "hospCrit":
        nchannels = nchannels_hospCrit
    if channelGrp == "Vaccinated":
        nchannels = nchannels_Vacc

    palette = sns.color_palette('Set1', len(nchannels))
    fig = plt.figure(figsize=(16, 8))
    fig.subplots_adjust(right=0.97, left=0.05, hspace=0.4, wspace=0.2, top=0.95, bottom=0.05)

    axes = [fig.add_subplot(2, 3, x + 1) for x in range(len(nchannels['channels1']))]

    for d, key in enumerate(nchannels.keys()):

        column_list = ['startdate', 'time']
        for channel in nchannels[key]:
            column_list.append(channel + f'_{grp}')

        df = load_sim_data(exp_name, region_suffix=f'_{grp}',  fname = 'trajectoriesDat.csv')
        df = df[df['date'].between(pd.Timestamp(first_day), pd.Timestamp(last_day))]


        channels = nchannels[key]
        if d == 0:
            label = "all"
        if d == 1:
            label = "detected"
            if channelGrp == "Vaccinated":
                label = "vaccinated"

        plot_on_fig(df, channels, axes, color=palette[d], label=label)
    axes[-1].legend()

    fname = 'channel_' + channelGrp + '_comparison'
    plt.savefig(os.path.join(plot_path, fname + '.png'))
    # plt.savefig(os.path.join(plot_path,'pdf', fname + '.pdf'), format='PDF')
    plt.show()

if __name__ == '__main__' :

    args = parse_args()
    stem = args.stem
    Location = args.Location

    datapath, projectpath, wdir, exe_dir, git_dir = load_box_paths(Location=Location)

    first_day = pd.Timestamp.today()- pd.Timedelta(30,'days')
    last_day = pd.Timestamp.today()+ pd.Timedelta(210,'days')

    exp_names = [x for x in os.listdir(os.path.join(wdir, 'simulation_output')) if stem in x]
    for exp_name in exp_names:
        sim_output_path = os.path.join(wdir, 'simulation_output', exp_name)
        plot_path = os.path.join(sim_output_path, '_plots')
        #compare_channels(channelGrp= "symp")
        #compare_channels(channelGrp= "infect")
        compare_channels(channelGrp= "hospCrit")
        compare_channels(channelGrp= "Vaccinated")

<file_sep>/data_processing/CDC Data MS.py
import os
import pandas as pd
import numpy as np
import sys
import seaborn as sns
import matplotlib.pyplot as plt
sys.path.append('../')
from load_paths import load_box_paths
datapath, projectpath, wdir,exe_dir, git_dir = load_box_paths()
from processing_helpers import *

""" 
df = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'Corona virus reports', 'il_cdc_thru_0811.csv'),low_memory=False)
print(df)
"""
"""Read in only relevant columns """
column_list =['icu_length',	'hosp_length', 'age_group','res_county','res_state','hosp_yn', 'icu_yn', 'death_yn']
df = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'Corona virus reports', 'il_cdc_thru_0811.csv'), usecols=column_list)
print(df)

"""Remove Missings and Unknowns """
df = df.dropna(subset=["hosp_length"])
df = df.dropna(subset=["age_group"])
df = df.dropna(subset=["death_yn"])
df = df[df['age_group'] != 'Unknown' ]
df = df[df['icu_yn'] != 'Unknown' ]
df = df[df['icu_yn'] != 'Missing' ]
print(df)
pd.crosstab(index=df['age_group'], columns='count')

### Look at median per age group
df = df[df['hosp_length'] !=0 ]
df_median = df.groupby(['age_group', 'death_yn'])['hosp_length'].agg([np.median]).reset_index()
print(df_median)
### Look at mean per age group
df = df[df['hosp_length'] !=0 ]
df_summary = df.groupby(['age_group', 'icu_yn'])['hosp_length'].agg([np.mean, CI_2pt5, CI_25, CI_50, CI_75, CI_97pt5]).reset_index()
df_summary = df_summary.sort_values(by=['icu_yn','age_group'])
print(df_summary)

"""Remove Missings and Unknowns """
df = df.dropna(subset=["hosp_length"])
df = df.dropna(subset=["age_group"])
df = df.dropna(subset=["death_yn"])
df = df[df['age_group'] != 'Unknown' ]
df = df[df['death_yn'] != 'Unknown' ]
df = df[df['death_yn'] != 'Missing' ]
print(df)
### Look at median per age group
df = df[df['hosp_length'] !=0 ]
df_median = df.groupby(['age_group', 'death_yn'])['hosp_length'].agg([np.median]).reset_index()
print(df_median)
### Look at mean per age group
df = df[df['hosp_length'] !=0 ]
df_summary = df.groupby(['age_group', 'death_yn'])['hosp_length'].agg([np.mean, CI_2pt5, CI_25, CI_50, CI_75, CI_97pt5]).reset_index()
df_summary = df_summary.sort_values(by=['death_yn','age_group'])
print(df_summary)


### Simple histogram, not age structured
plt.rcParams.update({'figure.figsize':(7,5), 'figure.dpi':100})
x = df['hosp_length']
plt.hist(x, bins=50)
plt.gca().set(title='hosp_length', ylabel='Frequency');
plt.show()
### Function for age structured plot
def plot_hist() :
    ## Get age groups
    agegroups = list(df['age_group'].unique())
    fig = plt.figure(figsize=(10, 6))
    palette = sns.color_palette('husl', 8)
    k = 0
    for c, age in enumerate(agegroups):
        ax = fig.add_subplot(3, 3, c + 1)
        mdf = df[df['age_group'] == age]
        ax.hist(mdf['hosp_length'], bins=50)
        ax.set_title(agegroups[c], y=0.8, fontsize=12)
        #ax.set(xlabel='hosp_length', ylabel='Frequency')
    return plt
## Generate plot and save
age_plot = plot_hist()
age_plot.savefig(os.path.join(git_dir,   'hosp_length_age_hist.png'))

agegroups = list(df['age_group'].unique())
fig = plt.figure(figsize=(10, 6))
for c, age in enumerate(agegroups):
    ax = fig.add_subplot(3, 3, c + 1)
    mdf = df[df['age_group'] == age]
    ax.hist(mdf['hosp_length'], bins=50)
    ax.set_title(agegroups[c], y=0.8, fontsize=12)
    #ax.set(xlabel='hosp_length', ylabel='Frequency')


import os
import pandas as pd
import numpy as np
import sys
import seaborn as sns
import matplotlib.pyplot as plt
sys.path.append('../')
from load_paths import load_box_paths
datapath, projectpath, wdir,exe_dir, git_dir = load_box_paths()

column_list =['icu_length',    'hosp_length', 'age_group','res_county','res_state','hosp_yn', 'icu_yn']
df = pd.read_csv(os.path.join(datapath, 'covid_IDPH', 'Corona virus reports', 'il_cdc_thru_0811.csv'), usecols=column_list)

def plot_hist() :
    ## Get age groups
    agegroups = list(df['age_group'].unique())
    fig = plt.figure(figsize=(10, 6))
    palette = sns.color_palette('husl', 8)
    k = 0
    for c, age in enumerate(agegroups):
        ax = fig.add_subplot(3, 3, c + 1)
        mdf = df[df['age_group'] == age]
        ax.hist(mdf['hosp_length'], bins=50)
        ax.set_title(agegroups[c], y=0.8, fontsize=12)
        #ax.set(xlabel='hosp_length', ylabel='Frequency')
    return plt

df = df.dropna(subset=["hosp_length"])
df = df.dropna(subset=["age_group"])
df = df[df['age_group'] != 'Unknown' ]

age_plot = plot_hist()





