## This job takes in the MIMIC-III CSV.gz files from the specified input bucket
## and converts them to Parquet format in the specified output bucket
## Author:  James Wiggins (wiggjame@amazon.com)
## Date:  6/3/19
## Revision: 1

import sys
from awsglue.transforms import *
from awsglue.utils import getResolvedOptions
from pyspark.context import SparkContext
from awsglue.context import GlueContext
from awsglue.job import Job
from pyspark.sql.functions import *
from pyspark.sql.types import *
import os.path
from os import path
import boto3

s3 = boto3.resource('s3')
glueContext = GlueContext(SparkContext.getOrCreate())
spark = glueContext.spark_session

# Specify MIMIC data input and output S3 buckets
mimiccsvinputbucket='mimic-iii-physionet'