#!/usr/bin/python

# Calculates statistics and plots the bag metrics from raw data

from __future__ import print_function
import csv
import os, sys
import numpy as np
import scipy as sp
import scipy.stats as sps
import pandas as pd
import re, shutil, tempfile
import itertools, operator
import subprocess
import Gnuplot
import Gnuplot.funcutils

###### Settings go here ######

searchAttrsList =   [
                        {   'groupby': ['Worker_Thread_Count', 'Static_Window_Size'],
                            'filter' : 'Fraction_of_Total_Window',
                            'model'  : 'Model',
                            'lpcount': 'Number_of_Objects',
                            'output' : 'threads_vs_staticwindow_key_fractionwindow_'  },

                        {   'groupby': ['Worker_Thread_Count', 'Fraction_of_Total_Window'],
                            'filter' : 'Static_Window_Size',
                            'model'  : 'Model',
                            'lpcount': 'Number_of_Objects',
                            'output' : 'threads_vs_fractionwindow_key_staticwindow_'   }
                    ]

'''
List of metrics available:

    Event_Commitment_Ratio
    Total_Rollbacks
    Simulation_Runtime_(secs.)
    Average_Memory_Usage_(MB)
    Event_Processing_Rate_(per_sec)
    Speedup_w.r.t._Sequential_Simulation
'''
metricList      =   [
                        {   'name'  : 'Event_Processing_Rate_(per_sec)',
                            'ystart': 0,
                            'yend'  : 1000000,
                            'ytics' : 100000    },

                        {   'name'  : 'Simulation_Runtime_(secs.)',
                            'ystart': 0,
                            'yend'  : 150,
                            'ytics' : 10        },

                        {   'name'  : 'Event_Commitment_Ratio',
                            'ystart': 1,
                            'yend'  : 2,
                            'ytics' : 0.1       },

                        {   'name'  : 'Speedup_w.r.t._Sequential_Simulation',
                            'ystart': 0,
                            'yend'  : 10,
                            'ytics' : 1         }
                    ]

rawDataFileName = 'bags'

statType        = [ 'Mean',
                    'CI_Lower',
                    'CI_Upper',
                    'Median',
                    'Lower_Quartile',
                    'Upper_Quartile'
                  ]


###### Don't edit below here ######

def mean_confidence_interval(data, confidence=0.95):
    # check the input is not empty
    if not data:
        raise RuntimeError('mean_ci - no data points passed')
    a = 1.0*np.array(data)
    n = len(a)
    m, se = np.mean(a), sps.sem(a)
    h = se * sps.t._ppf((1+confidence)/2., n-1)
    return m, m-h, m+h

def median(data):
    # check the input is not empty
    if not data:
        raise RuntimeError('median - no data points passed')
    return np.median(np.array(data))

def quartiles(data):
    # check the input is not empty
    if not data:
        raise RuntimeError('quartiles - no data points passed')
    sorts = sorted(data)
    mid = len(sorts) / 2
    if (len(sorts) % 2 == 0):
        # even
        lowerQ = median(sorts[:mid])
        upperQ = median(sorts[mid:])
    else:
        # odd
        lowerQ = median(sorts[:mid])  # same as even
        upperQ = median(sorts[mid+1:])
    return lowerQ, upperQ

def statistics(data):
    # check the input is not empty
    if not data:
        raise RuntimeError('statistics - no data points passed')

    mean = ci_lower = ci_upper = med = lower_quartile = upper_quartile = data[0]
    if len(data) > 1:
        mean, ci_lower, ci_upper = mean_confidence_interval(data)
        med = median(data)
        lower_quartile, upper_quartile = quartiles(data)

    statList = (str(mean), str(ci_lower), str(ci_upper), str(med), str(lower_quartile), str(upper_quartile))
    return ",".join(statList)

def sed_inplace(filename, pattern, repl):
    # For efficiency, precompile the passed regular expression.
    pattern_compiled = re.compile(pattern)

    # For portability, NamedTemporaryFile() defaults to mode "w+b" (i.e., binary
    # writing with updating). In this case, binary writing imposes non-trivial 
    # encoding constraints resolved by switching to text writing.
    with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_file:
        with open(filename) as src_file:
            for line in src_file:
                tmp_file.write(pattern_compiled.sub(repl, line))

    # Overwrite the original file with the temporary file in a
    # manner preserving file attributes (e.g., permissions).
    shutil.copystat(filename, tmp_file.name)
    shutil.move(tmp_file.name, filename)

def getIndex(aList, text):
    '''Returns the index of the requested text in the given list'''
    for i,x in enumerate(aList):
        if x == text:
            return i

def plot(data, fileName, title, subtitle, xaxisLabel, yaxisLabel, ystart, yend, ytics, linePreface):

    # Replace '_' with ' '
    g = Gnuplot.Gnuplot()
    multiLineTitle = title.replace("_", " ") + '\\n '+ subtitle.replace("_", " ")
    g.title(multiLineTitle)
    g("set terminal svg noenhanced background rgb 'white' size 1000,800 fname 'Helvetica' fsize 16")
    g("set key box outside center top horizontal font ',12' ")
    g("set autoscale xy")
    #g("set yrange [{0}:{1}]".format(unicode(ystart), unicode(yend)))
    #g("set ytics {}".format(unicode(ytics)))
    g("set grid")
    g.xlabel(xaxisLabel.replace("_", " "))
    g.ylabel(yaxisLabel.replace("_", " "))
    g('set output "' + fileName + '"')
    d = []
    for key in sorted(data[statType[0]]):
        result = Gnuplot.Data(  data['header'][key],data[statType[0]][key],\
                                data[statType[1]][key],data[statType[2]][key],\
                                with_="yerrorlines",title=linePreface+key   )
        d.append(result)
    g.plot(*d)

def plot_stats(dirPath, fileName, xaxisLabel, keyLabel, filterLabel, filterValue, model, lpCount):

    # Read the stats csv
    inFile = dirPath + 'stats/' + rawDataFileName + '/' + fileName + '.csv'
    reader = csv.reader(open(inFile,'rb'))
    header = reader.next()

    # Get Column Values for use below
    xaxis  = getIndex(header, xaxisLabel)
    kid    = getIndex(header, keyLabel)

    reader = sorted(reader, key=lambda x: int(x[xaxis]), reverse=False)
    reader = sorted(reader, key=lambda x: x[kid], reverse=False)

    for param in metricList:

        metric = param['name']
        ystart = param['ystart']
        yend   = param['yend']
        ytics  = param['ytics']

        outData = {'header':{}}

        # Populate the header
        for kindex, kdata in itertools.groupby(reader, lambda x: x[kid]):
            if kindex not in outData['header']:
                outData['header'][kindex] = []
            for xindex, data in itertools.groupby(kdata, lambda x: x[xaxis]):
                outData['header'][kindex].append(xindex)

        # Populate the statistical data
        for stat in statType:
            columnName  = metric + '_' + stat
            columnIndex = getIndex(header, columnName)
            if stat not in outData:
                outData[stat] = {}
            for xindex, data in itertools.groupby(reader, lambda x: x[xaxis]):
                for kindex, kdata in itertools.groupby(data, lambda x: x[kid]):
                    if kindex not in outData[stat]:
                        outData[stat][kindex] = []
                    value = [item[columnIndex] for item in kdata][0]
                    outData[stat][kindex].append(value)

        # Plot the statistical data
        title = model.upper() + ' model with ' + str("{:,}".format(lpCount)) + ' LPs'
        subtitle = 'key = ' + keyLabel
        outDir = dirPath + 'plots/' + rawDataFileName + '/'
        outFile = outDir + fileName + "_" + metric + '.svg'
        yaxisLabel = metric + '_(C.I._=_95%)'
        plot(outData, outFile, title, subtitle, xaxisLabel, yaxisLabel, ystart, yend, ytics, '')

        # Convert svg to pdf and delete svg
        outPDF = outDir + fileName + "_" + metric + '.pdf'
        subprocess.call(['inkscape', outFile, '--export-pdf', outPDF])
        subprocess.call(['rm', outFile])

def calc_and_plot(dirPath):

    # Load the sequential simulation time
    seqFile = dirPath + 'sequential.dat'
    if not os.path.exists(seqFile):
        print('Sequential data not available')
        sys.exit()
    seqFp = open(seqFile, 'r')
    seqCount, _, seqTime = seqFp.readline().split()
    seqFp.close()

    # Load data from csv file
    inFile = dirPath + rawDataFileName + '.csv'
    if not os.path.exists(inFile):
        print(rawDataFileName.upper() + ' raw data not available')
        sys.exit()

    data = pd.read_csv(inFile, sep=',')

    data['Event_Commitment_Ratio'] = \
            data['Events_Processed'] / data['Events_Committed']
    data['Total_Rollbacks'] = \
            data['Primary_Rollbacks'] + data['Secondary_Rollbacks']
    data['Event_Processing_Rate_(per_sec)'] = \
            data['Events_Processed'] / data['Simulation_Runtime_(secs.)']
    data['Speedup_w.r.t._Sequential_Simulation'] = \
            float(seqTime) / data['Simulation_Runtime_(secs.)']

    # Create the plots directory (if needed)
    outDir = dirPath + 'plots/'
    if not os.path.exists(outDir):
        os.makedirs(outDir)

    outName = outDir + rawDataFileName + '/'
    subprocess.call(['rm', '-rf', outName])
    subprocess.call(['mkdir', outName])

    # Create the stats directory (if needed)
    outDir = dirPath + 'stats/'
    if not os.path.exists(outDir):
        os.makedirs(outDir)

    outName = outDir + rawDataFileName + '/'
    subprocess.call(['rm', '-rf', outName])
    subprocess.call(['mkdir', outName])

    for searchAttrs in searchAttrsList:
        groupbyList = searchAttrs['groupby']
        filterName  = searchAttrs['filter']
        model       = searchAttrs['model']
        lpcount     = searchAttrs['lpcount']
        output      = searchAttrs['output']

        groupbyList.append(filterName)

        # Read unique values for the filter
        filterValues = data[filterName].unique().tolist()

        # Read the model name and LP count
        modelName = data[model].unique().tolist()
        lpCount   = data[lpcount].unique().tolist()

        for filterValue in filterValues:
            # Filter data for each filterValue
            filteredData = data[data[filterName] == filterValue]
            groupedData = filteredData.groupby(groupbyList)
            columnNames = list(groupbyList)

            # Generate stats
            result = pd.DataFrame()
            for param in metricList:
                metric = param['name']
                columnNames += [metric + '_' + x for x in statType]
                stats = groupedData.apply(lambda x : statistics(x[metric].tolist()))
                result = pd.concat([result, stats], axis=1)

            # Write to the csv
            fileName = output + str(filterValue)
            outFile = outName + fileName + '.csv'
            statFile = open(outFile,'w')
            for colName in columnNames:
                statFile.write(colName + ',')
            statFile.write("\n")
            statFile.close()
            result.to_csv(outFile, mode='a', header=False, sep=',')

            # Remove " from the newly created csv file
            # Note: It is needed since pandas package has an unresolved bug for
            # quoting arg which retains the double quotes for column attributes.
            sed_inplace(outFile, r'"', '')

            # Plot the statistics
            plot_stats( dirPath, fileName, groupbyList[0], groupbyList[1],
                            filterName, filterValue, modelName[0], lpCount[0] )


def main():
    dirPath = sys.argv[1]
    if not os.path.exists(dirPath):
        print('Invalid path to source')
        sys.exit()

    calc_and_plot(dirPath)

if __name__ == "__main__":
    main()


