#!/usr/bin/env python2

"""analyze.py: Reads and analyzes ecfs metadata db dump."""
__author__ = "Matthias Grawinkel"
__status__ = "Production"


import matplotlib
matplotlib.use('Agg')

from matplotlib import pyplot
from matplotlib import dates

import numpy as np

import sys
import os
import gzip
import time
import calendar
import datetime
import resource
import re
import math
import json

from collections import defaultdict
from collections import Counter

import hashlib

from file_size_groups import *

MONITOR_LINES=100000

# used for cdf plot
space_per_uid = Counter()
files_per_uid = Counter()


# used for file size histogram
num_files_per_size_category = Counter()
total_size_per_size_category = Counter()

# file_size_counter = Counter()

files_per_dir_counter = Counter()

stats = defaultdict(int)

# how many .grib files are there?
file_extension_cnt = Counter()
file_extension_size = Counter()

created_capacity_cnt = defaultdict(int)
created_capacity_size = defaultdict(int)

modified_capacity_cnt = defaultdict(int)
modified_capacity_size = defaultdict(int)


# this one will get HUUUUUGE
file_map = dict()

def prettyfy(number):
    d = float(number)
    if d - int(d) > 0:
        return '{:,.3f}'.format(d)
    return '{:,d}'.format(int(d))

def to_gigabyte(val):
    return (float(val) / 1024 / 1024 / 1024)

def to_terabyte(val):
    return (float(val) / 1024 / 1024 / 1024 / 1024)

def to_petabyte(val):
    return (float(val) / 1024 / 1024 / 1024 / 1024 / 1024)

def to_millions(val):
    return (float(val) / 1000 / 1000)

def to_billions(val):
    return (float(val) / 1000 / 1000 / 1000)

class Timer():
    def __init__(self, s):
        self.s = s

    def __enter__(self):
        self.start = time.time()

    def __exit__(self, *args):
        print ("%s: %fs" % (self.s, (time.time() - self.start)))


def get_md5(s, hexdigest=False):
    # return s
    m = hashlib.md5()
    m.update(s.encode())
    if hexdigest:
        return m.hexdigest()
    else:
        return m.digest()



def days_between(ts1, ts2):
    """
     '20050620-004900',
     '19700101-000000',
     '20120615-075612',
    """
    if ts1 == '19700101-000000' or ts2 == '19700101-000000':
        return None

    t1 = datetime.datetime.strptime(ts1, "%Y%m%d-%H%M%S")
    t2 = datetime.datetime.strptime(ts2, "%Y%m%d-%H%M%S")
    return (t1 - t2).days


def load(source_file, max_lines=None,log_start="20140905-000000"):

    object_re = re.compile("^\s*([\d]+)\s*([\d]+)\s*([\d]+)\s*([\d]+)\s*([\d]+-[\d]+)\s*([\d]+-[\d]+)\s*([\d]+-[\d]+)\s*([a-zA-Z0-9/.\-]+)\s*x'([[a-zA-Z0-9/.\-]+)'\s*$")


    with gzip.open(source_file, 'rt') as source:
        t = time.time()
        plines = 0
        for line in source:
            plines += 1
            if plines % MONITOR_LINES == 0:
                print ("processed lines: %d, mem: %rMB, lines/s: %r: found files: %d" %
                 (plines,
                  float(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) / 1024 / 1024,
                  int(MONITOR_LINES / (time.time() - t)),
                  len(file_map)
                 )
                )
                t = time.time()

            if max_lines:
                if plines >= max_lines:
                    break

            m = object_re.match(line)

            if m:
                r = m.groups()

                size = int(r[0])
                uid = int(r[1])
                gid = int(r[2])
                cos = int(r[3])  # cosid is Class-of-Service in HPSS
                creation_time = str(r[4])
                read_time = str(r[5])
                mod_time = str(r[6])
                path = str(r[7])
                magic = str(r[8])   # $bitfileid,$objectid;?

                stats["total_files"] += 1
                stats["total_size"] += size
                stats["max_file_size"] = max(stats["max_file_size"], size)

                # for cdf
                files_per_uid[uid] += 1
                space_per_uid[uid] += size

                # for file size histogram
                g = get_file_size_group(size)
                num_files_per_size_category[g] += 1
                total_size_per_size_category[g] += size

                # can be used for average file sizes et al.
                # file_size_counter[size] += 1

                #file name
                file_name = os.path.basename(path)

                if file_name.__contains__('.'):
                    extension = file_name[file_name.rfind('.'):]
                else:
                    extension = "unknown"
                
                file_extension_cnt[extension] += 1
                file_extension_size[extension] += size

                # files per dir
                dir_id = get_md5(os.path.dirname(path))
                files_per_dir_counter[dir_id] += 1

                # access times
                file_age_days = days_between(log_start, creation_time)

                created_capacity_cnt[file_age_days] += 1
                created_capacity_size[file_age_days] += size
                
                last_read_days = days_between(log_start, read_time)
                if not last_read_days:
                    stats["unread_files_cnt"] += 1
                    stats["unread_files_size"] += size
                
                last_modified_days = days_between(log_start, mod_time)
                modified_capacity_cnt[file_age_days] += 1
                modified_capacity_size[file_age_days] += size

                file_db_entry = (size, file_age_days, last_read_days, last_modified_days)

                # this will get large, but is required for the more interesting plots.
                file_map[get_md5(path)] = file_db_entry

                # how many days ago did a directory see its last upload / download / change?
                # how many files exist that have not been accessed since X days / never been read at all?
                # file_age to capacity?

def generate_x_labels(log_start):
    
    x_vals = list()
    deltas = list()
    day_0 = datetime.datetime.strptime(log_start, "%Y%m%d-%H%M%S")

    deltas.append(0)
    x_vals.append(day_0)
    
    d = day_0.replace(day=1)
    for i in range(80):
        delta = (day_0 - d).days
        deltas.append(delta)
        # print (d, delta)
        d -= datetime.timedelta(days=10)
        d = d.replace(day=1)
        x_vals.append(d)

    return x_vals, deltas

def unaccessed_files_plot(target_graph_dir, log_start, source_type):

    print(str(datetime.datetime.now()), "start to prepare unaccessed_files_plot")
    
    # these metrics will be plotted.
    unaccessed_files_cnt = Counter()
    unmodified_files_cnt = Counter()
    unread_files_cnt = Counter()
    total_created_files = Counter()
    existing_never_read_files = Counter()

    # index to filemap contents
    CREATION_DAYS = 1
    READ_DAYS = 2
    MODIFIED_DAYS = 3

    x_val_dates, deltas = generate_x_labels(log_start)
    x_val_dates = dates.date2num(x_val_dates)

    for days in deltas:
        # print (days)

        for f in file_map.itervalues():
            # print (f)
            if f[CREATION_DAYS] > days:
                total_created_files[days] += 1
                # make sure the file actually exists

                if f[READ_DAYS] and f[MODIFIED_DAYS]:
                    if days < min(f[READ_DAYS], f[MODIFIED_DAYS]):
                        unaccessed_files_cnt[days] += 1

                    if days < f[MODIFIED_DAYS]:
                        unmodified_files_cnt[days] += 1

                    if days < f[READ_DAYS]:
                        unread_files_cnt[days] += 1

                else:
                    existing_never_read_files[days] += 1
                    # some files have no f[2] (read_time) value
                    if days < f[MODIFIED_DAYS]:
                        unaccessed_files_cnt[days] += 1
                        unmodified_files_cnt[days] += 1
    # print(json.dumps(unaccessed_files_cnt, sort_keys=True, indent=2))
    print(str(datetime.datetime.now()), "plotting unaccessed_files_plot")
    
    y_vals = defaultdict(list)
    
    ax_files = [(total_created_files, "Existing files", 'k', ':o'),
                     (unaccessed_files_cnt, "Unaccessed files", 'r', '-s'),
                     (unmodified_files_cnt,"Unmodified files", 'g', '--*'), 
                     (unread_files_cnt, "Unread files", 'b', '-.+'),
                     (existing_never_read_files, "Existing never read files", 'k', ':x')
                    ]

    for p in ax_files:
        for key in deltas:
            y_vals[p[1]].append(to_millions(p[0][key]))