
import os, sys, logging, datetime, multiprocessing, icebridge_common, shutil

import threading, time

import flight_list # Contains AN_FLIGHTS and GR_FLIGHTS

#===============================================================================
# Constants

COMPLETED_DATES_FILE = '/u/smcmich1/icebridge/upload_software/completed_dates.txt'
FAILED_DATES_FILE    = '/u/smcmich1/icebridge/upload_software/failed_dates.txt'
SOFTWARE_FOLDER      = '/u/smcmich1/icebridge/upload_software/'
SIPS_LOG_FOLDER      = os.path.join(SOFTWARE_FOLDER, 'logs')

INITIAL_UNPACK_FOLDER = '/nobackup/smcmich1/icebridge/nsidc_uploads/temp_tar_unpack'

ASP_PATH = '/u/smcmich1/programs/StereoPipeline-2.6.0-2018-02-06-x86_64-Linux/bin/'

EMAIL_ADDRESS = 'scott.t.mcmichael@nasa.gov'

# Controls access to the two DATES files.
FLIGHT_DATES_RW_LOCK = threading.Lock()

#===============================================================================
# Classes

class Date:

    def __init__(self, year, month, day):
        self.year  = int(year)
        self.month = int(month)
        self.day   = int(day)

    def __str__(self):
        return ('%04d%02d%02d' % (self.year, self.month, self.day))

    def yyyymmdd(self):
        return ('%04d%02d%02d' % (self.year, self.month, self.day))

    def yyyy_mm_dd(self):
        return ('%04d_%02d_%02d' % (self.year, self.month, self.day))

    def yyyy_mm(self):
        return ('%04d_%02d' % (self.year, self.month))

class Campaign:

    def __init__(self, site, year):
        self.site = site
        self.year = year

    def getAllDates(self):
        '''Return a list of all the valid dates for this campaign'''

        if self.site == 'AN':
            input_list = flight_list.AN_FLIGHTS
        else:
            input_list = flight_list.GR_FLIGHTS
        flights = [f for f in input_list if f.startswith(self.year)]

        dates = []
        for f in flights:
            year  = f[0:4]
            month = f[4:6]
            day   = f[6:8]
            dates.append(Date(year,month,day))

        return dates

    def __str__(self):
        '''Return the string representation'''
        return self.site + '_' + self.year



def upload_log_and_cleanup(dem_folder, ortho_folder, dem_summary_folder, ortho_summary_folder,
                           unpack_prefix, has_dem_summary, has_ortho_summary, 
                           dem_tarball, ortho_tarball, camera_tarball, summary_tarball,
                           remote_folder, date, logger):
    '''Called by the worker thread in UploadManager'''

    print 'Ready to upload folder ' + dem_folder
    print 'Ready to upload folder ' + ortho_folder
    #return # DEBUG!!!!

    try:
        # Upload the data
        logger.info('Beginning data upload for flight ' + str(date))
        uploadFolderToNSIDC(dem_folder,   'dem/'  +remote_folder, logger)
        uploadFolderToNSIDC(ortho_folder, 'ortho/'+remote_folder, logger)
        if has_dem_summary:
            uploadFolderToNSIDC(dem_summary_folder,   'dem/'  +remote_folder, logger)
        if has_ortho_summary:
            uploadFolderToNSIDC(ortho_summary_folder, 'ortho/'+remote_folder, logger)
        logger.info('Data upload finished for flight ' + str(date))

        success = has_dem_summary and has_ortho_summary

    except Exception as e:
        success = False
        logger.error('Caught exception for date ' + str(date) +'\n' + str(e))

    if success:

        # Log the input tarballs we used and whether we had all summary files.
        updateLogFile(COMPLETED_DATES_FILE, date, dem_tarball, ortho_tarball, 
                      camera_tarball, summary_tarball,
                      has_dem_summary, has_ortho_summary)
        subject = 'COMPLETED flight date: ' + str(date)
        logger.info(subject)

    else:
        updateLogFile(FAILED_DATES_FILE, date, dem_tarball, ortho_tarball, 
                      camera_tarball, summary_tarball,
                      has_dem_summary, has_ortho_summary)
        subject = 'FAILED to process flight date '+ str(date)
        logger.error(subject)

    sendEmail(EMAIL_ADDRESS, subject, 'NT')

    # Clean up the temporary folders
    #raise Exception('DEBUG')

    if success:
        logger.info('Ready to delete folders: ' + unpack_prefix)
        cmd = 'rm -rf ' + unpack_prefix + '*'
        logger.info(cmd)
        os.system(cmd)




class UploadManager():
    '''Class to keep uploading data in the background while the main process starts a new flight.'''


    def __init__(self):
        self._worker = None

    def __del__(self):
        self.cleanup()

    def cleanup(self):
        if self._worker != None:
            self._worker.join()
        self.worker = None

    def uploadFlight(self, dem_folder, ortho_folder, dem_summary_folder, ortho_summary_folder,
                     unpack_prefix, has_dem_summary, has_ortho_summary, 
                     dem_tarball, ortho_tarball, camera_tarball, summary_tarball,
                     remote_folder, date, logger):
        '''Upload the flight in a separate thread.  If another flight is still being uploaded,
           blocks until that upload is finished.'''

        # Block here until we are not busy with another upload.
        if self._worker != None:
            self._worker.join()

        # Set up a working thread with the information
        self._worker = threading.Thread(target=upload_log_and_cleanup,
                                        args=(dem_folder, ortho_folder, dem_summary_folder,
                                              ortho_summary_folder, unpack_prefix,
                                              has_dem_summary, has_ortho_summary,
                                              dem_tarball, ortho_tarball, 
                                              camera_tarball, summary_tarball,
                                              remote_folder, date, logger))
        # Let the worker thread run on its own
        self._worker.start()
               
        return




#===============================================================================
# Functions

def sendEmail(address, subject, body):
    '''Send a simple email from the command line'''
    # Remove any quotes, as that confuses the command line.
    subject = subject.replace("\"", "")
    body    = body.replace("\"", "")
    try:
        cmd = 'mail -s "' + subject + '" ' + address + ' <<< "' + body + '"'
        os.system(cmd)
    except Exception as e:
        print("Could not send mail.")


def getLatestTarFileMatchingDate(dirs, date):
    '''Find the most recent tar file containing a date in the given folders'''
    date_string = str(date)
    candidates = []
    for d in dirs:
        # Get all matching tar files (possibly multiple versions)
        tars = os.listdir(d)
        new_candidates = [f for f in tars if ('.tar' in f) and (date_string in f)]
        candidates = candidates + [os.path.join(d, f) for f in new_candidates]

        # Ignore files manually marked not to use!
        candidates = [c for c in candidates if 'old' not in c]

    if not candidates:
        raise Exception('No tarballs found for date ' + str(date))

    # The last file alphabetically is the latest one
    return sorted(candidates)[-1]

# Functions to find needed tarballs.
def findDemTarball(campaign, date):
    dirs = ['/u/smcmich1/icebridge/output', '/u/smcmich1/icebridge/oleg_dems']
    return getLatestTarFileMatchingDate(dirs, date)

def findOrthoTarball(campaign, date):
    dirs = ['/u/smcmich1/icebridge/ortho', '/u/smcmich1/icebridge/oleg_ortho']
    return getLatestTarFileMatchingDate(dirs, date)

def findCameraTarball(campaign, date):
    dirs = ['/u/smcmich1/icebridge/camera', '/u/smcmich1/icebridge/oleg_cameras']
    return getLatestTarFileMatchingDate(dirs, date)

def findSummaryTarball(campaign, date):
    dirs = ['/u/smcmich1/icebridge/summaries', '/u/smcmich1/icebridge/oleg_summaries']
    return getLatestTarFileMatchingDate(dirs, date)

def fetchTarballsFromTapes(campaign, date_list, logger):
    '''Request that all of the tarballs we will need for this run be loaded from tape.'''

    logger.info('Locating all the tarballs needed for ' + str(len(date_list)) + ' dates.')

    # Find all the tarballs we will need
    needed_files = []
    for date in date_list:
        try:
            dem_tarball     = findDemTarball    (campaign, date)
            ortho_tarball   = findOrthoTarball  (campaign, date)
            camera_tarball  = findCameraTarball (campaign, date)
            summary_tarball = findSummaryTarball(campaign, date)
            needed_files.append(dem_tarball)
            needed_files.append(ortho_tarball)
            needed_files.append(camera_tarball)
            needed_files.append(summary_tarball)
        except:
            logger.error('Error finding all tarballs for date: ' + str(date))