# Worker to be used for analysis of CIS, Cs and LAS
# - Christopher Tunnell, 2009 <tunnell@hep.uchicago.edu>
# - Brian Martin, 2009 <brian.thomas.martin@cern.ch>
# - Seb Viret, 2009 (viret@in2p3.fr)
# - Mikhail Makouski, 2009 <makouski@cern.ch>
# - Marco van Woerden, 2011 (mwoerden@gmail.com)
# - Joshua Montgomery, 2012 (Joshua.J.Montgomery@gmail.com)
# - Andrey Kamenshchikov, 2013 (akamensh@cern.ch)
# Even though its not FORTRAN, its good to beable to print....
#        1         2         3         4         5         6         7         8
#2345678901234567890123456789012345678901234567890123456789012345678901234567890
from __future__ import print_function
from src.GenericWorker import *
from src.cesium        import chanlists,csdb

import sys
if sys.version_info[:1][0]==3: # python 3
    mypath="/afs/cern.ch/user/t/tilebeam/offline/lib/python%d.%d/site-packages" % sys.version_info[:2]
    sys.path.append(mypath)
    import pymysql
    python3 = True
else: # python 2
#
    import _mysql,MySQLdb
    python3 = False

import os
import time,datetime

class Use(GenericWorker):
    "Select runs for TUCS to use."

    # THIS WORKER ASSIGNS EVENTS TO EACH REGION WITH A RUN NUMBER
    # AND A RUN TYPE. THERE WILL ONLY BE ONE EVENT PER RUN.

    def __init__(self, run, run2='', type='readout', region=None,
                 useDateProg=True, verbose=0, runType='all',
                 keepOnlyActive=True, filter=' ',
                 amp='15000', getLast = False, updateSpecial = True,
                 updateC10 = True, specialFile='./specialcells.txt',#akamensh
                 badmodulesFile='./badmodules.txt',
                 allowC10Errors = False, cscomment='', TWOInput= False,
                 preprocessing=False):

        ### PARSE RUNS SELECTION ###
        (run, run2, TWOInput) = self.Translate_Runs(run, run2, TWOInput)

        ### REGION SELECTION ###
        if region != None and len(region)>0:			# CHECK WHETHER REGION IS SPECIFIED
            if ',' in region: 					# IF REGION IS A LIST OF REGIONS
                self.region = region.split(',')			# SEPERATE THE DIFFERENT SPECIFIED REGIONS
            elif isinstance(region,str): 			# A SINGLE REGION IS SPECIFIED AS STRING
                self.region = [region]				# SAVE REGION AS A LIST
            elif isinstance(region,list):			# MULTIPLE REGIONS ARE SPECIFIED AS LIST
                self.region = region				# STORE LIST
            if self.region[0][0] == 'H': 			# LIST OF REGIONS IS IN CELL HASH FORM
                from workers.noise.NoiseWorker import ConvertCellHash2TucsHash
                self.region = [ ConvertCellHash2TucsHash(int(hash[1:])) for hash in self.region ]
        else:							# NO REGION IS SPECIFIED
            self.region     = None				# EQUAL REGION TO NONE

        print("Starting mysql service")
        if python3:
            self.pymysqldb = pymysql.connect(host='pcata007.cern.ch', user='reader')    # CONNECT TO DATABASE
#             with self.pymysqldb: # with stopped working in python3.8
                 ##setup pymysql cursor for executing queries (necessary in py3)
            self.pymysqldb_cursor = self.pymysqldb.cursor()
        else:
            self.mysqldb = _mysql.connect(host='pcata007.cern.ch', user='reader')	# CONNECT TO DATABASE
        print("done")
            ### VARIABLES ###
            # SOME USEFUL GLOBAL VARIABLES
        self.type = type
        self.runs           = []				# RUN LIST
        self.runsToBeAddedToRegions        = []				      # RUN LIST
        self.CsAtlasRunList = []         			# runnumber set

        self.rType          = runType				# STORE REQUESTED RUNTYPE
        self.fragStr        = {}				# DIGIFLAGS FOR EACH RUN NUMBER
        self.keepOnlyActive = keepOnlyActive			# ONLY KEEP THE ACTIVE DETECTOR PARTS

        # USEFUL CESIUM VARIABLES
        self.verbose        = verbose				# VERBOSE
        self.updateSpec     = updateSpecial			# SPECIFIED UPDATE
        self.allowC10Err    = allowC10Errors			# ARE ERRORS FOR C10 ALLOWED
        self.cscomment      = cscomment 			# CESIUM RUN/MAGNET DESCRIPTION
        self.updateC10      = updateC10                         # akamensh: inherited from old version
        self.specialFile    = specialFile                       # akamensh: inherited from old version
        self.badmodulesFile = badmodulesFile                    # akamensh: inherited from old version

        # USEFUL CHARGE INJECTION VARIABLES
        self.twoInputs      = TWOInput				# STORE WHETHER CIS RUNS SHOULD BE BETWEEN TWO DATES
        self.preprocessing  = preprocessing
        # USEFUL LASER VARIABLES
        self.filter         = filter				# STORE WHICH LASER FILTER IS REQUESTED
        self.amp            = amp				# STORE REQUESTED AMPERAGE

        prelimRunList = []

        if self.preprocessing:
            if python3:
                self.pymysqldb_cursor.execute("""SELECT run, type, date, digifrags FROM tile.comminfo WHERE run='%d'""" % (run[0]))
                d = self.pymysqldb_cursor.fetchone()                # STORE DATABASE RESULT
                rundate = d[2].strftime("%Y-%m-%d") if d != () else None
            else:
                self.mysqldb.query("""SELECT run, type, date, digifrags FROM tile.comminfo WHERE run='%d'""" % (run[0]))
                r = self.mysqldb.store_result()				# STORE DATABASE RESULT
                d = r.fetch_row()				# FETCH DATABASE RESULT
                rundate = d[0][2].split()[0] if d != () else None

            (run, run2, self.twoInputs)=self.Translate_Runs(rundate, run2, TWOInput, preprocessing)

        if isinstance(run, str):
            if useDateProg:                  # We received dates
                if self.twoInputs:
                    prelimRunList = self.dateProg(run,run2)
                else:                        # Only start date was given
                    prelimRunList = self.dateProg(run,)
            elif os.path.exists(run):        # We received a filename
                f = open(run)
                for line in f.readlines():
                    line = line.rstrip()
                    prelimRunList.append(int(line))

        elif isinstance(run, int):           # We received a single run as integer
            prelimRunList = [run]

        elif isinstance(run, (list, tuple)): # We received a list or tuple of runs (have to be ints!)
            for element in run:
                if isinstance(element, int):
                    prelimRunList.append(element)               # why only int when later transforming to ints?

        runs = list(map(int, prelimRunList))				# MAP/TYPECAST RUNS INTO INTEGERS LIST
        ### SELECT ONLY THE LAST RUN (SPECIAL OPTION) ###
        if getLast and isinstance(run, str): 			# CHECK THAT ONLY THE LAST RUN IS REQUESTED
            run_max = 0						# INITIALIZE LATEST RUN NUMBER
            for thisrun in runs:					# LOOP OVER ALL RUNS
                if thisrun > run_max:				# IF GREATEST RUN NUMBER SOFAR IS FOUND
                    run_max = thisrun				# STORE AS MAXIMAL RUN NUMBER
            runs = [run_max]					# STORE LAST RUN IN INTEGER RUN NUMBER LIST

        # CESIUM: constants for E3,E4 or other special cells may come from external text file
        if self.rType=='cesium':
            print(("specfile: ",self.specialFile))
            csdb.specfile=self.specialFile
            specval = csdb.readspecialces(0,'XXX',0,0)
        # CESIUM: EACH CHANNEL MAY HAVE IT'S OWN LIST OF RUNS
        # AS THE RUNS ARE NOT PARTITION WIDE
        for thisrun in runs:					# LOOP OVER RUNS
            if self.rType=='cesium':				# CESIUM IS REQUESTED
                date = None
                if thisrun>40000:					# IF RUN NUMBER IS AN ATLAS RUN NUMBER
                    if python3:
                        self.pymysqldb_cursor.execute("""SELECT date FROM tile.comminfo WHERE run>='%d' ORDER BY run DESC LIMIT 1""" % (thisrun))
                        d = self.pymysqldb_cursor.fetchone()            # STORE DATABASE RESULT
                    else:
                        self.mysqldb.query("""SELECT date FROM tile.comminfo WHERE run>='%d' ORDER BY run DESC LIMIT 1""" % (thisrun))
                        r = self.mysqldb.store_result()			# STORE DATABASE RESULT
                        d = r.fetch_row()				# FETCH DATABASE RESULT
                else:						# IF RUN NUMBER IS SPECIAL CESIUM RUN NUMBER
                    if python3:
                        self.pymysqldb_cursor.execute("SELECT time FROM tile.runDescr WHERE run=%i" % (thisrun))
                        d = self.pymysqldb_cursor.fetchone()
                    else:
                        self.mysqldb.query("SELECT time FROM tile.runDescr WHERE run=%i" % (thisrun))
                        r = self.mysqldb.store_result()
                        d = r.fetch_row()
                if d != ():					# IF RESULT IS NON-EMPTY
                    if python3:
                        date = d[0].strftime("%Y-%m-%d %H:%M:%S")
                    else:
                        date = d[0][0]
                self.runs.append([thisrun, self.rType, date])	# APPEND RUN NUMBER

            else:
                if python3:
                    self.pymysqldb_cursor.execute("""SELECT run, type, date, enddate, mbdata, digifrags FROM tile.comminfo WHERE run='%d'""" % (thisrun)) # added mbdata flag
                    d = self.pymysqldb_cursor.fetchone()                # STORE DATABASE RESULT
                else:
                    self.mysqldb.query("""SELECT run, type, date, enddate, mbdata, digifrags FROM tile.comminfo WHERE run='%d'""" % (thisrun)) # added mbdata flag
                    r = self.mysqldb.store_result()				# STORE DATABASE RESULT
                    d = r.fetch_row()				# FETCH DATABASE RESULT

                if d == ():                    # IF RESULT IS EMPTY
                    rundb, rtype, date_dt, enddate_dt, mbdata, digifrags = thisrun, None, None, None, None, None
                    #date = date_dt.strftime("%Y-%m-%d %H:%M:%S")
                    #enddate = enddate_dt.strftime("%Y-%m-%d %H:%M:%S")
                    print('not in database', thisrun)
                else:                        # IF RESULT IS NON-EMPTY
                    if python3:
                        rundb, rtype, date_dt, enddate_dt, mbdata, digifrags = d    #  TAKE THE FIRST RUN WITH A GIVEN RUN NUMBER
                        date = date_dt.strftime("%Y-%m-%d %H:%M:%S")
                        enddate = enddate_dt.strftime("%Y-%m-%d %H:%M:%S")
                    else:
                        rundb, rtype, date, enddate, mbdata, digifrags = d[0]    #  TAKE THE FIRST RUN WITH A GIVEN RUN NUMBER

                if runType == 'all' or runType == rtype or (mbdata == '1' and runType == 'MBias'): # if we want mbias runs, ask for mbdata==1
                    if date:
                        if enddate:
                            date=date+","+enddate
                    elif enddate:
                        date=enddate
                    self.runs.append([int(rundb), rtype, date])	# APPEND RUN IF RUN TYPE IS WITHIN SELECTION

                if self.keepOnlyActive and ( runType == rtype or rtype == None ):
                    if self.verbose or (digifrags and len(digifrags)/6 != 256):
                        print('In run', thisrun, ', modules in readout: ',len(digifrags)//6)
                    if digifrags == None:			# IF THERE ARE NO DIGIFRAGS AVAILABLE
                        self.keepOnlyActive = False		# TURN OFF FILTER FOR ACTIVE DETECTOR ELEMENTS
                    self.fragStr[int(thisrun)] = digifrags		# STORE DIGIFRAGS

        if python3:
            self.pymysqldb_cursor.close()
            self.pymysqldb.close()                    # CLOSE DATABASE CONNECTION
        else: # python 2
            self.mysqldb.close()					# CLOSE DATABASE CONNECTION

    def ProcessStart(self):
        if self.region == None or self.region == '':        # IF NO DETECTOR COMPONENT IS SPECIFIED
            print('Use: Using all the detector')        # PRINTOUT
        else:                            # IF DETECTOR COMPONENT IS SPECIFIED
            print('Use: Only using region(s)', self.region)    # PRINTOUT
        runsOfThisRuntype = run_list.getRunsOfType(self.rType)

        for run in sorted(self.runs):
            thisrun = None
            for arun in runsOfThisRuntype:

                if arun.runNumber == run[0]:
                    thisrun = arun
                    break
            if thisrun == None:
                print("run ",run)
                thisrun = Run(runNumber=run[0],runType=run[1],time=run[2],data={})
            self.runsToBeAddedToRegions.append(thisrun)

        # SPECIAL CESIUM
        if self.rType=='cesium':				# IF CESIUM RUNS ARE REQUESTED
            if python3:
                self.db = pymysql.connect(host='pcata007', user='reader', db='tile')
                self.c = self.db.cursor(pymysql.cursors.Cursor)     # CURSOR THAT RETURNS DATABASE ROWS AS TUPLES
                self.cc = self.db.cursor(pymysql.cursors.DictCursor) # CURSOR THAT RETURNS DATABASE ROWS AS DICTIONARIES
            else:
                self.db = MySQLdb.connect(host='pcata007', user='reader', db='tile')
                self.c = self.db.cursor(MySQLdb.cursors.Cursor)	 # CURSOR THAT RETURNS DATABASE ROWS AS TUPLES
                self.cc = self.db.cursor(MySQLdb.cursors.DictCursor) # CURSOR THAT RETURNS DATABASE ROWS AS DICTIONARIES

            self.Abad=[41,42] 					# ALLOW ERRORS FOR A16 IN SPECIAL PMTS WITH CUTOUTS
            self.MAbad=[36,61]					# ALLOW ERRORS FOR A16 IN SPECIAL MODULES WITH CUTOUTS
            self.Abad1=[7,8,11,12]                              # allow errors for A12, A13 in special modules with cutouts
            self.MAbad1=[36,37,60,61]
            if self.allowC10Err:				# CHECK IF ERRORS FOR C10 SHOULD BE ALLOWED
                self.Cbad=[5,6] 				# ALLOW ERRORS FOR C10 IN SPECIAL PMTS
            else:						# IF ERRORS FOR C10 ARE NOT ALLOWED
                self.Cbad=[0] 					# DO NOT ALLOW ERRORS FOR C10
            self.Dbad=[17,18] 					# ALLOW ERRORS FOR D5 IN SPECIAL PMTS
            self.MDbad=[15,18]					# ALLOW ERRORS FOR D5 IN SPECIAL MODULES EBA15 AND EBC18
            self.period=datetime.timedelta(days=7)		# STORE A PERIOD OF 7 DAYS IN UNIX TIME

    def ProcessRegion(self, region):
        # Check if we need to do something for this region
        useRegion = False
        if self.region:
            for reg in self.region:
                if reg in region.GetHash() or reg in region.GetHash(1):
                    useRegion = True
                    break
        else:
            useRegion = True

        # if not we stop here
        if not useRegion:
            return region

        # Cesium is always special
        if self.rType == 'cesium':
            num=region.GetNumber(1)
            if len(num)==3: # This is a channel
                module=region.GetParent()
#
##### THIS NEEDS FIXING FOR BEING ABLE TO HAVE MULTIPLE USE STATEMENTS
#
                if module.GetEvents()==set([]):             # No runs in the module?  Add them all
                    for run in self.runsToBeAddedToRegions:                # LOOP
                        self.FillModule(module,run)            # ADD RUN TO THE MODULE
#
##### END OF PART NEEDING FIXING
#
                for evt in module.GetEvents():        # LOOP OVER ALL EVENTS IN CURRENT MODULE
                    if num[2] in evt.data['PMT']:
                        region.AddEvent(Event(evt.run, data={'csRun':evt.data['csRun'],
                                                             'csRun_time':evt.data['csRun_time'],
                                                             'calibration':None}))

        else:                        # IF REQUESTED RUNS ARE NOT CESIUM
            for run in self.runsToBeAddedToRegions:            # LOOP
                if self.keepOnlyActive and not self.isActive(region.GetHash(), int(run.runNumber)):
                    if self.verbose:            # IF FILTER IS ON
                        print('Region not in readout, removing: ', region.GetHash())
                else:
                    if self.rType=='Las' and 'gain' not in region.GetHash():    # IF REGION IS NOT an adc
                        continue                         # SKIP REGION
                    data = {}                            # CREATE EMPTY DATA COLLECTION
                    data['region'] = region.GetHash()    # STORE CHANNEL POSITION IN
                    region.AddEvent(Event(run, data=data))
        return region

    def ProcessStop(self):					# DEFINE PROCESSSTOP
        #Create list of runs for use in other workers
        global Use_run_list
        global Use_date_list
        Use_run_list = []
        Use_date_list=[]
        for run in sorted(self.runs):
            Use_run_list.append(run[0])
            Use_date_list.append(run[2].split()[0])
        #print "Use_date_list=", Use_date_list
        #print Use_run_list

        if self.rType=='cesium':			# IF CESIUM DATABASE CONNECTIONS WERE OPENED
            self.c.close()					# CLOSE DATABASE CURSOR
            self.cc.close()					# CLOSE DATABASE CURSOR
            self.db.close()					# CLOSE DATABASE

   ### SOME FUNCTIONS THAT ARE CESIUM SPECIFIC ###
    def FillModule(self,region,run):			# DEFINE FILLMODULE
        num=region.GetNumber(1)					 # Get region numbers

        if len(num)==2: 					 # IF REGION IS A MODULE
            rlist=[run]                                          # STORE RUN NUMBER IN RUN LIST
            if run.runNumber>40000:                              # The case of an Atlas runNumber
                date=time.strptime(run.time,"%Y-%m-%d %H:%M:%S") # IF A DATE IS SPECIFIED
                rlist=self.GetCsRuns(date-self.period,date)	 # STORE RUN NUMBERS IN RUN LIST

            (pmtlist,dict)=self.GetCsDescr(rlist,region.GetParent().GetName(),num[1])

            if dict!=None:					 # IF THERE IS A DATABASE RESULT
#                print (pmtlist,dict)
                if run.runNumber>40000:				 # IF A DATE IS SPECIFIED
                    ATLASrun=run				 # STORE ATLAS RUN NUMBER
                else:						 # IF NO DATE IS SPECIFIED
                    ATLASrun=self.GetOneATLASRun(dict['time'])	 # GET ATLAS RUN NUMBER

                currentrun = None
                for arun in self.CsAtlasRunList:
##                    print datetime.datetime.strptime(arun.time,"%Y-%m-%d %H:%M:%S").year,dict['time'].year
##                    if (dict['time']<datetime.datetime.strptime(arun.time,"%Y-%m-%d %H:%M:%S")):
##                        arun.time=dict['time'].strftime("%Y-%m-%d %H:%M:%S")
                    if arun.runNumber == ATLASrun:
                        currentrun = arun

                if currentrun==None:
                    currentrun = Run( ATLASrun, 'cesium', dict['time'] , None )
                    self.CsAtlasRunList.append(currentrun)
                region.AddEvent( Event(currentrun, data={'csRun':dict['run'],'csRun_time':dict['time'],'PMT':pmtlist,'source':dict['source']}) )

    def GetOneATLASRun(self,time):
        if self.c.execute("SELECT run FROM comminfo WHERE date>'%s' and  run<9999999 ORDER BY date ASC LIMIT 1"%\
                          (time.strftime("%Y-%m-%d %H:%M:%S")))>=1:
            cl=self.c.fetchall()
            for r in cl:
                return r[0]

    def GetCsRuns(self, time1, time2):
        "Given start and end time, find all Cs runs in between"
        csruns=[]
        if self.c.execute( "SELECT run FROM runDescr WHERE time>'%s' AND time<'%s' ORDER BY time DESC" % \
                          (time1.strftime("%Y-%m-%d %H:%M:%S"),time2.strftime("%Y-%m-%d %H:%M:%S")) )>=1:
            cl=self.c.fetchall()
            for r in cl:
                if not r[0] in csruns:
                    csruns.append(r[0])
        return csruns

    def GetCsDescr(self,csruns,par,mod):
        for csrun in csruns:
            #print csrun,par,mod
            # here we have list of runs and look for run/par/mod in the database
            # as runs are sorted in desc. order we get the latest cs run
            if self.cc.execute("SELECT * FROM runDescr WHERE run=%i AND partition='%s' AND module=%i"%(csrun.runNumber,par,mod))!=1:
                continue
            dbdicts = self.cc.fetchall()
##            print csrun,par,mod,dbdicts
            #it should be only one record
            d=None
            for dict in dbdicts:
                dblist=self.if_ok(dict)
##                print csrun,par,mod,dict,dblist
                return (dblist,dict)

        # if we get here there were no Cs runs during the previous month
        return (None,None)

    def if_ok(self,dbdict):
        retlist=[]
        par=dbdict['partition']
        mod=dbdict['module']
        if self.cscomment!=dbdict['comment'] and self.cscomment!="ANY":
            return retlist
        run=dbdict['run']
        #print "run",dbdict['run']

        badpmt=[]
        try:
            lines = open(self.badmodulesFile,"r").readlines()
            for line in lines:
                fields = line.strip().split()
                #=== ignore empty and comment lines
                if not len(fields)          : continue
                if fields[0].startswith("#"): continue

                #=== read in fields
                rnn = int(fields[0])
                prt = fields[1]
                mdn = int(fields[2])
                pmn = int(fields[3])
                if rnn == run and prt == par and mdn == mod:
                    if pmn<1 or pmn>48:
                        print('module %s'%prt,'%02i'%mdn,'in run %i'%run,'is bad module for cesium, assuming all constants are bad')
                        return retlist
                    else:
                        badpmt.append(pmn)

        except Exception as e:
            if self.verbose: print('file %s not found i.e. everything should be good '%self.badmodulesFile ,par,"%02i"%mod)

        for pmt in range(1,49):
            specC10  = ( (par=='EBA' or par=='EBC') and ((mod>=39 and mod<=42) or (mod>=55 and mod<=58)) )
            specD4   = ( (par=='EBA' and mod==15) or (par=='EBC' and mod==18) )
            gapCrack = ( not specD4 and (par=='EBA' or par=='EBC') and (pmt==1 or pmt==2 or pmt==13 or pmt==14) ) or \
                       ( specD4 and (pmt==13 or pmt==14 or pmt==19 or pmt==20) )
            specC10  = ( specC10 and (pmt==5 or pmt==6) )
            normC10  = ( not specC10 and (par=='EBA' or par=='EBC') and (pmt==5 or pmt==6) )

            if csdb.readspecialces(0,par,mod,pmt)>-2.0:
                print('forcing update of special channel ',par,"%02i"%mod,"%02i"%pmt)
                retlist.append(pmt)
                continue
            if gapCrack:
                if self.updateSpec:
                    if self.verbose: print('forcing update of gap/crack ',par,"%02i"%mod,"%02i"%pmt)
                    retlist.append(pmt)
                else:
                    if self.verbose: print(('do not update gap/crack ',par,"%02i"%mod,"%02i"%pmt))
                continue
            if specC10:
                if self.updateSpec:
                    if self.verbose: print(('forcing update of special C10 ',par,"%02i"%mod,"%02i"%pmt))
                    retlist.append(pmt)
                else:
                    if self.verbose: print(('do not update special C10 ',par,"%02i"%mod,"%02i"%pmt))
                continue
            if normC10:
                if self.updateC10:
                    if self.verbose: print(('forcing update of normal C10 ',par,"%02i"%mod,"%02i"%pmt))
                    retlist.append(pmt)
                else:
                    if self.verbose: print(('do not update normal C10 ',par,"%02i"%mod,"%02i"%pmt))
                continue
            if par in chanlists.deadchan and mod in chanlists.deadchan[par] and pmt in chanlists.deadchan[par][mod]:
                if self.verbose: print(('dead channel: ',par,"%02i"%mod,"%02i"%pmt))
                continue
            if par in chanlists.unstabchan and mod in chanlists.unstabchan[par] and pmt in chanlists.unstabchan[par][mod]:
                if self.verbose: print(('unstable channel, using as is',par,"%02i"%mod,"%02i"%pmt))
                retlist.append(pmt)
                continue
            if par in chanlists.wrongchan and mod in chanlists.wrongchan[par] and pmt in chanlists.wrongchan[par][mod]:
                if self.verbose: print(('wrong channel, using new value',par,"%02i"%mod,"%02i"%pmt))
                retlist.append(pmt)
                continue
            if pmt in badpmt:
                if self.verbose: print(('cs integral in run %i'%run,'for this channel is wrong ',par,"%02i"%mod,"%02i"%pmt))
                continue
            if dbdict['statusPMT'+"%02i"%pmt]=='OK' or dbdict['statusPMT'+"%02i"%pmt]=='big ped RMS' :
                retlist.append(pmt)
                continue
##            if par=='LBC' and (mod in self.MLBCbad) \
##                   and dbdict['statusPMT'+"%02i"%pmt]=='low signal':
##                if self.verbose: print 'suppressing low signal ',par,"%02i"%mod,"%02i"%pmt
##                retlist.append(pmt)
##                continue
##            if par=='EBC' and (mod in self.MEBCbad):
##                if self.verbose: print dbdict['statusPMT'+"%02i"%pmt],par,"%02i"%mod,"%02i"%pmt
##                retlist.append(pmt)
##                continue
##            if par=='EBC' and (mod in self.MEBCbad) \
##                   and dbdict['statusPMT'+"%02i"%pmt]=='low signal':
##                if self.verbose: print 'suppressing low signal ',par,"%02i"%mod,"%02i"%pmt
##                retlist.append(pmt)
##                continue
            if par[0]=='E' and (pmt in self.Cbad or (pmt in self.Dbad and mod in self.MDbad and specD4) or (pmt in self.Abad and mod in self.MAbad)) \
                   and dbdict['statusPMT'+"%02i"%pmt]=='wrong number of peaks':
                if self.verbose: print(('suppressing  wrong number of peaks ',par,"%02i"%mod,"%02i"%pmt))
                retlist.append(pmt)
                continue
            if par[0]=='E' and ((pmt in self.Abad1 and mod in self.MAbad1)):
                if dbdict['statusPMT'+"%02i"%pmt]=='integrator malfunction':
                    if self.verbose: print(('suppressing integrator malfunction ',par,"%02i"%mod,"%02i"%pmt))
                    retlist.append(pmt)
                    continue
                if dbdict['statusPMT'+"%02i"%pmt]=='overflow':
                    if self.verbose: print(('suppressing overflow ',par,"%02i"%mod,"%02i"%pmt))
                    retlist.append(pmt)
                    continue
                if dbdict['statusPMT'+"%02i"%pmt]=='low signal':
                    if self.verbose: print(('suppressing low signal ',par,"%02i"%mod,"%02i"%pmt))
                    retlist.append(pmt)
                    continue

        return retlist

    def Translate_Runs(self, runs_string, runsset2, inputs, preprocessing=False):
        'translates the string form of the runs list into a form that USE.py can use'

        runs = runs_string
        if type(runs)== type(int()) or type(runs)== type(list()): # looks
            twoInputs=False
            runs_past2=runs
            print("Use runs input is a int or list")

        elif type(runs)== type(str()):
            print("Use runs input is a string")
            if not '-' in runs: # if there is NOT '-' in runs (its a date) use date -28days
                runs_past= runs
                runs_past2=runs+'-28 days'
                twoInputs=True
            elif runs[:1]=='-': # checks string to see if it is -x days
                 twoInputs=False
                 runs_past2=runs
            else: #                 print 'im assuming runs is xxxx-xx-xx'
                return (runs_string, runsset2, inputs)

        elif type(runs)== type(tuple()): # takes tuple of date + -x days and finds runs x amount of days behind given date
            print('runs type tuple')
            if '-' in runs[1]:
                runs_past= runs[0]
                runs_past2= runs[1] # ''.join(runs) VRAM interfered
                print(('past ',  runs_past))
                print(('past2 ',  runs_past2))
                twoInputs=True
            else: # takes the list of two dates and gets the runs inbetween them
                runs_past=runs[0]
                runs_past2=runs[1]
                print(('past ',runs[0]))
                print(('past2 ', runs[1]))
                twoInputs=True
#                print"Dumping plots for %s" % selected_region,
        else:
            twoInputs=False
            runs_past2=runs
            print('Runs is not a list string int or tuple and i dont know what to do with it')

        if twoInputs==True:
            return (runs_past2, runs_past, True)
        else:
            return (runs_past2, '', False)

    # function to get list of runs for given runtype from Use
    def GetRunList(self):
        return self.runs

    def dateProg(self, string, string2=''):			# CALL MYSQL TO GRAB RUN LIST BASED ON THE DATE

        ### GRAB DATE ###
        if ((self.twoInputs == False)):
            r = os.popen("date -d '%s' +%%Y-%%m-%%d" % string)	# OPEN OS PIPE TO GRAB THE DATE
            date = r.readline().strip('\n')			# READ THE DATE AND STORE
            print(date)
            r.close()						# CLOSE OS PIPE
        else: 							# IF TWO DATES ARE USED
            r = os.popen("date -d '%s' +%%Y-%%m-%%d" % string)	# OPEN OS PIPE TO GRAB THE DATE
            date = r.readline().strip('\n')			# READ THE DATE AND STORE
            print(date)
            r.close()						# CLOSE OS PIPE
            r = os.popen("date -d '%s' +%%Y-%%m-%%d" % string2)	# OPEN SECOND OS PIPE TO GRAB THE DATE
            date2 = r.readline().strip('\n')			# READ THE DATE AND STORE
            print(date2)
            r.close()						# CLOSE OS PIPE

        ### GRAB RUN LIST ###

        if self.rType=='Las':  # Special treatment for LASER
            if self.twoInputs:
                datequery = "date>='%s' and date<='%s'" % (date,date2)
            else:
                datequery = "date>'%s'" % (date)

            if self.filter == ' ':
                filterquery = "((lasfilter='6' and events>10000) or (lasfilter='8' and events>10000))"
            else:
                filterquery = "lasfilter='%s'" % self.filter
                if self.filter == '6':
                    filterquery = " ( lasfilter='6' and events>10000)"
                if self.filter == '8':
                    filterquery = " ( lasfilter='8' and events>10000)"
            query = "select run from tile.comminfo where %s and %s and lasreqamp='%s' and type ='Las' and not (recofrags like '%%005%%' or recofrags like '%%50%%' ) and beamfrags like '%%200017%%' and not setup='ATLAS' and comments is NULL " % (datequery,filterquery,self.amp)
            print("This query will be used to select the laser runs:")
            print(" ")
            print(query)
            print(" ")

        elif self.rType=='cesium':				# CHECK IF REQUESTED RUN TYPE IS CESIUM

            query = "select run from tile.runDescr where time>'%s' and module<65" % (date)
            if self.cscomment!='':
                query = "%s and comment='%s'" % (query, self.cscomment)
            if self.twoInputs:
                query = "%s and time<'%s'" % (query, date2)
            query = """%s""" % query

        elif self.rType=='CIS' and self.twoInputs == True:	# CHECK IF REQUESTED RUN TYPE IS CIS WITH TWO DATES
            query = """select run from tile.comminfo where date<'%s' and date>'%s' and comments is NULL""" % (date2,date)

        elif self.rType=='L1Calo' and self.twoInputs == True:
            query = """select run from tile.comminfo where date<'%s' and date>'%s'""" % (date2,date)

        else:							# IF NOT LASER, CS, L1CALO, OR CIS WITH TWO DATES
            query = """select run from tile.comminfo where run<9999999 and date>'%s'""" % (date)

#        print query
        runs = []						# CLEAR INTEGER RUN NUMBER LIST
        if python3:
            self.pymysqldb_cursor.execute(query)

            r2 = self.pymysqldb_cursor.fetchall()

            for run in r2:
                if run[0] not in runs:                # IF RUN NUMBER IS NOT ALREADY STORED
                    runs.append(run[0])                # APPEND RUN NUMBER TO RUN NUMBER LIST
        else: # python 2
            self.mysqldb.query(query)
            r2 = self.mysqldb.store_result()					# STORE DATABASE RESULT

            for run in r2.fetch_row(maxrows=0):			# FETCH DATABASE
                if run[0] not in runs:				# IF RUN NUMBER IS NOT ALREADY STORED
                    runs.append(run[0])				# APPEND RUN NUMBER TO RUN NUMBER LIST

        return runs						# RETURN INTEGER RUN NUMBER LIST

    def isActive(self,hash, run):				# CHECK IF REGION IS USED IN THIS RUN
        if run not in self.fragStr or not self.fragStr[run]:
            return True						# RETURN TRUE IF REGION IS USED
        hexStr = ''						# BINARY STRING TO CHECK
        if 'B' in hash:						# CHECK IF REGION IS SOMETHING SPECIFIC
            if   'LBA' in hash:					# CHECK IF REGION IS LBA
                hexStr = '0x1'					# STORE APPROPRIATE HEXADECIMALS
            elif 'LBC' in hash:					# CHECK IF REGION IS LBC
                hexStr = '0x2'					# STORE APPROPRIATE HEXADECIMALS
            elif 'EBA' in hash:					# CHECK IF REGION IS EBA
                hexStr = '0x3'					# STORE APPROPRIATE HEXADECIMALS
            elif 'EBC' in hash:					# CHECK IF REGION IS EBC
                hexStr = '0x4'					# STORE APPROPRIATE HEXADECIMALS
            if '_m' in hash:					# CHECK IF REGION IS FURTHER SPECIFIED
                ind = hash.index('m')+1				# ADD ONE TO MODULE INDEX
                modStr = hex(int(hash[ind:ind+2])-1).lstrip('0x')
                while len(modStr)<2:				# IF ADDRESS IS SHORTER THAN TWO
                    modStr = '0'+modStr				# ADD SOME ZEROS
                hexStr += modStr				# ADD THE ADDRESSES
            return (hexStr in self.fragStr[int(run)])		# RETURN WHETHER THE ADDRESS IN THE DIGIFRAGS
        elif hash == 'TILECAL':					# IF REGION IS JUST THE ENTIRE TILECAL
            return True						# RETURN TRUE
        else:							# IF REGION IS SOMETHING ELSE
            print (('Unknown hash: ',hash))			# TELL USER THIS CANNOT BE
            return False					# RETURN FALSE

    # function to remove some runs for creating
    # long-term CIS plots
    # edited by Andrew Smith on Jan 10, 2019 ancsmith@cern.ch
    # populate a new list 'goodruns' with
    # the runs we want to use and set self.runs = goodruns

    def RemoveBadRuns(self, badruns=[]):
        goodruns = []

        print("Bad runs in RemoveBadRuns")
        print(badruns)

        for run in self.runs:
            if not (run[0] in badruns):
                goodruns.append(run)
            else:
                pass
        self.runs = goodruns


import os
import os.path
import copy
from collections import OrderedDict
from decimal import *
import distutils.dir_util
import json
import threading
import boto3
import botocore
import time

from pipetree import settings
from pipetree.utils import attach_config_to_object
from pipetree.exceptions import ArtifactMissingPayloadError
from pipetree.artifact import Artifact

STAGE_COMPLETE = 'complete'
STAGE_IN_PROGRESS = 'in_progress'
STAGE_DOES_NOT_EXIST = 'does_not_exist'

class ArtifactBackend(object):
    def __init__(self, **kwargs):
        config = copy.copy(self.DEFAULTS)
        config.update(kwargs)
        self._validate_config()
        self._config = kwargs
        attach_config_to_object(self, config)

    def _validate_config(self):
        raise NotImplementedError

    def load_artifact(self, artifact):
        """
        Returns a fully instantiated artifact with an Item containing
        payload and metadata. This occurs
        iff one is found matching the provided hashes/properties
        of the given artifact object. Otherwise returns None.
        """
        cached_artifact = self._find_cached_artifact(artifact)
        if cached_artifact is None:
            return None
        else:
            cached_artifact.load_payload(self._get_cached_artifact_payload(
                cached_artifact))
            return cached_artifact

    def save_artifact(self, artifact):
        """
        Saves an artifact at every layer of the cache.

        Artifact must have all necessary metadata, including:
         - specific_hash
         - dependency_hash
         - definition_hash
        """
        raise NotImplementedError

    def log_pipeline_stage_run_complete(self, dependency_hash, definition_hash):
        """
        Record that the pipeline stage run for the given dependency hash and
        definition hash completed successfully.
        """
        raise NotImplementedError

    def pipeline_stage_run_status(self, dependency_hash, definition_hash):
        """
        Returns the status of a pipeline stage run
        Returns one of: "does_not_exist", "in_progress", "completed"
        """
        raise NotImplementedError

    def find_pipeline_stage_run_artifacts(self, dependency_hash, definition_hash):
        """
        Finds all artifacts for a given pipeline run.
        """
        raise NotImplementedError

    def _find_cached_artifact(self, artifact):
        """
        Tries to find a cached artifact matching the provided hashes/properties
        of the given artifact object.

        If only stage & item name are supplied, will return the newest artifact
        given the pruning ordering.

        Loads the metadata, but not the payload of an artifact.
        """
        raise NotImplementedError

    def _get_cached_artifact_payload(self, artifact):
        """
        Returns the payload for a given artifact, assuming that it
        has already been produced and is cached.
        """
        raise NotImplementedError

    def _get_cached_artifact_metadata(self, artifact):
        """
        Returns the artifact metadata for a given artifact, assuming that it
        has already been produced and is cached.
        """
        raise NotImplementedError

    def _sorted_artifacts(self, artifact):
        """
        Returns a sorted list of artifacts, based upon pruning ordering
        """
        raise NotImplementedError

class LocalArtifactBackend(ArtifactBackend):
    """
    Provide a local cache layer for artifacts.
    Intended to be composed with S3ArtifactBackend to provide local storage.

    Utilizes a global internal lock to ensure serial access to files.
    Since the majority of execution time is spent generating individual
    artifacts, this shouldn't impact performance tremendously at the moment.
    """
    DEFAULTS = {
        "path": "~/.pipetree/local_cache/",
        "metadata_file": "pipeline.meta"
    }

    def __init__(self, path=DEFAULTS['path'], **kwargs):
        super().__init__(path=path, **kwargs)
        if not os.path.exists(self.path):
            distutils.dir_util.mkpath(self.path)
        self.cached_meta = {}
        self._write_lock = threading.Lock()

    def _validate_config(self):
        return True

    def _relative_artifact_dir(self, artifact):
        """
        Returns the relative directory to house artifacts from a given
        stage and of a given item type.
        """
        it = "default"
        if artifact.item is not None and artifact.item.type is not None:
            it = artifact.item.type
        return os.path.join(artifact._pipeline_stage, it)

    def _relative_artifact_path(self, artifact):
        """
        Returns the relative path for an artifact that has a specified
        specific_hash, dependency_hash, definition_hash, stage and item type
        """
        return os.path.join(self._relative_artifact_dir(artifact),
                            artifact.get_uid())

    def save_artifact(self, artifact):
        """
        Saves an artifact locally on disk.

        Artifact must have all necessary metadata, including:
         - specific_hash
         - dependency_hash
         - definition_hash
        """
        # TODO: Check if the file exists. If it does, skip writing it out.
        if artifact.item is None or artifact.item.payload is None:
            raise ArtifactMissingPayloadError(stage=artifact._pipeline_stage)

        distutils.dir_util.mkpath(os.path.join(
            self.path,
            self._relative_artifact_dir(artifact)))

        with self._write_lock:
            with open(os.path.join(self.path,
                                   self._relative_artifact_path(artifact)),
                      'w') as f:
                f.write(artifact.serialize_payload())

        self._write_artifact_meta(artifact)
        self._record_pipeline_stage_run_artifact(artifact)

    def _load_item_meta(self, pipeline_stage, item_type):
        """
        Load the shared metadata for all artifacts of the
        given stage & item type
        """
        if item_type is None:
            item_type = "default"
        try:
            with open(os.path.join(
                    self.path,
                    pipeline_stage,
                    item_type,
                    self.metadata_file),
                      'r') as f:
                contents = json.load(f)
                return contents
        except FileNotFoundError:
            return {}

    def _write_artifact_meta(self, artifact):
        with self._write_lock:
            self._u_write_artifact_meta(artifact)

    def _u_write_artifact_meta(self, artifact):
        """
        Writes this artifact's metadata to a shared metadata file
        """
        distutils.dir_util.mkpath(os.path.join(
            self.path,
            self._relative_artifact_dir(artifact)))

        item_meta = self._load_item_meta(artifact._pipeline_stage,
                                         artifact.item.type)

        item_meta[artifact.get_uid()] = artifact.meta_to_dict()
        meta_key = artifact._pipeline_stage + str(artifact.item.type)
        self.cached_meta[meta_key] = item_meta

        with open(os.path.join(
                self.path,
                self._relative_artifact_dir(artifact),
                self.metadata_file),
                  'w') as f:
            json.dump(item_meta, f)

    def _find_cached_artifact(self, artifact):
        """
        Loads the metadata, but not the payload of an artifact.

        Tries to find a cached artifact matching the provided hashes/properties
        of the given artifact object.

        If only stage & item name are supplied, will return the newest artifact
        given the pruning ordering.
        """
        if artifact._specific_hash is not None or \
           artifact._dependency_hash is not None:
            item_meta = self._load_item_meta(artifact._pipeline_stage,
                                             artifact.item.type)
            if artifact.get_uid() in item_meta:
                artifact.meta_from_dict(item_meta[artifact.get_uid()])
                artifact._loaded_from_local_cache = True
                return artifact

            return None
        else:
            # Sort artifacts and return most recent
            sorted_artifacts = self._sorted_artifacts(artifact)
            if len(sorted_artifacts) == 0:
                return None
            sorted_artifacts[0]._loaded_from_local_cache = True
            return sorted_artifacts[0]
        raise NotImplementedError

    def _get_cached_artifact_payload(self, artifact):
        """
        Returns the payload for a given artifact, assuming that it
        has already been produced and is cached.
        """
        with open(os.path.join(self.path,
                               self._relative_artifact_path(artifact)),
                  'r') as f:
            return f.read()

    def _get_cached_artifact_metadata(self, artifact):
        """
        Returns the metadata for a given artifact, assuming that it
        has already been produced and is cached.
        """
        raise NotImplementedError

    def log_pipeline_stage_run_complete(self, stage_config, dependency_hash):
        with self._write_lock:
            self._u_log_pipeline_stage_run_complete(stage_config,
                                                      dependency_hash)

    def _u_log_pipeline_stage_run_complete(self, stage_config,
                                           dependency_hash):
        """
        Record that the pipeline stage run for the given dependency hash and
        definition hash completed successfully.
        """
        meta = self._get_pipeline_stage_run_meta(stage_config,
                                                 dependency_hash)

        meta['complete'] = True
        distutils.dir_util.mkpath(os.path.join(
            self.path,
            stage_config.name))
        with open(os.path.join(
                self.path,
                stage_config.name,
                self._pipeline_stage_run_filename(
                        dependency_hash,
                        stage_config.hash())),
                  'w') as f:
            json.dump(meta, f)

    def _pipeline_stage_run_filename(self, dependency_hash,
                                     definition_hash):
        return ("pipeline_stage_run_%s_%s" %
                (dependency_hash, definition_hash))

    def _record_pipeline_stage_run_artifact(self, artifact):
        with self._write_lock:
            self._u_record_pipeline_stage_run_artifact(artifact)

    def _u_record_pipeline_stage_run_artifact(self, artifact):
        """
        Record that the given artifact was produced during its corresponding
        pipeline stage run.
        """
        meta = self._get_pipeline_stage_run_meta(
            artifact._config,
            artifact._dependency_hash)

        if 'artifacts' not in meta:
            meta['artifacts'] = {}

        if 'dependency_hash' not in meta:
            meta['dependency_hash'] = artifact._dependency_hash

        uid = artifact.get_uid()
        if uid not in meta['artifacts']:
            meta['artifacts'][uid] = \
                {"item_type": artifact.item.type,
                 "specific_hash": artifact._specific_hash,
                 "uid": uid
                }
        else:
            print("Artifact %s already generated for run %s" %
                  (artifact.get_uid(), artifact._pipeline_stage))

        distutils.dir_util.mkpath(os.path.join(
            self.path,
            artifact._pipeline_stage))
        with open(os.path.join(
                self.path,
                artifact._pipeline_stage,
                self._pipeline_stage_run_filename(
                    artifact._dependency_hash,
                    artifact._definition_hash)),
                  'w') as f:
            json.dump(meta, f)

    def pipeline_stage_run_status(self, stage_config,
                                  dependency_hash):
        meta = self._get_pipeline_stage_run_meta(
            stage_config,
            dependency_hash)

        if meta == {}:
            return STAGE_DOES_NOT_EXIST
        elif 'complete' in meta:
            return STAGE_COMPLETE
        else:
            return STAGE_IN_PROGRESS

    def find_pipeline_stage_run_artifacts(self, stage_config,
                                          dependency_hash):
        """
        Finds all artifacts for a given pipeline run.
        """
        meta = self._get_pipeline_stage_run_meta(
            stage_config,
            dependency_hash)

        if 'artifacts' not in meta:
            return []
        else:
            res = []
            for uid in meta['artifacts']:
                artDict = meta['artifacts'][uid]
                art = Artifact(stage_config)
                art.item.type = artDict['item_type']
                art._specific_hash = artDict['specific_hash']
                res.append(self._find_cached_artifact(art))
            return res

    def _get_pipeline_stage_run_meta(self, stage_config,
                                     dependency_hash):
        """
        Load metadata for a given run of a pipeline stage
        """
        try:
            with open(os.path.join(
                    self.path,
                    stage_config.name,
                    self._pipeline_stage_run_filename(
                        dependency_hash,
                        stage_config.hash())),
                    'r') as f:
                contents = json.load(f)
                return contents
        except FileNotFoundError:
            return {}

    def _sorted_artifacts(self, artifact):
        """
        Returns a sorted list of artifacts, based upon pruning ordering
        """
        item_meta = self._load_item_meta(artifact._pipeline_stage,
                                         artifact.item.type)

        result = []
        for k in item_meta:
            result.append(item_meta[k])
        sorted_metadata = sorted(result, key=lambda x: x["creation_time"])

        sorted_artifacts = []
        for x in sorted_metadata:
            a = Artifact(artifact._config, artifact.item.type)
            a.meta_from_dict(x)
            sorted_artifacts.append(a)

        return sorted_artifacts

class S3ArtifactBackend(ArtifactBackend):
    """
    Provide an S3 + DynamoDB storage backend for generated artifacts
    and their metadata.
    """
    DEFAULTS = {
        "path": "~/.pipetree/local_cache/",
        "enable_local_caching": True,
        "metadata_file": "pipeline.meta",
        "aws_region": settings.AWS_REGION,
        "aws_profile": settings.AWS_PROFILE,
        "s3_bucket_name": settings.S3_ARTIFACT_BUCKET_NAME,
        "dynamodb_artifact_table_name": settings.DYNAMODB_ARTIFACT_TABLE_NAME,
        "dynamodb_stage_run_table_name": settings.DYNAMODB_STAGE_RUN_TABLE_NAME
    }

    def __init__(self, path=DEFAULTS['path'], **kwargs):
        super().__init__(path=path, **kwargs)
        self._localArtifactBackend = LocalArtifactBackend(path=path, **kwargs)

        try:
            self._session = boto3.Session(profile_name=self.aws_profile,
                                    region_name=self.aws_region)
        except botocore.exceptions.NoCredentialsError:
            self._session = boto3.Session(region_name=self.aws_region)

        self._stage_run_table = None
        self._artifact_meta_table = None
        self._setup_s3()
        self._setup_dynamo_db()

    def _setup_s3(self):
        self._s3_client = self._session.client('s3')
        try:
            self._s3_client.create_bucket(Bucket=self.s3_bucket_name,
                                          CreateBucketConfiguration={
                                              'LocationConstraint': self.aws_region})
        except botocore.exceptions.ClientError as err:
            # Bucket already created? Good to go.
            print("Bucket %s already created" % self.s3_bucket_name)
            pass

    def _setup_dynamo_db(self):
        """
        Setup local dynamodb resource

        Setup local dynamodb tables as well. We define non-indexed columns
        despite not needing them in table creation, for our own convenience.
        """
        self._dynamodb = self._session.resource('dynamodb')

        stage_run_keys = OrderedDict([
            ('stage_config_hash', 'HASH'),
            ('dependency_hash', 'RANGE')
        ])
        stage_run_fields = {
            'stage_config_hash': 'S',
            'dependency_hash': 'S',
            'stage_run_status': 'S',
            'metadata': 'S',
            'start_time': 'N',
            'end_time': 'N'
        }
        self._stage_run_table = self._dynamo_db_create_table(
            self.dynamodb_stage_run_table_name,
            stage_run_keys,
            stage_run_fields)

        artifact_meta_keys = {
            'artifact_uid': 'HASH',
        }
        artifact_meta_fields = {
            'artifact_uid': 'S',
            'artifact_meta': 'S',
            'creation_time': 'N'
        }
        self._artifact_meta_table = self._dynamo_db_create_table(
            self.dynamodb_artifact_table_name,
            artifact_meta_keys,
            artifact_meta_fields)

    def _dynamo_db_create_table(self, table_name, keys, fields,
                                write_units=1, read_units=1):
        keySchema = []
        attributeDefinitions = []
        for key in keys:
            keySchema.append({'AttributeName': key,
                              'KeyType': keys[key]})
        for key in fields:
            if key in keys:
                attributeDefinitions.append({'AttributeName': key,
                                             'AttributeType': fields[key]})
        try:
            table = self._dynamodb.create_table(
                TableName=table_name,
                KeySchema=keySchema,
                AttributeDefinitions=attributeDefinitions,
                ProvisionedThroughput={
                    'ReadCapacityUnits': read_units,
                    'WriteCapacityUnits': write_units
                }
            )
            table.meta.client.get_waiter('table_exists').wait(TableName=table_name)
            return table
        except botocore.exceptions.ClientError:
            # If the table already exists, load and return that table.
            return self._dynamodb.Table(table_name)

    def _validate_config(self):
        return True

    def _relative_artifact_dir(self, artifact):
        """
        Returns the relative directory to house artifacts from a given
        stage and of a given item type.
        """
        it = "default"
        if artifact.item is not None and artifact.item.type is not None:
            it = artifact.item.type
        return os.path.join(artifact._pipeline_stage, it)

    def _relative_artifact_path(self, artifact):
        """
        Returns the relative path for an artifact that has a specified
        specific_hash, dependency_hash, definition_hash, stage and item type
        """
        return os.path.join(self._relative_artifact_dir(artifact),
                            artifact.get_uid())

    def save_artifact(self, artifact):
        """
        Saves an artifact locally on disk and in an S3 Bucket

        Artifact must have all necessary metadata, including:
         - specific_hash
         - dependency_hash
         - definition_hash
        """
        if artifact.item is None or artifact.item.payload is None:
            raise ArtifactMissingPayloadError(stage=artifact._pipeline_stage)

        # Cache the output locally and use local file for S3 upload
        self._localArtifactBackend.save_artifact(artifact)

        # Upload to S3
        key = self.s3_artifact_key(artifact)
        local_file = os.path.join(self.path, key)
        self._s3_client.upload_file(local_file,
                                    self.s3_bucket_name,
                                    key)

        self._write_artifact_meta(artifact)

    def _write_artifact_meta(self, artifact):
        """
        Writes this artifact's metadata to local storage & dynamodb
        """
        if self.enable_local_caching:
            self._localArtifactBackend._write_artifact_meta(artifact)

        # Insert artifact meta into DynamoDB
        self._artifact_meta_table.put_item(
            Item={
                'artifact_uid': artifact.get_uid(),
                'artifact_meta': json.dumps(artifact.meta_to_dict()),
                'creation_time': Decimal(time.time())
            }
        )

        # Update pipeline stage meta
        stage_run_key = {
            'stage_config_hash': str(artifact._definition_hash),
            'dependency_hash': str(artifact._dependency_hash)
        }
        response = self._stage_run_table.get_item(Key=stage_run_key)

        if 'Item' not in response:
            # Create stage run meta
            self._stage_run_table.put_item(
                Item={
                    'stage_config_hash': artifact._definition_hash,
                    'dependency_hash': str(artifact._dependency_hash),
                    'stage_run_status': STAGE_IN_PROGRESS,
                    'metadata': json.dumps({'artifacts': [
                        {'uid': artifact.get_uid(),
                         'specific_hash': artifact._specific_hash,
                         'type': artifact.item.type}
                    ]}, sort_keys=True)
                }
            )
        else:
            # Update stage meta
            meta = json.loads(response['Item']['metadata'])
            meta['artifacts'].append({
                "uid": artifact.get_uid(),
                "type": artifact.item.type,
                "specific_hash": artifact._specific_hash
            })
            self._stage_run_table.update_item(
                Key=stage_run_key,
                UpdateExpression='SET metadata = :metaVal',#, stage_run_status= :status',
                # The condition expression ensures metadata hasn't changed,
                # effectively performing an atomic CAS
                ConditionExpression='metadata = :oldMetaVal',
                ExpressionAttributeValues={
                    ':oldMetaVal': response['Item']['metadata'],
                    ':metaVal': json.dumps(meta, sort_keys=True),
                    #':status': STAGE_IN_PROGRESS
                }
            )
            item = response['Item']

    def _find_cached_artifact(self, artifact):
        """
        Loads the metadata, but not the payload of an artifact.
        """

        # Attempt to load artifact locally
        if self.enable_local_caching:
            res = self._localArtifactBackend._find_cached_artifact(artifact)
            if res is not None:
                return res

        # Otherwise, query DynamoDB
        art_key = {
            'artifact_uid': artifact.get_uid()
        }
        response = self._artifact_meta_table.get_item(Key=art_key)
        if 'Item' not in response:
            return None
        else:
            artifact.meta_from_dict(json.loads(response['Item']['artifact_meta']))
            artifact._loaded_from_s3_cache = True
            return artifact
        return None

    def s3_artifact_key(self, artifact):
        return self._relative_artifact_path(artifact)

    def _get_cached_artifact_payload(self, artifact):
        """
        Returns the payload for a given artifact, assuming that it
        has already been produced and is cached on S3.
        """
        obj = self._s3_client.get_object(
            Bucket= self.s3_bucket_name,
            Key= self.s3_artifact_key(artifact)
        )
        return obj['Body'].read()

    def log_pipeline_stage_run_complete(self, stage_config,
                                           dependency_hash):
        """
        Record that the pipeline stage run for the given dependency hash and
        definition hash completed successfully.
        """
        # Log locally
        if self.enable_local_caching:
            self._localArtifactBackend.log_pipeline_stage_run_complete(
            stage_config, dependency_hash)

        # Update dynamo db
        stage_run_key = {
            'stage_config_hash': str(stage_config.hash()),
            'dependency_hash': str(dependency_hash)
        }
        self._stage_run_table.update_item(
            Key=stage_run_key,
            UpdateExpression='SET stage_run_status = :status',
            # The condition expression ensures metadata hasn't changed,
            # effectively performing an atomic CAS
            ExpressionAttributeValues={
                ':status': STAGE_COMPLETE
            }
        )

    def pipeline_stage_run_status(self, stage_config,
                                  dependency_hash):
        stage_run_key = {
            'stage_config_hash': str(stage_config.hash()),
            'dependency_hash': str(dependency_hash)
        }
        response = self._stage_run_table.get_item(Key=stage_run_key)

        if 'Item' not in response:
            return STAGE_DOES_NOT_EXIST
        else:
            return response['Item']['stage_run_status']

    def find_pipeline_stage_run_artifacts(self, stage_config,
                                          dependency_hash):
        """
        Finds all artifacts for a given pipeline run, loading their
        metadata.
        """
        # Update pipeline stage metaa
        stage_run_key = {
            'stage_config_hash': stage_config.hash(),
            'dependency_hash': dependency_hash,
        }

        response = self._stage_run_table.get_item(Key=stage_run_key)
        if 'Item' not in response:
            return None
        else:
            res = []
            meta = json.loads(response['Item']['metadata'])
            for obj in meta['artifacts']:
                art = Artifact(stage_config)
                art.item.type = obj['type']
                art._specific_hash = obj['specific_hash']
                art._dependency_hash = dependency_hash
                art._definition_hash = stage_config.hash()
                res.append(self._find_cached_artifact(art))
            return res

    def _get_pipeline_stage_run_meta(self, stage_config,
                                     dependency_hash):
        """
        Load metadata for a given run of a pipeline stage
        """
        try:
            with open(os.path.join(
                    self.path,
                    stage_config.name,
                    self._pipeline_stage_run_filename(
                        dependency_hash,
                        stage_config.hash())),
                    'r') as f:
                contents = json.load(f)
                return contents
        except FileNotFoundError:
            return {}

    def _sorted_artifacts(self, artifact):
        """
        Returns a sorted list of artifacts, based upon pruning ordering
        """
        raise NotImplementedError


#https://www.acmicpc.net/problem/14940
#14940번-쉬운 최단거리

from collections import deque
import sys
input = sys.stdin.readline
n,m = map(int,input().split())
graph = [list(map(int,input().split())) for _ in range(n)]
visited = [[-1]*m for _ in range(n)]
dx = [-1,1,0,0]
dy = [0,0,-1,1]

for i in range(n):
    for j in range(m):
        if graph[i][j]==2:
            q = deque([(i,j)])
            visited[i][j]=0
        if graph[i][j]==0:
            visited[i][j]=0

while q:
        x,y = q.popleft()
        for i in range(4):
            nx = dx[i] + x
            ny = dy[i] + y
            if 0<=nx<n and 0<=ny<m and visited[nx][ny]==-1 and graph[nx][ny]==1:
                visited[nx][ny]=visited[x][y]+1
                q.append((nx,ny))

for i in range(n):
    print(*visited[i],sep=' ')


# -*- coding: utf-8 -*-

import logging

import numpy as np
from keras import backend as K
from keras.engine import Layer, InputSpec
from keras.layers import Flatten
from keras import initializers, regularizers, constraints
import tensorflow as tf

logger = logging.getLogger(__name__)

def _softmax(x, axis):
    ''' Computes softmax along a specified axis. '''
    if K.backend() == 'tensorflow':
        import tensorflow as tf
        return tf.nn.softmax(x, axis)
    elif K.backend() == 'cntk':
        import cntk
        return cntk.softmax(x, axis)
    elif K.backend() == 'theano':
        # Theano cannot softmax along an arbitrary dim.
        # So, we will shuffle `dim` to -1 and un-shuffle after softmax.
        perm = np.arange(K.ndim(x))
        perm[axis], perm[-1] = perm[-1], perm[axis]
        x_perm = K.permute_dimensions(x, perm)
        output = K.softmax(x_perm)

        # Permute back
        perm[axis], perm[-1] = perm[-1], perm[axis]
        output = K.permute_dimensions(x, output)
        return output
    else:
        raise ValueError("Backend '{}' not supported".format(K.backend()))

class AttentionLayer(Layer):
    ''' Attention layer that computes a learned attention over input sequence.

    For details, see papers:
    - https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf
    - http://colinraffel.com/publications/iclr2016feed.pdf (fig 1)

    Input:
      x: Input tensor for shape `(..., timesteps, features)` where `features`
      must be static (known).

    Output:
      2D tensor of shape `(..., features)`, i.e., `timesteps` axis is attended
      over and reduced.
    '''

    def __init__(self,
                 kernel_initializer='he_normal',
                 kernel_regularizer=None,
                 kernel_constraint=None,
                 use_bias=True,
                 bias_initializer='zeros',
                 bias_regularizer=None,
                 bias_constraint=None,
                 use_context=True,
                 context_initializer='he_normal',
                 context_regularizer=None,
                 context_constraint=None,
                 attention_dim=None,
                 **kwargs):
        '''
        Args:
          attention_dim: The dimensionality of the inner attention calculating
            neural network.
            For input `(32, 10, 300)`, with `attention_dim` of 100, the output
            is `(32, 10, 100)`, i.e., the attended words are 100 dimensional.
            This is then collapsed via summation to `(32, 10, 1)` to indicate
            the attention weights for 10 words.
            If set to None, `features` dims are used as `attention_dim`.
            (Default value: None)
        '''
        if 'input_shape' not in kwargs and 'input_dim' in kwargs:
            kwargs['input_shape'] = (kwargs.pop('input_dim'),)

        super(AttentionLayer, self).__init__(**kwargs)
        self.kernel_initializer = initializers.get(kernel_initializer)
        self.kernel_regularizer = regularizers.get(kernel_regularizer)
        self.kernel_constraint = constraints.get(kernel_constraint)

        self.use_bias = use_bias
        self.bias_initializer = initializers.get(bias_initializer)
        self.bias_regularizer = regularizers.get(bias_regularizer)
        self.bias_constraint = constraints.get(bias_constraint)

        self.use_context = use_context
        self.context_initializer = initializers.get(context_initializer)
        self.context_regularizer = regularizers.get(context_regularizer)
        self.context_constraint = constraints.get(context_constraint)

        self.attention_dim = attention_dim
        self.supports_masking = True

    def build(self, input_shape):
        # if len(input_shape) < 3:
        if len(input_shape) != 3:
            raise ValueError("Expected input shape of "
                             "`(..., timesteps, features)`, found `{}`"
                             .format(input_shape))

        attention_dim = input_shape[-1] if self.attention_dim is None \
            else self.attention_dim
        self.kernel = self.add_weight(name='kernel',
                                      shape=(input_shape[-1], attention_dim),
                                      initializer=self.kernel_initializer,
                                      regularizer=self.kernel_regularizer,
                                      constraint=self.kernel_constraint)
        if self.use_bias:
            self.bias = self.add_weight(name='bias',
                                        shape=(attention_dim,),
                                        initializer=self.bias_initializer,
                                        regularizer=self.bias_regularizer,
                                        constraint=self.bias_constraint)
        else:
            self.bias = None
        if self.use_context:
            self.context_kernel = self.add_weight(
                name='context_kernel',
                shape=(attention_dim,),
                initializer=self.context_initializer,
                regularizer=self.context_regularizer,
                constraint=self.context_constraint)
        else:
            self.context_kernel = None
        super(AttentionLayer, self).build(input_shape)

    def call(self, inputs, mask=None):
        # inputs: [..., timesteps, features]
        # ut : [..., timesteps, attention_dim]
        ut = K.dot(inputs, self.kernel)
        if self.use_bias:
            ut = K.bias_add(ut, self.bias)
        ut = K.tanh(ut)
        if self.use_context:
            ut = ut * self.context_kernel

        # Collapse `attention_dim` to 1. This indicates the weight for each
        # timestep.
        ut = K.sum(ut, axis=-1, keepdims=True)
        # ut = K.sum(ut, axis=-1)

        # Convert those weights into a distribution along timestep axis.
        # i.e., sum of alphas along `timesteps` axis should be 1.
        self.at = _softmax(ut, axis=1)
        # self.at = K.expand_dims(K.softmax(ut), -1)
        if mask is not None:
            self.at *= K.cast(K.expand_dims(mask, -1), K.floatx())

        # Weighted sum along `timesteps` axis.
        return K.sum(inputs * self.at, axis=-2)

    def comput_mask(self, inputs, input_mask=None):
        # Do not pass the mask to the next layers
        return None

    def compute_output_shape(self, input_shape):
        return input_shape[0], input_shape[-1]
        # input_shape = list(input_shape)
        # input_shape[-2] = input_shape[-1]
        # return tuple(input_shape[:-1])

    def get_attention_tensor(self):
        if not hasattr(self, 'at'):
            raise ValueError('Attention tensor is available after calling this'
                             ' layer with an input')
        return self.at

    def get_config(self):
        config = {
            'kernel_initializer':
            initializers.serialize(self.kernel_initializer),
            'kernel_regularizer':
            regularizers.serialize(self.kernel_regularizer),
            'kernel_constraint':
            constraints.serialize(self.kernel_constraint),
            'bias_initializer':
            initializers.serialize(self.bias_initializer),
            'bias_regularizer':
            regularizers.serialize(self.bias_regularizer),
            'bias_constraint':
            constraints.serialize(self.bias_constraint),
            'context_initializer':
            initializers.serialize(self.context_initializer),
            'context_regularizer':
            regularizers.serialize(self.context_regularizer),
            'context_constraint':
            constraints.serialize(self.context_constraint)
        }
        base_config = super(AttentionLayer, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

class ConsumeMask(Layer):
    ''' Layer that prevents mask propagation. '''

    def compute_mask(self, inputs, input_mask=None):
        # Do not pass the mask to the next layer.
        return None

    def call(self, inputs, mask=None):
        return inputs

class GlobalKMaxPooling1D(Layer):
    ''' Global k-max pooling operartion for temporal data.

    # Input shape
      3D tensor with shape: `(batch_size, steps, features)`.

    # Output shape
      2D tensor with shape: `(batch_size, k * features).
    '''

    def __init__(self, k=1, **kwargs):
        super(GlobalKMaxPooling1D, self).__init__(**kwargs)
        self.k = k
        self.input_spec = InputSpec(ndim=3)

    def compute_output_shape(self, input_shape):
        return (input_shape[0], input_shape[2] * self.k)

    def call(self, inputs):

        # Swap last two dimensions since top_k will be applied along the last
        # dimension.
        shifted_inputs = tf.transpose(inputs, [0, 2, 1])

        # Extract top_k, returns two tensors [values, indices]
        top_k = tf.nn.top_k(shifted_inputs, k=self.k, sorted=False,
                            name=None)[0]
        return K.batch_flatten(top_k)

    def get_config(self):
        config = {'k': self.k}
        base_config = super(GlobalKMaxPooling1D, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


import pandas as pd

import math

EPS = 1e-9

class DecisionRule:
    """Decision rules of the kind `x >= a`"""

    def __init__(self, thrsh, getter, out_val, feature_name='bla'):
        self.feature_name = feature_name
        self.threshold = thrsh
        # obj => value
        self.getter = getter
        # 0 or 1
        self.out_val = out_val

    def decide(self, obj):
        if self.getter:
            val = self.getter(obj)
        else:
            val = obj[self.feature_name]

        if val >= self.threshold:
            return self.out_val
        return not self.out_val

    def __repr__(self):
        return 'DecisionRule({}, {}, {})'.format(self.feature_name,
                self.threshold, self.out_val)

class SplitWrapper:
    """Represents one split by DecisionRule"""

    def __init__(self):
        self.num_labels_by_class = [0, 0]

    def update(self, label, add):
        self.num_labels_by_class[label] += add

    def num_objects(self):
        ret = sum(self.num_labels_by_class)
        return EPS if ret == 0 else ret

    def calc_entropy(self):
        res = 0
        for i in range(2):
            cur_objects = self.num_labels_by_class[i]
            val = cur_objects / self.num_objects()
            if val != 0:
                res += val * math.log(val, 2)

        return -res

    def calc_gini(self):
        num_objects = sum(self.num_labels_by_class)
        res = 0

        for i in range(2):
            prob = self.num_labels_by_class[i] / self.num_objects()
            res += prob * (1 - prob)

        return res

    def __repr__(self):
        return 'SplitWrapper({})'.format(', '.join(map(str, self.num_labels_by_class)))

class ClassificationResultWrapper:

    def __init__(self):
        self.true_positive = 0
        self.false_negative = 0
        self.false_positive = 0
        self.true_negative = 0

        self.predicted_by_label = [SplitWrapper(), SplitWrapper()]
        self.total_predicted = SplitWrapper()

    def calc_precision(self):
        return self.true_positive / (self.true_positive + self.false_positive + EPS)

    def calc_recall(self):
        return self.true_positive / (self.true_positive + self.false_negative + EPS)

    def calc_true_positive_rate(self):
        return self.calc_recall()

    def calc_false_positive_rate(self):
        return self.false_positive / (self.false_positive + self.true_negative + EPS)

    def update(self, ground_truth, predicted, add=1):
        self.predicted_by_label[predicted].update(ground_truth, add)
        self.total_predicted.update(ground_truth, add)

        if ground_truth == predicted:
            if predicted == 1:
                self.true_positive += add
            else:
                self.true_negative += add
        else:
            if ground_truth == 1:
                self.false_negative += add
            else:
                self.false_positive += add

    def to_pareto_point(self):
        return (self.calc_recall(), self.calc_precision())

    def to_roc_curve_point(self):
        return (self.calc_false_positive_rate(), self.calc_true_positive_rate())

    def __eq__(self, other):
        return (self.true_positive == other.true_positive and
            self.true_negative == other.true_negative and
            self.false_positive == other.false_positive and
            self.false_negative == other.false_negative)

    def __hash__(self):
        return id(self)

def calc_results_for_rules_scanline(rules, data, getter):
    labels = data['label']
    num_unseen_labels = [0, 0]
    num_unseen_labels[1] = sum(labels)
    num_unseen_labels[0] = len(labels) - num_unseen_labels[1]
    total_labels = num_unseen_labels[:]

    answers = [ClassificationResultWrapper() for rule in rules]

    events = []
    for i in range(data.shape[0]):
        obj = data.iloc[i]
        value = getter(obj)
        events.append((value, 1, int(obj['label'])))

    for i, rule in enumerate(rules):
        events.append((rule.threshold, 0, i))

    events.sort()

    for ev in events:
        tp = ev[1]

        if tp == 0:
            # Rule
            rule = rules[ev[2]]
            result = answers[ev[2]]
            result.update(0, rule.out_val, num_unseen_labels[0])
            result.update(1, rule.out_val, num_unseen_labels[1])

            result.update(0, not rule.out_val, total_labels[0] - num_unseen_labels[0])
            result.update(1, not rule.out_val, total_labels[1] - num_unseen_labels[1])
        else:
            num_unseen_labels[ev[2]] -= 1

    return answers

def calc_result_for_rule_naive(dec_rule, data):
    res = ClassificationResultWrapper()

    for i in range(data.shape[0]):
        obj = data.iloc[i]
        predicted = dec_rule.decide(obj)
        ground_truth = obj['label']

        res.update(ground_truth, predicted)

    return res

def split_naive(data, rule):
    res = [[], []]

    for i in range(data.shape[0]):
        obj = data.iloc[i]

        predicted = rule.decide(obj)
        res[predicted].append(obj)

    res[0] = pd.DataFrame(res[0])
    res[1] = pd.DataFrame(res[1])
    return tuple(res)


class Node:
    def __init__(self, val=0, neighbors=None):
        self.val = val
        self.neighbors = neighbors if neighbors is not None else []

def cloneGraph(node: Node) -> Node:
    if node is None:
        return None
    if node.neighbors is None:
        return Node(node.val, None)

    hashmap = {}

    def dfs(node: Node):
        if node.val not in hashmap:
            hashmap[node.val] = Node(node.val, [])
            for nd in node.neighbors:
                dfs(nd)
                hashmap[node.val].neighbours.append(hashmap[nd.val])

    dfs(node)
    return hashmap[node.val]
# if __name__ == '__main__':
#     testcases = [["2", "1", "+", "3", "*"], ["4", "13", "5", "/", "+"],
#                  ["10", "6", "9", "3", "+", "-11", "*", "/", "*", "17", "+", "5", "+"]]
#     for case in testcases:
#         print(f"{case} -> {evalRPN(case)}")


from django.conf.urls import url, include
from rest_framework_nested import routers

from paranuara_api import views

router = routers.DefaultRouter()
router.register(r'companies', views.CompanyViewSet)
router.register(r'people', views.PersonViewSet)

friends_router = routers.NestedSimpleRouter(router, r'people', lookup='person')
friends_router.register('friends', views.FriendsViewSet)

urlpatterns = [
    url(r'^', include(router.urls)),
    url(r'^', include(friends_router.urls)),
]


#!/usr/bin/env python3
import gi
from tkinter import *
from tkinter import ttk
from tkinter import font
from picamera import PiCamera
import time
import datetime
import sys

global camera
global endTime1
global endTime2
global endTime3
global endTime4
global timerLength
global startTime1
global startTime2
global startTime3
global startTime4

def quit(*args):
    root.destroy()

def show_time():
    now = datetime.datetime.now()
    now = now - datetime.timedelta(microseconds=now.microsecond)
    if now == startTime1 or now == startTime2 or now == startTime3 or now == startTime4:
        camera.stop_preview()
    elif now == endTime1 or now == endTime2 or now == endTime3 or now == endTime4:
        time.sleep(10)
        camera.start_preview()
    elif now > startTime1 and now < endTime1:
        txt.set(endTime1 - now - fix)
    elif now > startTime2 and now < endTime2:
        txt.set(endTime2 - now - fix)
    elif now > startTime3 and now < endTime3:
        txt.set(endTime3 - now - fix)
    elif now > startTime4 and now < endTime4:
        txt.set(endTime4 - now - fix)

    # Trigger the countdown after 1000ms
    root.after(1000, show_time)

# Use tkinter lib for showing the clock
root = Tk()
root.attributes("-fullscreen", True)
root.configure(background='black')
root.bind("x", quit)
root.after(0, show_time)
endTime1 = datetime.datetime(2018, 11, 17, 18, 30, 0)
endTime2 = datetime.datetime(2018, 11, 17, 18, 40, 0)
endTime3 = datetime.datetime(2018, 11, 17, 18, 50, 0)
endTime4 = datetime.datetime(2018, 11, 17, 19, 0, 0)
timerLength = datetime.timedelta(minutes=5)
fix = datetime.timedelta(seconds=1)
startTime1 = endTime1 - timerLength
startTime2 = endTime2 - timerLength
startTime3 = endTime3 - timerLength
startTime4 = endTime4 - timerLength
camera = PiCamera()
camera.start_preview()
fnt = font.Font(family='Helvetica', size=200, weight='bold')
txt = StringVar()
lbl = ttk.Label(root, textvariable=txt, font=fnt, foreground="white", background="black")
lbl.place(relx=0.5, rely=0.5, anchor=CENTER)
root.mainloop()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import json
import subprocess

from string import maketrans

from PySide.QtGui import QApplication, QFileDialog, QSystemTrayIcon, QMenu, QIcon, QPixmap

class LastDirectory(object):
    LAST_DIRECTORY = 'last_directory'
    CONFIG_FILE    = 'config.json'

    @staticmethod
    def get():
        json_ = json.load(open(LastDirectory.CONFIG_FILE, mode='r+'))

        lastDir = json_[LastDirectory.LAST_DIRECTORY] if os.path.isdir(json_[LastDirectory.LAST_DIRECTORY]) else os.path.dirname(os.path.realpath(sys.argv[0]))

        if len(lastDir) == 0:
            raise SystemExit

        json_[LastDirectory.LAST_DIRECTORY] = lastDir
        with open(LastDirectory.CONFIG_FILE, 'w') as json_file:
            json.dump(json_, json_file, sort_keys=True, indent=4, separators=(',', ':'))

        return lastDir

    @staticmethod
    def set(file):
        json_ = json.load(open(LastDirectory.CONFIG_FILE, mode='r+'))

        lastDir = os.path.dirname(os.path.realpath(file))

        if len(lastDir) == 0:
            raise SystemExit

        json_[LastDirectory.LAST_DIRECTORY] = lastDir
        with open(LastDirectory.CONFIG_FILE, 'w') as json_file:
            json.dump(json_, json_file, sort_keys=True, indent=4, separators=(',', ':'))

def getFileNew():
    fileNames, fileFilters = QFileDialog.getOpenFileNames(dir=LastDirectory.get())
    if len(fileNames) != 0:
        LastDirectory.set(fileNames[0])

    full_ = ''
    for i in fileNames:
        full_ += i if fileNames[-1] == i else i + ';'

    global app
    clipboard = app.clipboard()
    clipboard.setText(str(full_).translate(maketrans('\\', '/')))

def getDirectoryNew():
    directory_ = QFileDialog.getExistingDirectory()
    global app
    clipboard = app.clipboard()
    clipboard.setText(str(directory_).translate(maketrans('\\', '/')))

def getToolFileName(tool, comment):
    pyside_rcc, _ = QFileDialog.getOpenFileName(caption=u"Открыть {0}.exe, {1}".format(tool, comment), filter='*.exe')
    return pyside_rcc

def getTool(tool, comment):
    CONFIG_FILE = 'config.json'

    json_ = json.load(open(CONFIG_FILE, mode='r+'))

    toolFileName = json_[tool] if os.path.exists(json_[tool]) else getToolFileName(tool, comment)

    if len(toolFileName) == 0:
        raise SystemExit

    json_[tool] = toolFileName
    with open(CONFIG_FILE, 'w') as json_file:
        json.dump(json_, json_file, sort_keys=True, indent=4, separators=(',', ':'))

    return toolFileName

def processQrcFile():
    qrcFileName, _ = QFileDialog.getOpenFileName(caption=u'Выбери qrc файл', filter='*.qrc', dir=LastDirectory.get())
    LastDirectory.set(qrcFileName)

    pyFileName, _ = QFileDialog.getSaveFileName(caption=u'Сохрани py файл, незабудь "resources.qrc" -> "resources_rc.qrc"', filter='*.py', dir=LastDirectory.get())
    LastDirectory.set(pyFileName)

    subprocess.Popen([getTool('pyside_rcc', u'он где то тут: Python27/Lib/site-packages/PySide/pyside-rcc.exe'), qrcFileName, '-o', pyFileName])

def processUiFile():
    qrcFileName, _ = QFileDialog.getOpenFileName(caption=u'Выбери UI файл', filter='*.ui', dir=LastDirectory.get())
    LastDirectory.set(qrcFileName)

    pyFileName, _ = QFileDialog.getSaveFileName(caption=u'Сохрани py файл', filter='*.py', dir=LastDirectory.get())
    LastDirectory.set(pyFileName)

    subprocess.Popen([getTool('pyside_uic', u'он где то тут: Python27/Scripts/pyside-uic.exe'), qrcFileName, '-o', pyFileName])

def main():
    global app
    app = QApplication(sys.argv)

    tray_menu = QMenu()

    tray_menu.addAction(QIcon(QPixmap('data/file.png')), u'Получить пути файлов', getFileNew)
    tray_menu.addAction(QIcon(QPixmap('data/directory.png')), u'Получить путь директории', getDirectoryNew)
    tray_menu.addSeparator()
    tray_menu.addAction(QIcon(QPixmap('data/qrc.png')), u'Преобразовать .qrc в .py', processQrcFile)
    tray_menu.addAction(QIcon(QPixmap('data/ui.png')), u'Преозразовать .ui в .py', processUiFile)
    tray_menu.addSeparator()
    tray_menu.addAction(QIcon(QPixmap('data/close.png')), u'Выход', app.quit)

    tray = QSystemTrayIcon()
    tray.setIcon(QIcon(QPixmap('data//piggy.png')))
    tray.setContextMenu(tray_menu)
    tray.show()

    return app.exec_()

if __name__ == '__main__':
    main()


import pandas as pd
# 划分验证集，保存格式  text[\t]label
from sklearn.model_selection import train_test_split

# 自定义数据集
import os
import codecs
import csv

from paddlehub.dataset.base_nlp_dataset import BaseNLPDataset

def prepare_data():
    # 转换编码
    def re_encode(path):
        with open(path, 'r', encoding='GB2312', errors='ignore') as file:
            lines = file.readlines()
        with open(path, 'w', encoding='utf-8') as file:
            file.write(''.join(lines))

    #re_encode('data/test.csv')
    #re_encode('data/train.csv')

    # 读取数据
    train_labled = pd.read_csv('data/train.csv', engine ='python')
    test = pd.read_csv('data/test.csv', engine ='python')

    print(train_labled.shape)
    print(test.shape)
    print(train_labled.columns)

    train_labled.head(3)
    test.head(3)

    train_labled = train_labled[train_labled['情感倾向'].isin(['-1','0','1'])]
    train_labled['微博中文内容'].str.len().describe()

    train_labled = train_labled[['微博中文内容', '情感倾向']]
    train, valid = train_test_split(train_labled, test_size=0.2, random_state=2020)
    train.to_csv('./data/train.txt', index=False, header=False, sep='\t')
    valid.to_csv('./data/valid.txt', index=False, header=False, sep='\t')

class MyDataset(BaseNLPDataset):
    """DemoDataset"""
    def __init__(self):
        # 数据集存放位置
        self.dataset_dir = "./data"
        super(MyDataset, self).__init__(
            base_path=self.dataset_dir,
            train_file="train.txt",
            dev_file="valid.txt",
            test_file="valid.txt",
            train_file_with_header=False,
            dev_file_with_header=False,
            test_file_with_header=False,
            # 数据集类别集合
            label_list=["-1", "0", "1"])

def fine_tune():
    print("print my dataset")
    dataset = MyDataset()
    for e in dataset.get_train_examples()[:3]:
        print("{}\t{}\t{}".format(e.guid, e.text_a, e.label))

    import paddlehub as hub
    module = hub.Module(name="ernie", module_dir="./model")

    # 构建Reader
    print("vocab_path:", module.get_vocab_path())
    print("sp_model_path:", module.get_spm_path())
    print("word_dict_path:", module.get_word_dict_path())
    reader = hub.reader.ClassifyReader(
        dataset=dataset,
        vocab_path=module.get_vocab_path(),
        sp_model_path=module.get_spm_path(),
        word_dict_path=module.get_word_dict_path(),
        max_seq_len=256)

    # finetune策略1
    strategy = hub.L2SPFinetuneStrategy(
        learning_rate=1e-4,
        optimizer_name="adam",
        regularization_coeff=1e-3)

    # 运行配置
    config = hub.RunConfig(
        use_cuda=True,
        num_epoch=10,
        batch_size=32,
        checkpoint_dir="hub_finetune_ckpt0431",
        strategy=strategy)

    # Finetune Task
    inputs, outputs, program = module.context(
        trainable=True, max_seq_len=256)

    # Use "pooled_output" for classification tasks on an entire sentence.
    pooled_output = outputs["pooled_output"]

    feed_list = [
        inputs["input_ids"].name,
        inputs["position_ids"].name,
        inputs["segment_ids"].name,
        inputs["input_mask"].name,
    ]

    cls_task = hub.TextClassifierTask(
            data_reader=reader,
            feature=pooled_output,
            feed_list=feed_list,
            num_classes=dataset.num_labels,
            config=config,
            metrics_choices=["f1"])

    # finetune
    run_states = cls_task.finetune_and_eval()

if __name__ == "__main__":
    prepare_data()
    fine_tune()


from amcu.automcu import AutoMCU
import pytest

# @pytest.fixture
def test_amcu():
    """A test function to check the package using pytest, using assert the required input will be tested within that function"""

    em_csvs = ["data/EM1.csv", "data/EM2.csv"]
    em_counts = [2]
    em_names = ["EM1", "EM2"]
    band_ranges = [[0, 1], [1, 2]]
    amcu = AutoMCU()

    output = amcu.apply_image("../data/input_img", "../data/output_img.tif")
    assert output.shape == (10, 1)