# SPEC and BLISS reader
# initially 2016-11-23
# Sam Tardif (samuel.tardif@gmail.com)
# Python3 compatibility by Nils Blanc
# Modified to store scan lines by L. Renversade (2018)
# added functions to read hdf5 file by J.S. Micha (2022)

""" this file in intended to be name logfilereader.py and be located in main lauetools folder"""

import copy
import os
import time
import numpy as np

try:
    import h5py
except ModuleNotFoundError:
    print('warning: h5py is missing. It is useful for playing with hdf5 for some LaueTools modules. Install it with pip> pip install h5py')

def ReadSpec(fname, scan, outputdate=False):
    """
    Procedure very based on that of Vincent Favre-Nicolin (ESRF) procedure

    :param scan: scan index (integer)
    return :
    spec command (str), dict of data (key=counter name, val= values), and optionnaly [date (str)]
    """
    f = open(fname, "r")

    print('ReadSpec of IOLaueTools.py')

    s = "#S %d" % scan
    title = 0

    bigmca = []
    # spec command with motors, steps and exposure
    while 1:
        title = f.readline()
        if s == title[0 : len(s)]:
            break
        if len(title) == 0:
            break
    print(title)
    # date -----------------
    date = 'unknown'
    s = "#D"
    while 1:
        line = f.readline()
        if s == line[0:len(s)]:
            date = line[3:]
            break
        if len(line) == 0:
            break
    # data   (dict with key= counter column name) -------------
    s = "#L"
    coltit = 0

    while 1:
        coltit = f.readline()
        if s == coltit[0 : len(s)]:
            break
        if len(coltit) == 0:
            break
    d = {}
    coltit = coltit.split()
    for i in list(range(1, len(coltit))):
        d[coltit[i]] = []

    ii = 0
    while 1:  # reading data
        l = f.readline()
        if len(l) < 2:
            break

        if l[:2] == "#C":
            # deal with#C Thu Feb 25 23:54:05 2021.  Erreur com with laueT.
            if 'Erreur com' in l:
                print('line error #C :', l)
                continue
            elif not l.startswith("#C tiltcomp:"):
                print("Scan aborted after %d point(s)" % ii)
                break
            else:
                print(l)
        if l[0] == "#":
            continue
        l = l.split()
        # print "l",l
        #         print "coltit", coltit
        # print "nb columns",len(coltit)-1
        if l[0] != "@A":
            for i in list(range(1, len(coltit))):
                d[coltit[i]].append(float(l[i - 1]))
        else:
            # print "reading mca data array for one point"
            bill = np.zeros((128, 16))  # 2048=128*16
            mcadata = []
            l[-1] = l[-1][:-1]
            mcadata.append(np.array(l[1:]))
            bill[0] = np.array(np.array(l[1:]), dtype=np.int16)
            # fist line has its first element = '@A'
            if ii % 10 == 0:
                print("%d" % ii)
            for k in list(range(1, 127)):  # first and last line off , each line contains 16 integers
                l = f.readline()

                l = l.split()
                l[-1] = l[-1][:-1]
                mcadata.append(np.array(l[:16]))
                # print "uihuihui ",k,"   ",array(l)
                bill[k] = np.array(np.array(l[:16]), dtype=np.int16)
            # last line doesn't finish with \
            l = f.readline()
            l = l.split()
            # print array(l)
            mcadata.append(np.array(l))
            bill[-1] = np.array(np.array(l), dtype=np.int16)

            # bill=array(mcadata,dtype=uint16)
            bigmca.append(np.ravel(bill))
            ii += 1

            d["mca"] = np.array(bigmca)

    nb = len(d[coltit[1]])  # nb of points
    # print "nb",nb
    for i in list(range(1, len(coltit))):
        a = np.zeros(nb, dtype=float)
        for j in list(range(nb)):
            a[j] = d[coltit[i]][j]
        d[coltit[i]] = copy.deepcopy(a)
    f.close()
    if outputdate:
        return title, d, date
    else:
        return title, d

class SpecFile:
    """
    Simple class to read SPEC and hdf5 file. The main purposes are :
    (i) read the header and initiate the relevant properties
    (ii) build a dictionary of the scan number and their binary position in the file
    The latter allows for faster subsequent scan reading
    Definition:
    -----------
    SpecFile(spec_file, verbose = False)
     > spec_file : string
     > scan_numbers : integer or as tuple/list/array of integer
    Attributes (typical):
    -----------
    comments........all comments
    date............scan start datestamp
    file............string of the SPEC file name
    scan_dict.......dictionary {scan number : binary position in file}
    Examples:
    --------
    # read the specfile
    In : sf = SpecFile('./lineup0.dat')

    author: S. Tardif
    """

    # dictionary definitions for handling the spec identifiers
    def __param__(self):
        return {'S': self.__scanline__,
                'D': self.__dating__,
                'T': self.__counting__,
                'G': self.__configurating__,
                'Q': self.__hkl__,
                'O': self.__motorslabeling__,
                'o': self.__motorslabelingnospace__,
                'P': self.__positioning__,
                'M': self.__marccdpath__,
                'N': self.__speccol__,
                'L': self.__counterslabeling__,
                'C': self.__commenting__,
                '@': self.__special__,
                'XIAFILE': self.__xiafilenaming__,
                'XIACALIB': self.__xiacalibrating__,
                'XIAROI': self.__xiaroidefining__}

    def __scanline__(self, l):
        self.number = int(l.split()[1])
        self.type = l.split()[2]
        self.args = l.split()[3:]
        self.command = l[3 + len(l.split()[1]) + 2:].rstrip()

    def __dating__(self, l):
        self.date = l[3:]

    def __counting__(self, l):
        self.ct = float(l.split()[1])
        self.ct_units = l.split()[2]

    def __configurating__(self, l):
        self.__config__ = self.__config__ + l[3:] + " "

    def __hkl__(self, l):
        self.Qo = l[3:].split()

    def __motorslabeling__(self, l):
        self.__motorslabels__ = self.__motorslabels__ + l[3:] + " "

    def __motorslabelingnospace__(self, l):
        self.__motorslabelsnospace__ = self.__motorslabelsnospace__ + l[3:] + " "

    def __positioning__(self, l):
        self.__positions__ = self.__positions__ + l[3:] + " "

    def __speccol__(self, l):
        self.N = int(l[3:])

    def __counterslabeling__(self, l):
        self.counters = l.split()[1:]

    def __commenting__(self, l):
        self.comments = self.comments + l[3:]

    def __marccdpath__(self, l):
        self.M = l.split()[1:]

    def __special__(self, l):
        self.__param__()[l[2:].split(' ')[0]](l)

    def __xiafilenaming__(self, l):
        self.xianame = l.split()[1:][0]
        if not hasattr(self, "xiaroi"):
            self.xiaroi = dict()

    def __xiacalibrating__(self, l):
        self.xiacalib = l.split()[1:][0]

    def __xiaroidefining__(self, l):
        if not hasattr(self, "xiaroi"):
            self.xiaroi = dict()
        self.xiaroi[l.split()[1]] = [int(l.split()[2]), int(l.split()[3]), int(l.split()[4]), int(l.split()[5]),
                                     int(l.split()[6])]

    def __init__(self, spec_file, verbose=False, filetype='spec', collectallscans=True,
                                                                onlywirescan=False,onlywire=False):
        # init a bunch of stuff that will also be used by the children class Scan
        self.file = spec_file

        self.filetype = filetype # 'spec'  # 'hdf5'

        self.__motorslabels__ = ""  # list of all motors in the experiment
        self.__motorslabelsnospace__ = ""  # list of all motors in the experiment
        self.__positions__ = ""  # list the values of all motors
        self.__config__ = ""  # list the values of the UB matrix config
        self.comments = ""
        # next attributes come with the reading of the file
        self.scan_dict = {}  # dictionary to store the position in the file of the scans
        self.cmd_list =  {}  # list to store scan line strings

        if filetype == 'spec':
            try:
                with open(spec_file, 'rU') as f:  # the U mode indicates universal line break, essential for accurate counting
                    # first read the file header (mostly comments and motors definition)
                    # up to the first scan (identified by a line starting with "#S"
                    reading_header = True
                    while reading_header:
                        position_in_file = f.tell()  # get the position AHEAD of the scan first line
                        l = f.readline()
                        if len(l) > 1:  # not an empty line
                            if l[:2] == '#S':
                                reading_header = False
                                scan_number = int(l.split()[1])
                                if verbose:
                                    print("after reading the header, found scan {} at location {}".format(scan_number,
                                                                                position_in_file))
                                self.scan_dict[scan_number] = position_in_file
                                self.cmd_list[scan_number] = l[3:-1]
                            else:
                                try:
                                    self.__param__()[l[1]](l)
                                except KeyError:
                                    if verbose: print("unprocessed line:" + l)
                    reading_file = True  # change this switch when the end of file is reached (i.e. read an empty string)
                    while reading_file:
                        position_in_file = f.tell()  # get the position AHEAD of the scan first line
                        l = f.readline()
                        if len(l) > 1:  # not an empty line
                            if l[:2] == '#S':
                                scan_number = int(l.split()[1])
                                if verbose:
                                    print("found scan {} at location {}".format(scan_number, position_in_file))
                                self.scan_dict[scan_number] = position_in_file
                                self.cmd_list[scan_number] = l[3:-1]
                        if l == "":
                            reading_file = False
            except IOError:
                print("could not find the file {}".format(spec_file))
        elif filetype == 'hdf5':
            selp, allp = getwirescan_from_hdf5file(spec_file, collectallscans=collectallscans,
                                                            onlywirescan=onlywirescan, onlymesh=onlywirescan,
                                                            verbose=0)
            self.scan_dict = get_scandict(selp)
            self.cmd_list = get_cmd_list(selp)

# helper function of listprops generated by getwirescan_from_hdf5file
def get_cmd_list(listprops):
    """ return dict of commands in the list (generated by getwirescan_from_hdf5file)"""
    return { elem[0]: elem[4] for elem in listprops}
def get_scandict(listprops):
    """ return dict of keys ... (only to fit the previous spec-related interface)
    (generated by getwirescan_from_hdf5file)"""
    return { elem[0]: elem[0] for elem in listprops}
def get_file(listprops):
    """ return master file corresponding to listprops (generated by getwirescan_from_hdf5file)"""
    return listprops[0][-1]

def get_date(listprops):
    """ return date of file corresponding to listprops (generated by getwirescan_from_hdf5file)"""
    return listprops[0][3]

def getmeshscan_from_hdf5file(filename):

    print("getmeshscan_from_hdf5file  %s"%filename)
    _,ext = filename.rsplit('.',1)
    headname, ffname = os.path.split(filename)

    os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
    with h5py.File(filename, 'r') as f:

        listkeys = [kk for kk in f.keys()]
        print('hdf5 keys',listkeys)
        nbkeys = len(listkeys)

        # if key =   #########_int.int  then it is a pointer to a file ########.h5
        # if key =   int.int    this file contains truly the data
        #  #########   =  collectionname_datasetname
        listprops = []  # selected props
        allprops = []

        idx_key=0
        while idx_key<nbkeys:
            _key = listkeys[idx_key]
            objlink = f.get(_key, getlink=True)
            props = None
            if isinstance(objlink, h5py._hl.group.ExternalLink):

                lowlevelpath = objlink.filename
                print('key = %s is External link to %s'%(_key,lowlevelpath))
                foundfile = findlowesthdf5file(lowlevelpath,mainfolder=headname)
                if foundfile:
                    # removing string before _interger.integer
                    _modified_key = _key.rsplit('_',1)[-1]
                    props, isselected = getscanprops_lowest_hdf5(foundfile, _modified_key, collectallscans=False, onlymesh=True)
            elif isinstance(objlink, h5py._hl.group.HardLink):
                print('key = %s is Hard link to '%(_key))
                props, isselected = getscanprops_lowest_hdf5(filename, _key, collectallscans=False, onlymesh=True)

            allprops.append(props)

            if isselected:
                listprops.append(props)
            idx_key+=1

    # print('listprops',listprops)
    if listprops == []:
        #wx.MessageBox('No mesh scan in the file: %s'%filename,'INFO')
        print('\n\n*******\nNo mesh scan in the file: %s\n**********\n'%filename)
        return []

    # sorting by increasing date
    ar_lp = np.array(listprops, dtype=object)
    s_ix=np.argsort(ar_lp[:,3])
    sortedlistprops = ar_lp[s_ix]
    #print('sortedlistprops',sortedlistprops)

    return sortedlistprops

def getwirescan_from_hdf5file(filename, verbose=0, collectallscans=True, onlywirescan=False, onlymesh=False):

    if collectallscans:
        onlymesh=True
        onlywirescan=True

    if verbose: print("getwirescan_from_hdf5file  %s"%filename)
    _,ext = filename.rsplit('.',1)
    headname, ffname = os.path.split(filename)

    os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
    with h5py.File(filename, 'r') as f:

        listkeys = [kk for kk in f.keys()]
        if verbose: print('filename: %s \n hdf5 keys:'%filename, listkeys)
        nbkeys = len(listkeys)

        # if key =   #########_int.int  then it is a pointer to a file ########.h5
        # if key =   int.int    this file contains truly the data
        #  #########   =  collectionname_datasetname
        listprops = []   # selected props
        allprops = []
        idx_key=0
        while idx_key<nbkeys:
            _key = listkeys[idx_key]
            objlink = f.get(_key, getlink=True)
            props = None
            isselected = False
            if isinstance(objlink, h5py._hl.group.ExternalLink):

                lowlevelpath = objlink.filename
                if verbose: print('key = %s is External link to %s'%(_key,lowlevelpath))
                foundfile = findlowesthdf5file(lowlevelpath,mainfolder=headname)
                if foundfile:
                    # removing string before _interger.integer
                    _modified_key = _key.rsplit('_',1)[-1]
                    props, isselected = getscanprops_lowest_hdf5(foundfile, _modified_key,collectallscans=collectallscans,
                                                                onlymesh=onlymesh, onlywirescan=onlywirescan)
            elif isinstance(objlink, h5py._hl.group.HardLink):
                if verbose: print('key = %s is Hard link to '%(_key))
                props, isselected = getscanprops_lowest_hdf5(filename, _key,collectallscans=collectallscans,
                                                                onlymesh=onlymesh, onlywirescan=onlywirescan)
            if props is not None:
                allprops.append(props)
            if isselected:
                listprops.append(props)
            idx_key+=1

    #print('allprops',allprops)
    if listprops == []:
        #wx.MessageBox('No mesh scan in the file: %s'%filename,'INFO')
        print('\n\n*******\nNo wire scan in the file: %s\n**********\n'%filename)
        return []

    # sorting by increasing date
    ar_lp = np.array(listprops, dtype=object)
    s_ix=np.argsort(ar_lp[:,3])
    sortedlistprops = ar_lp[s_ix]
    #print('sortedlistprops',sortedlistprops)

    # sorting by increasing date
    ar_lpall = np.array(allprops, dtype=object)
    s_ix2=np.argsort(ar_lpall[:,3])
    sortedlistallprops = ar_lpall[s_ix2]
    #print('sortedlistprops',sortedlistprops)

    return sortedlistprops, sortedlistallprops

def findlowesthdf5file(filename, mainfolder='.', verbose=0):
    """ find the lowest hdf5 file pointing
    To be improved to consider relative path
    """

    foundfile = None
    _, ffname = os.path.split(filename)
    absfolder = os.path.abspath(mainfolder)
    relativepath = os.path.join(absfolder,filename)
    if verbose:
        print('filename',filename)
        print('relativepath', relativepath)
        print('ffname',ffname)
        print('absfolder',absfolder)
    if os.path.exists(relativepath): # relative path from current folder
        foundfile = relativepath
    elif ffname in os.listdir(absfolder): #local folder
        foundfile = os.path.join(absfolder,ffname)
    if verbose:
        print('foundfile',foundfile)
    return foundfile

def getscanprops_lowest_hdf5(filename, key, collectallscans=True, onlymesh=False, onlywirescan=False, verbose=0):
    """ get scan props from hdf5 file and filter optionally wrt scan type (mesh or wirescan)"""
    #print('\n\nterminal hdf5 file')
    _,ext = filename.rsplit('.',1)
    headname, ffname = os.path.split(filename)

    os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
    with h5py.File(filename, 'r') as f:

        idx, postfix = key.split('.')
        #print('reading key %s'%key)
        if h5py.__version__<'3.0':
            #maybe [()] is enough without decoding
            scancommand = f['%s.%s'%(idx,postfix)]['title'].value
            startdate = f['%s.%s'%(idx,postfix)]['start_time'].value
        else:
            #print('%s.%s'%(idx,postfix))
            scancommand = f['%s.%s'%(idx,postfix)]['title'][()].decode('UTF-8')
            startdate = f['%s.%s'%(idx,postfix)]['start_time'][()].decode('UTF-8')
        props = None

        if verbose:
            print('scancommand',scancommand)
            print('startdate',startdate)
        isselected = False
        if onlymesh:
            if 'amesh' in scancommand:
                keyfilename = ffname[:-3]  #  removing .h5
                props=['%s_%s'%(keyfilename,idx),idx, postfix, startdate, '%s_%s %s'%(keyfilename, idx, scancommand), filename]
                isselected=True
        if onlywirescan:
            if 'zf' in scancommand or 'yf' in scancommand:
                keyfilename = ffname[:-3]  #  removing .h5
                props=['%s_%s'%(keyfilename,idx),idx, postfix, startdate, '%s_%s %s'%(keyfilename, idx, scancommand), filename]
                isselected=True
        if collectallscans:
            if 'loopscan' in scancommand or 'ascan' in scancommand or 'a2scan' in scancommand:
                keyfilename = ffname[:-3]  #  removing .h5
                props=['%s_%s'%(keyfilename,idx),idx, postfix, startdate, '%s_%s %s'%(keyfilename, idx, scancommand), filename]
                isselected=True
    return props, isselected

class Scan(SpecFile):
    """
    Simple class to read extract scans from SPEC files. All the parameters of the scan and the data are read
    and stored as attributes.

    Definition:
    -----------
    Scan(spec_file, scan_numbers, verbose = False)
     > spec_file as SpecFile object or string (in this case a SpecFile object will be instanced)
     > scan_numbers as integer or as tuple/list/array of integer

    Attributes:
    -----------
    <countername>...data in counter <countername> (see counters for description)
    number..........scan number of the first scan in the list
    scan_numbers....list of all the scans included
    type............scan type
    args............scan arguments (motor start stop npoints counting)
    date............scan start datestamp
    ct..............scan counting time per point
    Qo..............H K L position at start of scan
    M...............MarCCD image file path
    N...............number of counters
    counters........list of counters
    motors..........dictionary of motors and their initial position
    comments........all comments
    tstart, tend....starting and finishing time
    duration........duration in s
    time_per_point..duration per point
    Examples:
    --------
    # read the specfile
    In : sf = SpecFile('./lineup0.dat')
    # read the scan
    In : scan = Scan(sf, 265)

    # read a series of scans
    In : scan = Scan(sf, (265,266,270))
    In : scan = Scan(sf, arange(265,270))

    # learn about the motors
    In : scan.motors
    Out:
    {'Chi': -144.8913,
     'Phi': 176.9778,
     ...
     'xpr3z': -8.8}

    # extract the info on a particular motor
    In : scan.motors['tth']
    Out: 35.042

    # plot two counters vs each others
    In : plot(scan.th,scan.det/scan.IC1)

    author: L. Renversade
    """

    def __init__(self, spec_file, scan_numbers, verbose=False, filetype='spec'):
        if type(spec_file) == str:
            spec_file = SpecFile(spec_file)
        self.file = spec_file.file
        try:
            len_scan_number = len(scan_numbers)  # it is a list of scan
        except TypeError:
            scan_numbers = [scan_numbers, ]  # it is a simple scan, we make a len 0 list
        scan_number = scan_numbers[0]  # for all intents and purposes
        # recover the names of the motors from the header of the SPEC file (i.e. the SpecFile instance)
        self.__motorslabels__ = spec_file.__motorslabels__  # list of all motors in the experiment
        self.__motorslabelsnospace__ = spec_file.__motorslabelsnospace__  # list of all motors in the experiment
        # prepare the scan-specific attributes
        self.__positions__ = ""  # list the values of all motors
        self.__config__ = ""  # list the values of the UB matrix config
        self.scan_numbers = scan_numbers
        self.comments = ""

        with open(self.file, 'rU') as f:
            # read the first (and possibly only) scan in the list
            # now try to find the scan
            f.seek(spec_file.scan_dict[scan_number])
            l = f.readline()
            if verbose: print("reading scan " + l)

            # read the scan header
            while l[0] == '#':
                try:
                    self.__param__()[l[1]](l)
                except KeyError:
                    if verbose: print("unprocessed line:\n" + l)
                l = f.readline()

            # finally read the data (comments at the end are also read and added to the comment attribute)
            data = [list(map(float, l.split()))]
            l = f.readline()
            while l != '\n' and l != '':
                if l[0] == '#' and l[1] != 'C':
                    break
                if l[0] == '#':
                    try:
                        self.__param__()[l[1]](l)
                    except KeyError:
                        if verbose: print("unprocessed line:\n" + l)
                else:
                    data.append(list(map(float, l.split())))
                l = f.readline()

            # now get the data for each scan in the list
            if len(scan_numbers) > 0:
                for scan_number in scan_numbers[1:]:
                    # now try to find the scan
                    f.seek(spec_file.scan_dict[scan_number])
                    l = f.readline()
                    if verbose: print("reading scan " + l)

                    # check that we actually concatenate similar scans !
                    similar_scan = (l.split()[2] == self.type)
                    if len(self.args) > 1:
                        try:
                            similar_scan = similar_scan * (l.split()[3] == self.args[0])
                        except ValueError:
                            similar_scan = False
                    if len(self.args) > 5:
                        try:
                            similar_scan = similar_scan * (l.split()[6] == self.args[3])
                        except ValueError:
                            similar_scan = False

                    if similar_scan:
                        # read pass the scan header
                        while l[0] == '#':
                            l = f.readline()

                        # finally read the data (comments at the end are also read and added to the comment attribute)
                        data.append(list(map(float, l.split())))
                        l = f.readline()
                        while l != '\n' and l != '':
                            if l[0] == '#' and l[1] != 'C':
                                break
                            if l[0] == '#':
                                try:
                                    self.__param__()[l[1]](l)
                                except KeyError:
                                    if verbose: print("unprocessed line:\n" + l)
                            else:
                                data.append(list(map(float, l.split())))
                            l = f.readline()
                    else:
                        print("not all scans are the same type")

            # set the data as attributes with the counter name
            for i in range(len(self.counters)):
                # print(i,range(len(self.counters)),self, self.counters[i], np.asarray(data).shape, np.asarray(data), data)
                setattr(self, self.counters[i], np.asarray(data)[:, i])

            # make the motors/positions dictionary
            # usual case
            if len(self.__motorslabels__.split()) == len(self.__positions__.split()):
                self.motors = dict(zip(self.__motorslabels__.split(), list(map(float, self.__positions__.split()))))
            # when some motors names have spaces and there is a second line (small o) to describe them
            elif len(self.__motorslabelsnospace__.split()) == len(self.__positions__.split()):
                self.motors = dict(
                    zip(self.__motorslabelsnospace__.split(), list(map(float, self.__positions__.split()))))

            # TEST : attribute-like dictionary
            # removed due to conflicts when a motor was also a counter
            #		for motor in self.motors:
            #								setattr(self, motor, self.motors[motor])

            # small sanity check, sometimes N is diffrent from the actual number of columns
            # which is known to trouble GUIs like Newplot and PyMCA
            if self.N != len(self.counters):
                print("Watch out! There are %i counters in the scan but SPEC knows only N = %i !!" % (
                len(self.counters), self.N))

            if hasattr(self, 'Epoch'):
                self.tstart = self.Epoch[0]
                self.tend = self.Epoch[-1]
                self.duration = self.tend - self.tstart
                self.time_per_point = self.duration / len(self.Epoch)
# class Scan2D(Scan):
# def __init__(self, spec_file, scan_number_list, verbose = verbose):

class Scan_hdf5(SpecFile):
    """
    Simple class to read extract scans from bliss files. All the parameters of the scan and the data are read
    and stored as attributes.

    Definition:
    -----------
    Scan(spec_file, scan_numbers, verbose = False)
     > hdf5 file (string) created by BLISS
     > scan_keys   string  WARNING!  single key for the moment!

    # not up to date!

    Attributes:
    -----------
    <countername>...data in counter <countername> (see counters for description)
    number..........scan number of the first scan in the list
    scan_numbers....list of all the scans included
    type............scan type
    args............scan arguments (motor start stop npoints counting)
    date............scan start datestamp
    ct..............scan counting time per point
    Qo..............H K L position at start of scan
    M...............MarCCD image file path
    N...............number of counters
    counters........list of counters
    motors..........dictionary of motors and their initial position
    comments........all comments
    tstart, tend....starting and finishing time
    duration........duration in s
    time_per_point..duration per point
    Examples:
    --------
    # read the specfile
    In : sf = SpecFile('./lineup0.dat')
    # read the scan
    In : scan = Scan(sf, 265)

    # read a series of scans
    In : scan = Scan(sf, (265,266,270))
    In : scan = Scan(sf, arange(265,270))

    # learn about the motors
    In : scan.motors
    Out:
    {'Chi': -144.8913,
     'Phi': 176.9778,
     ...
     'xpr3z': -8.8}

    # extract the info on a particular motor
    In : scan.motors['tth']
    Out: 35.042

    # plot two counters vs each others
    In : plot(scan.th,scan.det/scan.IC1)

    author: J.S. Micha
    """
    def __init__(self, spec_file, scan_keys,collectallscans=True,
                    onlywirescan=False, onlymesh=False, verbose=False):
        if type(spec_file) == str:
            spec_file = SpecFile(spec_file, filetype='hdf5')
        self.file = spec_file.file
        try:  # WARNING single key for the moment
            len_scan_number = len(scan_keys)  # it is a list of scan
        except TypeError:
            scan_keys = [scan_keys, ]  # it is a simple scan, we make a len 0 list
        scan_number = scan_keys[0]  # for all intents and purposes
        # recover the names of the motors from the header of the SPEC file (i.e. the SpecFile instance)
        self.__motorslabels__ = spec_file.__motorslabels__  # list of all motors in the experiment
        self.__motorslabelsnospace__ = spec_file.__motorslabelsnospace__  # list of all motors in the experiment
        # prepare the scan-specific attributes
        self.__positions__ = ""  # list the values of all motors
        self.__config__ = ""  # list the values of the UB matrix config
        self.scan_keys = scan_keys

        self.comments = ""

        print('beginning of __init__ of scan_hdf5')
        selp, allp = getwirescan_from_hdf5file(spec_file.file, onlywirescan=onlywirescan, collectallscans=collectallscans, onlymesh=onlymesh)
        tit, data, posmotors, fullpath, scan_date= readdata_from_hdf5key(selp, scan_keys, outputdate=True)

        if verbose:
            print('tit',tit)
            print('posmotors',posmotors)
            print('scan_date',scan_date)
            print('data',data)
            print('fullpath', fullpath)
        # set the data as attributes with the counter name
        for key,val in data.items():

            setattr(self, key, val)

        self.fullpath = fullpath

        scanindex= int(scan_keys.rsplit('_',1)[-1])
        basepath = os.path.split(fullpath)[0]
        self.scanfolder = os.path.join(basepath, 'scan_%04d'%scanindex)
        self.motors = posmotors
        self.command = tit
        self.scan_date = scan_date

        # small sanity check, sometimes N is diffrent from the actual number of columns
        # which is known to trouble GUIs like Newplot and PyMCA
        # if self.N != len(self.counters):
        #     print("Watch out! There are %i counters in the scan but SPEC knows only N = %i !!" % (
        #     len(self.counters), self.N))

        if hasattr(self, 'epoch'):
            self.tstart = self.epoch[0]
            self.tend = self.epoch[-1]
            self.duration = self.tend - self.tstart
            self.time_per_point = self.duration / len(self.epoch)

def parsehdf5time(strtime):
    """ parse time in hdf5 from bliss file
    Return  ascii time, epoch time"""
    sd, _ = strtime.split('+')
    da, ho = sd.split('T')
    yy, mm, dd = da.split('-')
    hh, mi, sec = ho.split(':')
    starttime = time.strptime('%s %s %s %s:%s:%s'%(yy, mm, dd, hh, mi, sec[:2]), '%Y %m %d %H:%M:%S')
    return time.asctime(starttime), time.mktime(starttime)

def ReadHdf5_v2(fname, scan, outputdate=False):
    """extract data of a scan in a ESRF Bliss made hdf5 file

    NOT USED?

    :param fname: file object or string (path)
    :type fname: file object or string
    :param scan: scan index
    :type scan: integer
    :param outputdate: output starting date of the scan in ascii format, defaults to False
    :type outputdate: bool, optional
    """
    os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"

    with h5py.File(fname, 'r') as f:

        i_scan = '%d' % scan + '.1'
        print('i_scan', i_scan)
        print('fname', f)
        print('fname[i_scan]', f[i_scan])
        datasetnames = [key for key in f[i_scan]['measurement'].keys()]
        d = {}
        for _n in datasetnames:
            d[_n] = f[i_scan]['measurement'][_n][()]

        if h5py.__version__>='3':
            title = f[i_scan]['title'][()].decode('UTF-8')
            st = f[i_scan]['start_time'][()].decode('UTF-8')
            et = f[i_scan]['end_time'][()].decode('UTF-8')
        else:
            title = f[i_scan]['title'].value
            st = f[i_scan]['start_time'].value
            et = f[i_scan]['end_time'].value
        duration = parsehdf5time(et)[1] - parsehdf5time(st)[1]

        print('title', title)
        print('duration', duration)

        if not outputdate:
            return title, d
        else:
            return title, d, st

def readdata_from_hdf5key(listkeyprops, key, outputdate=False):
    """access to a low level hdf5 file and read data corresponding to the key

    :param listkeyprops: array of strings which lists all the keys (see getmeshscan_from_hdf5file() of plotmeshspecGUI.py). Each element is:
    props=['%s_%s'%(keyfilename,idx),idx, postfix, startdate, '%s_%s %s'%(keyfilename, idx, scancommand), filename]
    :type fname: file object or string
    :param key: scan id  (key in hdf5 file)
    :type key: string
    :param outputdate: output starting date of the scan in ascii format, defaults to False
    :type outputdate: bool, optional
    """
    # print('key',key)
    # print('listkeyprops[:,0]',listkeyprops[:,0])

    _ix = np.where(listkeyprops[:,0]==key)[0]

    # print('_ix',_ix)
    # print('listkeyprops[_ix]', listkeyprops[_ix])

    if len(_ix)==0:
        title ='Not Reachable'
        d={}
        st =''
    else:
        assert len(_ix)==1
        general_id, idx, postfix, startdate, commandtitle, fullpath = listkeyprops[_ix[0]]

        os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
        with h5py.File(fullpath, 'r') as f:

            i_scan = '%s.%s'%(idx, postfix)
            # print('i_scan', i_scan)
            # print('fname', f)
            # print('fname[i_scan]', f[i_scan])
            datasetnames = [key for key in f[i_scan]['measurement'].keys()]
            d = {}
            for _n in datasetnames:
                d[_n] = f[i_scan]['measurement'][_n][()]

            if h5py.__version__>='3':
                title = f[i_scan]['title'][()].decode('UTF-8')
                st = f[i_scan]['start_time'][()].decode('UTF-8')
                et = f[i_scan]['end_time'][()].decode('UTF-8')
            else:
                title = f[i_scan]['title'].value
                st = f[i_scan]['start_time'].value
                et = f[i_scan]['end_time'].value
            duration = parsehdf5time(et)[1] - parsehdf5time(st)[1]

            positionersnames = [key for key in f[i_scan]['instrument']['positioners'].keys()]
            posmotors={}
            for _n in positionersnames:
                posmotors[_n] = f[i_scan]['instrument']['positioners'][_n][()]

            print('title', title)
            print('duration', duration)

        if not outputdate:
            return title, d, posmotors, fullpath
        else:
            return title, d, posmotors, fullpath, st

def ReadHdf5(fname, scan, outputdate=False):
    """extract data of a scan in a ESRF Bliss made hdf5 file when key are formatted as int.1

    :param fname: file object or string (path)
    :type fname: file object or string
    :param scan: string for scan id  (key in hdf5 file)
    :type scan: integer
    :param outputdate: output starting date of the scan in ascii format, defaults to False
    :type outputdate: bool, optional
    """
    os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"

    with h5py.File(fname, 'r') as f:

        i_scan = '%d' % scan + '.1'
        print('i_scan', i_scan)
        print('fname', f)
        print('fname[i_scan]', f[i_scan])
        datasetnames = [key for key in f[i_scan]['measurement'].keys()]
        d = {}
        for _n in datasetnames:
            d[_n] = f[i_scan]['measurement'][_n][()]

        if h5py.__version__>='3':
            title = f[i_scan]['title'][()].decode('UTF-8')
            st = f[i_scan]['start_time'][()].decode('UTF-8')
            et = f[i_scan]['end_time'][()].decode('UTF-8')
        else:
            title = f[i_scan]['title'].value
            st = f[i_scan]['start_time'].value
            et = f[i_scan]['end_time'].value
        duration = parsehdf5time(et)[1] - parsehdf5time(st)[1]

        print('title', title)
        print('duration', duration)

        if not outputdate:
            return title, d
        else:
            return title, d, st


import sys
sys.path.append("../../src")

import numpy as np
from equation import NonConvexEquation
from solver import Solver
from matplotlib.pyplot import*

if __name__ == "__main__":
    eqn = NonConvexEquation()
    eqn.setup()
    print("Energy Function: ", eqn.w)
    print("Differentiated Equation: ", eqn.f)

    # define domain, note complex type
    x = np.linspace(0.0, 1.0, 200, dtype=np.complex)

    # intialize solution
    u = np.zeros_like(x, dtype=x.dtype)

    # solve
    solver = Solver(eqn, x, u)
    solver.maxiter = 3
    solver.dt = 1e10
    solver.run()
     # plot
    figure()
    subplot(211)
    plot(x, u, 'r-')
    xlabel('x')
    ylabel('u')
    gca().ticklabel_format(axis='y', style='sci', scilimits=(-1,1))
    subplot(212)
    plot(x, solver.u_x, 'r-')
    xlabel('x')
    ylabel('u\'(x)')
    gca().ticklabel_format(axis='y', style='sci', scilimits=(-1,1))
    savefig("solution.pdf")
    show()


import ctypes
import time
import os
sharedLib=os.environ['PY_API_DIR']+"/cpython.so"
_qpy = ctypes.CDLL(sharedLib)
_qpy.handle.argtypes = (ctypes.POINTER(ctypes.c_char),ctypes.c_int)
_qpy.sendTradeList.argtypes = (ctypes.c_int,ctypes.c_longlong,ctypes.POINTER(ctypes.c_char),ctypes.c_int,ctypes.c_float)
_qpy.sendQuoteList.argtypes = (ctypes.c_int,ctypes.c_longlong,ctypes.POINTER(ctypes.c_char),ctypes.c_int,ctypes.c_float,ctypes.c_int,ctypes.c_float)

class kdbQ:
    global qpy

    def __init__(self,host,port):
        '''
        Initalisation of python to q handle
        kdbQ class stores the host, port and the handle made
        ----------------------------------------------------
        Error trap might be needed but most of the exceptions are handled by c library
        '''
        print("Opening Handle to q process {}:{}".format(host,port))
        self.host=host
        self.port=port
        self.h=self.handle()

    def handle(self):
        '''
        Open handle to kdb process
        Returns handle number
        '''
        result = _qpy.handle(ctypes.c_char_p(self.host),ctypes.c_int(self.port))
        return int(result)

    def sendTradeMsg(self,time,sym,size,price):
        '''
        Send trade messages to tickerplant taking the following arguement of specific type
        time - long
        sym - string
        size - int
        price - float
        '''
        result = _qpy.sendTradeList(ctypes.c_int(self.h),ctypes.c_longlong(self.kdbTime),ctypes.c_char_p(sym),ctypes.c_int(size),ctypes.c_float(price))
        print (int(result))

    def sendQuoteMsg(self,time,sym,bidSize,bidPrice,askSize,askPrice):
        '''
        Send quote messages to tickerplant taking the following arguement of specific type
        time - long
        sym - string
        bidSize - int
        bidPrice - float
        askSize - int
        askPrice - float
        '''
        result = _qpy.sendQuoteList(ctypes.c_int(self.h),ctypes.c_longlong(self.kdbTime),ctypes.c_char_p(sym),ctypes.c_int(bidSize),ctypes.c_float(bidPrice),ctypes.c_int(askSize),ctypes.c_float(askPrice))
        print (int(result))

    def castTime(self,timeStr):
        '''
        Given a time string HH:MM:SS
        it gets converted into a kdb long int
        '''
        self.timeObj=time.strptime(timeStr, "%H:%M:%S")
        #time.struct_time(tm_year=1900, tm_mon=1, tm_mday=1, tm_hour=8, tm_min=59, tm_sec=47, tm_wday=0, tm_yday=1, tm_isdst=-1)
        self.kdbTime=((60*self.timeObj.tm_hour+self.timeObj.tm_min)*60+self.timeObj.tm_sec)*1000000000
        #print("tuple: {}".format(self.kdbTime))


# -*- coding: utf-8 -*-

import re

from platformcode import config, logger, platformtools
from core.item import Item
from core import httptools, scrapertools, servertools, tmdb

host = 'https://www1.movidy.mobi/'

# ~ por si viene de enlaces guardados
ant_hosts = ['https://movidy.mobi/', 'https://www.movidy.mobi/']

domain = config.get_setting('dominio', 'movidymobi', default='')

if domain:
    if domain == host: config.set_setting('dominio', '', 'movidymobi')
    elif domain in str(ant_hosts): config.set_setting('dominio', '', 'movidymobi')
    else: host = domain

perpage = 22

# ~ las series estructura diferente  '/browse?type=series'

def acciones(item):
    logger.info()
    itemlist = []

    domain_memo = config.get_setting('dominio', 'movidymobi', default='')

    if domain_memo: url = domain_memo
    else: url = host

    itemlist.append(Item( channel='actions', action='show_latest_domains', title='[COLOR moccasin][B]Últimos Cambios de Dominios[/B][/COLOR]', thumbnail=config.get_thumb('pencil') ))

    itemlist.append(Item( channel='helper', action='show_help_domains', title='[B]Información Dominios[/B]', thumbnail=config.get_thumb('help'), text_color='green' ))

    itemlist.append(item.clone( channel='domains', action='test_domain_movidymobi', title='Test Web del canal [COLOR yellow][B] ' + url + '[/B][/COLOR]',
                                from_channel='movidymobi', folder=False, text_color='chartreuse' ))

    if domain_memo: title = '[B]Modificar/Eliminar el dominio memorizado[/B]'
    else: title = '[B]Informar Nuevo Dominio manualmente[/B]'

    itemlist.append(item.clone( channel='domains', action='manto_domain_movidymobi', title=title, desde_el_canal = True, folder=False, text_color='darkorange' ))

    platformtools.itemlist_refresh()

    return itemlist

def do_downloadpage(url, post=None, headers=None, raise_weberror=True):
    # ~ por si viene de enlaces guardados
    for ant in ant_hosts:
        url = url.replace(ant, host)

    if '/fecha/' in url: raise_weberror = False

    data = httptools.downloadpage(url, post=post, headers=headers, raise_weberror=raise_weberror).data

    return data

def mainlist(item):
    return mainlist_pelis(item)

def mainlist_pelis(item):
    logger.info()
    itemlist = []

    itemlist.append(item.clone( action='acciones', title= '[B]Acciones[/B] [COLOR plum](si no hay resultados)[/COLOR]', text_color='goldenrod' ))

    itemlist.append(item.clone( title = 'Buscar película ...', action = 'search', search_type = 'movie', text_color = 'deepskyblue' ))

    itemlist.append(item.clone( title = 'Catálogo', action = 'list_all', url = host + 'pelis/peliculas/', search_type = 'movie' ))

    itemlist.append(item.clone( title = 'Más destacadas', action = 'list_all', url = host + 'pelis/', group = 'masd', search_type = 'movie' ))

    itemlist.append(item.clone( title = 'Por género', action = 'generos', search_type = 'movie' ))
    itemlist.append(item.clone( title = 'Por año', action = 'anios', search_type = 'movie' ))

    return itemlist

def generos(item):
    logger.info()
    itemlist=[]

    generos = [
       'aventura',
       'comedia',
       'crimen',
       'documental',
       'drama',
       'familia',
       'historia',
       'misterio',
       'romance',
       'suspense',
       'terror',
       'western'
       ]

    for genero in generos:
        url = host + 'pelis/category/' + genero + '/'

        itemlist.append(item.clone( action = 'list_all', title = genero.capitalize(), url = url, text_color = 'deepskyblue' ))

    return itemlist

def anios(item):
    logger.info()
    itemlist = []

    from datetime import datetime
    current_year = int(datetime.today().year)

    for x in range(current_year, 1979, -1):
        url = host + 'pelis/peliculas/fecha/' + str(x) + '/'

        itemlist.append(item.clone( title = str(x), url = url, action='list_all', text_color = 'deepskyblue' ))

    return itemlist

def list_all(item):
    logger.info()
    itemlist = []

    if not item.page: item.page = 0

    if item.post:
        data = do_downloadpage(item.url, item.post)
    else:
        data = do_downloadpage(item.url)

    if not item.group:
        if '
' in data: bloque = scrapertools.find_single_match(data, '
(.*?)
')
        elif '
' in data: bloque = scrapertools.find_single_match(data, '
(.*?)')
        else: bloque = data

    else:
        bloque = scrapertools.find_single_match(data, '
(.*?)
')

    matches = re.compile('<article(.*?)
', re.DOTALL).findall(bloque)

    num_matches = len(matches)

    for article in matches[item.page * perpage:]:
        url = scrapertools.find_single_match(article, ' href="([^"]+)"')
        title = scrapertools.find_single_match(article, '
(.*?)
').strip()

        if not url or not title: continue

        thumb = scrapertools.find_single_match(article, ' src="([^"]+)"')

        itemlist.append(item.clone( action = 'findvideos', url = url, title = title, thumbnail = thumb, contentType = 'movie', contentTitle = title, infoLabels = {'year': '-'} ))

        if len(itemlist) >= perpage: break

    tmdb.set_infoLabels(itemlist)

    if itemlist:
        buscar_next = True
        if num_matches > perpage:
            hasta = (item.page * perpage) + perpage
            if hasta < num_matches:
                itemlist.append(item.clone( title = 'Siguientes ...', page = item.page + 1, action='list_all', text_color='coral' ))
                buscar_next = False

        if buscar_next:
            url = host + 'pelis/wp-admin/admin-ajax.php'

            if not item.next_page:
                if not '"load-more-ajax"' in data: return itemlist
                item.next_page = 1

            item.next_page = item.next_page + 1

            post = {'action': 'action_sorting_post', 'number': 22, 'page': item.next_page, 'types': 'movie'}

            itemlist.append(item.clone( title = 'Siguientes ...', url = url, post = post, action = 'list_all', page = 0, next = item.next_page, text_color='coral' ))

    return itemlist

def findvideos(item):
    logger.info()
    itemlist=[]

    data = do_downloadpage(item.url)

    matches = scrapertools.find_multiple_matches(data, '<li data-playerid="(.*?)".*?/flags/(.*?).png.*?(.*?)')

    ses = 0

    for url, lang, qlty in matches:
        ses += 1

        if '/hqq.' in url or '/waaw.' in url or '/netu.' in url: continue
        elif '/player.cuevana.ac' in url: continue

        if '.mystream.' in url: continue

        if lang == 'es': lang = 'Esp'
        elif lang == 'la': lang = 'Lat'
        elif lang == 'mx': lang = 'Lat'
        elif lang == 'en': lang = 'Vose'

        servidor = servertools.get_server_from_url(url)
        servidor = servertools.corregir_servidor(servidor)

        if servertools.is_server_available(servidor):
            if not servertools.is_server_enabled(servidor): continue
        else:
            if not config.get_setting('developer_mode', default=False): continue

        url = servertools.normalize_url(servidor, url)

        if '/links.' in url:
            data = do_downloadpage(url)

            links = scrapertools.find_multiple_matches(data, '<li onclick="go_to_player.*?' + "'(.*?)'")

            for link in links:
                if '/hqq.' in link or '/waaw.' in link or '/netu.' in link: continue
                elif '/player.cuevana.ac' in link: continue

                servidor = servertools.get_server_from_url(link)
                servidor = servertools.corregir_servidor(servidor)

                link = servertools.normalize_url(servidor, link)

                other = ''

                if servidor == 'various': other = servertools.corregir_other(link)

                if not servidor == 'directo':
                    itemlist.append(Item (channel = item.channel, action = 'play', title = '', server = servidor, url = link, language = lang, quality = qlty, other = other ))

            continue

        other = ''

        if servidor == 'various': other = servertools.corregir_other(url)

        if not servidor == 'directo':
          itemlist.append(Item (channel = item.channel, action = 'play', title = '', server = servidor, url = url, language = lang, quality = qlty, other = other ))

    if not itemlist:
        if not ses == 0:
            platformtools.dialog_notification(config.__addon_name, '[COLOR tan][B]Sin enlaces Soportados[/B][/COLOR]')
            return

    return itemlist

def search(item, texto):
    logger.info()
    try:
        item.url = host + 'pelis/?s=' + texto.replace(" ", "+")
        return list_all(item)
    except:
        import sys
        for line in sys.exc_info():
            logger.error("%s" % line)
        return []


# QBApp/boomiAPIParseResults.py
"""
Created on Wed Jan 8 10:16:40 2020

Functions to parse all the boomi atomspnere API calls, retrieving always the fields:
    * count: (Integer).
    * error: (Bolean).
    * results: List of tuples of (id, name, other fields), usually sort by name (index 1)
    * additional info (List).
    * query token (String) used when Boomi Atomsphere API paginates a query (more than 100 rows on th results).

@author: jaimeHMol
"""
# TODO: REFACTOR: Implement the parse and error handling as methods of every object
#                 created on the requests.
# TODO: Improve performance of the recursion implemented for the pagination
# TODO: Unify XML Name Spaces

from . import boomiAPIBuildRequests

import xml.etree.ElementTree as ET
import logging

XMLNAMESPACES = {"xsi": "http://www.w3.org/2001/XMLSchema-instance",
                 "bns": "http://api.platform.boomi.com/"}

def environmentList(environmentsResponse):
    """ Parsing the list of environments received on the response input obtained from Boomi Atompshere API.
    """
    count = 0            # Mandatory
    error = False        # Mandatory
    results = []         # Mandatory
    additional = []      # Optional
    queryToken = ''      # Optional

    environmentsResponseXML = ET.fromstring(environmentsResponse)

    if environmentsResponseXML.tag != "error":

        count = environmentsResponseXML.get("numberOfResults")
        logging.info("*** [Info] total number of results (environments): " + count)

        i = 0
        for child in environmentsResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                results.append((i, child.get("name")))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        results.sort(key = lambda tup: tup[1])
        results.insert(0,("","Choose..."))

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + environmentsResponseXML[0].text)

    return count, error, results, additional, queryToken

def atomList(atomsResponse):
    """ Parsing the list of atoms received on the response input obtained from Boomi Atompshere API.
    """
    count = 0       # Mandatory
    error = False   # Mandatory
    results = []    # Mandatory
    additional = [] # Optional
    queryToken = '' # Optional

    atomsResponseXML = ET.fromstring(atomsResponse)

    # logging.info(atomsResponseXML.tag)
    if atomsResponseXML.tag != "error":

        count = atomsResponseXML.get('numberOfResults')
        logging.info("*** [Info] total number of results (atoms): " + count)

        i = 0
        for child in atomsResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                results.append((child.get("id"), child.get("name")))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        results.sort(key = lambda tup: tup[1])
        logging.info("result: " + str(results))
        results.insert(0,("","Choose..."))

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + atomsResponseXML[0].text)

    return count, error, results, additional, queryToken

def processList(request, processesResponse):
    """ Parsing the list of processes received on the response input obtained from Boomi Atompshere API.
    """
    count = 0            # Mandatory
    error = False        # Mandatory
    results = []         # Mandatory
    additional = []      # Optional
    queryToken = ''      # Optional
    accResults = []

    processesResponseXML = ET.fromstring(processesResponse)

    if processesResponseXML.tag != "error":

        count = processesResponseXML.get("numberOfResults")
        logging.info("*** [Info] total number of results (processes): " + count)

        i = 0
        for child in processesResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result": #HINT: Be carefull with the name spaces
                i = i + 1
                results.append((child.get("id"), child.get("name")))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        # Recursive block to retrieve all the results (using the queryToken)
        # and the pagination used by the Boomi Atomsphere API
        accResults = accResults + results
        if queryToken != '':
            # accResults.append(results)
            processesResponse = boomiAPIBuildRequests.queryPagination(request, "Process", queryToken)
            _, _, newResult, _, _ = processList(request, processesResponse)

            return count, error, accResults + newResult, additional, queryToken
        else:

            accResults.sort(key = lambda tup: tup[1])
            return count, error, accResults, additional, queryToken

        # logging.info("result: " + str(results))
    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + processesResponseXML[0].text)

    return count, error, results, additional, queryToken

def scheduleList(scheduleResponse):
    """ Parsing the list of schedules received on the response input obtained from Boomi Atompshere API.
    """
    count = 0            # Mandatory
    error = False        # Mandatory
    results = []         # Mandatory
    additional = []      # Optional
    queryToken = ''      # Optional

    schedulesResponseXML = ET.fromstring(scheduleResponse)
    if schedulesResponseXML.tag != "error":

        count = schedulesResponseXML.get("numberOfResults")

        i = 0
        for child in schedulesResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                if child.find("bns:Schedule", XMLNAMESPACES) is not None:
                    cadence = child.find("bns:Schedule", XMLNAMESPACES).get("minutes")
                else:
                    cadence = "NA"
                results.append((child.get("atomId"), child.get("processId"), cadence, child.get("id")))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        results.sort(key = lambda tup: tup[1])

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + schedulesResponseXML[0].text)

    return count, error, results, additional, queryToken

def scheduleStatusGet(scheduleGetResponse):
    """ Parsing the schedule status received on the response input obtained from Boomi Atompshere API.
    """
    error = False        # Mandatory
    result = ''          # Mandatory

    scheduleGetResponseXML = ET.fromstring(scheduleGetResponse)
    if scheduleGetResponseXML.tag != "error":
        if scheduleGetResponseXML.get("enabled") == "true":
            result = "Yes"
        elif scheduleGetResponseXML.get("enabled") == "false":
            result = "No"
    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + scheduleGetResponseXML[0].text)

    return error, result

def executionRecordList(executionResponse):
    """ Parsing the list of executions received on the response input obtained from Boomi Atompshere API.
    """
    count = 0             # Mandatory
    error = False         # Mandatory
    results = []           # Mandatory
    additional = []       # Optional
    queryToken = ''       # Optional

    executionsResponseXML = ET.fromstring(executionResponse)
    if executionsResponseXML.tag != "error":

        count = executionsResponseXML.get("numberOfResults")
        logging.info("*** [Info] total number of results (executions): " + count)

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + executionsResponseXML[0].text)
    pass

def deploymentList(deploymentResponse):
    """ Parsing the list of deployments received on the response input obtained from Boomi Atompshere API.
    """
    count = 0            # Mandatory
    error = False        # Mandatory
    results = []          # Mandatory
    additional = []      # Optional
    queryToken = ''      # Optional

    deploymentResponseXML = ET.fromstring(deploymentResponse)
    if deploymentResponseXML.tag != "error":

        count = deploymentResponseXML.get("numberOfResults")

        i = 0
        for child in deploymentResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                if child.find("bns:processId", XMLNAMESPACES) is not None:
                    procId = child.find("bns:processId", XMLNAMESPACES).text
                if child.find("bns:environmentId", XMLNAMESPACES) is not None:
                    environmentId = child.find("bns:environmentId", XMLNAMESPACES).text

                results.append((procId, environmentId))
            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        results.sort(key = lambda tup: tup[1])
        # logging.info("deploymentResult: " + str(results))

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + deploymentResponseXML[0].text)

    return count, error, results, additional, queryToken

def environmentAtomList(environmentAtomsResponse):
    """ Parsing the list of atoms and their related environments received on the response input obtained from Boomi Atompshere API.
    """
    count = 0       # Mandatory
    error = False   # Mandatory
    results = {}    # Mandatory
    additional = [] # Optional
    queryToken = '' # Optional

    environmentAtomsResponseXML = ET.fromstring(environmentAtomsResponse)

    # logging.info(atomsResponseXML.tag)
    if environmentAtomsResponseXML.tag != "error":

        count = environmentAtomsResponseXML.get('numberOfResults')
        logging.info("*** [Info] total number of results (environment-atoms): " + count)

        i = 0
        for child in environmentAtomsResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                #Devolver un diccionario
                v_atomId, v_environmentId = child.get("atomId"), child.get("environmentId")
                results[v_atomId] = v_environmentId

            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        # results.sort(key = lambda tup: tup[1])

    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + environmentAtomsResponseXML[0].text)

    return count, error, results, additional, queryToken

def ProcessEnvironmentAttachment(processEnvironmentAttachmentResponse):
    """ Parsing the Process-Environment attachment information received on the response input obtained from Boomi Atompshere API.
    """
    count = 0       # Mandatory
    error = False   # Mandatory
    results = ''    # Mandatory
    additional = [] # Optional
    queryToken = '' # Optional

    processEnvironmentAttachmentResponseXML = ET.fromstring(processEnvironmentAttachmentResponse)

    # logging.info(atomsResponseXML.tag)
    if processEnvironmentAttachmentResponseXML.tag != "error":

        count = processEnvironmentAttachmentResponseXML.get('numberOfResults')

        i = 0
        for child in processEnvironmentAttachmentResponseXML:
            if child.tag == "{http://api.platform.boomi.com/}result":
                i = i + 1
                results = child.get("id")

            elif child.tag == "{http://api.platform.boomi.com/}queryToken":
                queryToken = child.text

        # results.sort(key = lambda tup: tup[1])
    else:
        error = True
        raise Exception("There is an error on the Boomi atomsphere API response: " + processEnvironmentAttachmentResponseXML[0].text)

    return count, error, results, additional, queryToken


# Advent of code, day 15

# open file
input = open("advent15_input.txt", "r")
# input = open("advent15_test_input.txt", "r")
# input = open("advent15_test_input2.txt", "r")

input_array = []
# read string into array
for line in input:
    input_array.append(line)

def part1():

    answer = 0

    start_values = input_array[0].split(",")
    n = len(start_values)

    values = []
    for i in range(n):
        values.append(int(start_values[i]))

    for i in range(n, 2020):
        if (values[i-1] not in values[0:i-1]):
            values.append(0)
        else:
            # loop backwards and find the previous value
            loop_val = i-2
#             print(values)
            while values[i-1] != values[loop_val]:
                loop_val -= 1
            values.append(i - 1 - loop_val)

    print(values)

    answer = values[-1]

    return answer

def part2():

    answer = 0

    start_values = input_array[0].split(",")
    n = len(start_values)

    values_dict = dict()
    for i in range(n):
        last_value = int(start_values[i])
        values_dict[int(start_values[i])] = []
        values_dict[int(start_values[i])].append(i)

    for i in range(n, 30000000):
#         if (i % 10000 == 0):
#             print(i)
        if (len(values_dict[last_value]) == 1):
            if 0 not in values_dict.keys():
                values_dict[0] = []
            values_dict[0].append(i)
            last_value = 0
        else:
            # loop backwards and find the previous value
            new_value = values_dict[last_value][-1] - values_dict[last_value][-2]
            if new_value not in values_dict.keys():
                values_dict[new_value] = []
            values_dict[new_value].append(i)
            last_value = new_value

    answer = last_value

    return answer

print("Part 1 answer: ", part1())
print("Part 2 answer: ", part2())


import numpy as np

from yt.funcs import mylog

def link_galaxies_and_halos(obj):
    """Link galaxies and halos to one another.

    This function creates the links between galaxy-->halo and
    halo-->galaxy objects.  Is run during creation, and loading in of
    each CAESAR file.

    Parameters
    ----------
    obj : :class:`main.CAESAR`
        Object containing halos and galaxies lists.

    """
    if not obj._has_galaxies:
        return

    mylog.info('Linking galaxies and halos')

    # halos
    for halo in obj.halos:
        halo.galaxies = []
        for galaxy_index in halo.galaxy_index_list:
            halo.galaxies.append(obj.galaxies[galaxy_index])

    # galaxies
    for galaxy in obj.galaxies:
        if galaxy.parent_halo_index > -1:
            galaxy.halo = obj.halos[galaxy.parent_halo_index]
        else:
            galaxy.halo = None

def link_clouds_and_galaxies(obj):
    """Link clouds and galaxies to one another.

    This function creates the links between cloud-->galaxy and
    galaxy-->cloud objects.  Is run during creation, and loading in of
    each CAESAR file.

    Parameters
    ----------
    obj : :class:`main.CAESAR`
        Object containing halos and galaxies lists.

    """

    if not obj._has_clouds:
        return

    mylog.info('Linking clouds and galaxies')

    #galaxies
    for galaxy in obj.galaxies:
        galaxy.clouds = []
        for cloud_index in galaxy.cloud_index_list:
            galaxy.clouds.append(obj.clouds[cloud_index])

    for cloud in obj.clouds:
        if cloud.parent_galaxy_index > -1:
            cloud.galaxy = obj.galaxies[cloud.parent_galaxy_index]
        else:
            cloud.galaxy = None

def create_sublists(obj):
    """Create sublists of objects.

    Will create the sublists:
        - central_galaxies
        - satellite_galaxies
        - unassigned_galaxies (those without a halo)

    Parameters
    ----------
    obj : :class:`main.CAESAR`
        Object containing halos and galaxies lists already linked.

    """
    if not obj._has_galaxies:
        return

    mylog.info('Creating sublists')

    obj.central_galaxies   = []
    obj.satellite_galaxies = []

    # assign halo sub lists
    for halo in obj.halos:
        halo.central_galaxy = -1
        for galaxy in halo.galaxies:
            if galaxy.central:
                halo.central_galaxy = galaxy.GroupID
#             else:
#                 halo.satellite_galaxies.append(galaxy)

    # assign galaxy sub lists
    for galaxy in obj.galaxies:
        if galaxy.central and galaxy.halo is not None:
#             galaxy.satellites = galaxy.halo.satellite_galaxies
            obj.central_galaxies.append(galaxy)
        elif galaxy.halo is not None:
#             galaxy.satellites = []
            obj.satellite_galaxies.append(galaxy)
        else:
            if not hasattr(obj, 'unassigned_galaxies'):
                obj.unassigned_galaxies = []
            obj.unassigned_galaxies.append(galaxy)


import os
import yaml
import json

from _utils import create_dir

this_file = os.path.abspath(__file__)
contracts_dir = os.path.dirname(os.path.dirname(this_file)) + "/contracts"
sdk_dir = os.path.dirname(os.path.dirname(this_file)) + "/python-sdk"
keystore_dir = sdk_dir + "/bin/accounts"
config_dir = os.path.expanduser('~') + '/.fisco-wallet'
message_dir = config_dir + "/twoWayExchange"
custom_dir = config_dir + "/oneWayTransfer"

class Config:
	"""
	对客户端配置的抽象封装
	"""
	def __init__(self):

		self.config_dir = config_dir
		# self.config_dir = os.path.dirname(config_file)
		self.config_file = '/config.yaml'
		self.config_path = self.config_dir + self.config_file

		self.initial_config = {
			'group_num' : 2,
			'contracts_info' : {}
		}

		# Variables from configuration file. They will be initialized after load
		self.group_num = 0

		"""
		The format of contracts_info:
		{
			'crc20' : {
				'geth_1' : '0xd24ea07c7c9742b95f53836d531510748d65e6c0'
				'geth_2' : '0xd24ea07c7c9742b95f53836d531510748d65e6c0'
			}

		}
		"""
		self.contracts_info = {}

	def isConfig(self):
		"""Check if exists configuration on default path"""
		if os.path.isfile(self.config_path):
			return True
		else:
			return False

	def createEmptyConfig(self):
		create_dir(self.config_dir)
		with open(self.config_path, 'w+') as f:
			yaml.dump(self.initial_config, f)

	def loadConfig(self):
		"""Load configuration from .yaml file"""
		if not self.isConfig():
			self.createEmptyConfig()
			self.loadConfig()
		else:
			with open(self.config_path, 'r') as yaml_file:
				file = yaml.safe_load(yaml_file)
			for key, value in file.items():
				setattr(self, key, value)
		return self

	# future work
	def dumpConfig(self):
		config = dict(
			group_num = self.group_num,
			contracts_info = self.contracts_info
		)
		with open(self.config_path, 'w+') as f:
			yaml.dump(config, f)

	# future work
	def addURL(self, chain_name, network_id, chain_id, url):
		if chain_name in self.chains_info:
			self.chains_info[chain_name]['service'].append(url)
		else:
			self.chains_info[chain_name] = dict(
				network_id = network_id,
				chain_id = chain_id,
				service = [url]
			)

		self.dumpConfig()
		print("Add URL successful!")

	# future work
	def deleteURL(self, chain_name, url):
		if chain_name in self.chains_info:
			for i in range(0, len(self.chains_info[chain_name]['service'])):
				if self.chains_info[chain_name]['service'][i] == url:
					del(self.chains_info[chain_name]['service'][i])
					self.dumpConfig()
					print("Delete URL successful!")
					break
				print("The url doesn't exist!")
		else:
			print("This chain doesn't exist!")

	def updateContractInfo(self, contract_name, group_id, contract_addr):
		if contract_name in self.contracts_info:
			self.contracts_info[contract_name][group_id] = contract_addr
		else:
			self.contracts_info[contract_name] = {group_id : contract_addr}
		self.dumpConfig()
		print("Contract updated!")

	def deleteContractInfo(self, contract_name, group_id):
		if contract_name in self.contracts_info:
			if group_id in self.contracts_info[contract_name]:
				del(self.contracts_info[contract_name][group_id])
				self.dumpConfig()
				print("Contract Deleted!")
			else:
				print("The record doesn't exist!")
		else:
			print("The record doesn't exist!")

	# future work
	def showConfig(self):
		self.loadConfig()
		config = dict(
			group_num = self.group_num,
			contracts_info = self.contracts_info
		)
		return json.dumps(config, indent=4)


from utils.Component import Component
from utils.Dependency import Dependency

import os.path

class ProFTPd(Component):

    def __init__(self, config):
        super(ProFTPd, self).__init__('ProFTPd', os.path.dirname(__file__), config)

        self.download_url = 'ftp://ftp.proftpd.org/distrib/source/proftpd-%s.tar.gz' % self.version

        self.dependencies = [
            Dependency('ZLib', cFlags=['-I${INCLUDE_PATH}'], ldFlags=['-L${LIB_PATH}']),
            Dependency('Ncurses', cFlags=['-I${INCLUDE_PATH}'], ldFlags=['-L${LIB_PATH}']),
            Dependency('OpenSSL', cFlags=['-I${INCLUDE_PATH}'], ldFlags=['-L${LIB_PATH}']),
            Dependency('MySQL', cFlags=['-I${INCLUDE_PATH}'], ldFlags=['-L${LIB_PATH}'])
        ]

    def configureFlags(self):
        flags = super(ProFTPd, self).configureFlags()

        flags.extend([
            '--with-modules=mod_sql:mod_sql_mysql:mod_tls'
        ])

        return  flags


import PyATEMMax
from pynput.keyboard import Key, Listener, KeyCode
from pynput import keyboard
import win32gui, win32process, os
import sys, time, threading
from colored import fg

atemMini = PyATEMMax.ATEMMax()
atem4K = PyATEMMax.ATEMMax()

PGM = atem4K.previewInput[1].videoSource.value
PVW = atem4K.previewInput[1].videoSource.value

def setPGMandPVWtoCurrent():
    global PGM
    global PVW
    if PVW == 5:
        PVW = atem4K.previewInput[1].videoSource.value + atemMini.programInput[1].videoSource.value
    if PGM == 5:
        PGM = atem4K.programInput[1].videoSource.value + atemMini.programInput[1].videoSource.value

setPGMandPVWtoCurrent()

stop_flag = False
def PGM_and_PVW_has_chaned():
    global PVW
    global PGM
    while not stop_flag:
        last_PGM = atem4K.previewInput[1].videoSource.value
        last_PVW = atem4K.previewInput[1].videoSource.value

        # if last_PGM != PGM:
        #     PGM = last_PGM
        #     setPGMandPVWtoCurrent()
        # if last_PVW != PVW:
        #     PVW = last_PVW
        #     setPGMandPVWtoCurrent()

        if last_PVW != PVW or last_PGM != PGM:
            PVW = last_PVW
            PGM = last_PGM
            setPGMandPVWtoCurrent()
            #consoleText()

        print(11111)
        time.sleep(0.01)

PGM_PVW_Listener = threading.Thread(target=PGM_and_PVW_has_chaned)
PGM_PVW_Listener.start()

for i in range(100):
    print(99999)
    time.sleep(0.01)

stop_flag = True
#PGM_PVW_Listener.join()


a = {
    'x': 1,
    'y': 2,
    'z': 3
}

b = {
    'w': 10,
    'x': 11,
    'y': 2
}

# Find keys in common
keyComm = a.keys() & b.keys()
print(keyComm)

# Find keys in a that are not in b
keyNot = a.keys() - b.keys()
print(keyNot)

# Find (key, value) pairs in common
itemComm = a.items() & b.items()
print(itemComm)

# Make a new dictionary with certail keys removed
c = {key:a[key] for key in a.keys() - {'z', 'w'}}
print(c)

## -- End pasted text --
{'y', 'x'}
{'z'}
{('y', 2)}
{'y': 2, 'x': 1}


import numpy as np

def create_heatmaps(joints, output_size, visibility,  sigma=2):
    size = 6 * sigma + 3
    offset = 3 * sigma + 1
    x = np.arange(0, size, 1, float)
    y = x[:, np.newaxis]
    x0, y0 = 3 * sigma + 1, 3 * sigma + 1
    g = np.exp(- ((x - x0) ** 2 + (y - y0) ** 2) / (2 * sigma ** 2))

    joints = cut_coords(joints, output_size, 0)

    # create heatmap in a way that the gaussian blobs fit into it even if the keypoint is at the border: make it bigger
    heatmaps = np.zeros((joints.shape[0], output_size + size, output_size + size))

    # [0, ..., 0, ..., (num_joints-1), ..., (num_joints-1)] (size * size times)
    indices_joints = np.transpose(np.tile(np.arange(joints.shape[0]), (size ** 2, 1)), (1, 0)).flatten()
    # [[0, 1, ..., (size-1), 0, ..., (size-1)], ..., [...]] (num_joints times)
    cnt_indices = np.tile(np.arange(size), (joints.shape[0] * size, 1)).reshape((joints.shape[0], size ** 2))
    # size times x_(joint 0) + [0, 1, ..., (size-1)] then size times x_(joint 1) + [0, 1, ..., (size-1)] etc.
    indices_x = np.asarray((np.tile(joints[:, 0].flatten(), (size * size, 1)).transpose((1, 0)) + cnt_indices).flatten(), dtype=np.int32)
    # [[0, 1, ..., (size-1)], ..., [0, 1, ..., (size-1)]] num_joints times
    cnt_indices = np.tile(np.arange(size), (joints.shape[0], 1))
    # [size times y_(joint 0) + 0, size times y_(joint 0) + 1, ..., size times y_(joint 0) + (size-1), ..]
    indices_y = np.asarray(np.transpose(np.tile(np.transpose(np.tile(joints[:, 1].flatten(), (size, 1)), (1, 0)) + cnt_indices, (size, 1, 1)), (1, 2, 0)).flatten(), dtype=np.int32)

    # flatten g and repeat num_joints times
    g = np.tile(g.flatten(), joints.shape[0])
    # put g at correct positions
    heatmaps[indices_joints, indices_y, indices_x] = g
    # reduce size of heatmaps, we added a padding so that gaussian blobs fit also if keypoints are at the border, we remove the padding now
    heatmaps = heatmaps[:, offset:-(size - offset), offset:-(size - offset)]
    heatmaps = np.asarray(heatmaps, dtype=np.float32)

    # remove non visible joints
    heatmaps = np.where(np.expand_dims(np.expand_dims(visibility, axis=-1), axis=-1) == 1, heatmaps, np.zeros_like(heatmaps))

    return heatmaps

def cut_coords(coords, size, blob_width):
    coords[:, :2] = np.asarray(coords[:, :2], dtype=np.int32)  # rounding might be better, but let's stay close to the original code
    coords[coords[:, 0] >= size + blob_width] = 0
    coords[coords[:, 0] < - blob_width] = 0
    coords[coords[:, 1] >= size + blob_width] = 0
    coords[coords[:, 1] < - blob_width] = 0
    return coords


import csv
import sys
import numpy as np
from tqdm import tqdm
import base64
from src.utils import open_graph
import torch

csv.field_size_limit(sys.maxsize)

_TSV_FIELDNAMES = ["scanId", "viewpointId", "image_w", "image_h", "vfov", "features"]

def _convert_item(item):
    item["image_w"] = int(item["image_w"])  # pixels
    item["image_h"] = int(item["image_h"])  # pixels
    item["vfov"] = int(item["vfov"])  # degrees
    item["features"] = np.frombuffer(
        base64.b64decode(item["features"]), dtype=np.float32
    ).reshape(
        (-1, 2048)
    )  # 36 x 2048 region features
    return item

def load_feats():
    keys = []
    pano_feats = {}
    path = "data/ResNet-152-places365.tsv"
    with open(path, "rt") as fid:
        reader = csv.DictReader(fid, delimiter="\t", fieldnames=_TSV_FIELDNAMES)
        for item in tqdm(reader):
            item = _convert_item(item)
            key = item["scanId"] + "-" + item["viewpointId"]
            keys.append(key.encode())
            pano_feats[key.encode()] = item["features"]

    scanfile = "data/connectivity/scans.txt"
    scans = [d.strip() for d in open(scanfile).readlines()]
    max_nodes = 230

    for s in scans:
        if s == "B6ByNegPMKs":
            continue
        nodes = sorted(
            [
                n
                for n in open_graph(
                    "data/connectivity/",
                    s,
                ).nodes()
            ]
        )
        node_feats = torch.zeros(max_nodes, 36, 2048)
        for i, n in enumerate(nodes):
            key = s + "-" + n
            node_feats[i, :, :] = torch.tensor(pano_feats[key.encode()])
        torch.save(node_feats, "data/node_feats/" + s + ".pt")
        break

load_feats()


import pandas as pd
import numpy as np
import json
import os
import helper
from matplotlib import pyplot as plt

def calc_percentiles(df):
    """Calculate percentiles of pRRx/pRRx% parameters in AF and SR, return dict.

    Args:
        df (pd.DataFrame): DF with values of pRRx/pRRx% parameters

    Returns:
        (dict): Dictionary with values of percentiles in AF and SR
    """
    params = helper.get_params(df)
    # Parameter types (perc, ms)
    groups = [group for group in params if len(params[group]) > 0]
    perc_dict = {g: dict() for g in groups}
    for group in groups:
        features = params[group]
        perc_dict[group] = {f: {'AF': {}, 'SR': {}} for f in features}
        # Calculate percentiles (0-100)
        for i, feature in enumerate(features):
            for is_af, rhythm, in ((0, 'SR'), (1, 'AF')):
                val = df[df['is_af'] == is_af][feature].values
                perc = list(np.arange(101)/100)
                q = list(np.quantile(val, perc))
                perc_dict[group][feature][rhythm] = {
                    'centiles': perc,
                    'values': q}
    return perc_dict

def plot_distr(perc_dict, db, x_sec, fig_dir):
    """Plot medians, and 10-90 and 25-75 percentile ranges
    of pRRx/pRRx% parameters in AF and SR

    Args:
        perc_dict (dict): Dictionary with parameter names and percentiles in AF and SR
        db (str): Acronym of the database
        x_sec (int): Length of RR sequence [s]
        fig_dir (str): Write directory for images
    """
    for group in perc_dict.keys():
        ax = helper.default_fig()
        title = f'Quartiles, 10-th and 90-th percentile of {group}'
        if group == 'pRRx':
            xlabel = 'Threshold x [ms]'
        else:
            xlabel = 'Threshold x [%]'
        features = list(perc_dict[group].keys())
        x_thr = helper.get_x_thr(features)
        for rhythm, color in (('SR', 'tab:blue'), ('AF', 'tab:orange')):
            # Values of percentiles
            val = np.array([perc_dict[group][f][rhythm]['values'] for f in features])
            # Median line
            p50 = val[:, 50]
            helper.plot_line(
                ax, xval=x_thr, yval=p50, label=rhythm,
                xlim=(0, np.ceil(x_thr[-1]/5)*5),
                ylim=(0, 100), xlabel=xlabel, ylabel='[%]',
                title=title, color=color, marker='o', markersize=2,
                linewidth=2)
            # Bands: 25-75 and 10-90 percentile ranges
            for perc, lw, alpha in [(25, 1, 0.6), (10, 1, 0.3)]:
                p_low = val[:, perc]
                p_high = val[:, 100-perc]
                ax.plot(x_thr, p_low, '-o', markersize=1, color=color,
                        alpha=alpha, linewidth=lw)
                ax.plot(x_thr, p_high, '-o', markersize=1, color=color,
                        alpha=alpha, linewidth=lw)
                ax.fill_between(x_thr, p_low, p_high, alpha=alpha, color=color)
        ax.legend()
        fname = f"distr_{group}_{db}_vs_threshold_{x_sec}s.png"
        helper.save_fig(fig_dir, fname)
        plt.show()

if __name__ == '__main__':
    prrx_dir = '../data/processed'
    fig_dir = '../reports/images/distr'
    x_sec = 60
    for db in ['ltafdb', 'afdb']:
        df = helper.read_prrx(prrx_dir, db, x_sec)
        perc_dict = calc_percentiles(df)
        if not os.path.exists(fig_dir):
            os.makedirs(os.path.join(fig_dir))
        plot_distr(perc_dict, db, x_sec, fig_dir)


from fld_data_base import FldDataBase
from fld_header import FldHeader
import numpy as np
import re

# TODO: This uses tofile from the base class, which probably creates some intermediate byte arrays
# Might want to see if this is significant memory disadvantage...

class FldDataMemmap(FldDataBase):

    def __init__(self,
                 header: FldHeader,
                 readonly: bool = False,
                 coords: np.ndarray = None,
                 u: np.ndarray = None,
                 p: np.ndarray = None,
                 t: np.ndarray = None,
                 s: np.ndarray = None):

        super().__init__(header)

        self._readonly = readonly
        self._coords = np.array([], self.float_type)
        self._u = np.array([], self.float_type)
        self._p = np.array([], self.float_type)
        self._t = np.array([], self.float_type)
        self._s = np.array([], self.float_type)

        if coords is not None:
            self._coords = coords
        if u is not None:
            self._u = u
        if p is not None:
            self._p = p
        if t is not None:
            self._t = t
        if s is not None:
            self._s = s

        self._set_rdcode()

    @classmethod
    def fromfile(cls, filename: str, mode: str = "r"):

        # Validate mode
        if mode not in ('r', 'r+', 'w+', 'c'):
            raise ValueError(f"User specified mode was {mode}, but mode must be 'r', 'r+', 'w+', or 'c'")

        # Read header and get offset where field data begin
        h = FldHeader.fromfile(filename)
        nxyz = h.nx1 * h.ny1 * h.nz1
        offset = 136 + h.nelt * h.int_type.itemsize

        # Some convenience functions
        def notify(msg: str):
            print("[{}] : {}".format(filename, msg))

        def error(msg: str):
            raise Exception("[{}] : {}".format(filename, msg))

        def read_memmap(shape):
            nonlocal offset
            v = np.memmap(filename, dtype=h.float_type, offset=offset, mode=mode, shape=shape)
            offset += v.nbytes
            return v

        # Now parse everything!
        coords = None
        u = None
        p = None
        t = None
        s = None
        notify("Attempting to fields from rdcode {}".format(h.rdcode))
        code_list = [s.upper() for s in re.split(r'(\D\d*)', h.rdcode) if s]
        for code in code_list:
            # Coordinate data
            if code == 'X':
                notify("Located coordinates X")
                coords = read_memmap((h.nelt, h.ndims, nxyz))
            # Velocity field
            elif code == 'U':
                notify("Located velocity field U")
                u = read_memmap((h.nelt, h.ndims, nxyz))
            # Pressure field
            elif code == 'P':
                notify("Located pressure field P")
                p = read_memmap((h.nelt, nxyz))
            # Temperature field
            elif code == 'T':
                notify("Located temperature field T")
                t = read_memmap((h.nelt, nxyz))
            # Passive scalars
            elif code.startswith('S'):
                try:
                    nscalars = int(code[1:])
                except ValueError:
                    error("Couldn'_t parse number of passive scalar fields")
                else:
                    notify("Located {} passive scalar fields".format(nscalars))
                    s = read_memmap((nscalars, h.nelt, nxyz))
            else:
                error("Warning: Unsupported rdcode '{}'".format(code))

        return cls(header=h, readonly=(mode == 'r'), coords=coords, u=u, p=p, t=t, s=s)

    @FldDataBase.coords.setter
    def coords(self, other: np.ndarray):
        if self._readonly:
            raise AttributeError("FldDataMemmap in read-only mode does not support field assignment.")
        if other.size != 0 and other.shape != (self.nelt, self.ndims, self.nx1 * self.ny1 * self.nz1):
            raise ValueError("Incorrect shape for coords: coords.shape must equal (nelt, ndims,  nx1 * ny1 * nz1)")
        self._coords = other.astype(self.float_type)
        self._set_rdcode()

    @FldDataBase.u.setter
    def u(self, other: np.ndarray):
        if self._readonly:
            raise AttributeError("FldDataMemmap in read-only mode does not support field assignment.")
        if other.size != 0 and other.shape != (self.nelt, self.ndims, self.nx1 * self.ny1 * self.nz1):
            raise ValueError("Incorrect shape for u: u.shape must equal (nelt, ndims, nx1 * ny1 * nz1)")
        self._u = other.astype(self.float_type)
        self._set_rdcode()

    @FldDataBase.p.setter
    def p(self, other: np.ndarray):
        if self._readonly:
            raise AttributeError("FldDataMemmap in read-only mode does not support field assignment.")
        if other.size != 0 and other.shape != (self.nelt, self.nx1 * self.ny1 * self.nz1,):
            raise ValueError("Incorrect shape for p: p.shape must equal (nelt * nx1 * ny1 * nz1,)")
        self._p = other.astype(self.float_type)
        self._set_rdcode()

    @FldDataBase.t.setter
    def t(self, other: np.ndarray):
        if self._readonly:
            raise AttributeError("FldDataMemmap in read-only mode does not support field assignment.")
        if other.size != 0 and other.shape != (self.nelt, self.nx1 * self.ny1 * self.nz1,):
            raise ValueError("Incorrect shape for t: t.shape must equal ``(nelt * nx1 * ny1 * nz1,)``")
        self._t = other.astype(self.float_type)
        self._set_rdcode()

    @FldDataBase.s.setter
    def s(self, other: np.ndarray):
        if self._readonly:
            raise AttributeError("FldDataMemmap in read-only mode does not support field assignment.")
        if other.size != 0 and (
                len(other.shape) != 3 or other.shape[1:] != (self.nelt, self.nx1 * self.ny1 * self.nz1)):
            raise ValueError(
                "Incorrect shape for s: s.shape must equal (x, nelt, nx1 * ny1 * nz1) for arbitrary number of scalars x")
        self._s = other.astype(self.float_type)
        self._set_rdcode()


from copy import deepcopy
import numpy as np

def rmse(data_row, data_prediction):
    error = []
    if len(data_row) == len(data_prediction):
        data = list(zip(data_row, data_prediction))
        for row, prediction in data:
            error.append(np.sqrt(np.square(np.subtract(row, prediction)).mean()))
    return error

def spikeEfficiency(data):
    channels = deepcopy(data)
    efficiency = []
    for channel in channels:
        efficiency.append(1 - (np.sum(np.abs(channel)) / channel.shape[0]))
    return efficiency