# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

import bpy, os, subprocess, colorsys, multiprocessing, sys
from math import pi
from subprocess import PIPE, Popen, STDOUT

nproc = multiprocessing.cpu_count()

class LiVi_c(object):
    def __init__(self, lexport, prev_op):
        self.acc = lexport.scene.livi_calc_acc
        self.scene = bpy.context.scene
        if str(sys.platform) != 'win32':
            self.rm = "rm"
            if lexport.scene.livi_export_time_type == "0" or lexport.scene.livi_anim == "1":
                self.simlistn = ("illumout", "irradout", "dfout")
                self.simlist = (" |  rcalc  -e '$1=47.4*$1+120*$2+11.6*$3' ", " |  rcalc  -e '$1=$1' ", " |  rcalc  -e '$1=(47.4*$1+120*$2+11.6*$3)/100' ")
                self.unit = ("Lux", "W/m^2", "DF %", "Glare")
            else:
                self.simlistn = ("cumillumout", "cumirradout", "", "", "daout")
                self.simlist = (" |  rcalc  -e '$1=47.4*$1+120*$2+11.6*$3' ", " |  rcalc  -e '$1=$1' ")
                self.unit = ("Luxhours", "Wh/m^2", "", "", "DA %")
        if str(sys.platform) == 'win32':
            self.rm = "del"
            if lexport.scene.livi_export_time_type == "0"  or lexport.scene.livi_anim == "1":
                self.simlistn = ("illumout", "irradout", "dfout")
                self.simlist = (' |  rcalc  -e "\$1=47.4*\$1+120*\$2+11.6*\$3" ', " |  rcalc  -e '\$1=\$1' ", " |  rcalc  -e '\$1=(47.4*\$1+120*\$2+11.6*\$3)/100' ")
                self.unit = ("Lux", "W/m^2", "DF %", "Glare")
            else:
                self.simlistn = ("cumillumout", "cumirradout", "", "", "daout")
                self.simlist = (' |  rcalc  -e "\$1=47.4*\$1+120*\$2+11.6*\$3" ', " |  rcalc  -e '\$1=\$1' ")
                self.unit = ("Luxhours", "Wh/m^2", "", "", "DA %")
        try:
            if os.lstat(lexport.filebase+".rtrace").st_size == 0:
                if lexport.metric == "3":
                    self.rad_glare(lexport, prev_op)
                elif prev_op.name == "Radiance Preview":
                    self.rad_prev(lexport, prev_op)
                else:
                    prev_op.report({'ERROR'},"There are no calcsurf materials. Associate a 'calcsurf' material with an object.")
            else:
                if prev_op.name == "Radiance Preview":
                    self.rad_prev(lexport, prev_op)
                elif lexport.metric == "3":
                    self.rad_glare(lexport, prev_op)
                elif lexport.metric == "4":
                    self.dayavail(lexport ,prev_op)
                else:
                    self.rad_calc(lexport, prev_op)
        except:
            pass

    def rad_prev(self, lexport, prev_op):
        if os.path.isfile(lexport.filebase+"-0.poly"):
            cams = [obs for obs in (lexport.scene.objects) if obs.type == 'CAMERA']
            if len(cams) != 0:
                cang = cams[0].data.angle*180/pi
                vv = cang * bpy.context.scene.render.resolution_y/bpy.context.scene.render.resolution_x
                spotmatrix = cams[0].matrix_world
                subprocess.call("rvu -n "+str(nproc)+" -vv "+str(vv)+" -vh "+str(cang)+" -vd "+str(-spotmatrix[0][2])+" "+str(-spotmatrix[1][2])+" "+str(-spotmatrix[2][2])+" -vp "+str(cams[0].location[0])+" "+str(cams[0].location[1])+" "+str(cams[0].location[2])+" "+lexport.pparams(lexport.scene.livi_calc_acc)+" "+lexport.filebase+"-"+str(lexport.scene.frame_current)+".oct &", shell = True)
            else:
                prev_op.report({'ERROR'}, "There is no camera in the scene. Radiance preview will not work")
        else:
            prev_op.report({'ERROR'},"Missing export file. Make sure you have exported the scene.")

    def rad_calc(self, lexport, calc_op):
        lexport.clearscened()
        res = [[] for frame in range(0, bpy.context.scene.frame_end+1)]
        for frame in range(0, bpy.context.scene.frame_end+1):
            if os.path.isfile(lexport.filebase+"-"+str(frame)+".af"):
                subprocess.call(self.rm+" "+lexport.filebase+"-"+str(frame)+".af", shell=True)
            rtcmd = "rtrace -n "+str(nproc)+" -w "+lexport.sparams(self.acc)+" -h -ov -I -af "+lexport.filebase+"-"+str(frame)+".af "+lexport.filebase+"-"+str(frame)+".oct  < "+lexport.filebase+".rtrace "+self.simlist[int(lexport.metric)]+" | tee -a "+lexport.newdir+"/"+self.simlistn[int(lexport.metric)]+"-"+str(frame)+".res"
            rtrun = Popen(rtcmd, shell = True, stdout=PIPE, stderr=STDOUT)
            for line in rtrun.stdout:
               res[frame].append(float(line.rstrip()))
        self.resapply(res, lexport)

    def rad_glare(self, lexport, calc_op):
        scene = bpy.context.scene
        gfiles=[]
        cams = [obs for obs in (scene.objects) if obs.type == 'CAMERA']

        if len(cams) != 0:
            for frame in range(0, scene.frame_end+1):
                spotmatrix = cams[0].matrix_world
                glarecmd = "rpict -vth -vh 180 -vv 180 -x 800 -y 800 -vd "+str(-spotmatrix[0][2])+" "+str(-spotmatrix[1][2])+" "+str(-spotmatrix[2][2])+" -vp "+str(cams[0].location[0])+" "+str(cams[0].location[1])+" "+str(cams[0].location[2])+" "+lexport.sparams(self.acc)+" "+lexport.filename+"-"+str(frame)+".oct | evalglare -c glaretemp"+str(frame)+".hdr"
                glarerun = Popen(glarecmd, shell = True, stdout = PIPE)
                for line in glarerun.stdout:
                    if line.decode().split(",")[0] == 'dgp':
                        glaretext = line.decode().strip(':').replace(',', ' ').split(' ')
                        glaretf = open(lexport.filebase+".glare", "w")
                        glaretf.write('%02d' % lexport.simtimes[frame].day+"/"+'%02d' % lexport.simtimes[frame].month+" "+'%02d' % lexport.simtimes[frame].hour+":"+'%02d' % lexport.simtimes[frame].minute+"\n"+glaretext[0]+": "+str(round(float(glaretext[6]),3))+"\n"+glaretext[1]+": "+str(round(float(glaretext[7]),3))+"\n"+glaretext[2]+": "+str(round(float(glaretext[8]),3))+"\n"+glaretext[3]+": "+str(round(float(glaretext[9]),3))+"\n"+glaretext[4]+": "+str(round(float(glaretext[10]),3))+"\n"+glaretext[5]+": "+str(round(float(glaretext[11]),3)))
                        glaretf.close()
                        subprocess.call("cat "+lexport.filename+".glare | psign -h 32 -cb 0 0 0 -cf 40 40 40 | pcompos glaretemp"+str(frame)+".hdr 0 0 - 800 550 > glare"+str(frame)+".hdr", shell=True)
                        subprocess.call("rm glaretemp"+str(frame)+".hdr", shell=True)

                gfile={"name":"glare"+str(frame)+".hdr"}
                gfiles.append(gfile)

            bpy.ops.sequencer.image_strip_add( directory = lexport.newdir, \
                files = gfiles, \
                frame_start=0, \
                channel=2, \
                filemode=9)
        else:
            calc_op.report({'ERROR'}, "There is no camera in the scene. Create one for glare analysis")
        lexport.scene.livi_display_panel = 0

    def dayavail(self, lexport, calc_op):
        res = [[0] * lexport.reslen for frame in range(0, bpy.context.scene.frame_end+1)]
        vecvals = []
        wd = (7, 5)[int(lexport.scene.livi_calc_da_weekdays)]
        if os.path.splitext(os.path.basename(lexport.scene.livi_export_epw_name))[1] == ".hdr":
            skyrad = open(lexport.filebase+".whitesky", "w")
            skyrad.write("void glow sky_glow \n0 \n0 \n4 1 1 1 0 \nsky_glow source sky \n0 \n0 \n4 0 0 1 180 \nvoid glow ground_glow \n0 \n0 \n4 1 1 1 0 \nground_glow source ground \n0 \n0 \n4 0 0 -1 180\n\n")
            skyrad.close()
            vecfile = open(lexport.scene.livi_calc_vec_name, "r")
        else:
            vecfile = open(lexport.newdir+"/"+os.path.splitext(os.path.basename(lexport.scene.livi_export_epw_name))[0]+".vec", "r")
        for li, line in enumerate(vecfile):
            vecvals.append([float(x) for x in line.strip("\n").strip("  ").split(" ")])
        vecfile.close()
        for frame in range(0, bpy.context.scene.frame_end+1):
            print('help', lexport.newdir, self.simlistn, lexport.metric)
            print(lexport.newdir+"/"+self.simlistn[int(lexport.metric)]+"-"+str(frame)+".res")
            hours = 0
            subprocess.call("oconv -w "+lexport.lights(frame)+" "+lexport.filename+".whitesky "+lexport.mat(frame)+" "+lexport.poly(frame)+" > "+lexport.filename+"-"+str(frame)+".oct", shell = True)
            if not os.path.isdir(lexport.newdir+"/s_data"):
                    os.makedirs(lexport.newdir+"/s_data")
            subprocess.call("cat "+lexport.filebase+".rtrace | rtcontrib -h -I -fo -bn 146 -ab 3 -ad 4096 -lw 0.0003 -n "+str(nproc)+" -f tregenza.cal -b tbin -o "+lexport.newdir+"/s_data/sensor%d.dat -m sky_glow "+lexport.filename+"-"+str(frame)+".oct", shell = True)

            for l, readings in enumerate(vecvals):
                finalillu = []
                for i in range(0, 146):
                    if float(readings[0]) >= lexport.scene.livi_calc_dastart_hour and float(readings[0]) < lexport.scene.livi_calc_daend_hour and float(readings[1]) < wd:
                        sensfile = open(lexport.newdir+"/s_data/sensor"+str(i)+".dat", "r")
                        for j, line in enumerate(sensfile):
                            sens = line.strip("\n").split("\t")
                            senreading = 179*(0.265*float(sens[0])+0.67*float(sens[1])+0.065*float(sens[2]))
                            if i == 0:
                                finalillu.append(senreading*readings[i+2])
                                if j == 0:
                                    hours = hours + 1
                            else:
                                finalillu[j] = finalillu[j] + senreading*readings[i+2]
                        sensfile.close()
                for k, reading in enumerate(finalillu):
                    if reading > lexport.scene.livi_calc_min_lux:

                        res[frame][k] = res[frame][k] + 1

            for r in range(0, len(res[frame])):
                if hours != 0:
                    res[frame][r] = res[frame][r]*100/hours

            daresfile = open(lexport.newdir+"/"+self.simlistn[int(lexport.metric)]+"-"+str(frame)+".res", "w")
            for r in res[frame]:
                daresfile.write(str(round(r, 2))+"\n")
            daresfile.close()
        calc_op.report({'INFO'}, "Calculation is finished.")
        self.resapply(res, lexport)

    def resapply(self, res, lexport):
        maxres = []
        minres = []
        avres = []

        for frame in range(0, lexport.scene.frame_end+1):
            maxres.append(max(res[frame]))
            minres.append(min(res[frame]))
            avres.append(sum(res[frame])/len(res[frame]))

        lexport.scene['resav'] = avres
        lexport.scene['resmax'] = maxres
        lexport.scene['resmin'] = minres

        for frame in range(0, lexport.scene.frame_end+1):
            rgb = []
            j = 0
            for i in range(0, len(res[frame])):
                h = 0.75*(1-(res[frame][i]-min(lexport.scene['resmin']))/(max(lexport.scene['resmax']) + 0.01 - min(lexport.scene['resmin'])))
                rgb.append(colorsys.hsv_to_rgb(h, 1.0, 1.0))
            cno = 0
            for geo in lexport.scene.objects:
                bpy.ops.object.select_all(action = 'DESELECT')
                bpy.context.scene.objects.active = None
                try:
                    if geo['calc'] == 1:
                        bpy.context.scene.objects.active = geo
                        geo.select = True
                        if frame == 0:
                            while len(geo.data.vertex_colors) > 0:
                                bpy.ops.mesh.vertex_color_remove()

                        bpy.ops.mesh.vertex_color_add()
                        geo.data.vertex_colors[frame].name = str(frame)
                        vertexColour = geo.data.vertex_colors[frame].data

                        for face in geo.data.faces:
                            if "calcsurf" in str(geo.data.materials[face.material_index].name):
                                v = vertexColour[face.index]
                                if lexport.scene['cp'] == 0:
                                    v.color1 = rgb[j]
                                    v.color2 = rgb[j]
                                    v.color3 = rgb[j]
                                    if len(face.vertices) == 4:
                                        v.color4 = rgb[j]
                                    j = j + 1

                                if lexport.scene['cp'] == 1:
                                    for vert in face.vertices:
                                        if vert == face.vertices[0]:
                                            v.color1 = rgb[cno + [i for i, x in enumerate(geo['cverts']) if x == vert][0]]
                                        elif vert == face.vertices[1]:
                                            v.color2 = rgb[cno + [i for i, x in enumerate(geo['cverts']) if x == vert][0]]
                                        elif vert == face.vertices[2]:
                                            v.color3 = rgb[cno + [i for i, x in enumerate(geo['cverts']) if x == vert][0]]
                                        elif vert == face.vertices[3]:
                                            v.color4 = rgb[cno + [i for i, x in enumerate(geo['cverts']) if x == vert][0]]
                        cno = cno + len(geo['cverts'])

                except Exception as e:
                    print(e)

        for frame in range(0, lexport.scene.frame_end+1):
            bpy.ops.anim.change_frame(frame = frame)
            for geo in lexport.scene.objects:
                try:
                    if geo['calc'] == 1:
                        for vc in geo.data.vertex_colors:
                            if frame == int(vc.name):
                                vc.active = 1
                                vc.active_render = 1
                                vc.keyframe_insert("active")
                                vc.keyframe_insert("active_render")
                            else:
                                vc.active = 0
                                vc.active_render = 0
                                vc.keyframe_insert("active")
                                vc.keyframe_insert("active_render")
                except:
                    pass
        lexport.scene.livi_display_panel = 1
        bpy.ops.wm.save_mainfile(check_existing = False)


from flask import Flask, jsonify, request
from flask_cors import CORS, cross_origin
from handler.user import UserHandler
from handler.email import EmailHandler

# Activate
app = Flask(__name__)
# Apply CORS to this app
CORS(app)

@app.route('/')
def greeting():
    return 'Hello, this is the Email DB App 404!'

# USER SERVICES
@app.route('/404/user/create', methods=['POST'])
def insertNewUser():
    print("REQUEST: ", request.json)
    user_data = request.json
    return UserHandler().insertUser(user_data);

@app.route('/404/user/<int:id_user>/friend/add', methods=['POST'])
def addFriend(id_user):
    print("REQUEST: ", request.json)
    user_data = request.json
    return UserHandler().addFriend(id_user,user_data);

@app.route('/404/user/<int:id_user>/friend/remove', methods=['POST'])
def removeFriend(id_user):
    print("REQUEST: ", request.json)
    user_data = request.json
    return UserHandler().removeFriend(id_user,user_data);

@app.route('/404/user/getallusers', methods=['GET'])
def users_handler():
    if request.method == 'GET':
        return UserHandler().getAllUsers()
    else:
        return jsonify("Not Supported"), 405

@app.route('/404/user/<int:id_user>', methods=['GET'])
def user_by_id_handler(id_user):
    if request.method == 'GET':
        return UserHandler().getUserbyId(id_user)
    else:
        return jsonify("Not Supported"), 405

@app.route('/404/user/top10usersinbox', methods=['GET'])
def top10inbox_handler():
    if request.method == 'GET':
        return UserHandler().viewTop10UsersWithMoreEmailsOnInbox()
    else:
        return jsonify("Not Supported"), 405

@app.route('/404/user/top10usersoutbox', methods=['GET'])
def top10outbox_handler():
    if request.method == 'GET':
        return UserHandler().viewTop10UsersWithMoreEmailsOnOutbox()
    else:
        return jsonify("Not Supported"), 405

# END USER SERVICES

# EMAIL SERVICES

@app.route('/404/user/<int:id_user>/email/send', methods=['POST'])
def createNewEmail(id_user):
    print("REQUEST: ", request.json)
    email_data = request.json
    return EmailHandler().createEmail(email_data,id_user);

@app.route('/404/user/<int:id_user>/email/inbox', methods=['GET'])
def retreiveInbox(id_user):
    return EmailHandler().retreiveInbox(id_user);

@app.route('/404/user/<int:id_user>/email/inbox/search', methods=['GET'])
def searchInbox(id_user):
    args = request.args;
    field = args.get("field", default=None, type=str)
    value = args.get("value", default=None, type=str)
    return EmailHandler().searchInbox(id_user,field,value);

@app.route('/404/user/<int:id_user>/email/<int:id_email>/setcategory', methods=['POST'])
def setCategory(id_user,id_email):
    json = request.json
    return EmailHandler().setCategoryEmail(id_user,id_email,json);

@app.route('/404/user/<int:id_user>/email/outbox', methods=['GET'])
def retreiveOutbox(id_user):
    return EmailHandler().retreiveOutbox(id_user);

@app.route('/404/user/<int:id_user>/email/outbox/search', methods=['GET'])
def searchOutbox(id_user):
    args = request.args;
    field = args.get("field", default=None, type=str)
    value = args.get("value", default=None, type=str)
    return EmailHandler().searchOutbox(id_user,field,value);

@app.route('/404/user/<int:id_user>/email/inbox/delete', methods=['POST'])
def deleteInbox(id_user):
    email_delete = request.json
    return EmailHandler().deleteInbox(id_user,email_delete);

@app.route('/404/user/<int:id_user>/email/outbox/delete', methods=['POST'])
def deleteOutbox(id_user):
    email_delete = request.json
    return EmailHandler().deleteOutbox(id_user,email_delete);

@app.route('/404/user/<int:id_user>/email/outbox/update', methods=['POST'])
def updateOutbox(id_user):
    email_update = request.json
    return EmailHandler().updateOutbox(id_user,email_update);

@app.route('/404/user/<int:id_user>/email/inbox/<int:id_email>', methods=['GET'])
def viewInboxEmail(id_user,id_email):
    return EmailHandler().viewInboxEmail(id_user,id_email);

@app.route('/404/user/<int:id_user>/email/outbox/<int:id_email>', methods=['GET'])
def viewInboxOutbox(id_user,id_email):
    return EmailHandler().viewOutboxEmail(id_user,id_email);

@app.route('/404/user/<int:id_user>/email/inbox/<int:id_email>/reply', methods=['POST'])
def replyEmail(id_user,id_email):
    print("REQUEST: ", request.json)
    email_data = request.json
    return EmailHandler().createEmailReply(id_user,id_email,email_data);

@app.route('/404/email/getallemails', methods=['GET'])
def emails_handler():
    if request.method == 'GET':
        return EmailHandler().getAllEmails()
    else:
        return jsonify("Not Supported"), 405

@app.route('/404/email/<int:id_email>', methods=['GET'])
def email_by_id_handler(id_email):
    if request.method == 'GET':
        return EmailHandler().getEmailbyId(id_email)
    else:
        return jsonify("Not Supported"), 405

@app.route('/404/email/mostrecipients', methods=['GET'])
def recipients_handler():
    if request.method == 'GET':
        return EmailHandler().viewEmailMostRecipients()
    else:
        return jsonify("Not Supported"), 405

@app.route('/404/email/mostreplies', methods=['GET'])
def replies_handler():
    if request.method == 'GET':
        return EmailHandler().viewEmailMostReplies()
    else:
        return jsonify("Not Supported"), 405

# END EMAIL SERVICES

if __name__ == '__main__':
    app.run()

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


import sys

class Demultiplexor:
    PID_PAT = 0x0000
    PID_NIT = 0x0010
    PID_SDT = 0x0011

    TABLE_ID_PAT = 0x00
    TABLE_ID_PMT = 0x02
    TABLE_ID_NIT = 0x40
    TABLE_ID_SDT = 0x42

    DESCRIPTOR_NETWORK_NAME = 0x40
    DESCRIPTOR_TERRESTRIAL_DELIVERY_SYSTEM = 0x5A
    DESCRIPTOR_SERVICE = 0x48

    def __init__(self):
        self.packet_num = 0

        self.PAT_analysed = False
        self.NIT_analysed = False
        self.SDT_analysed = False

        self.PAT_data = dict()

        self.network_name = ''
        self.network_id = 0
        self.bandwith = ''
        self.constellation = ''
        self.code_rate = ''
        self.guard_interval = ''

        self.sdt_payload = bytearray()
        self.adt_add_data = False

    def parse_PAT(self, packet):
        table_id = packet.payload[0]
        # Check if table_id is OK
        if table_id != self.TABLE_ID_PAT:
            print("ERROR: Bad table_id in PAT.", file=sys.stderr)
            return False

        print("pusi: ", packet.payload_unit_start_ind)
        section_length = ((packet.payload[1] & 0xf) << 8) + packet.payload[2]
        transport_stream_id = (packet.payload[3] << 8) + packet.payload[4]
        version_number = packet.payload[5] & 0x3E
        current_next_indicator = packet.payload[5] & 0x1
        section_number = packet.payload[6]
        last_section_number = packet.payload[7]

        i = 0
        while i < section_length:
            index = 8 + i
            program_number = (packet.payload[index] << 8) + packet.payload[index+1]

            index += 2

            network_pid = ((packet.payload[index] & 0x1f) << 8) + packet.payload[index+1]

            self.PAT_data[program_number] = network_pid

            print(hex(network_pid))

            if program_number == 0:
                self.PID_NIT = network_pid
            i += 4

        # self.print_PAT(table_id, section_length, self.PAT_data, transport_stream_id, version_number, current_next_indicator,
        #                section_number, last_section_number)

        self.PAT_analysed = True
        return True

    def print_PAT(self, table_id, section_length, PAT_data, transport_stream_id='unknown',
                  version_number='unknown', curr_next_ind='unknown', section_num='unknown',
                  last_section_num='unknown'):
        print("table id:", table_id)
        print("section length:", section_length)
        print("transport stream id:", transport_stream_id)
        print("version number:", version_number)
        print("current next indicator:", curr_next_ind)
        print("section number:", section_num)
        print("last section number:", last_section_num)
        print("PROGRAMS:")
        for prog_num, network_pid in PAT_data.items():
            print("\tprogram number:", prog_num, ",(" + str(hex(prog_num)) + ")")
            print("\tnetwork pid:", network_pid, ",(" + str(hex(network_pid)) + ")")

    def parse_NIT(self, packet):
        table_id = packet.payload[0]
        if table_id != self.TABLE_ID_NIT:
            print("ERROR: Bad table_id in NIT.", file=sys.stderr)
            return False

        section_length = ((packet.payload[1] & 0xf) << 8) + packet.payload[2]
        self.network_id = (packet.payload[3] << 8) + packet.payload[4]
        version_number = packet.payload[5] & 0x3E
        section_number = packet.payload[6]
        last_section_number = packet.payload[7]
        network_descriptors_length = ((packet.payload[8] & 0xF) << 8) + packet.payload[9]
        skip = 10
        self.parse_descriptors(skip, network_descriptors_length, packet)
        skip += network_descriptors_length
        transport_stream_loop_length = ((packet.payload[skip] & 0xF) << 8) + packet.payload[skip+1]
        # print("transport len:", transport_stream_loop_length)
        skip += 2

        i = 0
        while i < transport_stream_loop_length:
            transport_stream_id = (packet.payload[skip] << 8) + packet.payload[skip+1]
            skip += 2
            original_network_id = (packet.payload[skip] << 8) + packet.payload[skip+1]
            skip += 2
            transport_descriptors_length = ((packet.payload[skip] & 0xF) << 8) + packet.payload[skip+1]
            skip += 2
            self.parse_descriptors(skip, transport_descriptors_length, packet)
            skip += transport_descriptors_length
            i += transport_descriptors_length + 6

        self.NIT_analysed = True
        return True

    def parse_SDT(self, packet):
        table_id = packet.payload[0]
        # if table_id != self.TABLE_ID_SDT:
        #     print("ERROR: Bad table_id in SDT.", file=sys.stderr)
        #     return False

        section_length = ((packet.payload[1] & 0xF) << 8) + packet.payload[2]

        analyse = False
        if packet.payload_unit_start_ind and len(self.sdt_payload) == 0:
            self.adt_add_data = True
        if packet.payload_unit_start_ind and len(self.sdt_payload) > 0:
            analyse = True
            self.adt_add_data = False
            transport_stream_id = (packet.payload[3] << 8) + packet.payload[4]

        if self.adt_add_data:
            self.sdt_payload += packet.payload
        if analyse:
            self.SDT_analysed = True
            skip = 0
            while skip < section_length:
                service_id = (self.sdt_payload[skip+11] << 8) + self.sdt_payload[skip+12]
                descriptors_loop_length = ((self.sdt_payload[skip+11+3] & 0xF) << 8) + self.sdt_payload[skip+11+4]
                skip += 5
                self.parse_service_desc(skip + 11, descriptors_loop_length, self.sdt_payload)
                skip += descriptors_loop_length
        return True

    def parse_PTM(self, packet):
        table_id = packet.payload[0]
        if table_id != self.TABLE_ID_PMT:
            print("ERROR: Bad table_id in PMT.", file=sys.stderr)
            return False

        section_length = ((packet.payload[2] & 0xF) << 8) + packet.payload[3]
        program_number = (packet.payload[4] << 8) + packet.payload[5]

        program_info_length = ((packet.payload[10] & 0xF) << 8) + packet.payload[11]

        skip = program_info_length + 12
        i = 0
        while i < section_length:
            elementary_pid = ((packet.payload[skip+i+1] & 0x1F) << 8) + packet.payload[skip+i+2]

    def check_PMT(self, packet_pid):
        # If PID of packet is NIT PID, skip this packet
        if packet_pid == self.PID_NIT:
            return False
        elif packet_pid in self.PAT_data.values():
            return True
        return False

    def parse_service_desc(self, offset, length, payload):
        service_provider_name = ''
        service_name = ''
        j = 0
        while j < length:
            desc_len = payload[offset+j+1]
            if payload[offset + j] == self.DESCRIPTOR_SERVICE:
                service_provider_name_length = payload[offset+j+3]
                for i in range(0, service_provider_name_length):
                    service_provider_name += chr(payload[offset+4+j+i])

                service_name_length = payload[offset+j+4+service_provider_name_length]
                for i in range(0, service_name_length):
                    service_name += chr(payload[offset+5+service_provider_name_length+i])
            return service_provider_name, service_name

    def parse_descriptors(self, offset, length, packet):
        j = 0
        while j < length:
            desc_len = packet.payload[offset+j+1]
            if packet.payload[offset + j] == self.DESCRIPTOR_NETWORK_NAME:
                network_name = ''
                for i in range(0, desc_len):
                    self.network_name += chr(packet.payload[offset+j+2+i])
            elif packet.payload[offset + j] == self.DESCRIPTOR_TERRESTRIAL_DELIVERY_SYSTEM:
                self.bandwith = self.get_bandwith(packet.payload[offset + j + 6] >> 5)
                self.constellation = self.get_constellation(packet.payload[offset + j + 7] >> 6)
                self.code_rate = self.get_code_rate(packet.payload[offset + j + 7] & 0x7)
                code_rate_lp_stream = packet.payload[offset + j + 8] & 0xE0
                self.guard_interval = self.get_guard_interval(packet.payload[offset + j + 8] & 0x18)

                # print('network name:', self.network_name)
                # print('network ID:', self.network_id)
                # print('bandwith:', self.bandwith)
                # print('constellation:', self.constellation)
                # print('guard interval:', self.guard_interval)
                # print('code_rate:', self.code_rate)
            j += desc_len + 2

    def get_bandwith(self, bit_field):
        if bit_field == 0:
            return '8 MHz'
        elif bit_field == 1:
            return '7 MHz'
        elif bit_field == 2:
            return '6 MHz'
        elif bit_field == 3:
            return '5 MHz'
        else:
            return 'Unknown'

    def get_constellation(self, bit_field):
        if bit_field == 0:
            return 'QPSK'
        elif bit_field == 1:
            return '16-QAM'
        elif bit_field == 2:
            return '64-QAM'
        else:
            return 'Unknown'

    def get_code_rate(self, bit_field):
        if bit_field == 0:
            return '1/2'
        elif bit_field == 1:
            return '2/3'
        elif bit_field == 2:
            return '3/4'
        elif bit_field == 3:
            return '5/6'
        elif bit_field == 4:
            return '7/8'
        else:
            return 'Unknown'

    def get_guard_interval(self, bit_field):
        if bit_field == 0:
            return '1/32'
        elif bit_field == 1:
            return '1/16'
        elif bit_field == 2:
            return '1/8'
        else:
            return '1/4'


from ipynb_strip_copy import *

def test_quick():
    file_in = 'test_case.ipynb'
    file_out0 = 'test_case0.ipynb'
    file_out1 = 'test_case1.ipynb'

    json_in = json_from_ipynb(file_in)
    json_out0 = json_from_ipynb(file_out0)
    json_out1 = json_from_ipynb(file_out1)

    suffix_json_dict = search_apply_json(json_in)

    assert json_out0 == suffix_json_dict['0']
    assert json_out1 == suffix_json_dict['1']

def test_pts():
    file_in = 'test_case_points.ipynb'
    json_dict = json_from_ipynb(file_in)

    pts_dict = count_points_from_json(json_dict)

    pts_dict_expect = {'part 1': 20.0,
                       'problem 2': 13.0,
                       'part 3.398234 the quick brown fox': 15.0,
                       'part 4.2 b': 12.0}

    assert pts_dict == pts_dict_expect


import platform
import h5py
from skimage import io
import tifffile as tiff
import numpy as np
import glob
# import sima
# import sima.segment
import time
import pdb
import cv2
import pickle
import re
import matplotlib.pyplot as plt
from skimage import io
#import sys
#import os

from tools.h5pyTools import h5pyTools
import tools.googleDocsAccess as googleDocsAccess
from tools.pyqtgraph.configfile import *
from ScanImageTiffReader import ScanImageTiffReader

class extractSaveData:
    def __init__(self, mouse):
        self.mouse = mouse
        self.h5pyTools = h5pyTools()

        # determine location of data files and store location
        if platform.node() == 'thinkpadX1' or platform.node() == 'thinkpadX1B':
            laptop = True
            self.analysisBase = '/media/HDnyc_data/'
            self.suite2pPath = '/home/mgraupe/.miniconda2/envs/suite2p/bin/python'
        elif platform.node() == 'otillo':
            laptop = False
            self.analysisBase = '/media/paris_data/'
            self.suite2pPath = '/home/mgraupe/anaconda3/envs/suite2p/bin/python'
        elif platform.node() == 'yamal' or platform.node() == 'cerebellum-HP' or platform.node() == 'andry-ThinkPad-X1-Carbon-2nd' or platform.node() == 'OptiPlex-7070':
            laptop = False
            self.analysisBase = '/media/HDnyc_data/'
            self.suite2pPath = '/home/andry/anaconda3/envs/suite2p/bin/python'
        elif platform.node() == 'bs-analysis':  # andry-ThinkPad-X1-Carbon-2nd
            laptop = False
            self.analysisBase = '/home/mgraupe/nyc_data/'
        else:
            print('Run this script on a server or laptop. Otherwise, adapt directory locations.')
            sys.exit(1)
        # print('test1')
        self.listOfAllExpts = googleDocsAccess.getExperimentSpreadsheet()
        # print(self.listOfAllExpts['201007_t00'])
        # extract recording dates of the specific animal
        dates = []
        for d in self.listOfAllExpts[self.mouse]['dates']:
            dates.append(d)

        if dates[0] >= '181018':
            self.dataBase = '/media/invivodata2/'
        else:
            self.dataBase = '/media/invivodata/'

        # check if directory is mounted
        if not os.listdir(self.dataBase):
            os.system('mount %s' % self.dataBase)

        if not os.listdir(self.analysisBase):
            os.system('mount %s' % self.analysisBase)

        self.dataPCLocation = OrderedDict(
            [('behaviorPC', 'behaviorPC_data/dataMichael/'), ('2photonPC', ('altair_data/dataMichael/' if int(self.mouse[:6]) >= 170829 else 'altair_data/experiments/data_Michael/acq4/')),
                ('', ('altair_data/dataMichael/' if int(self.mouse[:6]) >= 170829 else 'altair_data/experiments/data_Michael/acq4/')), ])

        self.computerDict = OrderedDict(
            [('behaviorPC', 'behaviorPC'), ('2photonPC', '2photonPC'),('', '2photonPC')])

        self.analysisLocation = self.analysisBase + 'data_analysis/in_vivo_cerebellum_walking/LocoRungsData/%s/' % mouse
        self.figureLocation = self.analysisBase + 'data_analysis/in_vivo_cerebellum_walking/LocoRungsFigures/%s/' % mouse

        if not os.path.isdir(self.analysisLocation):
            os.system('mkdir %s' % self.analysisLocation)
        if not os.path.isdir(self.analysisLocation):
            os.system('mkdir %s' % self.figureLocation)

        fName = self.analysisLocation + 'analysis.hdf5'
        # if os.path.isfile(fName):
        try:
            self.f = h5py.File(fName, 'a')
        except FileExistsError:
            print('hdf5 file is used by another process and cannot be accessed.')
            sys.exit(1)
        else:
            pass
        self.mouse = mouse

        # experiments stored under this name were recorded as sequence with a certain number of repetitions
        self.listOfSequenceExperiments = ['locomotionTriggerSIAndMotor', 'locomotionTriggerSIAndMotorJin', 'locomotionTriggerSIAndMotor60sec', 'locomotion_recording_setup2']

    ############################################################
    def __del__(self):
        try:
            self.f.flush()
        except:
            pass
        else:
            print('hdf5 file flushed!')
        # self.f.close()
        print('on exit')

    ############################################################
    def getMotioncorrectedStack(self, folder, rec, suffix):
        allFiles = []
        for file in glob.glob(self.analysisLocation + '%s_%s_%s*_%s.tif' % (self.mouse, folder, rec, suffix)):
            allFiles.append(file)
            print(file)
        if len(allFiles) > 1:
            print('more than one matching image file')
            sys.exit(1)
        else:
            motionCoor = np.loadtxt(allFiles[0][:-3] + 'csv', delimiter=',', skiprows=1)
            imStack = io.imread(allFiles[0])
        return (imStack, motionCoor, allFiles[0])

    ############################################################
    def saveRungMotionData(self, mouse, date, rec, rungPositions):
        rec = rec.replace('/', '-')
        pickle.dump(rungPositions, open(self.analysisLocation + '%s_%s_%s_rungPositions.p' % (mouse, date, rec), 'wb'))

    ############################################################
    def getRungMotionData(self, mouse, date, rec):
        rec = rec.replace('/', '-')
        rungPositions = pickle.load(open(self.analysisLocation + '%s_%s_%s_rungPositions.p' % (mouse, date, rec), 'rb'))
        return rungPositions

    ############################################################
    def extractRoiSignals(self, folder, rec, tifFile):

        self.simaPath = self.analysisLocation + '%s_%s_%s' % (self.mouse, folder, rec)
        print(self.simaPath)
        if os.path.isdir(self.simaPath + '.sima'):
            print('sima dir exists')
            dataSet = sima.ImagingDataset.load(self.simaPath + '.sima')
        else:
            print('create sima dir')
            sequences = [sima.Sequence.create('TIFF', tifFile)]
            dataSet = sima.ImagingDataset(sequences, self.simaPath, channel_names=['GCaMP6F'])

        img = dataSet.time_averages[0][:, :, 0]

        self.simaPath = self.simaPath + '.sima/'
        overwrite = True
        if os.path.isfile(self.simaPath + 'rois.pkl'):
            print('rois already exist')
            input_ = raw_input('rois traces exist already, do you want to overwrite? (type \'y\' to overwrite, any character if not) : ')
            if input_ != 'y':
                overwrite = False
        if overwrite:
            print('create rois with roibuddy')
            segmentation_approach = sima.segment.STICA(channel='GCaMP6F', components=1, mu=0.9,  # weighting between spatial - 1 - and temporal - 0 - information
                # spatial_sep=0.8
            )
            print('segmenting calcium image ... ', end=" ")
            dataSet.segment(segmentation_approach, 'GCaMP6F_signals', planes=[0])
            print('done')
            while True:
                input_ = raw_input('Please check ROIs in \'roibuddy\' (type \'exit\' to halt) : ')
                if input_ == 'exit':
                    sys.exit(1)
                else:
                    break
        dataSet = sima.ImagingDataset.load(self.simaPath)
        rois = dataSet.ROIs['GCaMP6F_signals']

        # Extract the signals.
        dataSet.extract(rois, signal_channel='GCaMP6F', label='GCaMP6F_signals')
        raw_signals = dataSet.signals('GCaMP6F')['GCaMP6F_signals']['raw']
        # dataSet.export_signals('example_signals.csv', channel='GCaMP6F',signals_label='GCaMP6F_signals')
        # pdb.set_trace()

        # dataSet.signals('GCaMP6F')['GCaMP6F_signals']['mean_frame']

        # pdb.set_trace()
        roiLabels = []
        # extrace labels
        for n in range(len(rois)):
            roiLabels.append(rois[n].label)  # print 'ROI label', n, rois[n].label

        return (img, rois, raw_signals)

    ############################################################
    def getRecordingsList(self, expDate='all', recordings='all'):
        if recordings == 'all910':
            recID = 'recs910'
        elif recordings == 'all820':
            recID = 'recs820'
        elif recordings == 'all':
            recID = ['recs910','recs820']

        self.config = readConfigFile('simplexAnimals.config')
        #pdb.set_trace()
        for i in range(len(self.config)):
            if self.config['%s' % i]['mouse'] == self.mouse:
                print('experiment dictionary of mouse exists')
                self.expDict = self.config['%s' % i]['days']
                dictExists = True
        #pdb.set_trace()
        folderRec = []
        if self.mouse in self.listOfAllExpts:
            # print mouse
            # print expDate, self.listOfAllExpts[mouse]['dates']
            expDateList = []
            # pdb.set_trace()
            # provide choice of which days to include in analysis
            if expDate == 'some':
                print('Dates when experiments where performed with animal %s :' % self.mouse)
                didx = 0
                for d in self.listOfAllExpts[self.mouse]['dates']:
                    print('  %s %d' % (d, didx))
                    didx += 1
                print('Choose the dates for analysis by typing the index, e.g, \'1\', or \'0,1,3,5\' : ', end='')
                daysInput = input()
                daysInputIdx = [int(i) for i in daysInput.split(',')]  # print(daysInputIdx,daysInputIdx[0],type(daysInputIdx))
            elif expDate == 'all910' or expDate == 'all820':
                didx = 0
                daysInputIdx = []
                for d in self.listOfAllExpts[self.mouse]['dates']:
                    if d in self.expDict.keys() :
                        if recID in self.expDict[d]:
                            daysInputIdx.append(didx)
                    didx+=1
            elif expDate == 'all':
                didx = 0
                daysInputIdx = []
                for d in self.listOfAllExpts[self.mouse]['dates']:
                    if d in self.expDict.keys() :
                        daysInputIdx.append(didx)
                    didx+=1
            ########################################################
            # generate list of days to analyze
            if expDate == 'some' or expDate=='all910' or expDate=='all820' or expDate == 'all':
                didx = 0
                for d in self.listOfAllExpts[self.mouse]['dates']:
                    if didx in daysInputIdx:
                        # print(d)
                        expDateList.append(d)
                    didx += 1
            else:
                expDateList.append(expDate)
            print('Selected dates :', expDateList)
            #pdb.set_trace()
            #####################################################
            # chose recordings
            if recordings == 'all910':
                print('All 910 recordings will be analyzed')
                recIdx = 0
                recInputIdx = []
                for eD in expDateList:
                    dataFolders = self.listOfAllExpts[self.mouse]['dates'][eD]['folders']
                    if 'recs910' in self.expDict[eD]:
                        idx910 = self.expDict[eD]['recs910']['recordings']
                    else:
                        idx910 = None
                    for fold in dataFolders:
                        #print(' ', fold)
                        self.dataLocation = self.dataBase + self.dataPCLocation[dataFolders[fold]['recComputer']] + fold + '/'
                        recList = self.getDirectories(self.dataLocation)
                        #print(recList)
                        if idx910 is not None:
                            recInputIdx.append(recIdx+idx910)
                        recIdx+=len(recList)
                #pdb.set_trace()
            elif recordings == 'all820':
                print('All 820 recordings will be analyzed')
                recIdx = 0
                recInputIdx = []
                for eD in expDateList:
                    dataFolders = self.listOfAllExpts[self.mouse]['dates'][eD]['folders']
                    if 'recs820' in self.expDict[eD]:
                        idx820 = self.expDict[eD]['recs820']['recordings']
                    else:
                        idx820 = None
                    for fold in dataFolders:
                        #print(' ', fold)
                        self.dataLocation = self.dataBase + self.dataPCLocation[dataFolders[fold]['recComputer']] + fold + '/'
                        recList = self.getDirectories(self.dataLocation)
                        #print(recList)
                        if idx820 is not None:
                            recInputIdx.append(recIdx+idx820)
                        recIdx+=len(recList)
                #pdb.set_trace()
            elif recordings == 'some':
                # first show all recordings for a given date
                print('Choose recording to analyze')
                recIdx = 0
                for eD in expDateList:
                    dataFolders = self.listOfAllExpts[self.mouse]['dates'][eD]['folders']
                    for fold in dataFolders:
                        print(' ', fold)
                        # self.dataLocation = (self.dataBase2 + fold + '/') if eD >= '181018' else (self.dataBase + fold + '/')
                        self.dataLocation = self.dataBase + self.dataPCLocation[dataFolders[fold]['recComputer']] + fold + '/'
                        #self.recordingMachine = self.computerDict[dataFolders[fold]['recComputer']]
                        #print(self.dataLocation)
                        if not os.path.exists(self.dataLocation):
                            #    print('experiment %s exists' % fold)
                            # else:
                            print('Problem, experiment does not exist')
                        # recList = OrderedDict()
                        recList = self.getDirectories(self.dataLocation)
                        for r in recList:
                            print('    %s  %s' % (r, recIdx))
                            recIdx += 1
                print('Choose the recordings for analysis by typing the index, e.g, \'1\', or \'0,1,3,5\' : ', end='')
                recInput = input()
                recInputIdx = [int(i) for i in recInput.split(',')]
            elif recordings == 'all':
                print('All 910 and 820 recordings will be analyzed')
                recIdx = 0
                recInputIdx = []
                for eD in expDateList:
                    dataFolders = self.listOfAllExpts[self.mouse]['dates'][eD]['folders']
                    if 'recs910' in self.expDict[eD]:
                        idx910 = self.expDict[eD]['recs910']['recordings']
                    else:
                        idx910 = None
                    if 'recs820' in self.expDict[eD]:
                        idx820 = self.expDict[eD]['recs820']['recordings']
                    else:
                        idx820 = None
                    for fold in dataFolders:
                        #print(' ', fold)
                        self.dataLocation = self.dataBase + self.dataPCLocation[dataFolders[fold]['recComputer']] + fold + '/'
                        recList = self.getDirectories(self.dataLocation)
                        #print(recList)
                        if idx910 is not None:
                            recInputIdx.append(recIdx+idx910)
                        if idx820 is not None:
                            recInputIdx.append(recIdx + idx820)
                        recIdx+=len(recList)
            else:
                recInputIdx = [int(i) for i in recordings.split(',')]
            print('list of recordings : ', recInputIdx)
            #
            # pdb.set_trace()
            # then compile a list the selected recordings
            recIdx = 0
            for eD in expDateList:
                # print(expDateList, self.listOfAllExpts[mouse]['dates'], len(self.listOfAllExpts[mouse]['dates']))
                dataFolders = self.listOfAllExpts[self.mouse]['dates'][eD]['folders']
                # print(eD, self.listOfAllExpts[mouse]['dates'],dataFolders)
                for fold in dataFolders:
                    # self.dataLocation = (self.dataBase2 + fold + '/') if eD >= '181018' else (self.dataBase + fold + '/')
                    # pdb.set_trace()
                    self.dataLocation = self.dataBase + self.dataPCLocation[dataFolders[fold]['recComputer']] + fold + '/'
                    self.recordingMachine = self.computerDict[dataFolders[fold]['recComputer']]
                    print(self.dataLocation)
                    if not os.path.exists(self.dataLocation):
                        # print('experiment %s exists' % fold)
                        # else:
                        print('Problem, experiment does not exist')
                    recList = self.getDirectories(self.dataLocation)
                    if recordings == 'all':
                        tempRecList = []
                        for r in recList:
                            # if r[:-4] == 'locomotionTriggerSIAndMotor' or r[:-4] == 'locomotionTriggerSIAndMotorJin' or r[:-4] == 'locomotionTriggerSIAndMotor60sec' :
                            if r[:-4] in self.listOfSequenceExperiments:
                                subFolders = self.getDirectories(self.dataLocation + '/' + r)
                                for i in range(len(subFolders)):
                                    if subFolders[i][0] == '0':
                                        tempRecList.append(r + '/' + subFolders[i])
                                    else:
                                        tempRecList.append(r)
                                        break
                            else:
                                tempRecList.append(r)
                        folderRec.append([fold, eD, tempRecList])  # folderRec.append([fold,eD,recList])
                    else: # recordings == 'some':
                        tempRecList = []
                        for r in recList:
                            # only add recordings which were previously selected
                            if recIdx in recInputIdx:
                                # if r[:-4] == 'locomotionTriggerSIAndMotor' or r[:-4] == 'locomotionTriggerSIAndMotorJin' or r[:-4] == 'locomotionTriggerSIAndMotor60sec':
                                if r[:-4] in self.listOfSequenceExperiments:
                                    subFolders = self.getDirectories(self.dataLocation + '/' + r)
                                    for i in range(len(subFolders)):
                                        if subFolders[i][0] == '0':
                                            tempRecList.append(r + '/' + subFolders[i])
                                        else:
                                            tempRecList.append(r)
                                            break
                                else:
                                    tempRecList.append(r)
                            recIdx += 1
                        folderRec.append([fold, eD, tempRecList])
        #pdb.set_trace()
        print('Data was recorded on %s' % self.recordingMachine)
        return (folderRec, dataFolders)

    ############################################################
    def getDirectories(self, location):
        # seqFolder = self.dataLocation + '/' + r
        subFolders = [os.path.join(o) for o in os.listdir(location) if os.path.isdir(os.path.join(location, o))]
        subFolders.sort()
        return subFolders

    ############################################################
    def checkIfDeviceWasRecorded(self, fold, eD, recording, device):
        # recLocation =  (self.dataBase2 + '/' + fold + '/' + recording + '/') if eD >= '181018' else (self.dataBase2 + '/' + fold + '/' + recording + '/')
        recLocation = self.dataBase + self.dataPCLocation[self.listOfAllExpts[self.mouse]['dates'][eD]['folders'][fold]['recComputer']] + fold + '/' + recording + '/'
        # print(recLocation,eD,int(eD))
        if os.path.exists(recLocation):
            print('%s constains %s , ' % (fold, recording), end=" ")
        else:
            print('Problem, recording does not exist')
        if device in ['CameraGigEBehavior', 'CameraPixelfly']:
            if int(eD) >= 191104:
                pathToFile = recLocation + device + '/' + 'video_000.ma'
            else:
                pathToFile = recLocation + device + '/' + 'frames.ma'
        elif device == 'PreAmpInput':
            pathToFile = recLocation + '%s.ma' % 'DaqDevice'
        elif device == 'frameTimes':
            pathToFile = recLocation + '%s/%s.ma' % ('CameraGigEBehavior', 'daqResult')
        elif device == 'SICaImaging':
            recLocation = self.dataBase + self.dataPCLocation[self.listOfAllExpts[self.mouse]['dates'][eD]['folders'][fold]['recComputer']] + fold + '/'
            print(recLocation)
            tiffList = glob.glob(recLocation + '*tif')
            tiffList.sort()
            print(tiffList)
            if len(tiffList) > 0:
                print('Ca imaging was acquired with ScanImage')
                return (True, tiffList, recLocation)
            else:
                print('No Ca imaging with ScanImage here.')
                return (False, [], None)
        else:
            pathToFile = recLocation + '%s.ma' % device
        print(pathToFile)

        if os.path.isfile(pathToFile):
            fData = h5py.File(pathToFile, 'r')
            try:
                kk = fData['data']
            except KeyError:
                print('Device %s was acquired but NO data exists' % device)
                return (False, None)
            else:
                print('Device %s was acquired' % device)
                return (True, fData)
        else:
            print('Device %s was NOT acquired' % device)
            return (False, None)

    ############################################################
    def checkIfPawPositionWasExtracted(self, fold, eD, recording,DLCinst):

        rec = recording.replace('/', '-')
        fName = self.analysisLocation + '%s_%s_%s_*%s.h5' % (self.mouse, fold, rec, DLCinst)
        fList = glob.glob(fName)
        #print(fName)
        #pdb.set_trace()
        if len(fList) > 1:
            print('more than one file exist matching the file pattern %s' % fName)
            return (False, None)
        elif len(fList) == 1:
            print('paw data extraced and saved in %s' % fList[0])
            return (True, fList[0])
        else:
            print('no extraced paw data found for %s' % fName)
            return (False, None)

    ############################################################
    def readRawData(self, fold, eD, recording, device, fData, readRawData=True):
        # recLocation = (self.dataBase2 + '/' + fold + '/' + recording + '/') if eD >= '181018' else (self.dataBase2 + '/' + fold + '/' + recording + '/')
        recLocation = self.dataBase + self.dataPCLocation[self.listOfAllExpts[self.mouse]['dates'][eD]['folders'][fold]['recComputer']] + fold + '/' + recording + '/'
        # print(recLocation)
        if device == 'RotaryEncoder':
            # data from activity monitor
            if len(fData['data']) == 1:
                angles = fData['data'][()][0]
            # data during high-res recording
            else:
                angles = fData['data'][4]
            times = fData['info/1/values'][()]
            try:
                startTime = fData['info/2/DAQ/ChannelA'].attrs['startTime']
            except:
                startTime = os.path.getctime(recLocation)
                monitor = True
            else:
                monitor = False
            return (angles, times, startTime, monitor)

        elif device == 'AxoPatch200_2':
            current = fData['data'][()][0]
            # pdb.set_trace()
            ephysTimes = fData['info/1/values'][()]
            # imageMetaInfo = self.readMetaInformation(recLocation)
            return (current, ephysTimes)

        elif device == 'Imaging':
            if readRawData:
                frames = fData['data'][()]
            else:
                frames = np.empty([2, 2])
            frameTimes = fData['info/0/values'][()]
            imageMetaInfo = self.readMetaInformation(recLocation)
            return (frames, frameTimes, imageMetaInfo)
        elif device == 'CameraGigEBehavior':
            print('reading raw GigE data ...')
            if readRawData:
                frames = fData['data'][()]
            else:
                frames = np.empty([2, 2])
            frameTimes = fData['info/0/values'][()]
            imageMetaInfo = self.readGigEMetaInformation(recLocation)
            print('done')
            return (frames, frameTimes, imageMetaInfo)
        elif device == 'CameraPixelfly':
            print('reading raw Pixelfly data ...', end=" ")
            if readRawData:
                frames = fData['data'][()]
            else:
                frames = np.empty([2, 2])
            frameTimes = fData['info/0/values'][()]
            xPixelSize = fData['info/3/pixelSize'].attrs['0']
            yPixelSize = fData['info/3/pixelSize'].attrs['1']
            xSize = fData['info/3/region'].attrs['2']
            ySize = fData['info/3/region'].attrs['3']
            imageMetaInfo = np.array([xSize * xPixelSize, ySize * yPixelSize, xPixelSize, yPixelSize])
            print('done')
            return (frames, frameTimes, imageMetaInfo)
        elif device == 'PreAmpInput' or device == 'frameTimes':
            values = fData['data'][()]
            valueTimes = fData['info/1/values'][()]
            return (values, valueTimes)

        elif device == 'pawTraces':
            pawF = h5py.File(fData, 'a')
            pawTraces = pawF['df_with_missing']['table'][()]
            pawTracesA = pawTraces.view((float, (len(pawTraces[0][1]) + 1)))
            pawTracesA[:, 0] = np.arange(len(pawTraces))

            pFileName = '%s*.pickle' % fData[:-3]
            pfList = glob.glob(pFileName)
            print(pfList)
            if len(pfList) == 1:
                #pdb.set_trace()
                pawMetaData = pickle.load(open(pfList[0], 'rb'))
            else:
                pawMetaData = None
            return (pawTracesA, pawMetaData)

    ############################################################
    def readMetaInformation(self, filePath):
        # convert to um
        conversion = 1.E6
        config = readConfigFile(filePath + '.index')
        pixWidth = config['.']['Scanner']['program'][0]['scanInfo']['pixelWidth'][0] * conversion
        pixHeight = config['.']['Scanner']['program'][0]['scanInfo']['pixelHeight'][0] * conversion
        dimensionXY = np.array(config['.']['Scanner']['program'][0]['roi']['size']) * conversion
        position = np.array(config['.']['Scanner']['program'][0]['roi']['pos']) * conversion

        if pixWidth == pixHeight:
            deltaPix = pixWidth
        else:
            print('Pixel height and width are not equal.')
            sys.exit(1)

        # print r'dimensions (x,y, pixelsize in um) : ', np.hstack((dimensionXY,deltaPix))
        return np.hstack((position, dimensionXY, deltaPix))  # self.h5pyTools.createOverwriteDS(dataGroup,dataSetName,hstack((dimensionXY,deltaPix)))

    ############################################################
    def readGigEMetaInformation(self, filePath):
        config = readConfigFile(filePath + '.index')
        starttime = config['.']['startTime']
        # pdb.set_trace()
        return starttime

    ############################################################
    def saveLEDPositionCoordinates(self, groupNames, coordinates):
        (test, grpHandle) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        # pdb.set_trace()
        self.h5pyTools.createOverwriteDS(grpHandle, 'LEDcoordinates', np.column_stack((coordinates[1], coordinates[2])), [['nLED', coordinates[0]], ['circleRadius', coordinates[3]]])

    ############################################################
    def readLEDPositionCoordinates(self, currentGroupName):
        temp = self.f[currentGroupName + '/LEDcoordinates'][()]
        posX = temp[:, 0]
        posY = temp[:, 1]
        nLED = self.f[currentGroupName + '/LEDcoordinates'].attrs['nLED']
        circleRadius = self.f[currentGroupName + '/LEDcoordinates'].attrs['circleRadius']
        coordinates = np.array([nLED, posX, posY, circleRadius], dtype=object)  # type is object to allow for different data structure of the entries, i.e., single number and array
        return coordinates

    ############################################################
    # foldersRecordings[f][0], foldersRecordings[f][1], foldersRecordings[f][2], r
    def checkForLEDPositionCoordinates(self, date, folder, recordings, r):
        # [foldersRecordings[f][0], foldersRecordings[f][2][r], 'behavior_video']
        currentGroupNames = [date, recordings[r], 'LEDinVideo']
        #pdb.set_trace()
        (currentGroupName, currentGrpHandle) = self.h5pyTools.getH5GroupName(self.f, currentGroupNames)
        # check if coordinates for current recording exist already
        try:
            currentLEDcoordinates = self.readLEDPositionCoordinates(currentGroupName)
        except KeyError:
            currentCoodinatesExist = False
        else:
            print('LED roi coordinates for current recording exist')
            currentCoodinatesExist = True
            return (currentCoodinatesExist, currentLEDcoordinates)
        if r > 0:
            previousGroupNames = [date, recordings[r - 1], 'LEDinVideo']
            (previousGroupName, previousGrpHandle) = self.h5pyTools.getH5GroupName(self.f, previousGroupNames)
            try:
                previousLEDcoordinates = self.readLEDPositionCoordinates(previousGroupName)  # self.f[previousGroupName+'/LEDcoordinates'][()]
            except KeyError:
                pass
            else:
                print('LED roi coordinates for previous recording exist')
                return (currentCoodinatesExist, previousLEDcoordinates)
        print('NO LED roi coordinates exist')
        return (currentCoodinatesExist, None)

    ############################################################
    def checkForErroneousFramesIdx(self, date, folder, recordings, r, determineAgain=False):
        # [foldersRecordings[f][0], foldersRecordings[f][2][r], 'behavior_video']
        currentGroupNames = [date, recordings[r], 'erroneousFrames']
        # print(currentGroupNames)
        # pdb.set_trace()
        try:
            (grpName, grpHandle) = self.h5pyTools.getH5GroupName(self.f, currentGroupNames)
            idxExc = self.f[grpName + '/idxToExclude'][()]
            # print(type(idxExc))
            idxExc = np.array(idxExc, dtype=int)  # check if coordinates for current recording exist already
        except KeyError:
            excludeIdxExist = False
            canBeUsed = True
            return (excludeIdxExist, None, canBeUsed)
        else:
            print('idx of erroneous frames exist')
            excludeIdxExist = True
            try:
                canBeUsed = self.f[grpName + '/canBeUsed'][()][0]
            except KeyError:
                canBeUsed = True
            if np.array_equal(idxExc, np.array([-1])):
                idxExc = np.array([], dtype=int)
            if determineAgain:
                return (False, idxExc,canBeUsed)
            else:
                return (excludeIdxExist, idxExc,canBeUsed)

    ############################################################
    def saveErroneousFramesIdx(self, groupNames, idxToExclude,canBeUsed=True):
        # [foldersRecordings[f][0], foldersRecordings[f][2][r], 'behavior_video']
        # print(groupNames)
        (grpName, grpHandle) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        # print(grpName,grpHandle)
        if len(idxToExclude) == 0:
            idxToExclude = np.array([-1])  # pdb.set_trace(
        self.h5pyTools.createOverwriteDS(grpHandle, 'idxToExclude', idxToExclude)
        self.h5pyTools.createOverwriteDS(grpHandle, 'canBeUsed', np.array([canBeUsed]))
        self.f.flush()
        print('saved successfully', idxToExclude)

    ############################################################
    # idxTimePoints, startEndExposureTime, startEndExposurepIdx, videoIdx, frameSummary, imageMetaInfo
    def saveBehaviorVideoTimeData(self, groupNames, idxTimePoints, startEndExposureTime, startEndExposurepIdx, videoIdx, frameSummary, imageMetaInfo):
        # framesDuringRecording, startEndFrameTime, startEndFrameIdx, imageMetaInfo)
        # self.saveBehaviorVideoData([date,rec,'behavior_video'], framesRaw,expStartTime, expEndTime, imageMetaInfo)
        (test, grpHandle) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        # self.h5pyTools.createOverwriteDS(grpHandle,'behaviorFrames',len(frames))
        # pdb.set_trace()
        self.h5pyTools.createOverwriteDS(grpHandle, 'indexVideo', videoIdx, ['startTime', imageMetaInfo])
        self.h5pyTools.createOverwriteDS(grpHandle, 'indexTimePoints', idxTimePoints)
        self.h5pyTools.createOverwriteDS(grpHandle, 'startEndExposureTime', startEndExposureTime)
        self.h5pyTools.createOverwriteDS(grpHandle, 'startEndExposurepIndex', startEndExposurepIdx)
        self.h5pyTools.createOverwriteDS(grpHandle, 'frameDropExcludeSummary', frameSummary)

    ############################################################
    def readBehaviorVideoTimeData(self, groupNames):
        # self.saveBehaviorVideoData([date,rec,'behavior_video'], framesRaw,expStartTime, expEndTime, imageMetaInfo)
        (grpName, grpHandle) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        videoIdx = self.f[grpName + '/indexVideo'][()]
        imageMetaInfo = self.f[grpName + '/indexVideo'].attrs['startTime']
        idxTimePoints = self.f[grpName + '/indexTimePoints'][()]
        startEndExposureTime = self.f[grpName + '/startEndExposureTime'][()]
        startEndExposurepIdx = self.f[grpName + '/startEndExposurepIndex'][()]
        frameSummary = self.f[grpName + '/frameDropExcludeSummary'][()]
        return (idxTimePoints, startEndExposureTime, startEndExposurepIdx, videoIdx, frameSummary, imageMetaInfo)

    ############################################################
    def getBehaviorVideoFrames(self, groupNames):
        (grpName, test) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        print(grpName)
        #pdb.set_trace()  [date,rec,'behavior_video']
        try:
            firstLastRecordedFrame = self.f[grpName + '/firstLastRecordedFrame'][()]
        except:  # before, first and last name was stored under a different name
            groupNames[2] = 'behavior_video'
            (grpName, test) = self.h5pyTools.getH5GroupName(self.f, groupNames)
            firstLastRecordedFrame = self.f[grpName + '/firstLastRecordedFrame'][()]
        else:
            pass

        return firstLastRecordedFrame

    ############################################################
    # self.saveBehaviorVideoData([date,rec,'behavior_video'],  framesRaw,videoIdx,startEndExposureTime, imageMetaInfo)
    # [foldersRecordings[f][0], foldersRecordings[f][2][r],'behavior_video']
    def saveBehaviorVideoFrames(self, groupNames,  framesRaw, videoIdx ):
        (grpName, grpHandle) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        #print(grpName)
        #self.h5pyTools.createOverwriteDS(grpHandle, 'startEndExposureTime', startEndExposureTime,['imageMetaInfo', imageMetaInfo])
        self.h5pyTools.createOverwriteDS(grpHandle, 'firstLastRecordedFrame', np.array((framesRaw[videoIdx[0]],framesRaw[videoIdx[-1]])))

    ############################################################
    def saveImageStack(self, frames, fTimes, imageMetaInfo, groupNames, motionCorrection=[]):
        (grpName, grpHandle) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        self.h5pyTools.createOverwriteDS(grpHandle, 'caImaging', frames)
        self.h5pyTools.createOverwriteDS(grpHandle, 'caImagingTime', fTimes)
        self.h5pyTools.createOverwriteDS(grpHandle, 'caImagingField', imageMetaInfo)
        if len(motionCorrection) > 1:
            self.h5pyTools.createOverwriteDS(grpHandle, 'motionCoordinates', motionCorrection)

    ############################################################
    def readImageStack(self, groupNames):
        (grpName, test) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        frames = self.f[grpName + '/caImaging'][()]
        fTimes = self.f[grpName + '/caImagingTime'][()]
        imageMetaInfo = self.f[grpName + '/caImagingField'][()]
        return (frames, fTimes, imageMetaInfo)

    ############################################################
    def saveWalkingActivity(self, angularSpeed, linearSpeed, wTimes, angles, aTimes, startTime, monitor, groupNames):
        (grpName, grpHandle) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        self.h5pyTools.createOverwriteDS(grpHandle, 'angularSpeed', angularSpeed, ['monitor', monitor])
        self.h5pyTools.createOverwriteDS(grpHandle, 'linearSpeed', linearSpeed)
        self.h5pyTools.createOverwriteDS(grpHandle, 'walkingTimes', wTimes, ['startTime', startTime])
        self.h5pyTools.createOverwriteDS(grpHandle, 'anglesTimes', np.column_stack((aTimes, angles)), ['startTime', startTime])

    ############################################################
    def getWalkingActivity(self, groupNames):
        (grpName, grpHandle) = self.h5pyTools.getH5GroupName(self.f, groupNames)
        print(grpName)
        angularSpeed = self.f[grpName + '/angularSpeed'][()]
        monitor = self.f[grpName + '/angularSpeed'].attrs['monitor']
        linearSpeed = self.f[grpName + '/linearSpeed'][()]
        wTimes = self.f[grpName + '/walkingTimes'][()]
        startTime = self.f[grpName + '/walkingTimes'].attrs['startTime']
        angleTimes = self.f[grpName + '/anglesTimes'][()]
        return (angularSpeed, linearSpeed, wTimes, startTime, monitor, angleTimes)

    ############################################################
    def getPawRungPickleData(self, date, rec):
        frontpawPos = pickle.load(open(self.analysisLocation + '%s_%s_%s_frontpawLocations.p' % (self.mouse, date, rec), 'rb'))
        hindpawPos = pickle.load(open(self.analysisLocation + '%s_%s_%s_hindpawLocations.p' % (self.mouse, date, rec), 'rb'))
        rungs = pickle.load(open(self.analysisLocation + '%s_%s_%s_rungPositions.p' % (self.mouse, date, rec), 'rb'))
        return (frontpawPos, hindpawPos, rungs)

    ############################################################
    def readMetaDataFileAndReadSetting(self, tiffFile, keyWord):
        metData = ScanImageTiffReader(tiffFile).metadata()
        #pdb.set_trace()
        keyWordIdx = metData.find(keyWord)
        splitString = re.split('=|\n', metData[keyWordIdx:])
        keyWordParameter = float(splitString[1])
        return keyWordParameter

    ############################################################
    def readTimeStampOfFrame(self, tiffFileObj, nFrame):
        # frameNumberAcqMode
        # numeric, number of frame, counted from beginning of acquisition mode
        #
        # frameNumberAcq
        # numeric, number of frame in current acquisition
        #
        # acqNumber
        # numeric, number of current acquisition
        #
        # epochAcqMode
        # string, time of the acquisition of the acquisiton of the first pixel in the current acqMode; format: output of datestr(now) '25-Jul-2014 12:55:21'
        #
        # frameTimestamp
        # [s] time of the first pixel in the frame passed since acqModeEpoch
        #
        # acqStartTriggerTimestamp
        # [s] time of the acq start trigger for the current acquisition
        #
        # nextFileMarkerTimestamp
        # [s] time of the last nextFileMarker recorded. NaN if no nextFileMarker was recorded

        desc = tiffFileObj.description(nFrame)
        keyWordIdx = desc.find('epoch')  # string, time of the acquisition of the acquisiton of the first pixel in the current acqMode; format: output of datestr(now) '25-Jul-2014 12:55:21'
        dateString = re.split('\[|\]', desc[keyWordIdx:])
        dateIdv = dateString[1].split()
        # print(dateIdv)
        unixStartTime = int(datetime.datetime(int(dateIdv[0]), int(dateIdv[1]), int(dateIdv[2]), int(dateIdv[3]), int(dateIdv[4]), int(float(dateIdv[5]))).strftime('%s'))
        #
        keyWordIdx = desc.find('frameTimestamps_sec')  # [s] time of the first pixel in the frame passed since acqModeEpoch
        splitString = re.split('=|\n', desc[keyWordIdx:])
        frameTimestamps = float(splitString[1])

        keyWordIdx = desc.find('acqTriggerTimestamps_sec')  # [s] time of the acq start trigger for the current acquisition
        splitString = re.split('=|\n', desc[keyWordIdx:])
        acqTriggerTimestamps = float(splitString[1])

        keyWordIdx = desc.find('frameNumberAcquisition')  # numeric, number of frame in current acquisition
        splitString = re.split('=|\n', desc[keyWordIdx:])
        frameNumberAcquisition = int(splitString[1])

        keyWordIdx = desc.find('acquisitionNumbers')  # numeric, number of current acquisition
        splitString = re.split('=|\n', desc[keyWordIdx:])
        acquisitionNumbers = int(splitString[1])

        unixFrameTime = unixStartTime + frameTimestamps
        # print(tiffFile,unixTime)
        return ([frameNumberAcquisition, acquisitionNumbers, unixStartTime, unixFrameTime, frameTimestamps, acqTriggerTimestamps])
    ############################################################
    def getRawCalciumImagingData(self, tiffList,saveDir):
        imagingData = []
        timeStamps = []
        for i in range(len(tiffList)):
            # data = ScanImageTiffReader(tiffPaths[i]).data()
            tiffFileObject = ScanImageTiffReader(tiffList[i])
            data = tiffFileObject.data()
            # pdb.set_trace()
            fN = np.shape(data)[0]
            # frameNumbers.append(fN)
            for n in range(fN):
                timeStamps.append(self.readTimeStampOfFrame(tiffFileObject, n))
            timeStampsASingle = np.asarray(timeStamps)
            imagingData.append([i, data, timeStampsASingle])
        timeStampsA = np.asarray(timeStamps)
        return (imagingData, timeStampsA)  # np.save(saveDir+'/suite2p/plane0/timeStamps.npy',timeStampsA)

    ############################################################
    def getAnalyzedCaImagingData(self, analysisLocation, tiffList):

        timeStamps = []
        for i in range(len(tiffList)):
            zF = self.readMetaDataFileAndReadSetting(tiffList[i], 'scanZoomFactor')
            if i == 0:
                zFold = zF
            else:
                if zF != zFold:
                    print('scanZoomFactor is not the same between recordings!')
            tiffFileObject = ScanImageTiffReader(tiffList[i])
            timeS = self.readTimeStampOfFrame(tiffFileObject, 0)
            timeStamps.append(timeS[3])
        #
        nDirs = 0
        for name in glob.glob(analysisLocation+'_suite2p*'):
            caAnalysisLocation = name
            print(name)
            nDirs+=1
        if nDirs > 1:
            print('There are more than one matching directory!')
            pdb.set_trace()
        #caAnalysisLocation = eSD.analysisLocation+foldersRecordings[f][0]+'_suite2p/'
        if os.path.isdir(caAnalysisLocation):
            ops = np.load(caAnalysisLocation + '/suite2p/plane0/ops.npy',allow_pickle=True)
            ops = ops.item()
            nframes = ops['nframes']
            meanImg = ops['meanImg']
            try:
                meanImgE = ops['meanImgE']
            except KeyError:
                print('no enhanced image available ... using normal image')
                meanImgE = np.copy(meanImg)
            # pdb.set_trace()
            return (nframes, meanImg, meanImgE, zF, timeStamps)
        else:
            print('Ca imaging data has not been analyzed wiht suite2p yet!')

    ############################################################
    def extractAndSaveCaTimeStamps(self, dataDir, saveDir, tiffPaths):

        (_,timeStampsA) = self.getRawCalciumImagingData(tiffPaths, saveDir)
        np.save(saveDir + '/suite2p/plane0/timeStamps.npy', timeStampsA)

    ############################################################
    def getCaImagingRoiData(self, caAnalysisLocation, tiffList):
        # frameNumbers = []
        # timeStamps = []
        # for i in range(len(tiffList)):
        #    data = ScanImageTiffReader(tiffList[i]).data()
        #    fN = np.shape(data)[0]
        #    frameNumbers.append(fN)
        #    for n in range(fN):
        #        timeStamps.append(self.readTimeStampOfRecording(tiffList[i],n))

        # pdb.set_trace()

        if os.path.isdir(caAnalysisLocation):
            timeStamps = np.load(caAnalysisLocation + '/suite2p/plane0/timeStamps.npy')
            F = np.load(caAnalysisLocation + '/suite2p/plane0/F.npy')
            Fneu = np.load(caAnalysisLocation + '/suite2p/plane0/Fneu.npy')
            ops = np.load(caAnalysisLocation + '/suite2p/plane0/ops.npy',allow_pickle=True)
            ops = ops.item()
            iscell = np.load(caAnalysisLocation + '/suite2p/plane0/iscell.npy')
            stat = np.load(caAnalysisLocation + '/suite2p/plane0/stat.npy',allow_pickle=True)

            # pdb.set_trace()
            nRois = np.arange(len(F))
            realCells = (iscell[:, 0] == 1)
            nRois = nRois[realCells]
            Fluo = F[realCells] - 0.7 * Fneu[realCells]  # substract neuropil data
            stat = stat[realCells]
            # pdb.set_trace()
            return (Fluo, nRois, ops, timeStamps, stat)

    ############################################################
    def saveTif(self, frames, mouse, date, rec, norm=None):
        img_stack_uint8 = np.array(frames, dtype=np.uint8)
        if norm:
            tiff.imsave(self.analysisLocation + '%s_%s_%s_ImageStack%s.tif' % (mouse, date, rec, norm), img_stack_uint8)
        else:
            tiff.imsave(self.analysisLocation + '%s_%s_%s_ImageStack.tif' % (mouse, date, rec), img_stack_uint8)

    ############################################################
    def readTif(self, frames, mouse, date, rec):
        img_stack_uint8 = np.array(frames[:, :, :, 0], dtype=np.uint8)
        tiff.imsave(self.analysisLocation + '%s_%s_%s_ImageStack.tif' % (mouse, date, rec), img_stack_uint8)

    ############################################################
    def readPawTrackingData(self, date, rec, DLCinstance):
        rec = rec.replace('/', '-')
        (grpName, grpHandle) = self.h5pyTools.getH5GroupName(self.f, [date, rec, 'pawTrackingData',DLCinstance])
        # pdb.set_trace()
        rawPawPositionsFromDLC = self.f[grpName + '/rawPawPositionsFromDLC'][()]
        # self.h5pyTools.createOverwriteDS(grpHandle, 'croppingParameters', np.array(cropping))
        croppingParameters = self.f[grpName + '/croppingParameters'][()]
        pawTrackingOutliers = []
        jointNamesFramesInfo = []
        pawSpeed = []
        rawPawSpeed = []
        cPawPos = []
        for i in range(4):
            pTTemp = self.f[grpName + '/pawTrackingOutliers%s' % i][()]
            pawTrackingOutliers.append(pTTemp)
            jNTemp = 'paw %s' % i #self.f[grpName + '/pawTrackingOutliers%s' % i].attrs['PawID']
            jointNamesFramesInfo.append(jNTemp)
            pStemp = self.f[grpName + '/clearedPawSpeed%s' % i][()]
            pawSpeed.append(pStemp)
            pPPtemp = self.f[grpName + '/clearedXYPos%s' % i][()]
            cPawPos.append(pPPtemp)
            rPStemp = self.f[grpName + '/rawPawSpeed%s' % i][()]
            rawPawSpeed.append(rPStemp)
            if i == 0:
                recStartTime = self.f[grpName + '/clearedPawSpeed%s' % i].attrs['recStartTime']

        return (rawPawPositionsFromDLC, pawTrackingOutliers, jointNamesFramesInfo, pawSpeed, recStartTime, rawPawSpeed, cPawPos, croppingParameters)

    ############################################################
    # savePawTrackingData(mouse,foldersRecordings[f][0],foldersRecordings[f][2][r],DLCinstance,pawTrackingOutliers,pawMetaData,startEndExposureTime,imageMetaInfo,generateVideo=False)
    def savePawTrackingData(self, mouse, date, rec, DLCinstance, pawPositions, pawTrackingOutliers, pawMetaData, startEndExposureTime, startTime, generateVideo=True):
        # pdb.set_trace()
        jointNames = pawMetaData['data']['DLC-model-config file']['all_joints_names']
        jointIdx = pawMetaData['data']['DLC-model-config file']['all_joints']
        cropping = pawMetaData['data']['cropping_parameters']
        print('cropping parameters', cropping)
        # pdb.set_trace()
        rec = rec.replace('/', '-')
        (test, grpHandle) = self.h5pyTools.getH5GroupName(self.f, [date, rec, 'pawTrackingData', DLCinstance])
        self.h5pyTools.createOverwriteDS(grpHandle, 'rawPawPositionsFromDLC', pawPositions)
        self.h5pyTools.createOverwriteDS(grpHandle, 'croppingParameters', np.array(cropping))
        timeArray = np.average(startEndExposureTime,axis=1) # use the 'middle' of the exposure time as time-point of the frame
        for i in range(4):
            pawMask = pawTrackingOutliers[i][3]
            #pdb.set_trace()
            rawPawSpeed = np.sqrt((np.diff(pawPositions[:, (i * 3 + 1)])) ** 2 + (np.diff(pawPositions[:, (i * 3 + 2)])) ** 2) / np.diff(timeArray)
            rawSpeedTime = (timeArray[:-1] + timeArray[1:]) / 2.
            clearedPawSpeed = np.sqrt((np.diff(pawPositions[:, (i * 3 + 1)][pawMask])) ** 2 + (np.diff(pawPositions[:, (i * 3 + 2)][pawMask])) ** 2) / np.diff(timeArray[pawMask])
            clearedPawXSpeed = np.diff(pawPositions[:, (i * 3 + 1)][pawMask]) / np.diff(timeArray[pawMask])
            clearedPawYSpeed = np.diff(pawPositions[:, (i * 3 + 2)][pawMask]) / np.diff(timeArray[pawMask])
            clearedSpeedTime = (timeArray[pawMask][:-1] + timeArray[pawMask][1:]) / 2.
            clearedPosIdx = np.arange(len(pawPositions))[pawMask]
            clearedXYPos = np.column_stack((timeArray[pawMask], pawPositions[:, (i * 3 + 1)][pawMask], pawPositions[:, (i * 3 + 2)][pawMask], clearedPosIdx))
            clearedSpeedIdx = np.array((clearedPosIdx[:-1] + clearedPosIdx[1:]) / 2., dtype=int)
            #pdb.set_trace()
            self.h5pyTools.createOverwriteDS(grpHandle, 'pawTrackingOutliers%s' % i, pawTrackingOutliers[i][3],[['PawID', jointNames[i]],['pawTrackingOutliers',np.array([ pawTrackingOutliers[i][1], pawTrackingOutliers[i][2]])]])#, pawTrackingOutliers[i][1], pawTrackingOutliers[i][2]]])
                                             #['PawID', [jointNames[i], pawTrackingOutliers[i][1], pawTrackingOutliers[i][2]]])
            self.h5pyTools.createOverwriteDS(grpHandle, 'rawPawSpeed%s' % i, np.column_stack((rawSpeedTime, rawPawSpeed)), ['recStartTime', startTime])
            self.h5pyTools.createOverwriteDS(grpHandle, 'clearedPawSpeed%s' % i, np.column_stack((clearedSpeedTime, clearedPawSpeed, clearedPawXSpeed, clearedPawYSpeed, clearedSpeedIdx)),
                                             ['recStartTime', startTime])
            self.h5pyTools.createOverwriteDS(grpHandle, 'clearedXYPos%s' % i, clearedXYPos)  # pdb.set_trace()
        if generateVideo:
            fps = 80
            width = 800
            heigth = 600
            colors = [(255, 0, 255), (255, 0, 0), (255, 255, 0), (0, 0, 255)]
            indicatorPositions = [(270, 15), (270, 35), (240, 35), (240, 15)]
            sourceVideoFileName = self.analysisLocation + '%s_%s_%s_raw_behavior.avi' % (mouse, date, rec)
            outputVideoFileName = self.analysisLocation + '%s_%s_%s_paw_tracking.avi' % (mouse, date, rec)
            fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')
            # fourcc = cv2.VideoWriter_fourcc('H','F','Y','U')
            # print('1',fourcc,fps,width,heigth)
            out = cv2.VideoWriter(outputVideoFileName, fourcc, fps, (width, heigth))
            source = cv2.VideoCapture(sourceVideoFileName)
            nFrame = 0
            while (source.isOpened()):
                ret, frame = source.read()
                if ret == True:
                    # cv2.putText(frame, 'time %s sec' % round(midFrameTimes[i],4), (10,20), cv2.QT_FONT_NORMAL, 0.6, color=(220, 220, 220))
                    for i in range(4):
                        # print(int(pawPositions[nFrame, 3 * i + 1]+0.5),int(pawPositions[nFrame, 3 * i + 2]+0.5))
                        cv2.drawMarker(frame, (cropping[0] + int(pawPositions[nFrame, 3 * i + 1] + 0.5), cropping[2] + int(pawPositions[nFrame, 3 * i + 2] + 0.5)), colors[i],
                                       cv2.MARKER_CROSS, 20, 2)
                        if nFrame in pawTrackingOutliers[i][3]:
                            cv2.circle(frame, indicatorPositions[i], 7, (0, 255, 0), -1)
                        else:
                            cv2.circle(frame, indicatorPositions[i], 7, (0, 0, 255), -1)
                    out.write(frame)
                else:
                    break
                # print(nFrame)
                nFrame += 1
            out.release()
            source.release()

    ############################################################
    # (mouse, foldersRecordings[f][0], foldersRecordings[f][2][r], framesDuringRecording, expStartTime, expEndTime, imageMetaInfo)
    def saveBehaviorVideo(self, mouse, date, rec, framesRaw, idxTimePoints, startEndExposureTime, startEndExposurepIdx, videoIdx, frameSummary, imageMetaInfo):
        # [foldersRecordings[f][0],foldersRecordings[f][2][r],'walking_activity']
        midFrameTimes = (startEndExposureTime[:, 0] + startEndExposureTime[:, 1]) / 2.
        #pdb.set_trace()
        # img_stack_uint8 = np.array(frames[:, :, :, 0], dtype=np.uint8)
        # tiff.imsave(self.analysisLocation + '%s_%s_%s_ImageStack.tif' % (mouse, date, rec), img_stack_uint8)
        # replace possible backslashes from subdirectory structure and
        rec = rec.replace('/', '-')
        videoFileName = self.analysisLocation + '%s_%s_%s_raw_behavior.avi' % (mouse, date, rec)
        # cap = cv2.VideoCapture(self.analysisLocation + '%s_%s_%s_behavior.avi' (mouse, date, rec))

        vLength = np.shape(framesRaw)[0]
        width = np.shape(framesRaw)[1]
        heigth = np.shape(framesRaw)[2]

        # print('number of frames :', vLength, width, heigth)
        fps = 80
        # pdb.set_trace()
        # Define the codec and create VideoWriter object
        # fourcc = cv2.VideoWriter_fourcc(*'DIVX')  # (*'XVID')
        # fourcc = cv2.VideoWriter_fourcc(*'MPEG')  # (*'XVID')
        # M J P G is working great !! 184 MB per video
        # H F Y U is working great !! 2.7 GB per video
        # M P E G has issues !! DON'T USE (frames are missing)
        # X V I D : frame 3001 missing and last nine frames are screwed
        # 0 (no compression) : frame 3001 missing last 2 frames are the same
        # fourcc = cv2.VideoWriter_fourcc('H','F','Y','U')
        fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')  # cv2.VideoWriter_fourcc(*'MPEG') # 'HFYU' is a lossless codec, alternatively use 'MPEG'
        out = cv2.VideoWriter(videoFileName, fourcc, fps, (width, heigth))
        # pdb.set_trace()
        for i in np.arange(len(videoIdx)):
            frame8bit = np.array(np.transpose(framesRaw[videoIdx[i]]), dtype=np.uint8)
            frame = cv2.cvtColor(frame8bit, cv2.COLOR_GRAY2BGR)
            cv2.putText(frame, 'time %s sec' % round(midFrameTimes[i], 4), (10, 20), cv2.QT_FONT_NORMAL, 0.6, color=(220, 220, 220))
            cv2.putText(frame, 'frame %04d / %s' % (videoIdx[i], (vLength - 1)), (10, 40), cv2.QT_FONT_NORMAL, 0.6, color=(220, 220, 220))
            out.write(frame)
        # Release everything if job is finished
        # cap.release()
        out.release()
        cv2.destroyAllWindows()

    ############################################################
    # (mouse, foldersRecordings[f][0], foldersRecordings[f][2][r], framesDuringRecording, expStartTime, expEndTime, imageMetaInfo)
    def saveBehaviorVideoWithCa(self, mouse, date, rec, framesRaw, expStartTime, expEndTime, imageMetaInfo, angles, aTimes):
        # [foldersRecordings[f][0],foldersRecordings[f][2][r],'walking_activity']
        # self.saveBehaviorVideoData([date,rec,'behavior_video'], framesRaw,expStartTime, expEndTime, imageMetaInfo)
        midFrameTimes = (expStartTime + expEndTime) / 2.
        # pdb.set_trace()
        # img_stack_uint8 = np.array(frames[:, :, :, 0], dtype=np.uint8)
        # tiff.imsave(self.analysisLocation + '%s_%s_%s_ImageStack.tif' % (mouse, date, rec), img_stack_uint8)
        # replace possible backslashes from subdirectory structure and
        rec = rec.replace('/', '-')
        videoFileName = self.analysisLocation + '%s_%s_%s_raw_behavior_withCa.avi' % (mouse, date, rec)
        # cap = cv2.VideoCapture(self.analysisLocation + '%s_%s_%s_behavior.avi' (mouse, date, rec))
        caImg = io.imread('/media/HDnyc_data/data_analysis/in_vivo_cerebellum_walking/LocoRungsData/190101_f15/2019.03.21_000_suite2p_reg/suite2p/plane0/reg_tif/AVG2_output_1-902C.tif')
        # pdb.set_trace()
        caImg = (caImg - np.min(caImg)) * 255. / (np.max(caImg) - np.min(caImg))
        # caImg = (caImg - 15.)*255./(30.-15.)
        vLength = np.shape(framesRaw)[0]
        width = np.shape(framesRaw)[1]
        heigth = np.shape(framesRaw)[2]

        print('number of frames :', vLength, width, heigth)
        fps = 250
        fBehavior = 200
        fCa = 15
        afterEvery = fBehavior / fCa
        # pdb.set_trace()
        ##########################################################
        # create walking dynamics figure instance
        fig0 = plt.figure(figsize=(6.56, 3.5))
        plt.subplots_adjust(left=0.18, right=0.95, top=0.97, bottom=0.23)
        # x1 = 0.
        # y1 = 0.
        ax0 = fig0.add_subplot(1, 1, 1)
        ax0 = plt.axes(xlim=(0, 30), ylim=(-10, 130))
        line0, = ax0.plot([], [], 'k-', lw=2)
        ax0.set_ylabel('position (cm)', fontsize=18)
        ax0.set_xlabel('time (s)', fontsize=18)

        ax0.spines['top'].set_visible(False)
        ax0.spines['right'].set_visible(False)
        #
        ax0.spines['bottom'].set_position(('outward', 10))
        ax0.xaxis.set_ticks_position('bottom')
        #
        # if xyInvisible[1]:
        # ax.spines['left'].set_visible(False)
        # ax.yaxis.set_visible(False)
        # else:
        ax0.spines['left'].set_position(('outward', 10))
        ax0.yaxis.set_ticks_position('left')

        ##########################################################
        # create calcium dynamics figure instance
        fig = plt.figure(figsize=(6.56, 3.5))
        plt.subplots_adjust(left=0.18, right=0.95, top=0.97, bottom=0.23)
        # x1 = 0.
        # y1 = 0.
        ax = fig.add_subplot(1, 1, 1)
        ax = plt.axes(xlim=(0, 30), ylim=(45, 170))
        line1, = ax.plot([], [], 'g-', lw=2)
        line2, = ax.plot([], [], 'b-', lw=2)
        line3, = ax.plot([], [], 'r-', lw=2)
        ax.set_ylabel('fluorescence (a.u.)', fontsize=18)
        ax.set_xlabel('time (s)', fontsize=18)

        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        #
        ax.spines['bottom'].set_position(('outward', 10))
        ax.xaxis.set_ticks_position('bottom')
        #
        # if xyInvisible[1]:
        # ax.spines['left'].set_visible(False)
        # ax.yaxis.set_visible(False)
        # else:
        ax.spines['left'].set_position(('outward', 10))
        ax.yaxis.set_ticks_position('left')
        # ax.set_ylabel('fluorescence (a.u.)')
        # Define the codec and create VideoWriter object
        # fourcc = cv2.VideoWriter_fourcc(*'DIVX')  # (*'XVID')
        # fourcc = cv2.VideoWriter_fourcc(*'MPEG')  # (*'XVID')
        # M J P G is working great !! 184 MB per video
        # H F Y U is working great !! 2.7 GB per video
        # M P E G has issues !! DON'T USE (frames are missing)
        # X V I D : frame 3001 missing and last nine frames are screwed
        # 0 (no compression) : frame 3001 missing last 2 frames are the same
        # fourcc = cv2.VideoWriter_fourcc('H','F','Y','U')
        fourcc = cv2.VideoWriter_fourcc('M', 'J', 'P', 'G')  # cv2.VideoWriter_fourcc(*'MPEG') # 'HFYU' is a lossless codec, alternatively use 'MPEG'
        out = cv2.VideoWriter(videoFileName, fourcc, fps, (width + 512, heigth + 350))
        nCa = 1
        ttime = []
        ffluo = [[], [], []]
        colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255)]
        rois = [[464, 452, 11], [329, 99, 11], [318, 235, 11],  # [446,389,11]
                ]
        nPos = 0
        for i in np.arange(len(framesRaw)):
            output = np.zeros((heigth + 350, width + 512, 3), dtype="uint8")
            frame8bit = np.array(np.transpose(framesRaw[i]), dtype=np.uint8)
            frame = cv2.cvtColor(frame8bit, cv2.COLOR_GRAY2BGR)
            # cv2.circle(frame, (100, 100), 50, (0, 255, 0), -1)
            cv2.putText(frame, 'time %s sec' % round(midFrameTimes[i], 4), (10, 20), cv2.QT_FONT_NORMAL, 0.6, color=(220, 220, 220))
            cv2.putText(frame, 'frame %04d / %s' % (i, (vLength - 1)), (10, 40), cv2.QT_FONT_NORMAL, 0.6, color=(220, 220, 220))
            output[0:heigth, 0:width, :] = frame
            ttime.append(midFrameTimes[i])

            #####################################
            mask0 = aTimes < midFrameTimes[i]
            line0.set_data(aTimes[mask0], angles[mask0] * 80. / 360.)
            fig0.canvas.draw()
            img0 = np.fromstring(fig0.canvas.tostring_rgb(), dtype=np.uint8, sep='')
            img0 = img0.reshape(fig0.canvas.get_width_height()[::-1] + (3,))
            # img is rgb, convert to opencv's default bgr
            img0 = cv2.cvtColor(img0, cv2.COLOR_RGB2BGR)
            output[600:, :656, :] = img0
            ####################################
            # treatment of ca image
            frameCa8bit = np.array(caImg[nCa - 1], dtype=np.uint8)
            frameCa = cv2.cvtColor(frameCa8bit, cv2.COLOR_GRAY2BGR)

            for n in range(len(rois)):
                cv2.circle(frameCa, (rois[n][0], rois[n][1]), rois[n][2], colors[n], 2)
            cv2.putText(frameCa, '50 um', (35, 480), cv2.QT_FONT_NORMAL, 0.55, color=(220, 220, 220))
            cv2.line(frameCa, (30, 490), (94, 490), (220, 220, 220), 4)
            output[44:(44 + 512), (width):(width + 512), :] = frameCa
            # plot roi and add to movie
            for n in range(len(rois)):
                mask = np.zeros(shape=frameCa.shape, dtype="uint8")
                cv2.circle(mask, (rois[n][0], rois[n][1]), rois[n][2], (255, 255, 255), -1)
                maskedImg = cv2.bitwise_and(src1=frameCa, src2=mask)
                ffluo[n].append(np.mean(maskedImg[maskedImg > 0]))

            #########################################
            line1.set_data(ttime, ffluo[0])
            line2.set_data(ttime, ffluo[1])
            line3.set_data(ttime, ffluo[2])
            fig.canvas.draw()
            img = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
            img = img.reshape(fig.canvas.get_width_height()[::-1] + (3,))
            # img is rgb, convert to opencv's default bgr
            img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)

            ##########################################
            output[600:, 656:(656 + 656), :] = img
            # pdb.set_trace()
            #
            out.write(output)
            if (i > nCa * afterEvery) and nCa < (len(caImg)):
                nCa += 1
                print(i, nCa, nCa * afterEvery)
        # Release everything if job is finished
        # cap.release()
        out.release()
        cv2.destroyAllWindows()


import sys

arr = [[0] * 101 for _ in range(101)]               # 좌표는 1이상 100이하

N = 4                                               # 좌표 입력 4줄
for _ in range(N):
    nums = list(map(int, sys.stdin.readline().split()))

    for i in range(nums[0], nums[2]):               # 입력된 좌표에 맞는 값에 1 넣어주기 (합집합이니까)
        for j in range(nums[1], nums[3]):
            arr[i][j] = 1

res = 0
for i in range(101):                                # 배열 돌면서 1 있을때 마다 +1
    for j in range(101):
        if arr[i][j] == 1:
            res += 1

print(res)

백준/Bronze/2669. 직사각형 네개의 합집합의 면적 구하기/직사각형 네개의 합집합의 면적 구하기.py

직사각형 네개의 합집합의 면적 구하기.py