from typing import Awaitable, Callable, T, Optional
from fastapi import HTTPException

from system.transporters.Result import Result
from system.util.HttpUtils import handleError, returnErrorCheckResolver

async def middlewareRunner(data, method, middleware: Optional[Callable[..., Awaitable[T]]]|None):
    result = Result()
    if middleware is not None:
        try:
            print("[ Middleware ] Starting Runner ", data , type(data) )
            result = await middleware(result,data)
            print("[ Middleware ] middleware result ", result)
            if(result.status=="success"):
                print("[ Middleware ] middleware success ",middleware, result)
                print("[ Middleware ] middleware returned ", result.data)
                result.build("middlewareData",result.data)
                data.build("data", result.data)
                data =  await method(data)
                print("[ Middleware ] method returned from middle ware with data ", method, data)
                if returnErrorCheckResolver(data):
                    return data
                result.build("data", data)
                return result
            elif(result.status=="error" and result.clientErrorMessage is not None and len(result.clientErrorMessage)  > 0 ):
                print("[ Middleware ] type of clienterrormessage ",type(result.clientErrorMessage))
                print("[ Middleware ] middleware error ", result)
                return result
            else:
                print("[ Middleware ] error getting result from middleware, check middleware function, it must return an object with status and  data if status is success, or error if status is an error ")
                return Result().build("clientErrorMessage",{"Middleware returned malformed response."}).build("status","error").build("error",{"Middleware function returned malformed, if status success please return data :  { object | dict | string } , if status error please return error: { object | dict | string } and clientErrorMessage:  { object | dict | string } "})

        except Exception as error:
            print("[ Middleware ] error in middleware runner with middleware as ... and method as ... ", middleware, method, error)
            return await handleError(error, "[ Middleware ] Error in middleware, Exception.")
    else:
        try:
            dataInner = await method(data)
            print("[ Middleware ] returned middleware runner, with middleware injected middleware as NONE INJECTED ",method, dataInner)
            result.build("data",dataInner)
            result.build("status","success")
            return dataInner
        except (Exception) as error:
            print("[ Middleware ]  error in middleware runner with middleware as NONE INJECTED ", middleware, method, error)
            return await handleError(error, "[ Middleware ] Error in middleware.")


from MageneApp.models import INDIV,FAM,SOURCES
from django.shortcuts import render
from pathlib import Path
import os
# Create your views here.
# 65 in char is A
# 90 in char is Z
#for i in range(65, 91):
#    print(chr(i), end=" ")
def patro(request):
    ListAlph=[]
    for i in range(65, 91):
        ListAlph.append(chr(i))
    if 'letter' in request.GET:
        ll=request.GET['letter']
        listenom=INDIV.objects.values('NOM').filter(NOM__startswith=ll).order_by('NOM').distinct()
        return render(request, "patro.html", {'letter_list': ListAlph, 'toto': listenom})
    else:
        return render(request, "patro.html", {'letter_list': ListAlph})

def index(request):
    BASE_DIR = Path(__file__).resolve().parent.parent
    toto=os.path.join(BASE_DIR, 'MageneApp','static')
    return render(request, "index.html", {'toto': toto,'titi': BASE_DIR})

def getMar(idMar,Genre):
    ret=''
    if Genre=='F':
     listeMar=FAM.objects.filter(INDF=idMar)
     for indF in listeMar:
         if indF.INDM_id is not None:
            x1=INDIV.objects.get(CODEI=indF.INDM_id)
            ret=' X (' + str(indF.DATEM if indF.DATEM is not None else '') + ') ' + str(x1.NOM) + ' ' + str(x1.PRENOM if x1.PRENOM is not None else '')
    else:
        listeMar=FAM.objects.filter(INDM=idMar).select_related('INDM')
        for indF in listeMar:
          if indF.INDF_id is not None:
            x1=INDIV.objects.get(CODEI=indF.INDF_id)
            ret=ret + ' X (' + str(indF.DATEM if indF.DATEM is not None else '') + ') ' + str(x1.NOM) + ' ' + str(x1.PRENOM if x1.PRENOM is not None else '')
    return ret

def affBranche(request):
    listeNom=[]
    if 'patro' in request.GET:
        patro = request.GET['patro']
        listenom=INDIV.objects.filter(NOM=patro).order_by('CODEPATRO')
        for rec in listenom:
            CODEF=FAM.objects.filter(INDM=rec.CODEI) | FAM.objects.filter(INDF=rec.CODEI)
            strNom=str(rec.NOM) + ' ' + str(rec.PRENOM if rec.PRENOM is not None else '')
            strDateN= '(°' + str(rec.DATEN if rec.DATEN is not None else '') + ')'
            strMar= getMar(rec.CODEI, rec.GENRE)
            tab=[]
            tab.append(rec.CODEPATRO)
            tab.append(strNom)
            tab.append(strDateN)
            tab.append(strMar)
            tab.append(CODEF[0].CODEF)

            listeNom.append(tab)
        return render(request, "affBranche.html", {'html': listeNom})

def getIndiCode(Fam,Genre):
    if Fam is not None:
        listeMar = FAM.objects.get(CODEF=Fam)
        if Genre=='F':
         ret=str(listeMar.INDF_id if listeMar.INDF_id is not None else '')
        else:
         ret=str(listeMar.INDM_id if listeMar.INDM_id is not None else '')
    else:
        ret=''
    return ret

def getIndObj(vInd):
    Ind=INDIV.objects.filter(CODEI=vInd)[0]
    return Ind

def getMariObj(vMar):
    Mar=FAM.objects.filter(CODEF=vMar)[0]
    return Mar
def getEnf(vCodef):
    EnfListe = INDIV.objects.filter(CODEF_id=vCodef)
    CodeFEnf = []
    for enf in EnfListe:
        TB1=[]
        CODEF = FAM.objects.filter(INDM=enf.CODEI) | FAM.objects.filter(INDF=enf.CODEI)
        TB1.append(CODEF[0].CODEF)
        TB1.append(enf.PRENOM)
        CodeFEnf.append(TB1)
    return CodeFEnf

def getListeMar(vInd,vFamExl):
    CodeFMar=[]
    listMar = FAM.objects.filter(INDM=vInd) | FAM.objects.filter(INDF=vInd)
    for fam in listMar:
        if str(fam.CODEF) != str(vFamExl):
            CodeFMar.append(fam.CODEF)
    return CodeFMar
def affArbre(request):
    listeMMar=[]
    listeFMar=[]
    if 'CODEF' in request.GET:
        CodeM=request.GET['CODEF']
        Mar=getMariObj(CodeM)
        EnfListe=getEnf(CodeM)
        MaleCode=getIndiCode(CodeM, 'M')
        CodeF21=-1
        if MaleCode != '':
            listeMMar=getListeMar(MaleCode,CodeM)
            MaleInfo=getIndObj(MaleCode)
            Fam = FAM.objects.filter(INDM_id=MaleCode)
            SRCN = SOURCES.objects.filter(CODES=MaleInfo.CODESN)
            SRCD = SOURCES.objects.filter(CODES=MaleInfo.CODESD)
            SRCM = SOURCES.objects.filter(CODES=Fam[0].CODESM)
            MaleCode21=getIndiCode(MaleInfo.CODEF_id, 'M')

            if MaleCode21 != '':
                MaleInfo21 = getIndObj(MaleCode21)
                Fam21 = FAM.objects.filter(INDM=MaleCode21)
                SRC21N= SOURCES.objects.filter(CODES=MaleInfo21.CODESN)
                SRC21D = SOURCES.objects.filter(CODES=MaleInfo21.CODESD)
                SRC21M = SOURCES.objects.filter(CODES=Fam21[0].CODESM)
                CodeF21 = Fam21[0].CODEF
                Mar21 = getMariObj(CodeF21)
                MaleCode31 = getIndiCode(MaleInfo21.CODEF_id, 'M')
                if MaleCode31 != '':
                   MaleInfo31 = getIndObj(MaleCode31)
                   Fam31 = FAM.objects.filter(INDM=MaleCode31)
                   SRC31N = SOURCES.objects.filter(CODES=MaleInfo31.CODESN)
                   SRC31D = SOURCES.objects.filter(CODES=MaleInfo31.CODESD)
                   SRC31M = SOURCES.objects.filter(CODES=Fam31[0].CODESM)
                   CodeF31 = Fam31[0].CODEF
                   Mar31 = getMariObj(CodeF31)
                else:
                    CodeF31 = -1
                    SRC31N = SOURCES
                    SRC31D = SOURCES
                    SRC31M = SOURCES
                    Mar31 = FAM
                    MaleInfo31=INDIV
                FemelleCode31 = getIndiCode(MaleInfo21.CODEF_id, 'F')
                if FemelleCode31 != '':
                    FemelleInfo31 = getIndObj(FemelleCode31)
                    Fam31 = FAM.objects.filter(INDF=FemelleCode31)
                    SRC31NF = SOURCES.objects.filter(CODES=FemelleInfo31.CODESN)
                    SRC31DF = SOURCES.objects.filter(CODES=FemelleInfo31.CODESD)
                    SRC31MF = SOURCES.objects.filter(CODES=Fam31[0].CODESM)
                    CodeF31F = Fam31[0].CODEF
                else:
                    FemelleInfo31 = INDIV
                    SRC31NF = SOURCES
                    SRC31DF = SOURCES
                    SRC31MF = SOURCES
                    CodeF31F=-1
            else:
                MaleInfo21=INDIV
                CodeF21 = -1
                SRC21N = SOURCES
                SRC21D = SOURCES
                SRC21M = SOURCES
                Mar21 = FAM
                MaleInfo31 = INDIV
                Mar31 = FAM
                CodeF31 = -1
                SRC31N = SOURCES
                SRC31D = SOURCES
                SRC31M = SOURCES
                FemelleInfo31 = INDIV
                SRC31NF = SOURCES
                SRC31DF = SOURCES
                SRC31MF = SOURCES
                CodeF31F = -1
                Mar31 = FAM
            FemelleCode21 = getIndiCode(MaleInfo.CODEF_id, 'F')
            if FemelleCode21 != '':
                FemelleInfo21 = getIndObj(FemelleCode21)
                Fam31 = FAM.objects.filter(INDF=FemelleCode21)
                SRC21NF = SOURCES.objects.filter(CODES=FemelleInfo21.CODESN)
                SRC21DF = SOURCES.objects.filter(CODES=FemelleInfo21.CODESD)
                SRC21MF = SOURCES.objects.filter(CODES=Fam31[0].CODESM)
                CodeF21F = Fam31[0].CODEF
                MaleCode32 = getIndiCode(FemelleInfo21.CODEF_id, 'M')
                if MaleCode32 != '':
                    MaleInfo32 = getIndObj(MaleCode32)
                    Fam32 = FAM.objects.filter(INDM=MaleCode32)
                    SRC32N = SOURCES.objects.filter(CODES=MaleInfo32.CODESN)
                    SRC32D = SOURCES.objects.filter(CODES=MaleInfo32.CODESN)
                    SRC32M = SOURCES.objects.filter(CODES=Fam32[0].CODESM)
                    CodeF32 = Fam32[0].CODEF
                    Mar32 = getMariObj(CodeF32)
                else:
                    MaleInfo32 = INDIV
                    SRC32N = SOURCES
                    SRC32D = SOURCES
                    Mar32 = FAM
                    SRC32M = SOURCES
                    CodeF32 = -1
                FemelleCode32 = getIndiCode(FemelleInfo21.CODEF_id, 'F')
                if FemelleCode32 != '':
                    FemelleInfo32 = getIndObj(FemelleCode32)
                    Fam31 = FAM.objects.filter(INDF=FemelleCode32)
                    SRC32NF = SOURCES.objects.filter(CODES=FemelleInfo32.CODESN)
                    SRC32DF = SOURCES.objects.filter(CODES=FemelleInfo32.CODESD)
                    SRC32MF = SOURCES.objects.filter(CODES=Fam31[0].CODESM)
                    CodeF32F = Fam31[0].CODEF
                else:
                    FemelleInfo32 = INDIV
                    Mar32 = FAM
                    SRC32NF = SOURCES
                    SRC32DF = SOURCES
                    SRC32MF = SOURCES
                    CodeF32F = -1
            else:
                FemelleInfo21=INDIV
                Mar21 = FAM
                SRC21NF = SOURCES
                SRC21DF = SOURCES
                SRC21MF = SOURCES
                CodeF21F = -1
                MaleInfo32 = INDIV
                Mar32 = FAM
                SRC32N = SOURCES
                SRC32D = SOURCES
                SRC32M = SOURCES
                CodeF32 = -1
                FemelleInfo32 = INDIV
                SRC32NF = SOURCES
                SRC32DF = SOURCES
                SRC32MF = SOURCES
                CodeF32F = -1
        else:
            MaleInfo=INDIV
            SRCN = SOURCES
            SRCD = SOURCES
            SRCM = SOURCES
            MaleInfo21 = INDIV
            Mar21 = FAM
            SRC21N = SOURCES
            SRC21D = SOURCES
            SRC21M = SOURCES
            CodeF21 = -1
            FemelleInfo21 = INDIV
            SRC21NF = SOURCES
            SRC21DF = SOURCES
            SRC21MF = SOURCES
            CodeF21F = -1
            MaleInfo31 = INDIV
            Mar31 = FAM
            SRC31N = SOURCES
            SRC31D = SOURCES
            SRC31M = SOURCES
            CodeF31 = -1
            FemelleInfo31 = INDIV
            SRC31NF = SOURCES
            SRC31DF = SOURCES
            SRC31MF = SOURCES
            CodeF31F = -1
            MaleInfo32 = INDIV
            Mar32 = FAM
            SRC32N = SOURCES
            SRC32D = SOURCES
            SRC32M = SOURCES
            CodeF32 = -1
            FemelleInfo32 = INDIV
            SRC32NF = SOURCES
            SRC32DF = SOURCES
            SRC32MF = SOURCES
            CodeF32F = -1
        FemelleCode=getIndiCode(CodeM, 'F')
        if FemelleCode != '':
            listeFMar = getListeMar(FemelleCode, CodeM)
            FemelleInfo = getIndObj(FemelleCode)
            Fam = FAM.objects.filter(INDF_id=FemelleCode)
            SRCNF = SOURCES.objects.filter(CODES=FemelleInfo.CODESN)
            SRCDF = SOURCES.objects.filter(CODES=FemelleInfo.CODESD)
            SRCMF = SOURCES.objects.filter(CODES=Fam[0].CODESM)
            if FemelleInfo.CODEF is not None:
               MaleCode22 = getIndiCode(FemelleInfo.CODEF_id, 'M')
            else:
                MaleCode22=''
            if MaleCode22 != '':
                MaleInfo22 = getIndObj(MaleCode22)
                Fam22 = FAM.objects.filter(INDM=MaleCode22)
                SRC22N = SOURCES.objects.filter(CODES=MaleInfo22.CODESN)
                SRC22D = SOURCES.objects.filter(CODES=MaleInfo22.CODESN)
                SRC22M = SOURCES.objects.filter(CODES=Fam22[0].CODESM)
                CodeF22 = Fam22[0].CODEF
                Mar22 = getMariObj(CodeF22)
                MaleCode33 = getIndiCode(MaleInfo22.CODEF_id, 'M')
                if MaleCode33 != '':
                    MaleInfo33 = getIndObj(MaleCode33)
                    Fam33 = FAM.objects.filter(INDM=MaleCode33)
                    SRC33N = SOURCES.objects.filter(CODES=MaleInfo33.CODESN)
                    SRC33D = SOURCES.objects.filter(CODES=MaleInfo33.CODESN)
                    SRC33M = SOURCES.objects.filter(CODES=Fam33[0].CODESM)
                    CodeF33 = Fam33[0].CODEF
                    Mar33 = getMariObj(CodeF33)
                else:
                    MaleInfo33 = INDIV
                    SRC33N = SOURCES
                    SRC33D = SOURCES
                    SRC33M = SOURCES
                    Mar33 = FAM
                    CodeF33=-1
                FemelleCode33 = getIndiCode(MaleInfo22.CODEF_id, 'F')
                if FemelleCode33 != '':
                    FemelleInfo33 = getIndObj(FemelleCode33)
                    Fam31 = FAM.objects.filter(INDF=FemelleCode33)
                    SRC33NF = SOURCES.objects.filter(CODES=FemelleInfo33.CODESN)
                    SRC33DF = SOURCES.objects.filter(CODES=FemelleInfo33.CODESD)
                    SRC33MF = SOURCES.objects.filter(CODES=Fam31[0].CODESM)
                    CodeF33F = Fam31[0].CODEF
                else:
                    FemelleInfo33 = INDIV
                    Mar33 = FAM
                    SRC33NF = SOURCES
                    SRC33DF = SOURCES
                    SRC33MF = SOURCES
                    CodeF33F = -1
            else:
                MaleInfo22 = INDIV
                SRC22N = SOURCES
                SRC22D = SOURCES
                SRC22M = SOURCES
                Mar22 = FAM
                CodeF22=-1
                MaleInfo33 = INDIV
                SRC33N = SOURCES
                SRC33D = SOURCES
                SRC33M = SOURCES
                CodeF33 = -1
                Mar33= FAM
                FemelleInfo33 = INDIV
                SRC33NF = SOURCES
                SRC33DF = SOURCES
                SRC33MF = SOURCES
                CodeF33F = -1
            if FemelleInfo.CODEF is not None:
                FemelleCode22 = getIndiCode(FemelleInfo.CODEF_id, 'F')
            else:
                FemelleCode22=''
            if FemelleCode22 != '':
                FemelleInfo22 = getIndObj(FemelleCode22)
                Fam31 = FAM.objects.filter(INDF=FemelleCode22)
                SRC22NF = SOURCES.objects.filter(CODES=FemelleInfo22.CODESN)
                SRC22DF = SOURCES.objects.filter(CODES=FemelleInfo22.CODESD)
                SRC22MF = SOURCES.objects.filter(CODES=Fam31[0].CODESM)
                CodeF22F = Fam31[0].CODEF
                MaleCode34 = getIndiCode(FemelleInfo22.CODEF_id, 'M')
                if MaleCode34 != '':
                    MaleInfo34 = getIndObj(MaleCode34)

                    Fam34 = FAM.objects.filter(INDM=MaleCode34)
                    SRC34N = SOURCES.objects.filter(CODES=MaleInfo34.CODESN)
                    SRC34D = SOURCES.objects.filter(CODES=MaleInfo34.CODESN)
                    SRC34M = SOURCES.objects.filter(CODES=Fam34[0].CODESM)
                    CodeF34 = Fam34[0].CODEF
                    Mar34 = getMariObj(CodeF34)
                else:
                    MaleInfo34 = INDIV
                    SRC34N = SOURCES
                    SRC34D = SOURCES
                    SRC34M = SOURCES
                    Mar34 = FAM
                    CodeF34=-1
                FemelleCode34 = getIndiCode(FemelleInfo22.CODEF_id, 'F')
                if FemelleCode34 != '':
                    FemelleInfo34 = getIndObj(FemelleCode34)
                    SRC34NF = SOURCES.objects.filter(CODES=FemelleInfo34.CODESN)
                    SRC34DF = SOURCES.objects.filter(CODES=FemelleInfo34.CODESD)
                    SRC34MF = SOURCES.objects.filter(CODES=Fam31[0].CODESM)
                    CodeF34F = Fam31[0].CODEF
                else:
                    FemelleInfo34 = INDIV
                    Mar34 = FAM
                    SRC34NF = SOURCES
                    SRC34DF = SOURCES
                    SRC34MF = SOURCES
                    CodeF34F = -1
            else:
                FemelleInfo22 = INDIV
                Mar22 = FAM
                SRC22NF = SOURCES
                SRC22DF = SOURCES
                SRC22MF = SOURCES
                CodeF22F = -1
                MaleInfo34 = INDIV
                SRC34N = SOURCES
                SRC34D = SOURCES
                SRC34M = SOURCES
                Mar34 = FAM
                CodeF34 = -1
                FemelleInfo34 = INDIV
                SRC34NF = SOURCES
                SRC34DF = SOURCES
                SRC34MF = SOURCES
                CodeF34F = -1
        else:
            FemelleInfo = INDIV
            SRCNF = SOURCES
            SRCDF = SOURCES
            SRCMF = SOURCES
            MaleInfo22 = INDIV
            Mar22 = FAM
            SRC22N = SOURCES
            SRC22D = SOURCES
            SRC22M = SOURCES
            CodeF22 = -1
            FemelleInfo22 = INDIV
            SRC22NF = SOURCES
            SRC22DF = SOURCES
            SRC22MF = SOURCES
            CodeF22F = -1
            MaleInfo33 = INDIV
            Mar33 = FAM
            SRC33N = SOURCES
            SRC33D = SOURCES
            SRC33M = SOURCES
            CodeF33= -1
            FemelleInfo33 = INDIV
            SRC33NF = SOURCES
            SRC33DF = SOURCES
            SRC33MF = SOURCES
            CodeF33F = -1
            MaleInfo34 = INDIV
            Mar34 = FAM
            SRC34N = SOURCES
            SRC34D = SOURCES
            SRC34M = SOURCES
            CodeF34 = -1
            FemelleInfo34 = INDIV
            SRC34NF = SOURCES
            SRC34DF = SOURCES
            SRC34MF = SOURCES
            CodeF34F = -1
    TABMALE=[]
    TABMALE.append(MaleInfo)
    TABMALE.append(MaleInfo21)
    TABMALE.append(MaleInfo22)
    TABMALE.append(MaleInfo31)
    TABMALE.append(MaleInfo32)
    TABMALE.append(MaleInfo33)
    TABMALE.append(MaleInfo34)
    TABFEMELLE = []
    TABFEMELLE.append(FemelleInfo)
    TABFEMELLE.append(FemelleInfo21)
    TABFEMELLE.append(FemelleInfo22)
    TABFEMELLE.append(FemelleInfo31)
    TABFEMELLE.append(FemelleInfo32)
    TABFEMELLE.append(FemelleInfo33)
    TABFEMELLE.append(FemelleInfo34)
    CodeFMale=[]
    CodeFMale.append(-1)
    CodeFMale.append(CodeF21)
    CodeFMale.append(CodeF22)
    CodeFMale.append(CodeF31)
    CodeFMale.append(CodeF32)
    CodeFMale.append(CodeF33)
    CodeFMale.append(CodeF34)
    CodeFFemale = []
    CodeFFemale.append(-1)
    CodeFFemale.append(CodeF21F)
    CodeFFemale.append(CodeF22F)
    CodeFFemale.append(CodeF31F)
    CodeFFemale.append(CodeF32F)
    CodeFFemale.append(CodeF33F)
    CodeFFemale.append(CodeF34F)
    CodeSRCMaleN = []
    CodeSRCMaleN.append(SRCN)
    CodeSRCMaleN.append(SRC21N)
    CodeSRCMaleN.append(SRC22N)
    CodeSRCMaleN.append(SRC31N)
    CodeSRCMaleN.append(SRC32N)
    CodeSRCMaleN.append(SRC33N)
    CodeSRCMaleN.append(SRC34N)
    CodeSRCFemaleN = []
    CodeSRCFemaleN.append(SRCNF)
    CodeSRCFemaleN.append(SRC21NF)
    CodeSRCFemaleN.append(SRC22NF)
    CodeSRCFemaleN.append(SRC31NF)
    CodeSRCFemaleN.append(SRC32NF)
    CodeSRCFemaleN.append(SRC33NF)
    CodeSRCFemaleN.append(SRC34NF)
    CodeSRCMaleD = []
    CodeSRCMaleD.append(SRCD)
    CodeSRCMaleD.append(SRC21D)
    CodeSRCMaleD.append(SRC22D)
    CodeSRCMaleD.append(SRC31D)
    CodeSRCMaleD.append(SRC32D)
    CodeSRCMaleD.append(SRC33D)
    CodeSRCMaleD.append(SRC34D)
    CodeSRCFemaleD = []
    CodeSRCFemaleD.append(SRCDF)
    CodeSRCFemaleD.append(SRC21DF)
    CodeSRCFemaleD.append(SRC22DF)
    CodeSRCFemaleD.append(SRC31DF)
    CodeSRCFemaleD.append(SRC32DF)
    CodeSRCFemaleD.append(SRC33DF)
    CodeSRCFemaleD.append(SRC34DF)
    CodeSRCMaleM = []
    CodeSRCMaleM.append(SRCM)
    CodeSRCMaleM.append(SRC21M)
    CodeSRCMaleM.append(SRC22M)
    CodeSRCMaleM.append(SRC31M)
    CodeSRCMaleM.append(SRC32M)
    CodeSRCMaleM.append(SRC33M)
    CodeSRCMaleM.append(SRC34M)
    CodeSRCFemaleM = []
    CodeSRCFemaleM.append(SRCMF)
    CodeSRCFemaleM.append(SRC21MF)
    CodeSRCFemaleM.append(SRC22MF)
    CodeSRCFemaleM.append(SRC31MF)
    CodeSRCFemaleM.append(SRC32MF)
    CodeSRCFemaleM.append(SRC33MF)
    CodeSRCFemaleM.append(SRC34MF)
    Mariages=[]
    Mariages.append(Mar)
    Mariages.append(Mar21)
    Mariages.append(Mar22)
    Mariages.append(Mar31)
    Mariages.append(Mar32)
    Mariages.append(Mar33)
    Mariages.append(Mar34)
    return render(request, "affArbre.html", {'Male': TABMALE, 'Femelle': TABFEMELLE, 'Mariage': Mariages, 'EnfListe': EnfListe, \
                                             'CodeF':CodeFMale,'SRCNM':CodeSRCMaleN,'SRCDM':CodeSRCMaleD,'SRCMM':CodeSRCMaleM, \
                                             'CodeFF': CodeFFemale, 'SRCNF': CodeSRCFemaleM, 'SRCDF': CodeSRCFemaleD,
                                             'SRCMF': CodeSRCFemaleM, 'ListeMMar': listeMMar ,'ListeFMar': listeFMar})


import cv2
import numpy as np
import sys

MAX_LEN = 50
WINDOW = 100
def rand_short_path(pts):
    dedupe = list(set(pts))
    ysort = sorted(pts, key=lambda p: p[1])
    imap = {i: pt for i, pt in enumerate(ysort)}

    segs = {}
    from_pts = set(range(len(ysort)))
    to_pts = set(range(len(ysort)))       # implement as boolean array
    while from_pts:
        i = from_pts.pop()
        xi, yi = imap[i]

        min_d, min_j, min_pt = sys.maxsize, 0, None
        for j in range(max(0, i-WINDOW), min(len(ysort), i+WINDOW+1)):
            if j not in to_pts or j == i:
                continue

            xj, yj = imap[j]
            dx = xi-xj
            dy = yi-yj
            distance = dx*dx+dy*dy
            if distance < min_d and distance < MAX_LEN:
                min_d = distance
                min_pt = (xj, yj)
                min_j = j
        if min_pt:
            segs[(xi, yi)] = min_pt
            to_pts.remove(min_j)

    curves = []
    from_pts = set(ysort)
    to_pts = set(ysort)
    while from_pts:
        pt = from_pts.pop()
        while pt not in segs:
            if from_pts:
                pt = from_pts.pop()
            else:
                break
        if not from_pts:
            break

        curve = []
        while pt in segs:
            if segs[pt] not in to_pts:
                break

            pt = segs[pt]
            curve.append(pt)
            to_pts.remove(pt)

            if pt not in from_pts:
                break
            from_pts.remove(pt)

        if len(curve):
            curves.append(curve)

    while len(curves) > 1:
        c = curves.pop()
        (cx1, cy1), (cx2, cy2) = c[0], c[-1]

        min_d1, min_d2, min_p1, min_p2 = sys.maxsize, sys.maxsize, None, None
        for p in curves:
            if not len(p):
                continue
            (px1, py1), (px2, py2) = p[0], p[-1]
            d1, d2 = (cx2-px1)**2+(cy2-py1)**2, (px2-cx1)**2+(py2-cy1)**2
            if d1 < min_d1:
                min_p1 = p
                min_d1 = d1
            if d2 < min_d2:
                min_p2 = p
                min_d2 = d2

        if d1 < d2:
            curves.remove(min_p1)
            c.extend(min_p1)
            curves.append(c)
        else:
            curves.remove(min_p2)
            min_p2.extend(c)
            curves.append(min_p2)

    return curves.pop()

cam = cv2.VideoCapture(0)
_, img = cam.read()
r,c,_ = img.shape
scl = 5
mask = cv2.circle(np.zeros((r//scl, c//scl), dtype=np.uint8),
                    (r//scl//2, c//scl//2), r//scl//2, [255,255,255], -1)
_, mask = cv2.threshold(mask, 127, 255, 0)

while cv2.waitKey(1) != 27:

    mini = cv2.resize(img, (c//scl,r//scl), interpolation=cv2.INTER_LINEAR)
    #cv2.imshow("source", mini)

    canny = mini.copy()
    canny = cv2.GaussianBlur(canny, (7,7), 0)
    #cv2.imshow("blur", canny)

    canny = cv2.Canny(canny, 127, 255)
    canny = cv2.bitwise_or(canny, canny, mask=mask)
    #cv2.imshow("canny", canny)

    cntimg = mini.copy()
    _, cnt, hier = cv2.findContours(canny, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
    cv2.drawContours(cntimg, cnt, -1, [0, 255, 0], 1)
    #cv2.imshow("contour", cntimg)

    pathimg = np.zeros_like(img)
    points = [(c[0][0],c[0][1]) for c in [j for i in cnt for j in i]]

    path = rand_short_path(points)
    for i in range(len(path)-1):
        (x1, y1),(x2, y2) = path[i], path[i+1]

        cv2.line(pathimg, (x1*scl, y1*scl), (x2*scl, y2*scl), (0,i*255//len(path),0), 2)
    cv2.imshow("path", pathimg)

    _, img = cam.read()


CWD = 'F:/zhang zemin/GSE146771_scanpy'
CWD2 = 'F:/scCancer_analysis/soup_seurat'

# load gene names
adata_ref = sc.read("{CWD}/ALL_scanpy_BBKNN.h5ad".format(CWD=CWD2), cache = True)
adata = sc.read("{CWD}/seurat_obj_immu.mtx".format(CWD=CWD), cache = True).T

adata.var_names = pd.read_csv("{CWD}/immu_genenames.csv".format(CWD=CWD)).iloc[:, 1]

# load cell names
adata.obs_names = pd.read_csv("{CWD}/immu_cellnames.csv".format(CWD=CWD)).iloc[:, 1]

metadata = pd.read_csv("{CWD}/meta.csv".format(CWD=CWD2), index_col = 0)
adata_ref.obs = metadata

var_names = adata_ref.var_names.intersection(adata.var_names)
adata_ref = adata_ref[:, var_names]
adata = adata[:, var_names]

adata

sc.pp.normalize_total(adata, target_sum=1e4)
sc.pp.log1p(adata)

sc.tl.ingest(adata, adata_ref, obs='cell_label')


def main():
    n, m, k = map(int, input().split())

    paths_dict = {}
    for _ in range(m):
        u, v, x = map(int, input().split())

        if u in paths_dict:
            paths_dict[u].append((v, x))
        else:
            paths_dict[u] = [(v, x)]

        if v in paths_dict:
            paths_dict[v].append((u, x))
        else:
            paths_dict[v] = [(u, x)]

    if k > 0:
        bakeries = set(map(int, input().split()))
    else:
        bakeries = set()

    min_length = float("inf")
    for bakery in bakeries:
        if bakery in paths_dict:
            for adjacent_city in paths_dict[bakery]:
                if not adjacent_city[0] in bakeries:
                    min_length = min(min_length, adjacent_city[1])

    print(min_length if min_length != float("inf") else -1)

if __name__ == "__main__":
    main()

# correct


#!/usr/bin/env python
# -*- coding:utf-8 -*

'''
 Gestion des modules MPC3008
	pour création entrée analogique
	sur un rpiduino_io
		- Rpi
		- pcduino

	Gestion du protocole SPI : hardware spi

	Wiring :
		        _ _
			   | U |
		CH0 ---|   |--- Vdd : 2.7V - 5V
		CH1 ---|   |--- Vref
		CH2 ---|   |--- 0V (circuit analogique)
		CH3 ---|   |--- CLK : pin_clock
		CH4 ---|   |--- Dout : pin_miso
		CH5 ---|   |--- Din : pin_mosi
		CH6 ---|   |--- CS : pin_cs
		CH7 ---|   |--- 0V (circuit digital)
		       |___|

	CH0-CH7 : analog inputs

 AUTEUR : FredThx

 Projet : rpiduino_io

'''
#TODO :
#	-	creer une sur-classe SPI
#	-	se poser la question de la fréquence (ici, pas de time.sleep) : fréquence non maitrisée

from rpiduino_io import *
from f_spi import *

class mcp300x_hspi_io(device_io, hard_spi_client):
	'''Classe pour convertisseur analogique/numérique MCP3004/MCP3008
	'''
	def __init__(self, bus, device, vref = 3.3):
		'''Initialisation
			bus		:	spi bus (0|1)
			device	:	no spi client (0 : CS = CE0 = GPIO08 | 1 : CS = CE1 = GPIO07)
			Vref	:	ref voltage
		'''
		hard_spi_client.__init__(self, bus, device)
		self.pin = {}
		self.vref = vref

class mcp3004_hspi_io(mcp300x_hspi_io):
	'''Classe pour convertisseur analogique/numérique MCP3004
	'''
	def __init__(self, bus=0, device=0, vref = 3.3):
		'''Initialisation
			bus		:	spi bus (0|1)
			device	:	no spi client
			Vref			:	ref voltage
		'''
		mcp300x_hspi_io.__init__(self, bus, device, vref)
		for i in range(4):
			self.pin[i] = mcp300x_hspi_pin(self, i)

class mcp3008_hspi_io(mcp300x_hspi_io):
	'''Classe pour convertisseur analogique/numérique MCP3008
	'''
	def __init__(self, bus=0, device=0, vref = 3.3):
		'''Initialisation
			bus		:	spi bus
			device	:	no spi client
			Vref			:	ref voltage
		'''
		mcp300x_hspi_io.__init__(self, bus, device,  vref)
		for i in range(8):
			self.pin[i] = mcp300x_hspi_pin(self, i)

class mcp300x_hspi_pin(analog_pin_io):
	'''Une pin analogique sur mcp3004/3008
	'''
	def __init__(self, mcp300x_ship, chanel):
		'''Initialisation
			mcp300x_ship	:	a mcp300x_io
			chanel			:	n° of the input chanel
		'''
		self.ship = mcp300x_ship
		self.chanel = chanel

	def get(self):
		''' get the raw value of the input chanel
		'''
		result = self.ship.xfer2([1,(8+self.chanel)<<4,0])
		result = ((result[1]&3) << 8) + result[2]
		return result

	def get_voltage(self):
		''' get the voltage on the input chanel
		'''
		return self.get() * self.ship.vref / 1024.

	@property
	def vref(self):
		return self.ship.vref

#########################################################
#                                                       #
#		EXEMPLE                                         #
#                                                       #
#########################################################

if __name__ == '__main__':
	import time
	pc = rpiduino_io()
	mcp3008 = mcp3008_hspi_io()
	ch0 = mcp3008.pin[7]
	while True:
		print('Voltage : %.2f Volts' % ch0.get_voltage())
		time.sleep(0.5)


r""" Parametric-AAA

"""

from itertools import product
import numpy as np
from copy import deepcopy as copy
import scipy.linalg
from iterprinter import IterationPrinter

from .aaa import RationalBarycentric

def _build_cauchy(x,y):
	with np.errstate(divide = 'ignore', invalid = 'ignore'):
		C = 1./(np.tile(x.reshape(-1,1), (1,len(y))) - np.tile(y.reshape(1,-1), (len(x),1)))
		# If we have a divide by zero error, we reason that for that row,
		# only the matching column is non-zero
		for row in np.argwhere(~np.all(np.isfinite(C), axis = 1)):
			C[row] = np.zeros(C.shape[1])
			C[row, np.argmin(np.abs(x[row] - y)).flatten()] = 1
	return C

def eval_paaa(Xe, X, y, I, b, basis, order):
	d = X.shape[1]

	C = np.ones((len(Xe),1), dtype = X.dtype)
	for i in range(d):
		Ct = _build_cauchy(Xe[:, i], basis[i])
		C = np.vstack([C[:,j] * Ct[:,k] for j, k in product(range(C.shape[1]), range(Ct.shape[1]))]).T

	num = np.einsum('ij,j...->i...', C, [bi*yi for bi, yi in zip(b,y[order])] )
	denom = np.einsum('ij,j...->i...', C, b)

	with np.errstate(divide = 'ignore', invalid = 'ignore'):
		return np.einsum('i...,i->i...', num, 1./denom)

def paaa(X, y, verbose = True, maxiter = 100, tol = None):
	r"""

	"""
	X = np.array(X)
	y = np.array(y)
	assert X.shape[0] == y.shape[0], "X and y must have the same first dimension"

	if tol is None:
		tol = 1e-10

	d = X.shape[1]
	r = np.zeros(y.shape)

	degree = np.zeros(d, dtype = int)

	if verbose:
		printer = IterationPrinter(it = '4d', degree = f'{2+3*d:d}s', norm = '20.10e', cond = '10.2e')
		printer.print_header(it = 'iter', degree = 'degree', norm = 'norm mismatch', cond = 'cond #')
		printer.print_iter(norm = np.linalg.norm(y))
	# indices of points where we interpolate
	I = np.zeros(y.shape[0], dtype = bool)

	mismatch = np.copy(y)
	best = None
	for it in range(maxiter):
		# compute the pointwise-residual
		residual = np.sum(np.abs(mismatch), axis = tuple(range(1,len(y[0].shape)+1)))
		# zero out points we've already identified so we don't re-evaluate
		residual[I] = 0

		# find the point with the largest residual and add that to the interpolation set
		i = np.argmax(residual)
		I[i] = True

		# determine points in tensor-product barycentric basis
		basis = [np.array(list(set(X[I,i]))).reshape(-1) for i in range(d)]
		degree = np.array([len(b)-1 for b in basis], dtype = int)

		# With this algorithm we need to "complete the square"
		# by adding all terms of the form
		I = np.all( [np.any([X[:,i] == bi for bi in basis[i]], axis = 0) for i in range(d)], axis = 0)
		#print("I", np.argwhere(I).flatten())
		assert np.prod([len(b) for b in basis]) == np.sum(I), "Missing points in tensor product grid"

		# Build the multivariate Cauchy matrix
		C = np.ones((np.sum(~I),1), dtype = X.dtype)
		for i in range(d):
			Ct = _build_cauchy(X[~I, i], basis[i])
			C = np.vstack([C[:,j] * Ct[:,k] for j, k in product(range(C.shape[1]), range(Ct.shape[1]))]).T

		# Build the Loewner matrix associated with each input
		Lten = []
		# The issue is we need to match the ordering of I to that in C
		order = [ int(np.argwhere( (X == xx).all(axis=1))) for xx in product(*basis)]
		for idx in np.ndindex(y[0].shape):
			Lten.append( (C.T * y[(~I,*idx)]).T - C*y[(order,*idx)] )
		L = np.vstack(Lten)

		# Compute coefficients for denominator polynomial
		U, s, VH = scipy.linalg.svd(L, full_matrices = False, compute_uv = True, overwrite_a = True)
		b = VH.conj().T[:,-1]

		if len(s) >= 2:
			with np.errstate(divide='ignore',invalid='ignore'):
				cond = s[0]*np.sqrt(2)/(s[-2] - s[-1])
		else:
			cond = None

		r = eval_paaa(X, X, y, I, b, basis, order)
		if np.all(np.isfinite(r)):
			best = (np.copy(I), np.copy(b), copy(basis), copy(order))
			assert np.all(np.isclose(r[I], y[I])), "Constructed rational approximant failed to correctly interpolate"
		else:
			if verbose:
				print("Encountered nan in evaluation")
			break

		mismatch = y - r

		mismatch_norm = np.linalg.norm(mismatch)

		if verbose:
			degree_str = '(' + ','.join([f'{di:2d}' for di in degree]) + ')'
			printer.print_iter(it = it, norm = mismatch_norm, degree = degree_str, cond = cond)

		if mismatch_norm < tol:
			if verbose: print('terminated due to small mismatch')
			break

	return best

class ParametricAAARationalApproximation(RationalBarycentric):
	def __init__(self, tol = None, verbose = True, maxiter = 100):
		self.tol = None
		self.verbose = verbose
		self.maxiter = maxiter
		self.degree = None

	def fit(self, X, y):
		self.X = np.copy(X)
		self.y = np.copy(y)
		self._I, self._b, self._basis, self._order = paaa(X, y, verbose = self.verbose, maxiter = self.maxiter)
		self.degree = np.array([len(bi)-1 for bi in self._basis], dtype = int)

	def __call__(self, X):
		return eval_paaa(X, self.X, self.y, self._I, self._b, self._basis, self._order)

	@property
	def interpolation_points(self):
		return np.array(list(product(*self._basis)))


config = {}
config['batch_size'] = 50 #number of samples taken per each update
config['hidden_size'] = 128 # number of hidden units per layer
config['num_layers'] = 2
config['learning_rate'] = 0.002
config['learning_rate_decay'] = 0.97 # set to 0 to not decay learning

config['decay_rate'] = 0.95
config['step_clipping'] = 1.0
config['dropout'] = 0

config['model'] = 'gru'  # 'rnn' or 'gru' 'lstm'
config['nepochs'] = 1000

config['seq_length'] = 50
config['hdf5_file'] = 'input.hdf5'
config['source_file'] = 'post.txt'
config['target_file'] = 'cmnt.txt'
config['train_size'] = 0.95
config['save_path'] = '{0}_best.pkl'.format(config['model'])
config['load_path'] = '{0}_saved.pkl'.format(config['model'])
config['last_path'] = '{0}_last.pkl'.format(config['model'])


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import yfinance as yf
from keras.models import load_model
import streamlit as st

def show_stock_price_prediction():

    plt.rcParams['font.sans-serif'] = ['Taipei Sans TC Beta']

    plt.style.use("dark_background")

    st.title('股價預測')

    user_input = st.text_input('輸入股票代號', '2330.TW')
    selected_year = st.slider('選擇期間（年）', 1, 10, 3)

    if st.button('預測股價'):

        df = yf.download(user_input, period=f'{selected_year}y')

        st.subheader(f'至今{selected_year}年期間的股票資料')
        st. write(df.describe())

        st.subheader('收盤價走勢')
        fig = plt.figure(figsize = (12,6))
        plt.plot(df.Close, 'b', label="收盤價")
        plt.legend(loc="upper left")
        st.pyplot(fig)

        # Splitting Data into Training and Testing

        data_training = pd.DataFrame(df['Close'][0:int(len(df)*0.70)])
        data_testing = pd.DataFrame(df['Close'][int(len(df)*0.70): int(len(df))])

        from sklearn.preprocessing import MinMaxScaler
        scaler = MinMaxScaler(feature_range=(0,1))

        data_training_array = scaler.fit_transform(data_training)

        #Load my model
        model = load_model('lstm_model_2330.h5', compile = False)

        #Testing Part

        past_100_days = data_training.tail(100)
        # final_df = past_100_days.append(data_testing, ignore_index=True)
        final_df = pd.concat([past_100_days, data_testing], ignore_index=True)
        input_data = scaler.fit_transform(final_df)

        x_test = [] # 產生一個名叫x_test的list
        y_test = [] # 產生一個名叫y_test的list

        for i in range(100, data_training_array.shape[0]):
            x_test.append(data_training_array[i-100: i])
            y_test.append(data_training_array[i, 0])

        x_test, y_test = np.array(x_test), np.array(y_test)
        y_predicted = model.predict(x_test)
        scaler = scaler.scale_

        scaler_factor = 1/scaler[0]
        y_predicted = y_predicted * scaler_factor
        y_test = y_test * scaler_factor

        #Final Graph

        st.subheader('使用LSTM模型預測收盤價走勢')
        fig2 = plt.figure(figsize=(12,6))
        plt.plot(y_test, 'b', label = '原收盤價')
        plt.plot(y_predicted, 'r', label = '預測收盤價')
        plt.xlabel('時間')
        plt.ylabel('收盤價')
        plt.legend()
        st.pyplot(fig2)


"""
Python file containing the models.

Need to add support for adding in weights to the encoder model.
"""

import os
import time
import enum

import cv2
import torch
import face_alignment
import torch.nn as nn
import torch.nn.functional as F

import cheapfake.contrib.ResNetSE34L as audio_model
import cheapfake.lipnet.models as lipnet

# Uncomment the lines below if the files exist.
lipnet_options = __import__("lipnet_config")
face_alignment_options = __import__("face_alignment_config")

# Delete the line below if the files above exist.
#face_alignment_options = 0

class NetworkType(enum.Enum):
    face_alignment = 1
    lipnet = 2
    resnet_audio = 3

class FeaturesEncoder(nn.Module):
    """Implementation of a feature encoder that creates embeddings that fit into a multi-layer perceptron.

    """

    def __init__(self, network_type=NetworkType.face_alignment):
        """Instantiates a new FeaturesEncoder network.

        Parameters
        ----------
        network_type : cheapfake.contrib.models.NetworkType instance, optional
            The type of network to instantiate, by default NetworkType.face_alignment. The supported options are NetworkType.face_alignment, NetworkType.lipnet, and NetworkType.resnet_audio.

        """
        super(FeaturesEncoder, self).__init__()

        assert network_type.value in (1, 2, 3), "Network architecture not supported."

        self.network_type = network_type

        if network_type.value == 1:
            self.conv1 = nn.Conv2d(2, 4, kernel_size=3, stride=1, padding=1)
            self.batchnorm1 = nn.BatchNorm2d(4)
            self.maxpool1 = nn.MaxPool2d(kernel_size=3, stride=4)
            #self.conv2 = nn.Conv2d(4, 4, kernel_size=3, stride=1, padding=1)
            #self.batchnorm2 = nn.BatchNorm2d(4)
            #self.maxpool2 = nn.MaxPool2d(kernel_size=2, stride=2)
            self.flatten = nn.Flatten()
            self.fc1 = nn.Linear(1292, 256)
            self.relu = nn.ReLU(inplace=True)
        elif network_type.value == 2:
            self.conv1 = torch.nn.Conv2d(1, 4, kernel_size=3, stride=1, padding=1)
            self.batchnorm1 = torch.nn.BatchNorm2d(4)
            self.maxpool1 = torch.nn.MaxPool2d(kernel_size=3, stride=4)
            #self.conv2 = torch.nn.Conv2d(16, 25, kernel_size=3, stride=1, padding=1)
            #self.batchnorm2 = torch.nn.BatchNorm2d(25)
            #self.maxpool2 = torch.nn.MaxPool2d(kernel_size=2, stride=2)
            self.flatten = torch.nn.Flatten()
            self.fc1 = torch.nn.Linear(9728, 256)
            #self.fc2 = torch.nn.Linear(
            #    int((25 * 18 * 128) / 4), int((25 * 18 * 128) / 16)
            #)
            #self.fc3 = torch.nn.Linear(
            #    int((25 * 18 * 128) / 16), int((25 * 18 * 128) / 64)
            #)
            #self.fc4 = torch.nn.Linear(int((25 * 18 * 128) / 64), 256)
            self.relu = torch.nn.ReLU(inplace=True)

    def forward(self, x):
        """Performs a forward pass of the input ``x`` through the encoder network.

        Parameters
        ----------
        x : torch.Tensor instance
            Torch tensor containing the input to the network.

        Returns
        -------
        x : torch.Tensor instance
            Torch tensor containing the embedded features.

        """
        if self.network_type.value == 1:
            x = self.conv1(x)
            x = self.batchnorm1(x)
            x = self.maxpool1(x)
            #x = self.conv2(x)
            #x = self.batchnorm2(x)
            #x = self.maxpool2(x)
            x = self.flatten(x)
            #print(x.shape)
            x = self.fc1(x)
            x = self.relu(x)
        elif self.network_type.value == 2:
            x = self.conv1(x)
            x = self.batchnorm1(x)
            x = self.maxpool1(x)
            #x = self.conv2(x)
            #x = self.batchnorm2(x)
            #x = self.maxpool2(x)
            x = self.flatten(x)
            #print(x.shape)
            x = self.fc1(x)
            x = self.relu(x)
            #x = self.fc2(x)
            #x = self.relu(x)
            #x = self.fc3(x)
            #x = self.relu(x)
            #x = self.fc4(x)
            #x = self.relu(x)
        return x

class AugmentedFAN(nn.Module):
    """Augmented Face Alignment Network (AFAN) that includes a feature embedding network that acts on the output of FAN.

    """

    def __init__(self, device=torch.device("cpu"), num_modules=1, verbose=True):
        """Instantiates a new AugmentedFAN object.

        Pre-trained weights are automatically loaded.

        Parameters
        ----------
        device : torch.device instance
            The device on which all procedures are carried out, by default torch.device("cpu").
        num_modules : int, optional
            The number of modules, by default 1.
        verbose : bool, optional
            If True then verbose output will be printed onto the system console.

        """
        super(AugmentedFAN, self).__init__()

        assert isinstance(device, torch.device)
        assert isinstance(num_modules, int)
        assert isinstance(verbose, bool)

        if hasattr(face_alignment_options, "landmarks_type"):
            assert face_alignment_options.landmarks_type in [
                "2D",
                "3D",
            ], "Landmarks must either be in 2D or 3D."
            self.landmarks_type = face_alignment_options.landmarks_type
        else:
            self.landmarks_type = "2D"

        if self.landmarks_type == "2D":
            landmarks_type = face_alignment.LandmarksType._2D
        else:
            landmarks_type = face_alignment.LandmarksType._3D

        if hasattr(face_alignment_options, "face_detector"):
            assert face_alignment_options.face_detector in [
                "sfd",
                "dlib",
            ], "Only S3FD and DLib are supported as face detection schemes."
            self.face_detector = face_alignment_options.face_detector
        else:
            self.face_detector = "sfd"

        self.device = device
        device = "cuda" if "cuda" in self.device.type else "cpu"
        self.num_modules = num_modules
        self.verbose = verbose

        self.face_alignment_model = face_alignment.FaceAlignment(
            landmarks_type=self.landmarks_type,
            device=device,
            face_detector=self.face_detector,
            verbose=self.verbose,
        )
        self.encoder_model = FeaturesEncoder(
            network_type=NetworkType.face_alignment
        ).to(self.device)

    def _load_encoder_weights(self):
        pass

    def forward(self, x):
        """Performs a forward pass of ``x`` through the network.

        Parameters
        ----------
        x : torch.Tensor instance
            Torch tensor containing the input to the network.

        Returns
        -------
        landmarks : numpy.ndarray or torch.Tensor instance
            Numpy array or Torch tensor containing the xy-coorddinates of the detected facial landmarks.
        face_embeddings : numpy.ndarray or torch.Tensor instance
            Numpy array or Torch tensor containing the embeddings of the facial features returned by the FAN.

        """
        landmarks = list()
        face_embeddings = list()
        for batch in x:
            landmark = self.face_alignment_model.get_landmarks_from_batch(
                batch.to(self.device)
            )

            # Compute the FAN embeddings.
            # Need to check for multiple landmarks before you put it in to a Torch tensor.
            landmark = torch.Tensor(landmark)
            if landmark.shape[1] != 1:
                landmark = landmark[:, :1, :, :]
            landmarks.append(landmark)
            landmark = landmark.permute(1, 3, 0, 2).float().to(self.device)
            # landmark = landmark[None, :, :, :].float().to(self.device)

            face_embedding = self.encoder_model(landmark.to(self.device))
            face_embeddings.append(face_embedding)

        landmarks = torch.stack(landmarks)
        landmarks = landmarks.squeeze(axis=2).float().to(self.device)

        face_embeddings = torch.stack(face_embeddings).float().to(self.device)

        return landmarks, face_embeddings
        # return landmarks

class AugmentedLipNet(nn.Module):
    """Augmented LipNet that includes a feature embedding network that acts on the output of LipNet.

    """

    def __init__(
        self, device=torch.device("cpu"), dropout_rate=0.5, num_modules=1, verbose=True
    ):
        """Instantiates a new AugmentedLipNet object.

        Parameters
        ----------
        device : torch.device instance
            The device on which all procedures are carried out, by default torch.device("cpu").
        dropout_rate : float, optional
            The dropout rate applied after each convolutional layer, by default 0.5.
        verbose : bool, optional
            If True then progress of the forward pass is printed to the system console, by default True.

        """
        super(AugmentedLipNet, self).__init__()

        assert isinstance(device, torch.device)
        assert isinstance(dropout_rate, float)
        assert isinstance(verbose, bool)
        assert (
            dropout_rate > 0 and dropout_rate <= 1
        ), "Dropout rate must be a proper probability."

        self.device = device
        self.dropout_rate = dropout_rate
        self.verbose = verbose

        self.lipnet_model = lipnet.LipNet(
            dropout_rate=self.dropout_rate, verbose=self.verbose
        ).to(self.device)
        self.encoder_model = FeaturesEncoder(network_type=NetworkType.lipnet).to(
            self.device
        )

        if hasattr(lipnet_options, "weights"):
            self._load_lipnet_weights()
        else:
            print("[INFO] Pre-trained weights were not loaded.")

    def _load_lipnet_weights(self):
        pretrained_dict = torch.load(lipnet_options.weights, map_location=self.device)
        pretrained_dict["fully_connected.weight"] = pretrained_dict.pop("FC.weight")
        pretrained_dict["fully_connected.bias"] = pretrained_dict.pop("FC.bias")

        model_dict = self.lipnet_model.state_dict()
        pretrained_dict = {
            k: v
            for k, v in pretrained_dict.items()
            if k in model_dict.keys() and v.size() == model_dict[k].size()
        }
        missed_params = [
            k for k, v in model_dict.items() if not k in pretrained_dict.keys()
        ]
        if self.verbose:
            print(
                "Loaded parameters / Total parameters: {}/{}".format(
                    len(pretrained_dict), len(model_dict)
                )
            )

            model_dict.update(pretrained_dict)
            self.lipnet_model.load_state_dict(model_dict)
        #else:
        #    print(
        #        "[WARNING] Invalid path to pre-trained weights, and thus no weights will be loaded."
         #   )

    def _load_encoder_weights(self):
        pass

    def forward(self, x):
        """Performs a forward pass of the input ``x`` through the network.

        Parameters
        ----------
        x : torch.Tensor instance
            Torch tensor containing the input to the network.

        Returns
        -------
        lipnet_embedding : torch.Tensor instance
            Torch tensor containing the learned embedding from LipNet.

        """
        x = self.lipnet_model(x)
        x = x.permute(1, 0, 2)
        x = x[None, :, :, :].permute(1, 0, 2, 3)
        lipnet_embedding = self.encoder_model(x)
        lipnet_embedding = lipnet_embedding[:, None, :].float().to(self.device)

        return lipnet_embedding

    @staticmethod
    def _find_bounding_box(landmarks, tol=(2, 2, 2, 2)):
        """Computes the minimum bounding box containing ``landmarks``, with tolerance (in pixels) in the left, right, top, and bottom directions.

        Parameters
        ----------
        landmarks : torch.Tensor instance
            Torch tensor containing the xy-coordinates of the facial landmarks.
        tol : tuple, optional
            The tolerance (in pixels) in each direction (left, top, right, bottom) for the cropping operation, by default (2, 2, 2, 2).

        Returns
        -------
        bbox : tuple (of ints)
            Tuple (min_x, min_y, max_x, max_y) containing the coordinates of the minimum bounding box with tolerance (in pixels) in the left, top, right, and bottom directions.

        """
        assert isinstance(tol, tuple)
        assert len(tol) == 4, "Only four values can be specified for the tolerance."

        x_coords, y_coords = zip(*landmarks)
        bbox = (
            min(x_coords) - tol[0],
            min(y_coords) - tol[1],
            max(x_coords) + tol[2],
            max(y_coords) + tol[3],
        )
        bbox = tuple([int(item) for item in bbox])

        return bbox

    @staticmethod
    def _find_bounding_box_batch(batch_landmarks, tol=(2, 2, 2, 2)):
        """Computes the minimum bounding box containing ``batch_landmarks``, with tolerances (in pixels) in the left, right, top, and bottom directions, for a batch of images.

        It is assumed that the batch dimension is the first dimension.

        Parameters
        ----------
        batch_landmarks : torch.Tensor instance
            Torch tensor containing a batch of xy-coordinates of the facial landmarks.
        tol : tuple, optional
            The tolerance (in pixels) in each direction (left, top, right, bottom) for the cropping operation, by default (2, 2, 2, 2).

        Returns
        -------
        batch_bbox : list (of tuples)
            List containing tuples of xy-coordinates of the bounding boxes for the batch of facial landmarks.

        """
        batch_bbox = list()
        batch_landmarks = batch_landmarks[:, 48:68]
        for landmark in batch_landmarks:
            batch_bbox.append(AugmentedLipNet._find_bounding_box(landmark, tol))

        return batch_bbox

    @staticmethod
    def _crop_lips(frames, landmarks, tol=(2, 2, 2, 2), channels_first=True):
        """Crops the lip area from a batch of frames.

        Parameters
        ----------
        frames : torch.Tensor instance
            Torch tensor instance containing frames for cropping the lip area.
        landmarks : torch.Tensor instance
            Torch tensor containing the xy-coordinates of the facial landmarks.
        tol : tuple, optional
            The tolerance (in pixels) in each direction (left, top, right, bottom) for the cropping operation, by default (2, 2, 2, 2)
        channels_first : bool, optional
            If True then the input and output are assumed to have the channel dimension come before the spatial dimensions.

        Returns
        -------
        cropped_frames : torch.Tensor instance
            Torch tensor containing the cropped lip areas.

        """
        assert isinstance(tol, tuple)
        assert len(tol) == 4, "Only four values can be specified for the tolerance."
        assert isinstance(channels_first, bool)

        if channels_first:
            frames = frames.permute(0, 2, 3, 1)

        bboxes = AugmentedLipNet._find_bounding_box_batch(
            batch_landmarks=landmarks, tol=tol
        )

        output_shape = (frames.shape[0], 64, 128, 3)
        extracted_lips = torch.empty(output_shape)
        for idx, (bbox, frame) in enumerate(zip(bboxes, frames)):
            extracted_lip = frame[bbox[1] : bbox[3], bbox[0] : bbox[2], :]
            extracted_lips[idx] = torch.from_numpy(
                cv2.resize(
                    extracted_lip.cpu().numpy(),
                    dsize=(128, 64),
                    interpolation=cv2.INTER_CUBIC,
                )
            )

        return extracted_lips

    @staticmethod
    def _crop_lips_batch(
        batch_frames, batch_landmarks, tol=(2, 2, 2, 2), channels_first=True
    ):
        """Crops the lip area for a batch of batch of frames.

        It is assumed that the batch dimension is the first dimension.

        Parameters
        ----------
        batch_frames : torch.Tensor instance
            Torch tensor containing the batch of frames.
        batch_landmarks : torch.Tensor instance
            Torch tensor containing the batch of xy-coordinates of the facial landmarks.
        tol : tuple, optional
            The tolerance (in pixels) in each direction (left, top, right, bottom) for the cropping operation, by default (2, 2, 2, 2)
        channels_first : bool, optional
            If True then the input and output are assumed to have the channel dimension come before the spatial dimensions.

        Returns
        -------
        batch_extracted_lips : torch.Tensor instance
            Torch tensor containing a batch of cropped lip areas.

        """
        assert isinstance(tol, tuple)
        assert len(tol) == 4, "Only four values can be specified for the tolerance"
        assert isinstance(channels_first, bool)

        output_shape = (
            batch_frames.shape[0],
            batch_frames.shape[1],
            64,
            128,
            batch_frames.shape[2],
        )
        batch_extracted_lips = torch.empty(output_shape)
        for idx, (frames, landmarks) in enumerate(zip(batch_frames, batch_landmarks)):
            #print(landmarks.shape)
            batch_extracted_lips[idx] = AugmentedLipNet._crop_lips(
                frames, landmarks, tol=tol, channels_first=channels_first
            )

        return batch_extracted_lips

class AugmentedResNetSE34L(nn.Module):
    """Augmented ResNetSE34L that includes a feature embedding network that acts on the output of ResNetSE34L.

    """

    def __init__(self, num_out=256, device=torch.device("cpu"), verbose=True, **kwargs):
        """Instantiates a new AugmentedResNetSE34L object.

        Parameters
        ----------
        num_out : int, optional
            The number of output features, by default 256.
        device : torch.device instance
            The device on which all procedures are carried out, by default torch.device("cpu").
        verbose : bool, optional
            If True then verbose output is printed to the system console.
        **kwargs
            Arguments passed onto i2ai.mmid.models.ResNetSE34L.

        """
        super(AugmentedResNetSE34L, self).__init__()

        assert isinstance(num_out, int)
        assert isinstance(device, torch.device)
        assert isinstance(verbose, bool)

        self.num_out = num_out
        self.device = device
        self.verbose = verbose

        self.resnet_model = audio_model.ResNetSE34L(nOut=num_out)
        self.resnet_model = self.resnet_model.to(device)

        # No encoder model for now...

    def forward(self, x):
        """Performs a forward pass of the input ``x`` through the network.

        Parameters
        ----------
        x : torch.Tensor instance
            Torch tensor containing the input to the network.

        Returns
        -------
        x : torch.Tensor instance
            Torch tensor containing the audio embeddings.

        """
        x = self.resnet_model(x)

        return x

class MultimodalClassifier(nn.Module):
    """Implementation of the Multimodal Classifier network.

    """

    def __init__(self, device=torch.device("cpu"), verbose=True):
        """Instantiates a new MultimodalClassifier object.

        Parameters
        ----------
        device : torch.device instance
            The device on which all procedures are carried out, by default torch.device("cpu").
        verbose : bool, optional
            If True then verbose output is printed to the system console.

        """
        super(MultimodalClassifier, self).__init__()

        assert isinstance(device, torch.device)
        assert isinstance(verbose, bool)

        self.device = device
        self.verbose = verbose

        # Build the network. Expected input is (batch_size, 3, 1, 256).
        self.conv1 = nn.Conv2d(3, 4, kernel_size=3, stride=1, padding=1)
        self.batchnorm1 = nn.BatchNorm2d(4)
        self.maxpool1 = nn.MaxPool2d(kernel_size=1, stride=2)
        self.conv2 = nn.Conv2d(4, 4, kernel_size=3, stride=2, padding=1)
        self.batchnorm2 = nn.BatchNorm2d(4)
        self.maxpool2 = nn.MaxPool2d(kernel_size=1, stride=2)
        self.flatten = nn.Flatten()
        self.fc1 = nn.Linear(128, 2)
        #self.fc2 = nn.Linear(64, 32)
        #self.fc3 = nn.Linear(32, 2)
        self.relu = nn.ReLU(inplace=True)
        self.softmax = nn.Softmax()

    def forward(self, x):
        """Performs a forward pass of the input ``x`` through the network.

        Parameters
        ----------
        x : torch.Tensor instance
            Torch tensor containing the input to the network.

        Returns
        -------
        prediction : torch.Tensor instance
            Class label representing whether the video is real or fake.

        """
        x = self.conv1(x)
        x = self.batchnorm1(x)
        x = self.maxpool1(x)
        x = self.conv2(x)
        x = self.batchnorm2(x)
        x = self.maxpool2(x)
        x = self.flatten(x)
        #x = self.relu(x)
        x = self.fc1(x)
        #x = self.relu(x)
        #x = self.fc2(x)
        #x = self.relu(x)
        #x = self.fc3(x)
        #x = self.relu(x)

        prediction = self.softmax(x)

        return prediction


from sys import exit
from subprocess import Popen, PIPE, STDOUT, TimeoutExpired

class Ropper:

    def __init__(self, binary, input, job, ropchain, bad_chars):
        self.binary = binary
        self.input = input
        self.job = job
        self.logger = job.logger
        self.ropchain = ropchain
        self.bad_chars = bad_chars

    def run(self, timeout):
        cmd = ["ropper", "--single", "--nocolor", "--file", self.binary, "--chain", "execve"]
        if self.bad_chars:
            cmd += ["--badbytes", self.bad_chars]
        self.logger.debug("Run ropper: {}".format(" ".join(cmd)))
        process = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT)

        try:
            stdout = process.communicate(timeout=timeout)[0]
            self.logger.debug("ropper output:")
            self.logger.debug(stdout.decode(errors='ignore'))
        except TimeoutExpired:
            process.kill()
            self.logger.critical("FAIL TIMEOUT")
            exit(3)

        if process.returncode != 0:
            self.logger.error("Compilation ERROR with {} (ropper)".format(process.returncode))
            exit(1)

        lines = stdout.splitlines()
        n = len(lines)
        ropchain_generator = [b"from __future__ import print_function"]
        for i, line in enumerate(lines):
            if line == b"from struct import pack":
                n = i
            if i >= n:
                if line == b"print rop":
                    ropchain_generator.append(b"print(rop, end='')")
                    break
                else:
                    ropchain_generator.append(line)

        script_path = "{}.ropper.script".format(self.binary)
        with open(script_path, 'wb') as script:
            script.write(b"\n".join(ropchain_generator))

        script_cmd = ["/usr/bin/python2", script_path]
        with open(self.ropchain, "wb") as ropchain_output:
            script_p = Popen(script_cmd, stdout=ropchain_output, stderr=PIPE)
            try:
                stderr = script_p.communicate(timeout=timeout)[1]
                self.logger.debug("ropper script output:")
                self.logger.debug(stderr.decode(errors='ignore'))
            except TimeoutExpired:
                script_p.kill()
                self.logger.critical("FAIL TIMEOUT")
                exit(3)

            if script_p.returncode != 0:
                self.logger.error("Compilation ERROR with {} (ropper script)".format(script_p.returncode))
                exit(1)


from portal.plugins.kinesisresponder.kinesis_responder import KinesisResponder
import json
import urllib
from django.conf import settings
from s3_mixin import S3Mixin, FileDoesNotExist
from vs_mixin import VSMixin
import logging
from gnmvidispine.vs_item import VSItem, VSNotFound
from datetime import datetime
import portal.plugins.gnmatomresponder.constants as const
import re

logger = logging.getLogger(__name__)

#Still need: holding image. this is more likely to come from the launch detection side than the atom side.

extract_extension = re.compile(r'^(?P<basename>.*)\.(?P<extension>[^\.]+)$')
multiple_underscore_re = re.compile(r'_{2,}')
make_filename_re = re.compile(r'[^\w\d\.]')

class MasterImportResponder(KinesisResponder, S3Mixin, VSMixin):
    def get_project_collection(self, content):
        """
        Gets a populated VSCollection reference to the project collection mentioned in content
        :param content: parsed json message including the 'projectId' key
        :return:
        """
        from .exceptions import NotAProjectError

        try:
            project_collection = self.get_collection_for_id(content['projectId'])
        except VSNotFound:
            try:
                logger.warning(u"Invalid parent collection {0} specified for atom {1} ({2}). Falling back to default.".format(
                    content['projectId'], content['atomId'],
                    content.get('title','(unknown title)').encode("UTF-8","backslashescape")
                ))
            except UnicodeDecodeError:
                pass
            except UnicodeEncodeError:
                pass
            project_collection = None
        except NotAProjectError:
            try:
                logger.warning(u"Collection {0} specified for atom {1} ({2}) is not a Project. Falling back to default.".format(
                    content['projectId'], content['atomId'],
                    content.get('title','(unknown title)').encode("UTF-8","backslashescape")
                ))
            except UnicodeDecodeError:
                pass
            except UnicodeEncodeError:
                pass
            project_collection = None

        if project_collection is None:
            collection_id = getattr(settings,'ATOM_RESPONDER_DEFAULT_PROJECTID',None)
            if collection_id is None:
                raise RuntimeError("Unable to get a project ID for atom {0}, and no default is set".format(content['atomId']))
            else:
                project_collection = self.get_collection_for_id(collection_id)

        return project_collection

    def update_pluto_record(self, item_id, project_id):
        import traceback
        try:
            from portal.plugins.gnm_masters.signals import master_external_update

            master_external_update.send(sender=self.__class__, item_id=item_id, project_id=project_id)
        except ImportError as e:
            logger.error("Unable to signal master update: {0}".format(e))
        except Exception as e:
            logger.error("An error happened when outputting master_external_create signal: {0}".format(traceback.format_exc()))

    def get_or_create_master_item(self, atomId, title, project_collection, user):
        master_item = self.get_item_for_atomid(atomId)

        created = False

        if master_item is None:
            if title is None:
                raise RuntimeError("Title field not set for atom {0}.".format(atomId))
            if user is None:
                logger.warning("User field not set for atom {0}.".format(atomId))
                user_to_set="unknown_user"
            else:
                user_to_set=user
            master_item = self.create_placeholder_for_atomid(atomId,
                                                             title=title,
                                                             user=user_to_set,
                                                             parent=project_collection
                                                             )
            logger.info("Created item {0} for atom {1}".format(master_item.name, atomId))
            created = True
        return master_item, created

    def process(self,record, approx_arrival, attempt=0):
        """
        Process a message from the kinesis stream.  Each record is a JSON document which contains keys for atomId, s3Key,
        projectId.  This will find an item with the given atom ID or create a new one, get a signed download URL from
        S3 for the media and then instruct Vidsipine to import it.
        Rather than wait for the job to complete here, we return immediately and rely on receiving a message from VS
        when the job terminates.
        :param record: JSON document in the form of a string
        :param approx_arrival:
        :param attempt: optional integer showing how many times this has been retried
        :return:
        """
        from media_atom import request_atom_resend, HttpError
        from tasks import timed_request_resend
        content = json.loads(record)

        logger.info(content)

        #We get two types of message on the stream, one for incoming xml the other for incoming media.
        if content['type'] == const.MESSAGE_TYPE_MEDIA or content['type'] == const.MESSAGE_TYPE_RESYNC_MEDIA:
            project_collection = self.get_project_collection(content)
            if 'user' in content:
                atom_user = content['user']
            else:
                atom_user = None
            master_item, created = self.get_or_create_master_item(content['atomId'], content['title'], project_collection, atom_user)

            return self.import_new_item(master_item, content, parent=project_collection)
        elif content['type'] == const.MESSAGE_TYPE_PAC:
            logger.info("Got PAC form data message")
            record = self.register_pac_xml(content)
            self.ingest_pac_xml(record)
            logger.info("PAC form data message complete")
        elif content['type'] == const.MESSAGE_TYPE_PROJECT_ASSIGNED:
            logger.info("Got project (re-)assignment message: {0}".format(content))

            master_item = self.get_item_for_atomid(content['atomId'])
            if master_item is not None:
                logger.info("Master item for atom already exists at {0}, assigning".format(master_item.name))
                self.assign_atom_to_project(content['atomId'], content['commissionId'], content['projectId'], master_item)
            else:
                logger.warning("No master item exists for atom {0}.  Requesting a re-send from media atom tool".format(content['atomId']))
                try:
                    request_atom_resend(content['atomId'], settings.ATOM_TOOL_HOST, settings.ATOM_TOOL_SECRET)
                except HttpError as e:
                    if e.code == 404:
                        if attempt >= 10:
                            logger.error("{0}: still nothing after 10 attempts. Giving up.".format(content['atomId']))
                            raise
                        logger.warning("{0}: Media atom tool responded with a 404 on attempt {1}: {2}. Retrying in 60s.".format(content['atomId'], attempt, e.content))
                        timed_request_resend.apply_async(args=(record, approx_arrival),
                                                         kwargs={'attempt': attempt+1},
                                                         countdown=60)
                    else:
                        logger.exception("{0}: Could not request resync".format(content['atomId']))

            logger.info("Project (re-)assignment complete")
        else:
            raise ValueError("Unrecognised message type: {0}".format(content['type']))

    def register_pac_xml(self, content):
        """
        Start the import of new PAC data by registering it in the database.
        :param content: JSON message content as received from atom tool
        :return: the database model instance
        """
        from models import PacFormXml

        (record, created) = PacFormXml.objects.get_or_create(atom_id=content['atomId'], defaults={'received': datetime.now()})
        if not created:
            logger.info("PAC form xml had already been delivered for {0}, over-writing".format(content['atomId']))

        record.completed = None
        record.status = "UNPROCESSED"
        record.last_error = ""
        record.pacdata_url = "s3://{bucket}/{path}".format(bucket=content['s3Bucket'], path=content['s3Path'])
        record.save()
        return record

    def check_for_old_finished_jobs(self, vs_item_id):
        from models import ImportJob

        jobs = ImportJob.objects.filter(item_id=vs_item_id).filter(status='FINISHED').count()

        return jobs > 0

    def check_key(self, key, vs_item_id):
        from models import ImportJob

        jobs = ImportJob.objects.filter(item_id=vs_item_id).filter(s3_path=key).count()

        return jobs > 0

    def check_for_processing(self, vs_item_id):
        from models import ImportJob

        jobs = ImportJob.objects.filter(item_id=vs_item_id).filter(processing=True).count()

        return jobs > 0

    def import_new_item(self, master_item, content, parent=None):
        from models import ImportJob, PacFormXml
        from pac_xml import PacXmlProcessor
        from mock import MagicMock
        if not isinstance(master_item, VSItem) and not isinstance(master_item, MagicMock): raise TypeError #for intellij
        from portal.plugins.kinesisresponder.sentry import inform_sentry

        vs_item_id = master_item.get("itemId")

        if vs_item_id is None:
            vs_item_id = master_item.name

        old_finished_jobs = self.check_for_old_finished_jobs(vs_item_id)

        old_key = self.check_key(content['s3Key'], vs_item_id)

        if old_finished_jobs is True and old_key is True:
            logger.info('A job for item {0} has already been successfully completed. Aborting.'.format(vs_item_id))
            inform_sentry('A job for item {0} has already been successfully completed. Aborting.'.format(vs_item_id), {
                "master_item": master_item,
                "content": content,
                "parent": parent
            })
            return

        processing_job = self.check_for_processing(vs_item_id)

        if processing_job is True:
            logger.info('Job for item {0} already in progress. Aborting.'.format(vs_item_id))
            inform_sentry('Job for item {0} already in progress. Aborting.'.format(vs_item_id), {
                "master_item": master_item,
                "content": content,
                "parent": parent
            })
            return

        safe_title = content.get('title','(unknown title)').encode("UTF-8","backslashescape").decode("UTF-8")

        #using a signed URL is preferred, but right now VS seems to have trouble ingesting it.
        #so, we download instead and ingest that. get_s3_signed_url is left in to make it simple to switch back
        #download_url = self.get_s3_signed_url(bucket=settings.ATOM_RESPONDER_DOWNLOAD_BUCKET, key=content['s3Key'])
        downloaded_path = self.download_to_local_location(bucket=settings.ATOM_RESPONDER_DOWNLOAD_BUCKET,
                                                          key=content['s3Key'],
                                                          #this is converted to a safe filename within download_to_local_location
                                                          filename=content.get('title', None)) #filename=None => use s3key instead

        download_url = "file://" + urllib.quote(downloaded_path)

        logger.info(u"{n}: Ingesting atom with title '{0}' from media atom with ID {1}".format(safe_title, content['atomId'], n=master_item.name))
        try:
            logger.info(u"{n}: Download URL is {0}".format(download_url, n=master_item.name))
        except UnicodeEncodeError:
            pass
        except UnicodeDecodeError:
            pass

        job_result = master_item.import_to_shape(uri=download_url,
                                                 essence=True,
                                                 shape_tag=getattr(settings,"ATOM_RESPONDER_SHAPE_TAG","lowres"),
                                                 priority=getattr(settings,"ATOM_RESPONDER_IMPORT_PRIORITY","HIGH"),
                                                 jobMetadata={'gnm_source': 'media_atom'},
                                                 )
        logger.info(u"{0} Import job is at ID {1}".format(vs_item_id, job_result.name))

        master_item.set_metadata({const.GNM_ASSET_FILENAME: downloaded_path})

        if parent is not None:
            logger.info(u"{0}: Adding to collection {1}".format(vs_item_id, parent.name))
            master_item.name = vs_item_id
            parent.addToCollection(master_item)
            logger.info(u"{0}: Done".format(vs_item_id))
        else:
            logger.error(u"{0}: No parent collection specified for item!".format(vs_item_id))

        self.update_pluto_record(vs_item_id, parent.name)
        #make a note of the record. This is to link it up with Vidispine's response message.
        record = ImportJob(item_id=vs_item_id,
                           job_id=job_result.name,
                           status='STARTED',
                           started_at=datetime.now(),
                           user_email=content.get('user',"Unknown user"),
                           atom_id=content['atomId'],
                           atom_title=content.get('title', "Unknown title"),
                           s3_path=content['s3Key'],
                           processing=True)
        previous_attempt = record.previous_attempt()
        if previous_attempt:
            record.retry_number = previous_attempt.retry_number+1
            logger.info(u"{0} Import job is retry number {1}".format(vs_item_id, record.retry_number))
        record.save()

        try:
            logger.info(u"{n}: Looking for PAC info that has been already registered".format(n=vs_item_id))
            pac_entry = PacFormXml.objects.get(atom_id=content['atomId'])
            logger.info(u"{n}: Found PAC form information at {0}".format(pac_entry.pacdata_url,n=vs_item_id))
            proc = PacXmlProcessor(self.role_name, self.session_name)
            proc.link_to_item(pac_entry, master_item)
        except PacFormXml.DoesNotExist:
            logger.info(u"{n}: No PAC form information has yet arrived".format(n=vs_item_id))

    def ingest_pac_xml(self, pac_xml_record):
        """
        Master process to perform import of pac data
        :param pac_xml_record: instance of PacFormXml model
        :return:
        """
        from pac_xml import PacXmlProcessor

        vsitem = self.get_item_for_atomid(pac_xml_record.atom_id)
        if vsitem is None:
            logger.warning("No item could be found for atom ID {0}, waiting for it to arrive".format(pac_xml_record.atom_id))
            return

        proc = PacXmlProcessor(self.role_name,self.session_name)

        #this process will call out to Pluto to do the linkup once the data has been received
        return proc.link_to_item(pac_xml_record, vsitem)

    def assign_atom_to_project(self, atomId, commissionId, projectId, master_item):
        """
        (re)-assigns the given master to a project.
        If the project is not associated with the given commission, warns but does not fail.
        :param atomId:
        :param commissionId:
        :param projectId:
        :return:
        """

        current_project_id = master_item.get(const.PARENT_COLLECTION)
        if current_project_id is not None:
            logger.warning("Re-assigning master {0} to project {1} so removing from {2}".format(master_item.name, projectId, current_project_id))
            current_project_ref = self.get_collection_for_id(current_project_id)
            current_project_ref.removeFromCollection(master_item)

        new_project_ref = self.get_collection_for_id(projectId)
        expected_commission_id = new_project_ref.get(const.PARENT_COLLECTION)
        if expected_commission_id!=commissionId:
            logger.warning("Project {0} belongs to commission {1}, but media atom thinks it belongs to commission {2}. Continuing anyway".format(projectId, expected_commission_id, commissionId))

        logger.info("Adding master {0} to collection {1}".format(master_item.name, new_project_ref.name))
        new_project_ref.addToCollection(master_item)
        logger.info("Setting up project fields for {0}".format(master_item.name))
        self.set_project_fields_for_master(master_item,parent_project=new_project_ref)
        logger.info("Telling gnm_masters about updates for {0}".format(master_item.name))
        self.update_pluto_record(master_item.name, projectId)


# -*- coding: utf-8 -*-
"""
pytorch_neural_network_binary_classification.ipynb
## Neural Network Classification with PyTorch
## 1. Make Custom Classification Dataset and Get It Ready
"""

from sklearn.datasets import make_circles

# Number of samples, let's get 1000 samples
n_samples = 1000

# Create Circles
X, y = make_circles(n_samples=n_samples,
                    noise=0.03,
                    random_state=42)

# Make DataFrame of Circle Data
import pandas as pd

dataFrame = pd.DataFrame({'X1': X[:, 0],
                          'X2': X[:, 1],
                          'Label': y})

dataFrame.head(10)

from matplotlib import pyplot as plt
plt.scatter(x=X[:, 0],
            y=X[:, 1],
            c=y)

"""
## 1.1 Turn Data into Tensors and Create Train & Test Splits
"""

# Turn data into tensors
import torch
X = torch.from_numpy(X).type(torch.float)
y = torch.from_numpy(y).type(torch.float)

# Split data into training and tests sets
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

"""
## 2. Building a Model

To do so, we want to:

1. Setup device agonistic code so our code will run on an accelerator (GPU) if there is one
2. Construct a model (by subclassing nn.Module)
3. Define a loss function and optimizer
4. Create a training and test loop
"""

import torch
from torch import nn

device = 'cuda' if torch.cuda.is_available() else 'cpu'

"""
Now we've setup device agnostic code, let's create a model that:

1. Subclasses nn.Module (almost all models in PyTorch subclass nn.Module)
2. Create 2 nn.Linear() layers that are capable of handling the shapes of our data
3. Defines a forward() method that outlines the forward pass (or forward computation) of the model
4. Instatiate an instance of our model class and send it to the target device
"""

# 1. Construct a model that subclasses nn.Module
class CircleModel(nn.Module):
  def __init__(self):
    super().__init__()
    # 2. Create 2 nn.Linear layers capable of handling the shapes of our data
    self.input_layer = nn.Linear(in_features=2,
                                 out_features=64,
                                 bias=True) # takes in 2 features and upscales to 64 features

    self.first_hidden_layer = nn.Linear(in_features=64,
                                  out_features=1,
                                  bias=True) # takes in 64 features from previous layer and outputs a single feature (same shape as y)

  # 3. Define a forward() method that outlines the forward pass
  def forward(self, x:torch.Tensor) -> torch.Tensor:
    return self.first_hidden_layer(self.input_layer(x)) # x -> input_layer ->  first_hidden_layer -> output

# 4. Instantiate an instance of our model class and send it to the target device
model = CircleModel().to(device)
model

# Let's replicate the model above using nn.Sequential()
sequential_model = nn.Sequential(
    nn.Linear(in_features=2, out_features=64, bias=True),
    nn.Linear(in_features=64, out_features=1, bias=True)
).to(device)

sequential_model

# Setup the Loss Function and Optimizer
loss_func = nn.BCEWithLogitsLoss() # BCEWithLogitsLoss = sigmoid activation function built-in
optimizer = torch.optim.SGD(params=model.parameters(),
                            lr=0.01)

# Evaluation Function
def accuracy_func(y_pred, y_true):
  correct = torch.eq(y_true, y_pred).sum().item()
  return ((correct/len(y_true))*100)

"""
## 3. Build a Train & Test Loop
"""

torch.cuda.manual_seed(42)
torch.manual_seed(42)
epochs = 100

for epoch in range(epochs):

  # Train Mode
  model.train()

  # Send the Data Target Device
  X_train, X_test = X_train.to(device), X_test.to(device)
  y_train, y_test = y_train.to(device), y_test.to(device)

  # 1. Forward Pass
  train_logits = model.forward(X_train).squeeze()
  train_pred = torch.round(torch.sigmoid(train_logits)) # turn logits -> pred probs -> pred labels

  # 2. Calculate Train Loss/Accuracy
  train_loss = loss_func(train_logits, y_train)
  train_accuracy = accuracy_func(train_pred, y_train)

  # 3. Zero Grad with Optimizer
  optimizer.zero_grad()

  # 4. Backpropagation with Loss
  train_loss.backward()

  # 5. Optimizer step (gradient descent)
  optimizer.step()

  # Test Mode
  model.eval()
  with torch.inference_mode():
    # 1. Farward Pass
    test_logits = model.forward(X_test).squeeze()
    test_pred = torch.round(torch.sigmoid(test_logits))

    # 2. Calculate Test Loss/Accuracy
    test_loss = loss_func(test_logits, y_test)
    test_accuracy = accuracy_func(test_pred, y_test)

  if epoch % 5 == 0:
    print(f'Epoch: {epoch} | Training Accuracy {train_accuracy:.2f} | Training Loss: {train_loss:.2f} | Testing Accuracy {test_accuracy:.2f} | Testing Loss: {test_loss:.2f}')

"""
## From the metrics it looks like our model isn't learning anything...
"""

import requests
from pathlib import Path

# Download helper functions
if Path("helper_functions.py").is_file():
  print("helper_functions.py already exists, skipping download")
else:
  print("Downloading helper_functions.py")
  request = requests.get("https://raw.githubusercontent.com/UygarKAYA/DeepLearning/main/utils/helper_functions.py")
  with open("helper_functions.py", "wb") as f:
    f.write(request.content)

from utils.helper_functions import plot_decision_boundary

# Plot decision boundary of the model
plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
plt.title("Train")
plot_decision_boundary(model, X_train, y_train)
plt.subplot(1, 2, 2)
plt.title("Test")
plot_decision_boundary(model, X_test, y_test)

"""
## 4. Improving a model (from a model perspective)

* Add more layers - give the model more chances to learn about patterns in the * data
* Add more hidden units - go from 5 hidden units to 10 hidden units
* Fit for longer
* Changing the activation functions
* Change the learning rate
* Change the loss function

"""

class CircleModelV1(nn.Module):
  def __init__(self):
    super().__init__()
    self.input_layer = nn.Linear(in_features=2, out_features=10)
    self.first_hidden_layer = nn.Linear(in_features=10, out_features=10)
    self.second_hidden_layer = nn.Linear(in_features=10, out_features=1)

  def forward(self, x:torch.Tensor) -> torch.Tensor:
    return self.second_hidden_layer(self.first_hidden_layer(self.input_layer(x)))

improvedModel = CircleModelV1().to(device)
improvedModel

# Create a Loss Function
loss_func = nn.BCEWithLogitsLoss()
# Create a Optimizer
optimizer = torch.optim.SGD(params=improvedModel.parameters(),
                            lr=0.001)

torch.manual_seed(42)
torch.cuda.manual_seed(42)

epochs = 250
for epoch in range(epochs):
  # Training Step
  improvedModel.train()

  train_logits = improvedModel.forward(X_train).squeeze()
  train_prediction = torch.round(torch.sigmoid(train_logits))

  train_loss = loss_func(train_logits, y_train)
  train_accuracy = accuracy_func(train_prediction, y_train)

  optimizer.zero_grad()

  train_loss.backward()

  optimizer.step()

  # Testing Step
  improvedModel.eval()
  with torch.inference_mode():
    test_logits = improvedModel.forward(X_test).squeeze()
    test_predictions = torch.round(torch.sigmoid(test_logits))

    test_loss = loss_func(test_logits, y_test)
    test_accuracy = accuracy_func(test_predictions, y_test)

  if epoch % 25 == 0:
    print(f'Epoch: {epoch} | Training Accuracy {train_accuracy:.2f} | Training Loss: {train_loss:.2f} | Testing Accuracy {test_accuracy:.2f} | Testing Loss: {test_loss:.2f}')

from utils.helper_functions import plot_decision_boundary

# Plot decision boundary of the model
plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
plt.title("Train")
plot_decision_boundary(improvedModel, X_train, y_train)
plt.subplot(1, 2, 2)
plt.title("Test")
plot_decision_boundary(improvedModel, X_test, y_test)

"""
## 4.1 The missing piece: non-linearity
### 4.1.1 Building a model with non-linearity
"""

# Build a model with non-linear activation functions

class NonLinearModel(nn.Module):
  def __init__(self):
    super().__init__()
    self.input_layer = nn.Linear(in_features=2, out_features=10)
    self.first_hidden_layer = nn.Linear(in_features=10, out_features=10)
    self.second_hidden_layer = nn.Linear(in_features=10, out_features=1)
    self.relu = nn.ReLU()

  def forward(self, x:torch.Tensor) -> torch.Tensor:
    return self.second_hidden_layer(self.relu(self.first_hidden_layer(self.relu(self.input_layer(x)))))

improvedModelV1 = NonLinearModel().to(device)
improvedModelV1

loss_func = nn.BCEWithLogitsLoss()
optimizer = torch.optim.SGD(params=improvedModelV1.parameters(),
                            lr=0.01)

torch.manual_seed(42)
torch.cuda.manual_seed(42)

epochs = 250
for epoch in range(epochs):
  # Training Step
  improvedModelV1.train()

  train_logits = improvedModelV1.forward(X_train).squeeze()
  train_prediction = torch.round(torch.sigmoid(train_logits))

  train_loss = loss_func(train_logits, y_train)
  train_accuracy = accuracy_func(train_prediction, y_train)

  optimizer.zero_grad()

  train_loss.backward()

  optimizer.step()

  # Testing Step
  improvedModelV1.eval()
  with torch.inference_mode():
    test_logits = improvedModelV1.forward(X_test).squeeze()
    test_predictions = torch.round(torch.sigmoid(test_logits))

    test_loss = loss_func(test_logits, y_test)
    test_accuracy = accuracy_func(test_predictions, y_test)

  if epoch % 25 == 0:
    print(f'Epoch: {epoch} | Training Accuracy {train_accuracy:.2f} | Training Loss: {train_loss:.2f} | Testing Accuracy {test_accuracy:.2f} | Testing Loss: {test_loss:.2f}')

from utils.helper_functions import plot_decision_boundary

# Plot decision boundary of the model
plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
plt.title("Train")
plot_decision_boundary(improvedModelV1, X_train, y_train)
plt.subplot(1, 2, 2)
plt.title("Test")
plot_decision_boundary(improvedModelV1, X_test, y_test)