import binascii
from enum import Enum, auto

class Network(Enum):
    MAIN_PRIVATE = auto()
    MAIN_PUBLIC = auto()
    TEST_PRIVATE = auto()
    TEST_PUBLIC = auto()

    def is_private(self) \
            -> bool:
        if self is Network.MAIN_PRIVATE or self is Network.TEST_PRIVATE:
            return True
        elif self is Network.MAIN_PUBLIC or self is Network.TEST_PUBLIC:
            return False
        else:
            raise Exception("Unknown Network")

    def is_public(self) \
            -> bool:
        return not self.is_private()

    def is_testnet(self) \
            -> bool:
        return self is Network.TEST_PRIVATE or self is Network.TEST_PUBLIC

    def invert(self) \
            -> 'Network':
        """
        Inverts Public to Private and vice-versa
        """

        if self is Network.MAIN_PRIVATE:
            return Network.MAIN_PUBLIC
        elif self is Network.MAIN_PUBLIC:
            return Network.MAIN_PRIVATE
        elif self is Network.TEST_PRIVATE:
            return Network.TEST_PUBLIC
        elif self is Network.TEST_PUBLIC:
            return Network.TEST_PRIVATE
        else:
            raise Exception("Unknown Network")

# Only BIP-44, P2PKH (xprv, xpub)
# Others: BIP-49 'P2WPKH-nested-in-P2SH' (yprv, ypub); BIP-84 'P2WPKH' (zprv, zpub)
VERSION_BYTES = {
    Network.MAIN_PRIVATE: binascii.unhexlify('0488ade4'),  # xprv
    Network.MAIN_PUBLIC: binascii.unhexlify('0488b21e'),  # xpub
    Network.TEST_PRIVATE: binascii.unhexlify('04358394'),  # tprv
    Network.TEST_PUBLIC: binascii.unhexlify('043587cf')  # tpub
}

def get_network_from_bytes(version_bytes: bytes) \
        -> Network:
    if version_bytes == VERSION_BYTES[Network.MAIN_PRIVATE]:
        return Network.MAIN_PRIVATE
    elif version_bytes == VERSION_BYTES[Network.MAIN_PUBLIC]:
        return Network.MAIN_PUBLIC
    elif version_bytes == VERSION_BYTES[Network.TEST_PRIVATE]:
        return Network.TEST_PRIVATE
    elif version_bytes == VERSION_BYTES[Network.TEST_PUBLIC]:
        return Network.TEST_PUBLIC
    else:
        raise Exception(f'Invalid version_bytes: {version_bytes.hex()}')

def get_private_version_byte_of(network: Network) \
        -> bytes:
    if network.is_testnet():
        return VERSION_BYTES[Network.TEST_PRIVATE]
    else:
        return VERSION_BYTES[Network.MAIN_PRIVATE]

def get_public_version_byte_of(network: Network) \
        -> bytes:
    if network.is_testnet():
        return VERSION_BYTES[Network.TEST_PUBLIC]
    else:
        return VERSION_BYTES[Network.MAIN_PUBLIC]


import numpy as np
import matplotlib
import math
matplotlib.use("Agg")
from matplotlib import pyplot as plt
#from scipy.optimize import curve_fit
from lmfit import Model, Parameters
from scipy import special
import os
from os.path import expanduser
import sys
sys.float_info.max

matplotlib.rc('xtick', labelsize=15)
matplotlib.rc('ytick', labelsize=15)

###########################################################################
## Global variables
###########################################################################
Pop = 8
TotalTimePoint = 9
OutputFilePath = "/mnt/c/Users/Ruby/Google Drive/Lab Project/AllPopReadCounts/20190227_NSF_Grant/P"+str(Pop)+"/0307/"
if not os.path.exists(OutputFilePath):
    os.makedirs(OutputFilePath)
BarcodeCountPath = "/mnt/c/Users/Ruby/Google Drive/Lab Project/AllPopReadCounts/SourceFile/"

bluish_green = "#009E73"

def read_MeanFitness(Pop):
    InputFilePath = "/mnt/c/Users/Ruby/Google Drive/Lab Project/AllPopReadCounts/Pop"+str(Pop)+"/"
    FilePathName = InputFilePath+"P"+str(Pop)+"_It5_MeanFitness.txt"
    file = open(FilePathName,"r")
    triplets=file.read().split()
    file.close()
    y=np.array(triplets, dtype=np.float32)
    return y

def read_Fitness(Pop):
    InputFilePath = "/mnt/c/Users/Ruby/Google Drive/Lab Project/AllPopReadCounts/Pop"+str(Pop)+"/"
    FilePathName = InputFilePath+"P"+str(Pop)+"_It5_RelativeFitness.txt"
    file = open(FilePathName,"r")
    triplets=file.read().split()
    file.close()
    y=np.array(triplets, dtype=np.float32)
    #print(y.shape)
    return np.transpose(np.reshape(y,(-1,TotalTimePoint-1)))

def read_Adpative_LNs(Pop):
    InputFilePath = "/mnt/c/Users/Ruby/Google Drive/Lab Project/AllPopReadCounts/Pop"+str(Pop)+"/"
    FilePathName = InputFilePath+"P"+str(Pop)+"_It5_ListAdaptiveBarcodeIndex.txt"
    file = open(FilePathName,"r")
    triplets=file.read().split()
    file.close()
    y=np.array(triplets, dtype=np.int32)
    return y

def readBarcodeCount(Pop):
    global BarcodeCountPath, TotalTimePoint
    FileName = "Pop"+str(Pop)+"BowtieReadCounts"
    FilePathName = BarcodeCountPath +FileName+".txt"
    file = open(FilePathName,"r")
    triplets=file.read().split()
    file.close()
    A=np.array(triplets, dtype=np.int32)
    return  np.transpose(np.reshape(A,(-1,TotalTimePoint)))

def get_Barcode_Freq(BarcodeCountsArray):  # BarcodeCountsArray[t][bc] Add 0.5 to zero count to aviod divergent of log10(freqeuncy)
    AllCountsArray = [sum(BarcodeCountsArray[i]) for i in range(0,len(BarcodeCountsArray))]
    #BarcodeFreqArray =  np.asarray([[(count+ int(count==0)*0.5)/AllCountsArray[t] for count in BarcodeCountsArray[t]] for t in range(0,len(BarcodeCountsArray))])
    BarcodeFreqArray =  np.asarray([[count/AllCountsArray[t]+ int(count==0)*10**-8 for count in BarcodeCountsArray[t]] for t in range(0,len(BarcodeCountsArray))])
    return BarcodeFreqArray, AllCountsArray

def plot_CleanAdaptive_BCFeq(Pop,time_end,ListAdaptive,ListCleanAdaptive):
    global OutputFilePath, TotalTimePoint
    # read barcode count
    BarcodeCountsArray = readBarcodeCount(Pop)
    # get Barcode Freq
    AllCountsArray = np.sum(BarcodeCountsArray,axis=1)
    BarcodeFreqArray =  [[(count/AllCountsArray[t]+ int(count==0)*10**-8) for count in BarcodeCountsArray[t]] for t in range(0,TotalTimePoint)]
    # plot Frequency for Adaptive Lineages
    plt.figure()
    x = np.asarray([2*i for i in range(TotalTimePoint)])
    plt.ylabel("log10 Barcode Freqeuncy")
    plt.xlabel("Time (cycle)")
    plt.title("Adaptive Lineages, Red:" +str(np.size(ListCleanAdaptive))+" Grey:"  +str(np.size(ListAdaptive)-np.size(ListCleanAdaptive)) )
    plt.xlim(0,np.max(x)+1)
    for bc in ListAdaptive:
        y = [np.log10(BarcodeFreqArray[t][bc]) for t in range(0,len(x))]
        if bc in ListCleanAdaptive:
            plt.plot(x,y,color='red',alpha=0.3, linewidth = 0.3)
        else:
            plt.plot(x,y,color='black',alpha=1, linewidth = 0.3)
    plt.savefig(OutputFilePath+"Adaptive_Lineages_Clean_Pop"+str(Pop)+".png")
'''
def plot_DEF_shift(Pop,ListCleanAdaptive,t,t_mid,freq,Med):
    finess = read_Fitness(Pop)
    binwidth = 0.1
    NLs = np.size(ListCleanAdaptive)
    #data = finess[t][ListCleanAdaptive]
    data = finess[t_mid][ListCleanAdaptive]
    data2 = finess[t+1][ListCleanAdaptive]
    #BarcodeCountsArray = readBarcodeCount(Pop)
    #freq = BarcodeFreqArray[t][ListCleanAdaptive]
    ds = data2 - data
    Mean = np.mean(ds)
    Med = np.median(ds)
    Var = np.var(ds)
    # plot without shift
    plt.figure()
    bins = np.arange(min(data), max(data) + binwidth, binwidth)
    #plt.hist(data,bins,label="cycle"+str(2*t+1),color= 'blue', alpha=0.3)
    plt.hist(data,bins,label="cycle"+str(2*t_mid+1),color= 'blue', alpha=0.3)
    bins = np.arange(min(data2), max(data2) + binwidth, binwidth)
    plt.hist(data2,bins,label="cycle"+str(2*t+3),color= 'green', alpha=0.3)
    plt.legend(loc='best')
    plt.xlabel("fitness (1/cycle)")
    plt.title("Adaptive Lineages, NLs="+str(NLs))
    plt.savefig(OutputFilePath+"CleanAdaptive_DFE_Pop"+str(Pop)+"_T"+str(t)+".png")
    # plot scatter
    plt.figure()
    line_max = max(np.max(data),np.max(data2))
    line_min = min(np.min(data),np.min(data2))
    line = np.arange(line_min, line_max, 0.01)
    plt.plot(line,line, 'k--')
    plt.plot(line,line+Med, 'r--')
    colormap = plt.cm.jet(np.linspace(0,1,101))
    colmax = -1
    colmin = -4
    colbin = (colmax-colmin)/100
    for bc in range(np.size(ListCleanAdaptive)):
        s = np.log10(freq[bc])
        if s>=colmax:
            s= colmax
        elif s< colmin:
            s = colmin
        colorIndx = (int)((s-colmin)/colbin)
        plt.scatter(data[bc],data2[bc], s=50, facecolors='none', edgecolors=colormap[colorIndx])
    plt.title("Pop "+str(Pop) + ",  NLs="+str(np.size(ListCleanAdaptive)))
    plt.xlabel("s  cycle " + str(2*t_mid+1))
    plt.ylabel("s  cycle " + str(2*t+3))
    plt.savefig(OutputFilePath+"CleanAdaptive_Pop"+str(Pop)+"_s"+str(t_mid)+"_s"+str(t+1)+".png")
    # plot with shift
    data2 = data2 - Med
    plt.figure()
    bins = np.arange(min(data), max(data) + binwidth, binwidth)
    plt.hist(data,bins,label="cycle"+str(2*t_mid+1),color= 'blue', alpha=0.3)
    bins = np.arange(min(data2), max(data2) + binwidth, binwidth)
    plt.hist(data2,bins,label="cycle"+str(2*t+3),color= 'green', alpha=0.3)
    plt.legend(loc='best')
    plt.xlabel("fitness (1/cycle)")
    plt.title("Adaptive Lineages, NLs="+str(NLs))
    plt.savefig(OutputFilePath+"CleanAdaptive_DFE_Shift_Pop"+str(Pop)+"_T"+str(t)+".png")
    # plot ds
    plt.figure()
    bins = np.arange(min(ds), max(ds) + binwidth, binwidth)
    plt.hist(ds,bins,label="NLs="+str(NLs),color= 'black', alpha=0.4)
    plt.xlabel("s(cycle "+str(2*t+3)+") - s(cycle "+str(2*t_mid+1)+")")
    plt.title("Mean "+str(Mean)+" Median "+str(Med)+" Var "+str(Var))
    plt.legend(loc='best')
    plt.savefig(OutputFilePath+"CleanAdaptive_ds_Dist_Pop"+str(Pop)+"_T"+str(t)+".png")
    plt.close('all')

def DFE_shift(Pop,t_mid):
    global TotalTimePoint
    Adaptive_NLs = read_Adpative_LNs(Pop)
    BarcodeCountsArray = readBarcodeCount(Pop)
    BarcodeFreqArray, AllCountsArray = get_Barcode_Freq(BarcodeCountsArray)
    finess = read_Fitness(Pop)
    mean_fitness = read_MeanFitness(Pop)
    Median_ds = []
    Var_ds = []
    for t in range(t_mid,TotalTimePoint-2):
        #listNonzeros = np.nonzero(BarcodeCountsArray[t]>10)[0]
        listNonzeros = np.nonzero(BarcodeCountsArray[t_mid]>100)[0]
        listNonzeros2 = np.nonzero(BarcodeCountsArray[t+1]>100)[0]
        ListCleanAdaptive = list(set(listNonzeros)&set(listNonzeros2)&set(Adaptive_NLs))
        #ds = finess[t+1][ListCleanAdaptive] - finess[t][ListCleanAdaptive]
        ds = finess[t+1][ListCleanAdaptive] - finess[t_mid][ListCleanAdaptive]
        Median_ds.append(np.median(ds))
        Var_ds.append(np.var(ds))
        freq = BarcodeFreqArray[t_mid][ListCleanAdaptive]
        plot_DEF_shift(Pop,ListCleanAdaptive,t,t_mid,freq,np.median(ds))
    #t0=3
    #sum_Ds = [np.sum(Median_ds[t0:t]) for t in range(t0,np.size(Median_ds)+1)]
    Alignment_mean_fitness = [m for m in mean_fitness[0:t_mid+1]]
    for t in range(t_mid+1,TotalTimePoint-1):
        #Alignment_mean_fitness[t] = Alignment_mean_fitness[t]-sum_Ds[t-t0]
        Alignment_mean_fitness.append(mean_fitness[t]-Median_ds[t-t_mid-1])
    plt.figure()
    x = np.asarray([1+2*i for i in range(np.size(mean_fitness))])
    plt.plot(x,Alignment_mean_fitness,color='blue',marker='o',ls='solid',label='new mean fitness')
    plt.plot(x,mean_fitness,color='black',marker='o',ls='solid',label='old mean fitness')
    plt.legend(loc='best')
    plt.xlabel('Time (cycle)')
    plt.ylabel('Mean Fitness (1/cycle)')
    plt.savefig(OutputFilePath+"AlignmingMeanFitness_Pop"+str(Pop)+".png")

'''

'''
def find_Adaptives(Pop,t_mid,D_fitness,freq):
    CV_CUT = 0.25
    Freq_CUT = -2.5
    S_CUT = 0
    BarcodeFreqArray = freq
    #listlarge = np.nonzero(np.min(BarcodeCountsArray[t_mid-2:t_mid+1],axis=0)>Count_CUT)[0]
    #listlarge = np.nonzero(np.log10(np.min(BarcodeFreqArray[t_mid-2:t_mid+1],axis=0))>Freq_CUT)[0]
    listlarge = np.nonzero(np.log10(BarcodeFreqArray[t_mid+1])>Freq_CUT)[0]
    fitness = [read_Fitness(Pop)[t] - D_fitness[t] for t in range(t_mid-2,t_mid+1)]
    #fitness =
    s_Mean = np.mean(fitness, axis=0)
    s_STD = np.std(fitness, axis=0)
    s_CV= np.abs(np.asarray([std/mean for std,mean in zip(s_STD,s_Mean)]))
    CV_freq = np.asarray([cv/f*10**-3 for cv,f in zip(s_CV,freq[t_mid])])
    #print(CV_freq.shape)
    listCV = np.nonzero(CV_freq < CV_CUT)[0]  #np.nonzero(s_CV < CV_CUT)[0]
    listMean = np.nonzero(s_Mean > S_CUT )[0]
    listADP = list(set(listlarge))#list(set(listCV) & set(listMean) & set(listlarge))
    print(np.size(listADP))

    plt.figure()
    #plt.plot(s_Mean,s_CV,'k.',markersize=2)
    #plt.plot(s_Mean[listlarge],s_CV[listlarge],'b.',markersize=2)
    plt.plot(s_Mean[listlarge],CV_freq[listlarge],'b.',markersize=2)
    #plt.plot(s_Mean[listlarge],np.exp(-1*s_CV[listlarge]),'b.',markersize=2)
    #plt.plot(np.log10(freq[t_mid][listlarge]),s_CV[listlarge],'b.',markersize=2)
    #plt.plot(s_Mean[listADP],s_CV[listADP],'r.',markersize=3)
    plt.plot(s_Mean[listADP],CV_freq[listADP],'r.',markersize=3)
    #plt.plot(np.log10(freq[t_mid][listADP]),s_CV[listADP],'r.',markersize=3)
    #plt.plot(s_Mean[listADP],np.exp(-1*s_CV[listADP]),'r.',markersize=3)
    plt.ylim(0,2)
    plt.xlim(0,1.5)
    plt.xlabel("Mean of fitness")
    #plt.xlabel("log10 Barcode Freq")
    plt.ylabel("abs CV of fitness")
    #plt.ylabel("Exp -| CV of fitness|")
    plt.title("P"+str(Pop)+" Cycle"+str(2*t_mid-3)+"-"+str(2*t_mid+1))
    plt.savefig(OutputFilePath+"P"+str(Pop)+"_s2CV_Cycle"+str(2*t_mid+1)+".png")
    #plt.savefig(OutputFilePath+"P"+str(Pop)+"_s2ExpCV_Cycle"+str(2*t_mid+1)+".png")
    plt.close('all')

    return listADP
'''
def find_largeLNs(Pop,t_mid,freq):
    Freq_CUT = -3.0
    listlarge = np.nonzero(np.log10(freq[t_mid+1])>Freq_CUT)[0]
    listADP = list(set(listlarge))
    return listADP

def plot_S1S2(Pop,list,t1,t2,D_fitness,Freq):
    global OutputFilePath
    s1 = read_Fitness(Pop)[t1][list] - D_fitness[t1]
    s2 = read_Fitness(Pop)[t2][list] - D_fitness[t2]
    freq = Freq[t1][list]
    ds = np.asarray([i-j for i,j in zip(s2,s1)])
    Med = np.median(ds)
    Mean = np.mean(ds)
    WeightMean = np.average(ds,weights=freq)
    line_max = max(np.max(s1),np.max(s2))
    line_min = min(np.min(s1),np.min(s2))
    line = np.arange(line_min, line_max, 0.01)
    plt.figure()
    plt.plot(line,line, 'k--')
    plt.plot(line,line+WeightMean, 'r--')
    #plt.plot(line,line+Med, 'r--')
    colormap = plt.cm.jet(np.linspace(0,1,101))
    colmax = -1
    colmin = -4
    colbin = (colmax-colmin)/100
    for bc in range(np.size(list)):
        s = np.log10(freq[bc])
        if s>=colmax:
            s= colmax
        elif s< colmin:
            s = colmin
        colorIndx = (int)((s-colmin)/colbin)
        plt.scatter(s1[bc],s2[bc],s=20, facecolors='none', edgecolors=colormap[colorIndx])
    plt.xlabel('Fitness cycle '+str(2*t1+1))
    plt.ylabel('Fitness cycle '+str(2*t2+1))
    plt.title('P'+str(Pop)+"  NLs="+str(np.size(list))+" WM "+str(WeightMean))#+ ", weighted mean "+str(WeightMean))
    plt.savefig(OutputFilePath+"P"+str(Pop)+"_Cycle_"+str(2*t1+1)+"-"+str(2*t2+1)+".png")
    return WeightMean
    #return Med
def plot_new_mean_fitness(Pop,D_fitness,t_mid):
    global OutputFilePath
    mean_fitness = read_MeanFitness(Pop)
    x = [2*t+1 for t in range(np.size(mean_fitness))]
    new_mean_fitness = mean_fitness - D_fitness
    print(new_mean_fitness)
    plt.figure()
    plt.plot(x,mean_fitness,color='black',marker='o',ls='solid',label='old mean fitness')
    plt.plot(x[0:t_mid+1],new_mean_fitness[0:t_mid+1],color='red',marker='o',ls='solid',label='new mean fitness')
    plt.xlabel("Time (cycle)")
    plt.title("P"+str(Pop)+"  Mean Fitness (1/cycle)")
    plt.legend(loc='best')
    plt.savefig(OutputFilePath+"P"+str(Pop)+"_MeanFitness_Cycle_"+str(2*t_mid+1)+".png")

##### Plot Mean Fitness

T_ini = 3

D_mean_fitness = [0 for i in range(TotalTimePoint-1)]

BarcodeCountsArray = readBarcodeCount(Pop)

#Adaptive_NLs = read_Adpative_LNs(Pop)
#listsameADP = list(set(listADP)& set(Adaptive_NLs))
#BarcodeCountsArray = readBarcodeCount(Pop)
#AllCountsArray = [sum(BarcodeCountsArray[i]) for i in range(0,len(BarcodeCountsArray))]

BarcodeFreqArray, AllCountsArray = get_Barcode_Freq(BarcodeCountsArray)

#listADP = [0 for i in range(T_ini,TotalTimePoint-1)]
#listADP[0] = find_Adaptives(Pop,T_ini,D_mean_fitness,BarcodeFreqArray)
listlargeLNs = find_largeLNs(Pop,T_ini,BarcodeFreqArray)
#listADP_all = listADP[0]
for t in range(T_ini+1,TotalTimePoint-1):
    #D_mean_fitness[t] = plot_S1S2(Pop,listADP[t-1-T_ini],t-1,t,D_mean_fitness,BarcodeFreqArray)
    D_mean_fitness[t] = plot_S1S2(Pop,listlargeLNs,t-1,t,D_mean_fitness,BarcodeFreqArray)
    plot_new_mean_fitness(Pop,D_mean_fitness,t)
    #print(D_mean_fitness)
    listlargeLNs = find_largeLNs(Pop,t,BarcodeFreqArray)
    #listADP[t-T_ini] = find_Adaptives(Pop,t,D_mean_fitness,BarcodeFreqArray)
    #listADP_all.extend(listADP[t-T_ini])
    #listADP_all = list(set(listADP_all))

#print(np.size(listADP_all))

np.save(OutputFilePath+"P"+str(Pop)+"_Dx.npy", D_mean_fitness)

#BarcodeFreqArray, AllCountsArray = get_Barcode_Freq(BarcodeCountsArray)
'''
D_fitness = np.load(OutputFilePath+"P"+str(Pop)+"_Dx.npy")

old_fitness = read_Fitness(Pop)
new_fitness = [old_fitness[t] - D_fitness[t] for t in range(np.size(D_fitness))]

new_mean_fitness = read_MeanFitness(Pop) - D_fitness
print(new_mean_fitness)
cal_mean_fitness = [sum([(f1+f2)*s/2 for f1,f2,s in zip(BarcodeFreqArray[t],BarcodeFreqArray[t+1],new_fitness[t])]) for t in range(TotalTimePoint-1)]
#cal_mean_fitness = [sum([f1*s for f1,s in zip(BarcodeFreqArray[1+t],new_fitness[t])]) for t in range(TotalTimePoint-1)]
print(cal_mean_fitness)

plt.figure()
x = [2*t+1 for t in range(np.size(new_mean_fitness))]
plt.plot(x,new_mean_fitness,color='red',marker='o',ls='solid',label='new mean fitness')
plt.plot(x,cal_mean_fitness,color='goldenrod',marker='o',ls='solid',label='mean fitness direct calculation')
plt.legend(loc='best')
plt.xlabel("Time (cycle)")
plt.title("P"+str(Pop) + "  Mean Fitness (1/cycle)")
plt.savefig(OutputFilePath+"P"+str(Pop)+"_MeanFitness_check.png")
'''
'''
a = np.load(OutputFilePath+"P"+str(Pop)+"_newADPs_dS_2.npz")

D_fitness = a['Dx']
listADP = a['ADP']

listADP_all = []
for idx in range(listADP.shape[0]):
    listADP_all.extend(listADP[idx])

listADP_all = list(set(listADP_all))

old_fitness = read_Fitness(Pop)
new_fitness = [old_fitness[t] - D_fitness[t] for t in range(np.size(D_fitness))]

ADP_maxS = []
ADP_aveS = []
for bc in listADP_all:
    max_s = 0
    ave_s = 0
    normalized = 0
    for idx in range(listADP.shape[0]):
        if bc in listADP[idx]:
            s = new_fitness[idx+T_ini][bc]
            max_s = max(max_s,s)
            ave_s = ave_s + s
            normalized = normalized +1
    ave_s = ave_s/normalized
    ADP_maxS.append(max_s)
    ADP_aveS.append(ave_s)

plt.figure()
binwidth = 0.05
bins = np.arange(min(ADP_maxS), max(ADP_maxS) + binwidth, binwidth)
plt.hist(ADP_maxS,bins)
plt.title("NLs="+str(np.size(listADP_all)))
plt.savefig(OutputFilePath+"P"+str(Pop)+"DFE_ADP_max.png")

plt.figure()
binwidth = 0.05
bins = np.arange(min(ADP_aveS), max(ADP_aveS) + binwidth, binwidth)
plt.hist(ADP_aveS,bins)
plt.title("NLs="+str(np.size(listADP_all)))
plt.savefig(OutputFilePath+"P"+str(Pop)+"DFE_ADP_ave.png")

plt.figure()
binwidth = 0.05
ADP_T13 = new_fitness[6][listADP[3]]
bins = np.arange(min(ADP_T13), max(ADP_T13) + binwidth, binwidth)
plt.hist(ADP_T13,bins)
plt.title("NLs="+str(np.size(ADP_T13)))
plt.savefig(OutputFilePath+"P"+str(Pop)+"DFE_ADP_Cycle13.png")

plt.figure()
binwidth = 0.05
ADP_T11 = new_fitness[5][listADP[2]]
bins = np.arange(min(ADP_T11), max(ADP_T11) + binwidth, binwidth)
plt.hist(ADP_T11,bins)
plt.title("NLs="+str(np.size(ADP_T11)))
plt.savefig(OutputFilePath+"P"+str(Pop)+"DFE_ADP_Cycle11.png")
'''

'''
t_mid = t_mid + 1
listADP = find_Adaptives(Pop,t_mid,D_mean_fitness,BarcodeFreqArray)
D_mean_fitness[t_mid+1] = plot_S1S2(Pop,listADP,t_mid,t_mid+1,D_mean_fitness,BarcodeFreqArray)
plot_new_mean_fitness(Pop,D_mean_fitness,t_mid)
print(D_mean_fitness)
#print(np.size(listADP),np.size(Adaptive_NLs),np.size(listsameADP))
'''

'''
plt.figure()
plt.xlabel('Time (cycle)')
plt.ylabel('Log10 BC frequency')
plt.title("P"+str(Pop)+"  Grey: old-ADP, Blue: new-ADP, Red: both-ADP ("+str(np.size(listsameADP))+")")
x = [2*t for t in range(t_mid-2,t_mid+2)]
for bc in Adaptive_NLs:
    #count = [BarcodeCountsArray[t][bc] for t in range(t_mid-2,t_mid+2)]
    y = [np.log10(BarcodeCountsArray[t][bc]/AllCountsArray[t]+(10**-8)*(BarcodeCountsArray[t][bc]==0)) for t in range(t_mid-2,t_mid+2)]
    plt.plot(x,y,color='grey',linewidth=0.1,alpha=1)
for bc in listADP:
    #count = [BarcodeCountsArray[t][bc] for t in range(t_mid-2,t_mid+2)]
    y = [np.log10(BarcodeCountsArray[t][bc]/AllCountsArray[t]+(10**-8)*(BarcodeCountsArray[t][bc]==0)) for t in range(t_mid-2,t_mid+2)]
    plt.plot(x,y,color='blue',linewidth=0.2,alpha=1)
for bc in listsameADP:
    #count = [BarcodeCountsArray[t][bc] for t in range(t_mid-2,t_mid+2)]
    y = [np.log10(BarcodeCountsArray[t][bc]/AllCountsArray[t]+(10**-8)*(BarcodeCountsArray[t][bc]==0)) for t in range(t_mid-2,t_mid+2)]
    plt.plot(x,y,color='red',linewidth=0.2,alpha=1)
plt.savefig(OutputFilePath+"P"+str(Pop)+"_ADP_LNs_Cycle"+str(2*t_mid+1)+".png")
'''

#DFE_shift(Pop,t_mid)
#Adaptive_NLs = read_Adpative_LNs(Pop)

#for Pop in range(1,11):
#    DFE_shift(Pop,t_mid)

'''
s1 = read_Fitness(Pop)[0:time_end+1]

s1_mean = np.mean(s1, axis=0)
s1_tend = s1[time_end]
s1_SEM = np.std(s1, axis=0)/(s1.shape[0])**0.5  # standard error of mean
s1_CV = np.abs(np.asarray([sem/mean*(s1.shape[0])**0.5 for sem,mean in zip(s1_SEM, s1_mean)]))
s1_max = np.max(s1[1:], axis=0)
s1_sort = np.sort(s1[1:], axis=0)
s1_sort = s1_sort[::-1] # reverse (decending order)
s1_max2 = np.mean(s1_sort[0:2], axis=0)

listCVsmall = np.nonzero(s1_CV<1)[0]
listPositiveS = np.nonzero(s1_mean>0)[0]

BarcodeCountsArray = readBarcodeCount(Pop)
listNonzeros = np.nonzero(np.min(BarcodeCountsArray,axis=0)>0)[0]
#ListCleanAdaptive = list(set(listCVsmall)&set(listPositiveS)&set(Adaptive_NLs))
ListCleanAdaptive = np.asarray(list(set(Adaptive_NLs)&set(listNonzeros)))#list(set(listNonzeros)&set(Adaptive_NLs))
print(np.size(ListCleanAdaptive),np.size(Adaptive_NLs))
#plot_CleanAdaptive_BCFeq(Pop,time_end,Adaptive_NLs,ListCleanAdaptive)
print(ListCleanAdaptive[0:10])
BarcodeFreqArray, AllCountsArray = get_Barcode_Freq(BarcodeCountsArray)
print(BarcodeFreqArray.shape)
s1 = read_Fitness(Pop)
for t in range(TotalTimePoint-2):
    list1 = np.nonzero(BarcodeCountsArray[t]>10)[0]
    list2 = np.nonzero(BarcodeCountsArray[t+1]>10)[0]
    ListCleanAdaptive = list(set(Adaptive_NLs)&set(listNonzeros))
    x = s1[t][ListCleanAdaptive]
    y = s1[t+1][ListCleanAdaptive]
    freq = BarcodeFreqArray[t][ListCleanAdaptive]
    colormap = plt.cm.jet(np.linspace(0,1,101))
    colmax = -2
    colmin = -6
    colbin = (colmax-colmin)/100
    line_max = max(np.max(x),np.max(y))
    line_min = min(np.min(x),np.min(y))
    line = np.arange(line_min, line_max, 0.01)
    plt.figure()
    plt.plot(line,line, 'k--')
    for bc in range(np.size(ListCleanAdaptive)):
        s = np.log10(freq[bc])
        if s>=colmax:
            s= colmax
        elif s< colmin:
            s = colmin
        colorIndx = (int)((s-colmin)/colbin)
        plt.scatter(x[bc],y[bc], s=50, facecolors='none', edgecolors=colormap[colorIndx])
    plt.title("Pop "+str(Pop) + ",  NLs="+str(np.size(ListCleanAdaptive)))
    plt.xlabel("s  cycle " + str(2*t+1))
    plt.ylabel("s  cycle " + str(2*t+3))
    plt.savefig(OutputFilePath+"CleanAdaptive_Pop"+str(Pop)+"_s"+str(t)+"_s"+str(t+1)+".png")
    plt.close('all')

'''
'''
binwidth = 0.1

finess = read_Fitness(Pop)

for t in range(TotalTimePoint-2):
    plt.figure()
    data = finess[t][ListCleanAdaptive]
    bins = np.arange(min(data), max(data) + binwidth, binwidth)
    plt.hist(data,bins,label="t"+str(t),color= 'blue', alpha=0.3)
    data = finess[t+1][ListCleanAdaptive]
    bins = np.arange(min(data), max(data) + binwidth, binwidth)
    plt.hist(data,bins,label="t"+str(t+1),color= 'green', alpha=0.3)
    plt.legend(loc='best')
    plt.savefig(OutputFilePath+"CleanAdaptive_Dist_Pop"+str(Pop)+"_T"+str(t)+".png")

Ds = []
Var_Ds = []
binwidth = 0.1

for t in range(TotalTimePoint-2):
    data = finess[t][ListCleanAdaptive]
    data2 = finess[t+1][ListCleanAdaptive]
    ds = data2 - data
    plt.figure()
    bins = np.arange(min(ds), max(ds) + binwidth, binwidth)
    plt.hist(ds,bins,label="t"+str(t),color= 'black', alpha=0.3)
    plt.xlabel("ds")
    Ds.append(np.median(ds))
    Var_Ds.append(np.var(ds))
    plt.title("Mean "+str(np.mean(ds))+" Median "+str(np.median(ds))+" Var "+str(np.var(ds)))
    plt.savefig(OutputFilePath+"CleanAdaptive_Dist_ds_Pop"+str(Pop)+"_T"+str(t)+".png")
print("Ds",Ds)
plt.close('all')
#dx=[]
for t in range(TotalTimePoint-2):
    plt.figure()
    data = finess[t][ListCleanAdaptive]
    #mean_s1 = np.mean(data)
    bins = np.arange(min(data), max(data) + binwidth, binwidth)
    plt.hist(data,bins,label="t"+str(t),color= 'blue', alpha=0.3)
    data2 = finess[t+1][ListCleanAdaptive]
    #mean_s2 = np.mean(data2)
    #dx.append(mean_s2 - mean_s1)
    #data = data2 - dx[t]
    data = data2 - Ds[t]
    bins = np.arange(min(data), max(data) + binwidth, binwidth)
    plt.hist(data,bins,label="t"+str(t+1),color= 'green', alpha=0.3)
    plt.legend(loc='best')
    plt.title("median(s(t+1))-median(s(t))="+str(Ds[t]))
    plt.savefig(OutputFilePath+"CleanAdaptive_Dist_Alignment_Pop"+str(Pop)+"_T"+str(t)+".png")
#print("dx ",dx)
plt.close('all')
t0=3
#sum_dx = [np.sum(dx[t0:t]) for t in range(t0,np.size(dx)+1)]
Ds_new = [0,0]
for t in range(0,np.size(Ds)):
    if Var_Ds[t] < 0.5:
        Ds_new.append(Ds[t])
    else:
        Ds_new.append(0)

sum_Ds = [np.sum(Ds[t0:t]) for t in range(t0,np.size(Ds)+1)]
print("sum Ds",sum_Ds)

mean_fitness = read_MeanFitness(Pop)
print("Mean Fitness",mean_fitness)
Alignment_mean_fitness = [m for m in mean_fitness]
for t in range(t0,TotalTimePoint-1):
    #Alignment_mean_fitness[t] = Alignment_mean_fitness[t]+sum_dx[t-t0]
    Alignment_mean_fitness[t] = Alignment_mean_fitness[t]-sum_Ds[t-t0]
print("Align Mean Fitness",Alignment_mean_fitness)
np.save(OutputFilePath+"Pop"+str(Pop)+"_Alignming_MeanFitness.npy",Alignment_mean_fitness)
x = np.asarray([1+2*i for i in range(np.size(mean_fitness))])
print(x)
plt.plot(x,Alignment_mean_fitness,color='blue',marker='o',ls='solid',label='Alignming x')
#mean_fitness = read_MeanFitness(Pop)
plt.plot(x,mean_fitness,color='black',marker='o',ls='solid',label='x')
plt.legend(loc='best')
plt.xlabel('Time (cycle)')
plt.ylabel('Mean Fitness (1/cycle)')
plt.savefig(OutputFilePath+"AlignmingMeanFitness_Pop"+str(Pop)+".png")

'''
'''
plt.figure()
plt.plot(s1_mean,s1_SEM,color = bluish_green, marker='.', ls='None')
plt.xlabel("Mean (fintess) over time")
plt.ylabel("Standard Error of Mean SEM (fitness)")
plt.title("Population "+str(Pop)+", NLs="+str(np.size(Adaptive_NLs)))
plt.savefig(OutputFilePath+"AdaptiveFitness_SEM2Mean_Pop"+str(Pop)+".png")
'''
'''
plt.figure()
plt.plot(s1_mean[Adaptive_NLs],s1_CV[Adaptive_NLs],color = bluish_green, marker='.', ls='None')
dash_x = np.arange(0,2,0.1)
dash_y = np.arange(0,5,0.1)
plt.plot(dash_x,[2 for i in range(np.size(dash_x))],'k--')
plt.plot([0 for i in range(np.size(dash_y))],dash_y,'k--')
plt.xlabel("Mean (fintess) over time")
plt.ylabel("Coefficient of Variation CV (fitness)")
plt.title("Population "+str(Pop)+", NLs="+str(np.size(Adaptive_NLs)))
plt.xlim(-0.5,1.5)
plt.ylim(0,5)
plt.savefig(OutputFilePath+"AdaptiveFitness_CV2Mean_Pop"+str(Pop)+".png")
'''
'''
plt.figure()
binwidth = 0.1
data = s1_mean[ListCleanAdaptive]
bins = np.arange(min(data), max(data) + binwidth, binwidth)
plt.hist(data,bins, label="T_ave S",color='blue', alpha = 0.3)
data = s1_tend[ListCleanAdaptive]
bins = np.arange(min(data), max(data) + binwidth, binwidth)
plt.hist(data,bins,label="cycle7 S",color='red', alpha = 0.3)
data = s1_max2[ListCleanAdaptive]
bins = np.arange(min(data), max(data) + binwidth, binwidth)
plt.hist(data,bins,label="max2 S",color='black', alpha = 0.3)
plt.legend(loc='best')
plt.xlabel("Fitness (1/cycle)")
plt.title("Population "+str(Pop)+" NLs="+str(np.size(ListCleanAdaptive)))
plt.savefig(OutputFilePath+"CleanAdaptive_DFE_Pop"+str(Pop)+"_compare3_max2"+".png")
'''

'''
Pop = 1
y1 = read_MeanFitness(Pop)
Pop = 6
y2 = read_MeanFitness(Pop)

x = np.asarray([1+2*i for i in range(np.size(y1))])
plt.figure()
plt.xlim(0,np.max(x)+1)
plt.ylim(-0.05,0.8)
plt.plot(x,y1,color='black',marker='o',ls='solid',label='Mono-culture')
plt.plot(x,y2,color=bluish_green,marker = 'o',ls= 'solid',label='Co-culture')

for Pop in range(2,6):
    y = read_MeanFitness(Pop)
    plt.plot(x,y,color='black',marker='o',ls='solid')
for Pop in range(7,11):
    y = read_MeanFitness(Pop)
    plt.plot(x,y,color=bluish_green,marker='o',ls='solid')

plt.xlabel('Time (cycle)',size=15)
plt.legend(loc='best',fontsize=15)
plt.title('Mean Fitness (1/cycle)',size=20)
plt.savefig(OutputFilePath+"MeanFitness_1&8.png")
'''
'''
fig, ax = plt.subplots()
width = 0.35       # the width of the bars
barX = [0,1]
barY = [y1[-1], y2[-1]]

ax.bar(barX[0],barY[0],width, color="black")
ax.bar(barX[1],barY[1],width, color=bluish_green)
ax.set_xticks([idx+width/2 for idx in barX])
ax.set_xticklabels(['Mono-culture', 'Co-culture'],fontsize=20)
ax.set_xlim(-0.3,1.7)
ax.set_title("Cycle-9 Mean Fintess (1/cycle)",size=20)
plt.savefig(OutputFilePath+"MeanFitness_Cycle9_1&6.png")
'''


#!/usr/bin/env python3
# -*- coding: utf-8 -*
from enum import Enum, unique

__author__ = 'Mr.Huo'

# 常量
# month类型的枚举类
Month = Enum('Month', ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'))

# 如果需要更精确地控制枚举类型，可以从Enum派生出自定义类
@unique
class Weekday(Enum):
    Sun = 0
    Mon = 1
    Tue = 2
    Wed = 3
    Thu = 4
    Fri = 5
    Sat = 6

def main():
    for name, member in Month.__members__.items():
        print(name, '=>', member, ',', member.value)
    for day, member in Weekday.__members__.items():
        print(day, '=>', member, ',', member.value)
    print(Weekday.Wed,'=>',Weekday.Wed.value)
    print(Weekday['Tue'],'=>',Weekday['Tue'].value)
    print(Weekday(1),'=>',Weekday(1).value)

if __name__ == '__main__':
    main()


# TODO: get info on available graphs
import os
import random
import zipfile

import igraph as ig
import numpy as np
import pandas as pd

PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
DATA_PATH = os.path.join(PATH, 'processed_graphs')
MSIGDB_PATH = os.path.join(PATH, 'raw_graphs/msigdb_v2023.1.Hs_json_files_to_download_locally.zip')

def get_available_graphs():
    files = os.listdir(DATA_PATH)
    return files

def get_available_msigdb():
    f = zipfile.ZipFile(MSIGDB_PATH)
    files = f.namelist()
    f.close()
    return files

def load_graph(filename):
    files = os.listdir(DATA_PATH)
    if filename not in files:
        # if filename not in files, try opening it as a path
        if not os.path.exists(filename):
            raise FileNotFoundError()
    else:
        filename = os.path.join(DATA_PATH, filename)
    if 'tsv' in filename:
        df = pd.read_csv(filename, sep='\t')
    else:
        df = pd.read_csv(filename)
    return df

def _load_msigdb(data, object_category='BiologicalProcess', object_id_prefix='MSigDB',
        predicate='participates_in'):
    """
    Returns a pandas edge table and a graph
    """
    import gene_names
    new_entries = []
    for k, v in data.items():
        gene_set_name = k
        source = v['exactSource']
        ref = v['pmid']
        genes = v['geneSymbols']
        try:
            gene_ids = gene_names.get_ids(genes)
        except:
            gene_ids = []
            for g in genes:
                try:
                    gene_id = gene_names.get_ids([g])
                    gene_ids.append(gene_id)
                except:
                    continue
        for gene_id, symbol in zip(gene_ids, genes):
            entry = {}
            entry['subject_category'] = 'Gene'
            entry['subject_id_prefix'] = 'NCBIGene'
            entry['subject_id'] = gene_id
            entry['subject_name'] = symbol
            entry['predicate'] = predicate
            entry['object_category'] = object_category
            entry['object_id_prefix'] = object_id_prefix
            entry['object_id'] = gene_set_name
            entry['object_name'] = gene_set_name
            entry['Primary_Knowledge_Source'] = source
            entry['Knowledge_Source'] = v['collection']
            entry['publications'] = ref
            new_entries.append(entry)
    return pd.DataFrame(new_entries)

def load_msigdb(name):
    """Load an msigdb graph as a dataframe. Input name is one of the filenames returned by get_available_msigdb."""
    import json
    f = zipfile.ZipFile(MSIGDB_PATH)
    m = f.getinfo(name)
    json_file = f.open(m)
    data = json.load(json_file)
    df = _load_msigdb(data)
    f.close()
    return df

def df_to_networkx(df, directed=False):
    """
    Converts a panda dataframe to a networkx Graph (or DiGraph), with node and edge attributes.
    """
    import networkx as nx
    create_using = nx.Graph
    if directed:
        create_using = nx.DiGraph
    df['subject_id_full'] = df['subject_id_prefix'] + '::' + df['subject_id'].astype(str)
    df['object_id_full'] = df['object_id_prefix'] + '::' + df['object_id'].astype(str)
    graph = nx.from_pandas_edgelist(df, source='subject_id_full', target='object_id_full',
            edge_attr=['predicate',
                       'Primary_Knowledge_Source',
                       'Knowledge_Source',
                       'publications'],
            create_using=create_using)
    node_attributes = {}
    for _, row in df.iterrows():
        if row['subject_id'] not in node_attributes:
            node_attributes[row['subject_id_full']] = {
                    'id': row['subject_id'],
                    'id_prefix': row['subject_id_prefix'],
                    'name': row['subject_name'],
                    'category': row['subject_category']}
        if row['object_id'] not in node_attributes:
            node_attributes[row['object_id_full']] = {
                    'id': row['object_id'],
                    'id_prefix': row['object_id_prefix'],
                    'name': row['object_name'],
                    'category': row['object_category']}
    nx.set_node_attributes(graph, node_attributes)
    return graph

def df_to_graph(df, directed=False):
    """
    Converts a panda dataframe to an igraph.Graph (or DiGraph), with node and edge attributes.
    """
    df['subject_id_full'] = df['subject_id_prefix'] + '::' + df['subject_id'].astype(str)
    df['object_id_full'] = df['object_id_prefix'] + '::' + df['object_id'].astype(str)
    # reorder?
    edges_df = df[['subject_id_full', 'object_id_full', 'predicate', 'Primary_Knowledge_Source', 'Knowledge_Source', 'publications']]
    graph = ig.Graph.DataFrame(edges_df, directed=directed, use_vids=False)
    node_attributes = {}
    for i, row in df.iterrows():
        if row['subject_id'] not in node_attributes:
            node_attributes[row['subject_id_full']] = {
                    'id': row['subject_id'],
                    'id_prefix': row['subject_id_prefix'],
                    'feature_name': row['subject_name'],
                    'category': row['subject_category']}
        if row['object_id'] not in node_attributes:
            node_attributes[row['object_id_full']] = {
                    'id': row['object_id'],
                    'id_prefix': row['object_id_prefix'],
                    'feature_name': row['object_name'],
                    'category': row['object_category']}
    for v in graph.vs:
        attributes = node_attributes[v['name']]
        v.update_attributes(attributes)
    return graph

def get_nodes_table(graph):
    """
    Returns a Pandas DataFrame of the nodes.
    """
    rows = []
    for v in graph.vs:
        row = {'id': v['name']}
        row.update(v.attributes())
        rows.append(row)
    return pd.DataFrame(rows)

def get_names_to_ids(graph, category=None):
    """Returns a dict mapping node names to IDs (ignoring prefixes and categories so on)"""
    names_to_ids = {}
    for v in graph.vs:
        if category is not None and v['category'] != category:
            continue
        names_to_ids[v['feature_name']] = v['name']
    return names_to_ids

def get_names_to_ids_networkx(graph, category=None):
    """Returns a dict mapping node names to IDs (ignoring prefixes and categories so on)"""
    names_to_ids = {}
    for n, attrs in graph.nodes.items():
        if category is not None and attrs['category'] != category:
            continue
        names_to_ids[attrs['name']] = n
    return names_to_ids

def get_spoke_categories(graph):
    return set(attrs['category'] for attrs in graph.vs)

def get_spoke_sources(graph):
    return set(attrs['source'] for attrs in graph.vs)

def spoke_identifiers_to_ids(graph, category, source=None):
    """
    Returns a mapping from SPOKE identifiers to IDs.

    category: 'Protein', 'Gene', 'Compound', 'Disease', etc
    source: 'KEGG', ...
    """
    identifiers_to_ids = {}
    for v in graph.vs:
        attrs = v.attributes()
        if 'category' in attrs and attrs['category'] == category:
            if source is None or ('source' in attrs and attrs['source'] == source):
                identifiers_to_ids[attrs['identifier']] = v['name']
    return identifiers_to_ids

def spoke_identifiers_to_ids_networkx(graph, category, source=None):
    """
    Returns a mapping from SPOKE identifiers to IDs.

    category: 'Protein', 'Gene', 'Compound', 'Disease', etc
    source: 'KEGG', ...
    """
    identifiers_to_ids = {}
    for n, attrs in graph.nodes.items():
        if 'category' in attrs and attrs['category'] == category:
            if source is None or ('source' in attrs and attrs['source'] == source):
                identifiers_to_ids[attrs['identifier']] = n
    return identifiers_to_ids

def get_category_ids_to_nodes(graph, category):
    """
    Returns a dict that maps from identifiers in the specified category to graph node indices, for graphs that are not SPOKE.
    """
    identifiers_to_ids = {}
    for v in graph.vs:
        if v['category'] == category:
            identifiers_to_ids[v['id']] = v.index
    return identifiers_to_ids

def largest_component(graph):
    "Returns a subgraph containing the largest connected component of the given graph."
    components = graph.connected_components()
    sizes = components.sizes()
    largest_component = 0
    largest_component_size = 0
    for i, c in enumerate(sizes):
        if c > largest_component_size:
            largest_component_size = c
            largest_component = i
    subgraph = components.subgraph(largest_component)
    return subgraph

def nodes_in_category(graph, category, attr_name='category'):
    "Returns all nodes that are within a given category, as a list of igraph.Vertex objects."
    nodes_in_category = []
    for v in graph.vs:
        attrs = v.attributes()
        if attr_name in attrs and attrs[attr_name] == category:
            nodes_in_category.append(v)
    return nodes_in_category

def nodes_in_category_networkx(graph, category, attr_name='category'):
    nodes_in_category = []
    for n, attrs in graph.nodes.items():
        if attr_name in attrs and attrs[attr_name] == category:
            nodes_in_category.append(n)
    return nodes_in_category

def random_nodes_in_category(graph, category, n_nodes):
    """
    Returns a list of random node ids in the given category.
    """
    nodes_in_category = []
    for v in graph.vs:
        attrs = v.attributes()
        if 'category' in attrs and attrs['category'] == category:
            nodes_in_category.append(attrs['name'])
    return random.sample(nodes_in_category, n_nodes)

def random_nodes(graph, n_nodes):
    """
    Returns a list of random node ids.
    """
    return random.sample([v['name'] for v in graph.vs], n_nodes)

def random_nodes_in_category_networkx(graph, category, n_nodes):
    """
    Returns a list of random spoke ids in the given category.
    """
    nodes_in_category = []
    for n, attrs in graph.nodes.items():
        if 'category' in attrs and attrs['category'] == category:
            nodes_in_category.append((n, attrs['identifier']))
    return random.sample(nodes_in_category, n_nodes)

# random nodes with similar degree distributions? investigative bias - constrain null model to be similar to the problem. We could select random nodes among the nodes that are in the general set...
def degree_sample(graph, node_list, n_samples, dist_or_probs):
    """
    Degree-based node sampling, to sample nodes such that they approximately match the given degree distribution.

    Args:
        graph - an igraph.Graph
        node_list - a list of vertices to be sampled from
        n_samples - the number of points to sample
        dist_or_probs - Either a scipy.stats distribution that has a pdf function (could probably use a kernel density estimation), or a list/array of probabilities over node_list
    """
    prob_vals = []
    # lol
    if hasattr(dist_or_probs, '__getitem__'):
        prob_vals = dist_or_probs
    else:
        dist = dist_or_probs
        prob_vals = dist.pdf(graph.degree(node_list))
        prob_vals = np.array(prob_vals)
        prob_vals = prob_vals/prob_vals.sum()
    sampled_nodes = np.random.choice(node_list, size=n_samples, replace=False, p=prob_vals)
    return sampled_nodes

# subgraph
# randomize the graph... randomly shuffle the nodes/edges?
# shuffle the edges in the network, look at the set again and again. preserve the degree...

# TODO: estimate from whole graph
def randomize_graph(G):
    """
    Randomizes the given graph using edge swapping while preserving the degree distribution.

    Args:
    G (networkx.Graph): Input graph.

    Returns:
    networkx.Graph: Randomized graph.
    """
    G_rand = G.copy()
    edges = list(G_rand.edges)
    nodes = list(G_rand.nodes)
    n = len(edges)

    for _ in range(n):
        # Pick two edges and swap
        edge1 = random.choice(edges)
        edge2 = random.choice(edges)


import os
import tweepy
from datetime import datetime, date, timedelta
from dateutil.relativedelta import relativedelta
import json
import requests
from bs4 import BeautifulSoup
import lxml
from linebot.models import (
    FlexSendMessage, BubbleContainer, BoxComponent, TextComponent, SeparatorComponent, URIAction
)
import html5lib

consumer_key = os.getenv('TWITTER_CONSUMER_KEY')
consumer_secret = os.getenv('TWITTER_CONSUMER_SECRET')
access_token = os.getenv('TWITTER_ACCESS_TOKEN')
access_token_secret = os.getenv('TWITTER_ACCESS_TOKEN_SECRET')
bearer_token = os.getenv('TWITTER_ACCESS_TOKEN_SECRET')

def create_contents(news_tweets):
    contents = []
    for tweet in news_tweets:
        response = requests.get(tweet.entities['urls'][0]['expanded_url'])
        # soup = BeautifulSoup(response.text, "lxml")
        soup = BeautifulSoup(response.text, "html5lib")
        if soup.title is None:
            news_title = "タイトルが取得できませんでした"
        else:
            news_title = soup.title.string
        news_url = tweet.entities['urls'][0]['expanded_url']
        contents.append(
            {
                'title': news_title,
                'favorite_count': tweet.favorite_count,
                'news_url': news_url,
            }
        )
        print("------------------------------------------------------------")
    if not contents:
        return []
    print(contents)

    sorted_contents = sorted(
        contents,
        key=lambda x: x['favorite_count'],
        reverse=True,
    )

    titles = []
    unique_contents = []
    for c in sorted_contents:
        if c['title'] not in titles:
            titles.append(c['title'])
            unique_contents.append(c)
    print(unique_contents)
    return unique_contents[0:5]

def design_message(contents):
    body_comps = []
    for t in contents:
        header_text = TextComponent(
            type="text",
            text=t['title'],
            weight='bold',
            size='xs',
            margin='md',
            gravity='center',
            wrap=True,
            action=URIAction(uri=t['news_url']),
        )

        box = BoxComponent(
            type='box',
            layout='vertical',
            contents=[
                header_text,
                SeparatorComponent(
                    type='separator',
                ),
            ]
        )
        body_comps.append(box)

    header = BoxComponent(
        type="box",
        layout="vertical",
        background_color="#F06161FF",
        contents=[
            TextComponent(
                type="text",
                text="〜 まゆニュース 〜",
                weight="bold",
                color="#FFFFFFFF",
            )
        ]
    )
    body_text_box = BoxComponent(
        type="box",
        layout='vertical',
        contents=body_comps,
    )
    body = BoxComponent(
        type="box",
        layout='vertical',
        contents=[
            body_text_box,
        ],
    )
    footer = BoxComponent(
        type="box",
        layout="vertical",
        background_color="#F06161FF",
    )
    bubble = BubbleContainer(
        type="bubble",
        direction='ltr',
        header=header,
        body=body,
        footer=footer,
    )
    message = FlexSendMessage(alt_text="まゆニュース", contents=bubble)
    return [message]

def search_news():
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = tweepy.API(auth)

    now = datetime.utcnow()
    twelve_hours = relativedelta(hours=12, minutes=1)
    half_day = datetime.strftime(now - twelve_hours, f"%Y-%m-%d_%H:%M:%S")

    q = f"#松岡茉優 OR 松岡茉優 filter:news exclude:retweets since:{half_day}"

    news_tweets = tweepy.Cursor(
        api.search_tweets,
        q=q,
        tweet_mode='extended',
        include_entities=True,
        result_type='mixed',
        count=30
    ).items()

    contents = create_contents(news_tweets)
    if not contents:
        return []
    return design_message(contents)

if __name__ == "__main__":
    search_news()


"""

# About this code.

## How to Install

- Move all the code to a new directory.
- Every file knows its dependencies and so can
  be executed directly using `python3 xx.py`.
- Look at the _okX.py_ files for samples of how to use this code.

## How to Configure

This file is loaded by everything else. It creates a global `my`
holding system-wide settings.

Also controlled here is the random number seed[^seed]. The idiom
`my =Defaults().reset()` will reset everything back to the defaults,
including the seed.  If that is not desired, just use `my=Defaults()`.

Note that, often, this code  read rows of data where the first row
contains the names of the columns. Those names can contain magic
symbols denoting special properties of the columns. For a list of
those properties, see `ignore, less, more, klass`, below.

"""
import random

class Defaults:
  def __init__(i):
    ## system settings (do not change)
    # misc
    i.inf     = 10**32
    i.tiny    = 1/i.inf
    i.private = "_"
    # characters in data,header
    i.ignore = "?"   # column to ignore
    i.less   = "<"   # a numeric goal, to be minimized
    i.more   = ">"   # a numeric goal, to goal to maximize
    i.klass  = "!"   # a symbolic class, to be predicted for

    # other stuff
    i.seed = 235324971  # from random.org
    # hyper parameter settings
    # sampling
    i.keep   = 128
    # chops
    i.cohen  = 0.3 # 0.5 0.4 0.3 0.2
    i.ncohen = 1/7 # 2/9 2/8 1/7 1/6
    i.bins   = 16
    i.simplerBy = 0.01
    # read data in 'eras' of size i.era
    i.era    = 512
  def reset(i,seed=None):
    random.seed(seed or i.seed)
    return i

my= Defaults().reset()

"""

## How to Test

Each file can be loaded and tested independently .
To enable that,  each file begins with a set of `import` statements that describe all its dependencies.

Some files `X.py` have demo/test code in `okX.py`

- The demo/test code uses the unit test engine in `ok.py`. This test engine was strongly inspired by
  Kent Beck's most excellent [making making](https://www.youtube.com/watch?v=nIonZ6-4nuU)  video.
- To test the whole system, run `python3 ok.py`. When loaded as a top-level module,
  this code loads all the "okX.py" files, then runs the `ok()` functions.

## How to Follow this Code's Conventions

All these classes start with an uppercase letters.

All this  code is laid out to be read on mobile devices; hence:

- All code is 60 characters wide.
- This code  does not use `self` but rather `i` to point back to the current instance..
- All tabs are expanded to two spaces.

Any sub-class of `Pretty` (defined in the `lib` module) will print itself as key=value
pairs inside curly braces.

If a class has derived properties, call the method that derives the property (say)
`xxx0` then wrap the class in `@memos`. This will automatically make `xxx` a property
of instances of that class. Further, that method will be memo-ed such that if
the property is requested 100 times, it will get computed once and cached (inside `i._memo`)
and returned 100 times from the cache. If a method makes some change that makes
the memos stale, wrap that method in `@fresh`.

## How to Document

To document this code, add in Markdown comments within multi-line Python quotes.

- All the files get converted into Markdown and rendered by Jekyll.
- To avoid clashes between documentation and code files,
  the former have a dash in their name (e.g. _about-book.md_).
- So to extend this code, do not write code files with dashes in the name.

[^seed]: Computers don’t generate truly random numbers—they are deterministic, which means that they operate by a set of rules. You can mimic randomness by specifying a set of rules. For example, “take a number x, add 900 +x, then subtract 52.” In order for the process to start, you have to specify a starting number, x (the seed).

## Comprehension Questions:

```
     1	@relation weather
     2
     3	@attribute outlook {sunny, overcast, rainy}
     4	@attribute temperature real
     5	@attribute humidity real
     6	@attribute windy {TRUE, FALSE}
     7	@attribute play {yes, no}
     8
     9	@data
    10	sunny, 85, 85, false, no
    11	sunny, 72, 95, false, no
    12	sunny, 80, 90, true,  no
    13	rainy, 65, 70, true,  no
    14	rainy, 71, 91, true,  no
    15	overcast, 83, 86, false, yes
    16	rainy, 70, 96, false, yes
    17	rainy, 68, 80, false, yes
    18	overcast, 64, 65, true, yes
    19	sunny, 69, 70, false, yes
    20	rainy, 75, 80, false, yes
    21	sunny, 75, 70, true, yes
    22	overcast, 72, 90, true, yes
    23	overcast, 81, 75, false, yes
```

0. For the above data, rewrite it using our magic headers `ignore,less,more,klass`
   (hint: replace likes 1 to 9 with one row with the column names). See if you can
   ignore the _humidity_ column. And try to minimize _temperature_.
1. For all the hyperparamaters listed above, search the code for (e.g.) `my.bins`.
     - Try to work out what the parameter does.
     - Try to work out what  would happen if the hyperparameter doubled or halved
2. All the above hyperparamaters were set via ``engineering judgment''; i.e. We guessed.
   If the results of AI tools is controlled by so many guesses, what does that say
   about the trustworthiness of these tools.
3. Read section 3 in the [DUO paper](https://arxiv.org/pdf/1812.01550.pdf).
   Answer the following questions.
     - What is hyperparameter optimization?
     - What are its benefits and weaknesses?
     - The hyperparameter optimizers themselves are controlled
       by their own magic parameters.
       How slow/fast might hyper-hyperparameter optimization be?

"""


"""A class that implements a generalized linear model.

Author:
    Ilias Bilionis

Date:
    8/14/2013
"""

__all__ = ['GeneralizedLinearModel']

import numpy as np
from ._function import Function

class GeneralizedLinearModel(Function):

    """A class that implements a generalized linear model."""

    # The basis (must be a best.maps.Function)
    _basis = None

    # The weight matrix/vector
    _weights = None

    # The square root of the covariance matrix.
    _sigma_sqrt = None

    # The noise precision
    _beta = None

    @property
    def basis(self):
        return self._basis

    @property
    def weights(self):
        return self._weights

    @weights.setter
    def weights(self, val):
        val = np.array(val)
        val = np.atleast_2d(val)
        if self.basis.num_output == 1 and not val.shape[0] == 1:
            val = val.T
        if val.shape[1] == 1:
            val = val.reshape((val.shape[0], ))
        self._weights = val

    @property
    def sigma_sqrt(self):
        return self._sigma_sqrt

    @sigma_sqrt.setter
    def sigma_sqrt(self, val):
        val = np.array(val)
        assert val.ndim == 2
        assert val.shape[1] == self.basis.num_output
        self._sigma_sqrt = val

    @property
    def beta(self):
        return self._beta

    @beta.setter
    def beta(self, val):
        val = float(val)
        assert val > 0.
        self._beta = val

    def __init__(self, basis, weights=None, sigma_sqrt=None,
                 beta=1e99,
                 num_output=1,
                 name='Generalized Linear Model'):
        """Initialize the object.

        Arguments:
            basis       ---     A basis for the model. It must be a
                                best.maps.Function.

        Keword Arguments:
            weights     ---     The weight matrix vector. It is a weight
                                matrix if basis.num_output > 1. If None,
                                then it is initialized to zero and its
                                dimensionality is found by looking at
                                num_output. It should be a
                                self.num_output x self.basis.num_output
                                matrix.
            sigma_sqrt  ---     The square root of the covariance
                                function.
            beta        ---     The noise precision.
            num_output  ---     The number of output dimensions of the
                                generalized linear model. If weights is
                                not None, then it is ignored.
            name        ---     A name for the model.
        """
        assert isinstance(basis, Function)
        self._basis = basis
        if weights is None:
            weights = np.zeros((basis.num_output, num_output))
        if sigma_sqrt is None:
            sigma_sqrt = np.zeros((basis.num_output, basis.num_output))
        self.sigma_sqrt = sigma_sqrt
        self.beta = beta
        num_hyp = np.prod(weights.shape)
        super(GeneralizedLinearModel, self).__init__(self.basis.num_input,
                                                     num_output,
                                                     num_hyp=0,
                                                     name=name)
        self.weights = weights

    def __call__(self, x, hyp=None):
        """Evaluate the mean at x."""
        return np.dot(self.basis(x), self.weights)

    def d(self, x, hyp=None):
        """Evaluate the derivative of the mean at x."""
        return np.dot(self.basis.d(x), self.weights)

    def get_predictive_covariance(self, x):
        """Evaluate the predictive covariance matrix at x."""
        tmp = np.dot(self.sigma_sqrt, self.basis(x).T)
        return np.dot(tmp.T, tmp) + np.eye(tmp.shape[1]) / self.beta

    def get_predictive_variance(self, x):
        """Evaluate the predictive variance at x."""
        D = np.dot(self.sigma_sqrt, self.basis(x).T)
        return (np.einsum('ij, ij->j', D, D) + np.ones(self.num_output)
                / self.beta)


import subprocess
import sys
from pathlib import Path

import nox  # type: ignore

PYTHON_ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
PYTHON_DEFAULT_VERSION = "3.8"
DOC_DEPENDENCIES = [".", "jinja2", "mkdocs", "mkdocs-material"]
LINT_DEPENDENCIES = [
    "black==19.10b0",
    "flake8",
    "flake8-bugbear",
    "mypy",
    "check-manifest",
    "packaging>=20.0",
    "isort",
]
# Packages whose dependencies need an intact system PATH to compile
# pytest setup clears PATH.  So pre-build some wheels to the pip cache.
PREBUILD_PACKAGES = {
    "all": ["jupyter==1.0.0"],
    "macos": ["black==20.8b1"],
    "unix": [],
    "win": [],
}

# Set nox options
if sys.platform == "win32":
    # build_docs fail on Windows, even if `chcp.com 65001` is used
    nox.options.sessions = ["tests", "lint"]
else:
    nox.options.sessions = ["tests", "lint", "build_docs"]
nox.options.reuse_existing_virtualenvs = True

def prebuild_wheels(session, prebuild_dict):
    if sys.platform == "darwin":
        platform = "macos"
    elif sys.platform == "win32":
        platform = "win"
    else:
        platform = "unix"

    prebuild_list = prebuild_dict.get("all", []) + prebuild_dict.get(platform, [])

    session.install("wheel")
    wheel_dir = Path(session.virtualenv.location) / "prebuild_wheels"
    wheel_dir.mkdir(exist_ok=True)

    for prebuild in prebuild_list:
        session.run("pip", "wheel", f"--wheel-dir={wheel_dir}", prebuild, silent=True)

def has_changes():
    status = (
        subprocess.run(
            "git status --porcelain", shell=True, check=True, stdout=subprocess.PIPE
        )
        .stdout.decode()
        .strip()
    )
    return len(status) > 0

def get_branch():
    return (
        subprocess.run(
            "git rev-parse --abbrev-ref HEAD",
            shell=True,
            check=True,
            stdout=subprocess.PIPE,
        )
        .stdout.decode()
        .strip()
    )

def on_master_no_changes(session):
    if has_changes():
        session.error("All changes must be committed or removed before publishing")
    branch = get_branch()
    if branch != "master":
        session.error(f"Must be on 'master' branch. Currently on {branch!r} branch")

@nox.session(python=PYTHON_ALL_VERSIONS)
def tests(session):
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    prebuild_wheels(session, PREBUILD_PACKAGES)
    session.install("-e", ".", "pytest", "pytest-cov")
    tests = session.posargs or ["tests"]
    session.run("pytest", "--cov=pipx", "--cov-report=", *tests)
    session.notify("cover")

@nox.session(python=PYTHON_ALL_VERSIONS)
def test_all_packages(session):
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    session.install("-e", ".", "pytest")
    tests = session.posargs or ["tests"]
    session.run(
        "pytest", "-v", "--tb=no", "--show-capture=no", "--all-packages", *tests
    )

@nox.session
def cover(session):
    """Coverage analysis"""
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    session.install("coverage")
    session.run("coverage", "report", "--show-missing", "--fail-under=70")
    session.run("coverage", "erase")

@nox.session(python=PYTHON_DEFAULT_VERSION)
def lint(session):
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    session.install(*LINT_DEPENDENCIES)
    files = [str(Path("src") / "pipx"), "tests", "scripts"] + [
        str(p) for p in Path(".").glob("*.py")
    ]
    session.run("isort", "--check", "--diff", "--profile", "black", *files)
    session.run("black", "--check", *files)
    session.run("flake8", *files)
    session.run("mypy", *files)
    session.run("check-manifest")
    session.run("python", "setup.py", "check", "--metadata", "--strict")

@nox.session(python=PYTHON_ALL_VERSIONS)
def develop(session):
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    session.install(*DOC_DEPENDENCIES, *LINT_DEPENDENCIES)
    session.install("-e", ".")

@nox.session(python=PYTHON_DEFAULT_VERSION)
def build(session):
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    session.install("build")
    session.run("rm", "-rf", "dist", "build", external=True)
    session.run("python", "-m", "build")

@nox.session(python=PYTHON_DEFAULT_VERSION)
def publish(session):
    on_master_no_changes(session)
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    session.install("twine")
    build(session)
    print("REMINDER: Has the changelog been updated?")
    session.run("python", "-m", "twine", "upload", "dist/*")

@nox.session(python=PYTHON_DEFAULT_VERSION)
def build_docs(session):
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    session.install(*DOC_DEPENDENCIES)
    session.env[
        "PIPX__DOC_DEFAULT_PYTHON"
    ] = "typically the python used to execute pipx"
    session.run("python", "scripts/generate_docs.py")
    session.run("mkdocs", "build")

@nox.session(python=PYTHON_DEFAULT_VERSION)
def publish_docs(session):
    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
    session.install(*DOC_DEPENDENCIES)
    build_docs(session)
    session.run("mkdocs", "gh-deploy")

@nox.session(python=PYTHON_DEFAULT_VERSION)
def watch_docs(session):
    session.install(*DOC_DEPENDENCIES)
    session.run("mkdocs", "serve")

@nox.session(python=PYTHON_DEFAULT_VERSION)
def pre_release(session):
    on_master_no_changes(session)
    if session.posargs:
        new_version = session.posargs[0]
    else:
        new_version = input("Enter new version: ")
    session.run("python", "scripts/pipx_prerelease.py", new_version)
    session.run("git", "--no-pager", "diff", external=True)
    print("")
    session.log(
        "If `git diff` above looks ok, execute the following command:\n\n"
        f"    git commit -a -m 'Pre-release {new_version}.'\n"
    )

@nox.session(python=PYTHON_DEFAULT_VERSION)
def post_release(session):
    on_master_no_changes(session)
    session.run("python", "scripts/pipx_postrelease.py")
    session.run("git", "--no-pager", "diff", external=True)
    print("")
    session.log(
        "If `git diff` above looks ok, execute the following command:\n\n"
        "    git commit -a -m 'Post-release.'\n"
    )


import numpy as np
import sympy as sp

class Programa_Newton:
    def procedimiento_newton(self,f1,f2,x,y,iteraciones):

        def F(vector):
            x = vector[0]
            y = vector[1]
            valf1 = eval(f1)
            valf2 = eval(f2)
            vector = np.array([valf1,valf2])
            return vector

        def dF(vector):
            x1,y1,z,w,sin,cos,tan,cot,sec,csc,asin,acos,atan,acot,asec,acsc = sp.symbols('x y z w sin cos tan cot sec csc asin acos atan acot asec acsc')
            deriv1x = sp.diff(f1,x1)
            deriv1y = sp.diff(f1,y1)
            deriv2x = sp.diff(f2,x1)
            deriv2y = sp.diff(f2,y1)

            x = self.valorx
            y = self.valory

            dx1 = eval(str(deriv1x))
            dy1 = eval(str(deriv1y))
            dx2 = eval(str(deriv2x))
            dy2 = eval(str(deriv2y))

            vector = np.array([[dx1,dy1],[dx2,dy2]])

            return vector

        N = int(iteraciones) + 1
        vector = np.array([x,y])

        lista_resultados = [""]

        for k in range(N):
            vectorold = vector
            self.valorx = vector[0]
            self.valory = vector[1]

            Jinv = np.linalg.inv(dF(vector))

            vector = vector - np.dot(Jinv,F(vector))

            e = np.linalg.norm(vector - vectorold)

            cadena_resultados = ("Iteracion: " + str(k) + " |   Resultado: " + str(vector) + "    |   Error: " + str(e))
            lista_resultados.append(cadena_resultados)

            if e <= 0.0:
                break

        return lista_resultados


"""Functions used to create the input matrix of the simple neural network
from the cropped image detected using polygons"""

import os
from PIL import Image
import numpy
import cv2

HELP_COMPONENTS = {"white": [[[0, 0, 100], [255, 100, 255]]], "black": [[[0, 0, 0], [255, 100, 100]]]}
SCALE_G = 0.5

def image2array(img):
    """
    Returns the image numpy array
    :param img: string of image path
    :return: numpy array of pixel intensities
    """
    """transforms image in ppm format to array"""
    img = Image.open(img)
    return numpy.array(img.getdata(), numpy.uint8).reshape(img.size[1], img.size[0], 3)

def show_image(img, title, scale=SCALE_G):
    """
    Shows the images
    :param img: image array
    :param title: string of the window title
    :param scale: float
    :return: None
    """
    newx, newy = int(img.shape[1] * scale), int(img.shape[0] * scale)
    cv2.namedWindow(title, cv2.WINDOW_NORMAL)
    cv2.resizeWindow(title, (newx, newy))
    cv2.imshow(title, img)
    cv2.waitKey()
    cv2.destroyAllWindows()

def detect_image_component(image, components, show=False):
    """
    Takes an image and shows the mask of each components on components list
    :param image: image array
    :param components: component dictionnary
    :param show: bool, True if the image is shown
    :return: mask result
    """

    # convert BGR to HSV
    hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
    # resulting list
    res = []
    # use of the threshold technique to select the mask for each component
    for component in components.keys():
        s = hsv.shape
        thresh = components[component]
        mask = numpy.zeros((s[0], s[1]))
        for threshold in thresh:
            lower_comp = numpy.array(threshold[0])
            higher_comp = numpy.array(threshold[1])
            mask += cv2.inRange(hsv, lower_comp, higher_comp)
        if show:
            show_image(mask, component)
        res.append(mask)
    return res

def reshape_masks(masks, category_number):
    """
    Concatenation of masks
    :param masks: mask list
    :param category_number: int
    :return: conctenated masks
    """
    """transforms the list of image_masks into a vector to vector"""
    final_mask = numpy.zeros((category_number, 1))
    for mask in masks:
        final_mask = numpy.concatenate((numpy.reshape(mask.T, [mask.shape[0] * mask.shape[1], 1]), final_mask), axis=0)
    return final_mask

def normalisation(image_path, category_number, components=HELP_COMPONENTS):
    """
    Normalizes image pixel intensites into values between  0 and 1
    :param image_path: string path of the image
    :param category_number: int
    :param components: dictionnary containing the interval of each mask
    :return: the normalized vector
    """
    matrix_img = image2array(image_path)
    masks = detect_image_component(matrix_img, components)
    vector = reshape_masks(masks, category_number)
    normalized_vector = vector / 255
    return normalized_vector

def load_input_with_shuffle(directory, first_image_path, category_number, categories):
    """
    Shuffles and normalizes the columns of the input matrix
    :param directory: string of the directory path
    :param first_image_path: string of the path of the first image in the directory
    :param category_number: int
    :param categories: list of categories
    :return: normalized and shuffled input matrix
    """
    final_matrix = normalisation(first_image_path, category_number)
    i = 0
    for root, dirs, files in os.walk(directory):
        for filename in files:
            if i != 0:
                new_img = normalisation(directory + '\\' + filename, category_number)
                final_matrix = numpy.concatenate((final_matrix, new_img), axis=1)
            i += 1
    compteur = 0
    for root, dirs, files in os.walk(directory):
        for filename in files:
            for i in range(category_number):
                if categories[i] in filename:
                    final_matrix[(final_matrix.shape[0] - category_number + i), compteur] = 1
            compteur += 1
    final_matrix = final_matrix.T
    numpy.random.shuffle(final_matrix)
    return final_matrix.T

# Test
# print(load_input_with_shuffle(r"C:\Users\Antonio\Documents\Projet_Autonomous_Driving\Dataset_global\Dataset_entrainement_triangle\trianglesred_entrainement",
#       r"C:\Users\Antonio\Documents\Projet_Autonomous_Driving\Dataset_global\Dataset_entrainement_triangle\trianglesred_entrainement\attention49.jpg",
#       3)[1568:1571,0:20])
# #
# 1571, 354)
# (input.shape[0] - CATEGORY_NUMBER):input.shape[0], :


import torch
import numpy as np
import pdb

from engineer.utils.filter import OneEuroFilter
from engineer.utils.transformation import batch_rodrigues, batch_axisang2quat

def compute_angle(axisang):
    axisang_norm = torch.norm(axisang + 1e-8, p=2, dim=1)
    angle = torch.unsqueeze(axisang_norm, -1)

    return angle

def smooth_poses(pred_pose, min_cutoff=0.004, beta= 1.5, d_cutoff = 1.):
    # min_cutoff: Decreasing the minimum cutoff frequency decreases slow speed jitter
    # beta: Increasing the speed coefficient(beta) decreases speed lag.

    one_euro_filter = OneEuroFilter(
        torch.zeros_like(pred_pose[0]),
        pred_pose[0],
        min_cutoff=min_cutoff,
        beta=beta,
        d_cutoff = d_cutoff
    )
    pred_pose = pred_pose.detach().clone()
    pred_pose_hat = torch.zeros_like(pred_pose)
    # initialize
    pred_pose_hat[0] = pred_pose[0]

    for idx, pose in enumerate(pred_pose[1:]):
        idx += 1
        # Note that the original VIBE has some bug, I fix it
        if torch.abs(pred_pose[idx] - pred_pose_hat[idx-1]).sum(-1).max()>0.5:
            reverse_pose = 1+(2*torch.pi - 2*compute_angle(pose))/(compute_angle(pose))
            mask = torch.abs(pred_pose[idx] - pred_pose_hat[idx-1]).sum(-1)>0.5
            pose = pose.detach()
            pose[mask] = -pose[mask]
            pose[mask] *= reverse_pose[mask]

        t = torch.ones_like(pose) * idx
        pose = one_euro_filter(t, pose)
        pred_pose_hat[idx] = pose

    return pred_pose_hat


import logging
import unittest
from pathlib import Path

from eotile.eotile_module import main as eomain
from eotile.eotiles.eotiles import create_tiles_list_eo, get_tile, write_tiles_bb
from eotile.eotiles.get_bb_from_tile_id import get_tiles_from_tile_id, tile_id_matcher
from eotile.eotiles.utils import build_nominatim_request, input_matcher

class TestEOTile(unittest.TestCase):
    def test_tile_list_utils_s2(self):
        aux_data_dirpath = Path("eotile/data/aux_data")
        filename_tiles_s2 = aux_data_dirpath / "s2_no_overlap.gpkg"
        ls2 = create_tiles_list_eo(
            filename_tiles_s2,
            Path("tests/test_data/illinois.shp"),
        )
        self.assertEqual(len(ls2), 33)

        self.assertTrue(get_tile(ls2, "15TXH") is not None)

        self.assertTrue(get_tile(ls2, "15TXF") is not None)

    def test_tile_list_utils_l8(self):
        aux_data_dirpath = Path("eotile/data/aux_data")
        filename_tiles_l8 = aux_data_dirpath / "l8_tiles.gpkg"
        l8 = create_tiles_list_eo(
            filename_tiles_l8,
            Path("tests/test_data/illinois.shp"),
        )
        self.assertEqual(len(l8), 18)
        self.assertTrue(get_tile(l8, "25030") is not None)

    def test_read_write_tiles_bb(self):
        aux_data_dirpath = Path("eotile/data/aux_data")
        filename_tiles_l8 = aux_data_dirpath / "l8_tiles.gpkg"
        ll8 = create_tiles_list_eo(
            filename_tiles_l8,
            Path("tests/test_data/illinois.shp"),
        )
        write_tiles_bb(ll8, Path("/tmp/test_read_write.shp"))

        self.assertTrue(get_tile(ll8, "25030") is not None)

    def test_input_matcher(self):
        polygon = "POLYGON((1 1,5 1,5 5,1 5,1 1))"
        mpoly = "MULTIPOLYGON(((1 1,5 1,5 5,1 5,1 1),(2 2,2 3,3 3,3 2,2 2)),((6 3,9 2,9 4,6 3)))"

        bbox1 = "['36.9701313', '42.5082935', '-91.5130518', '-87.0199244']"
        bbox2 = "'36.9701313', '42.5082935', '-91.5130518', '-87.0199244'"
        bbox3 = "'36.9701313','42.5082935','-91.5130518','-87.0199244'"

        location1 = "Toulouse"
        location2 = "Nowhere"
        location3 = "France"

        tile_id1 = "31TCJ"
        tile_id2 = "199030"

        file1 = "/tmp"
        file2 = "/dev/null"

        test_list = [
            polygon,
            mpoly,
            bbox1,
            bbox2,
            bbox3,
            location1,
            location3,
            tile_id1,
            tile_id2,
            file1,
            file2,
        ]

        with self.assertRaises(ValueError):
            input_matcher(location2)

        out_list = []
        for elt in test_list:
            out_list.append(input_matcher(elt))

        self.assertListEqual(
            out_list,
            [
                "wkt",
                "wkt",
                "bbox",
                "bbox",
                "bbox",
                "location",
                "location",
                "tile_id",
                "tile_id",
                "file",
                "file",
            ],
        )

    def test_tile_id_list_test(self):
        tile_id_list_2 = "31TCJ, 31TCF"
        tile_id_list_3 = "199030, 199029, 197031"
        out_list = []
        for elt in [tile_id_list_2, tile_id_list_3]:
            out_list.append(input_matcher(elt))
        self.assertListEqual(out_list, ["tile_id", "tile_id"])

    def test_id_matcher(self):
        test_id_srtm = "N02W102"
        test_id_cop = "S02W102"
        test_id_s2 = "18SWJ"
        test_id_l8 = "12033"
        test_id_srtm5x5 = "srtm_37_04"
        self.assertEqual(tile_id_matcher(test_id_l8), [False, True, False, False])
        self.assertEqual(tile_id_matcher(test_id_s2), [True, False, False, False])
        self.assertEqual(tile_id_matcher(test_id_cop), [False, False, True, False])
        self.assertEqual(tile_id_matcher(test_id_srtm), [False, False, True, False])
        self.assertEqual(tile_id_matcher(test_id_srtm5x5), [False, False, False, True])

    def test_get_tiles_from_tile_id(self):
        aux_data_dirpath = Path("eotile/data/aux_data")
        output_s2, output_l8, output_dem, output_srtm5x5 = get_tiles_from_tile_id(
            ["31TCJ"], aux_data_dirpath, False, False, dem=True, srtm5x5=True
        )
        self.assertEqual(len(output_s2), 1)
        self.assertEqual(len(output_l8), 4)
        self.assertEqual(len(output_dem), 4)
        self.assertEqual(len(output_srtm5x5), 1)

        output_s2, output_l8, output_dem, output_srtm5x5 = get_tiles_from_tile_id(
            ["200035"], aux_data_dirpath, False, False, dem=True, srtm5x5=True
        )
        self.assertEqual(len(output_s2), 8)
        self.assertEqual(len(output_l8), 1)

    def test_main_module(self):
        output_s2, output_l8, output_dem, output_srtm5x5 = eomain(
            "-74.657, 39.4284, -72.0429, 41.2409",
            no_l8=False,
            no_s2=False,
            dem=True,
            srtm5x5=True,
        )
        self.assertEqual(len(output_s2), 12)
        self.assertEqual(len(output_l8), 9)
        self.assertEqual(len(output_dem), 7)
        self.assertEqual(len(output_srtm5x5), 2)

    def test_main_module_2(self):
        output_s2, output_l8, output_dem, output_srtm5x5 = eomain(
            "tests/test_data/illinois.shp",
            no_l8=False,
            no_s2=False,
            dem=True,
            srtm5x5=True,
        )
        self.assertEqual(len(output_s2), 33)
        self.assertEqual(len(output_l8), 18)
        self.assertEqual(len(output_dem), 27)
        self.assertEqual(len(output_srtm5x5), 4)

    def test_main_module_3(self):
        output_s2, output_l8, output_dem, output_srtm5x5 = eomain(
            "Toulouse",
            no_l8=False,
            no_s2=False,
            dem=True,
            srtm5x5=True,
            threshold=0.1,
        )
        self.assertEqual(len(output_s2), 1)
        self.assertEqual(len(output_l8), 2)
        self.assertEqual(len(output_dem), 1)
        self.assertEqual(len(output_srtm5x5), 1)

    def test_main_module_4(self):
        output_s2, output_l8, output_dem, output_srtm5x5 = eomain(
            "31TCJ",
            no_l8=False,
            no_s2=False,
            dem=True,
            srtm5x5=True,
            min_overlap=0.1,
        )
        self.assertEqual(len(output_s2), 1)
        self.assertEqual(len(output_l8), 3)
        self.assertEqual(len(output_dem), 4)
        self.assertEqual(len(output_srtm5x5), 0)

    def test_build_nominatim_request(self):
        self.assertTrue(
            abs(
                build_nominatim_request(None, "Toulouse", "0.1").area
                - 0.013155945340939995
            )
            < 0.005
        )

if __name__ == "__main__":
    logging.basicConfig(filename="test_eotile.log", level=logging.INFO)
    unittest.main()


# -*- coding: utf-8 -*-
"""
Created on Sun May  3 12:21:14 2020

@author: Anang Kunaefi

description:
    text classification t1 (task1), t2 (task2), t3 (task3)
"""

import pandas as pd
import numpy as np
import pickle
import nltk
from nltk.tokenize import word_tokenize
from nltk import pos_tag
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.parse.stanford import StanfordParser
from sklearn.preprocessing import LabelEncoder
from collections import defaultdict
from nltk.corpus import wordnet as wn
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import StandardScaler, MinMaxScaler, FunctionTransformer
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer
from sklearn import model_selection, naive_bayes, svm
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, confusion_matrix, classification_report
import seaborn as sns
import matplotlib.pyplot as plt
import re
import string
import os
import pickle
from pprint import pprint
from custom_vectorizer import CustomVectorizer, PunctVectorizer
from datetime import datetime
# from contractions import contraction

'''
Supporting Class and functions for features union
'''
class TextSelector(BaseEstimator, TransformerMixin):
    def __init__(self, field):
        self.field = field
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.field]

class NumberSelector(BaseEstimator, TransformerMixin):
    def __init__(self, field):
        self.field = field
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[[self.field]]

def print_stats(preds, target, labels, sep='-', sep_len=40, fig_size=(10,8)):
    print('Accuracy = %.3f' % metrics.accuracy_score(target, preds))
    print(sep*sep_len)
    print('Classification report:')
    print(metrics.classification_report(target, preds))
    print(sep*sep_len)
    print('Confusion matrix')
    cm=metrics.confusion_matrix(target, preds)
    cm = cm / np.sum(cm, axis=1)[:,None]
    sns.set(rc={'figure.figsize':fig_size})
    sns.heatmap(cm,
        xticklabels=labels,
        yticklabels=labels,
           annot=True, cmap = 'YlGnBu')
    plt.pause(10)

def get_dataset(task):

    # get the dataset after text preprocessing

    # get feature after text preprocessing

    if task == 1:
        df_feat = pd.read_pickle('../data/feature_extraction_task1.pkl')
        df = df_feat.filter(['lemmatized_review','category',
                        'total_words','punctuation','num_np','num_vp',
                        'num_md','grammar','indicator'],axis=1)

    elif task == 2:
        df_feat = pd.read_pickle('../data/feature_extraction_task2.pkl')
        df = df_feat.filter(['lemmatized_review','category',
                        'total_words','punctuation','num_np','num_vp',
                        'num_md','grammar','indicator'],axis=1)
        invalid_columns = df[df.category > 3].index
        df.drop(invalid_columns,inplace=True)

    elif task == 3:
        df_feat = pd.read_pickle('../data/feature_extraction_task3.pkl')
        df = df_feat.filter(['lemmatized_review','decision_category',
                        'total_words','punctuation','num_np','num_vp',
                        'num_md','grammar','indicator'],axis=1)

    # jaga2 jika ada null values
    null_index = df[df['lemmatized_review'] == ""].index
    df.drop(null_index,inplace=True)

    null_columns=df.columns[df.isnull().any()]
    # df.drop(null_columns,inplace=True)
    df.dropna(how='any',inplace=True)
    print(df[null_columns].isnull().sum())

    return df

# END of class and function

# SETTING
TASK = 3
start = datetime.now()

'''
# 1. The corpus
# The corpus is already processed in feature_extraction,
# and saved in the format of pandas dataframe, then pickled
# column: lemmatized_review, total_words, num_of_np, num_of_vp, num_of_md
'''
print('getting the data...')
df_reviews = get_dataset(TASK)
print(df_reviews.columns.tolist())

# Prepare train and test data#
if TASK == 1:
    combined_features = ['lemmatized_review','total_words','punctuation', 'num_np',
                         'num_vp','num_md','grammar','indicator']
    column_label = 'category'
elif TASK == 2:
    combined_features = ['lemmatized_review','total_words','punctuation', 'num_np',
                         'num_vp','num_md','grammar','indicator']
    column_label = 'category'
elif TASK == 3:
    combined_features = ['lemmatized_review','total_words',
                         'punctuation','num_np','num_vp',
                         'num_md','grammar','indicator']
    column_label = 'decision_category'

Train_X, Test_X, Train_Y,  Test_Y = model_selection.train_test_split(df_reviews[combined_features],df_reviews[column_label],test_size=0.2, random_state= 0)

'''
# Step 2. Use ML Algorithm Using Pipeline
# Gabungkan feature
'''

# all features combined
all_feats = FeatureUnion([
        ('text2', Pipeline([
            ('colext', TextSelector('lemmatized_review')),
            ('tfidf', TfidfVectorizer(min_df=.0025, max_df=0.25, ngram_range=(1,3)))
        ])),
        ('text', Pipeline([
            ('colext', TextSelector('lemmatized_review')),
            ('vect', CustomVectorizer()),
            ('tfidf', TfidfTransformer()),
        ])),
        ('text3', Pipeline([
            ('colext', TextSelector('lemmatized_review')),
            ('vect', CountVectorizer()),
        ])),
        ('words', Pipeline([
            ('wordext', NumberSelector('total_words')),
            ('wscaler', MinMaxScaler()),
        ])),
        # ('punct', Pipeline([
        #     ('colext', TextSelector('punctuation')),
        #     ('punct',PunctVectorizer())
        # ])),
        ('nouns', Pipeline([
            ('nounext', NumberSelector('num_np')),
            ('wscaler', MinMaxScaler()),
        ])),
        ('verbs', Pipeline([
            ('verbext', NumberSelector('num_vp')),
            ('wscaler', StandardScaler()),
        ])),
        ('modals', Pipeline([
                ('modalext', NumberSelector('num_md')),
            ('wscaler', StandardScaler()),
        ])),
])

# structural features
structural_feats = FeatureUnion([
        ('words', Pipeline([
            ('wordext', NumberSelector('total_words')),
            ('wscaler', MinMaxScaler()),
        ])),
])

# lexical features
lexical_feats = FeatureUnion([
        ('text2', Pipeline([
            ('colext', TextSelector('lemmatized_review')),
            ('tfidf', TfidfVectorizer(min_df=.0025, max_df=0.25, ngram_range=(1,3)))
        ])),
        ('text', Pipeline([
            ('colext', TextSelector('lemmatized_review')),
            ('vect', CustomVectorizer()),
            ('tfidf', TfidfTransformer()),
        ])),
        ('text3', Pipeline([
            ('colext', TextSelector('lemmatized_review')),
            ('vect', CountVectorizer()),
        ])),
        # original
        # ('text2', Pipeline([
        #     ('colext', TextSelector('lemmatized_review')),
        #     ('tfidf', TfidfVectorizer(min_df=.0025, max_df=0.25, ngram_range=(1,3)))
        # ])),
])

# punctuation features
punctuation_feats = FeatureUnion([
        ('text', Pipeline([
            ('colext', TextSelector('punctuation')),
            ('punct',PunctVectorizer())
        ])),
])

# contextual features
contextual_feats = FeatureUnion([
        ('nouns', Pipeline([
            ('nounext', NumberSelector('num_of_np')),
            ('wscaler', MinMaxScaler()),
        ])),
        ('verbs', Pipeline([
            ('verbext', NumberSelector('num_of_vp')),
            ('wscaler', MinMaxScaler()),
        ])),
        ('modals', Pipeline([
            ('modalext', NumberSelector('num_of_md')),
            ('wscaler', MinMaxScaler()),
        ])),
])

# lexical plus contextual
lexcon_feats = FeatureUnion([
        ('text', Pipeline([
            ('colext', TextSelector('text_final')),
            ('tfidf', TfidfVectorizer(min_df=.0025, max_df=0.25, ngram_range=(1,3)))
        ])),
        ('nouns', Pipeline([
            ('nounext', NumberSelector('num_of_np')),
            ('nscaler', MinMaxScaler()),
        ])),
        ('verbs', Pipeline([
            ('verbext', NumberSelector('num_of_vp')),
            ('vscaler', MinMaxScaler()),
        ])),
        ('modals', Pipeline([
            ('modalext', NumberSelector('num_of_md')),
            ('mscaler', MinMaxScaler()),
        ])),
])

# Classifier
print('defining classifier...')
rf_classifier = RandomForestClassifier(n_estimators=1000)
nb_classifier = naive_bayes.GaussianNB()
lr_classifier = LogisticRegression(solver ='liblinear')
svm_classifier = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')

pipe = Pipeline([('feats', lexical_feats),
                 ('clf', svm_classifier)
                 ])

print('training...')
pipe.fit(Train_X, Train_Y)
predicts = pipe.predict(Test_X)

print_stats(Test_Y, predicts, pipe.classes_)
print(confusion_matrix(Test_Y, predicts))
print(classification_report(Test_Y, predicts))
print('Hasil Task ', TASK)
print('Accuracy: ', accuracy_score(predicts,Test_Y)*100)
print('Precision = ', precision_score(Test_Y, predicts, average="macro"))
print('Recall score = ', recall_score(Test_Y, predicts, average="macro"))
print('F1 score = ', f1_score(Test_Y, predicts, average="macro"))

# end time
end = datetime.now()
print('total time = ', end - start)

# save the best model
filename = '../data/best_classifier_model.pk'
with open('filename','wb') as f:
    pickle.dump(pipe,f)


import numpy as np
import cv2
import logging
from PIL import Image
import pytesseract
from contouring import isolate_text_boxes, boxes_to_contours

# Setup for the debug logger
logging.basicConfig(format='%(asctime)s - %(filename)s - %(funcName)s - %('
                          'message)s', level=logging.DEBUG,
                    filename='debug.txt', filemode='w')
logger = logging.getLogger('logger')

def get_partial_image(box, cv_img):
    """Returns new array with only part of original image.

    Parameters:
    box (list(tuple)): List of box corner coordinates.
    cv_img (OpenCV image): OpenCV image to take a part of.

    Return:
    (numpy array): New array of box from cv_img.
    """
    min_x, min_y = box[0]
    max_x, max_y = box[1]
    return np.array(cv_img[min_y : max_y, min_x : max_x])

def get_text(img):
    """Returns text from a PIL image.

    Parameter:
    img (PIL image): PIL image to get text from.

    Return:
    (str): String of text form img.
    """
    return pytesseract.image_to_string(img)

def get_text_from_boxes(boxes, cv_img):
    """Returns a location-to-text dictionary of text found inside the
    boxes in the image (where location is considered to be the center of
    the box).

    Parameters:
    boxes (list(box)): List of box coordinates.
    cv_img (OpenCV image): OpenCV image to get text from.

    Return:
    box_to_text (dictionary(tuple : str)): Dictionary containing
    coordinates of center of box in boxes and text from box.
    """
    box_to_text = dict()

    for box in boxes:
        partial_img = get_partial_image(box, cv_img)
        partial_img = Image.fromarray(partial_img)
        location = (np.amin(box, axis=0) + np.amax(box, axis=0)) / 2
        location = tuple(location.astype(int))
        text = get_text(partial_img)
        if text:
            box_to_text[location] = text
    return box_to_text

def extract_text(img, padding=10, dist_threshold=12, debug=False):
    """Isolates text boxes found in the OpenCV image, and then uses
    Tesseract to extract text from each box, returning a
    location-to-text dictionary.

    Parameters:
    img (OpenCV image): OpenCV image to extract text from.
    padding (int): Number of pixels of paddinf on img.
    dist_threshold (int): Threshold of number of pixels to determine
    whether boxes are too close.

    Return:
    box_to_test (dictionary(tuple : str)): Dictionary containing
    coordinates of center of text boxes and text from text boxes.
    """
    text_boxes = isolate_text_boxes(img, padding=padding,
                                    dist_threshold=dist_threshold)

    if debug:
        img_temp = np.array(img)
        img_temp = cv2.drawContours(img_temp, boxes_to_contours(text_boxes),
                                    -1, (0,0,255), 2)
        output = 'image_with_text_boxes.jpg'
        cv2.imwrite(output, img_temp)
        logger.debug(f'Written file {output}')

    box_to_text = get_text_from_boxes(text_boxes,
                                      cv2.cvtColor(img, cv2.COLOR_BGR2GRAY))
    return box_to_text


import urllib.request
import cv2
import numpy as np
import time
import csv
from datetime import datetime

font = cv2.FONT_HERSHEY_SIMPLEX
bottomLeftCornerOfText = (10,600)
fontScale = 1
fontColor = (0,0,0)
thickness = 2
lineType = 2

url = 'http://192.168.4.1/capture'

# Calibration left thermometer
#temperature value 1
t1_F1 = 70
#pixel location of temperature in y axis 1
t1_P1 = 245
#temperature value 2
t1_F2 = 40
#pixel location of temperature in y axis 2
t1_P2 = 552

# Calibration right thermometer (if used)
t2_F1 = 70
t2_P1 = 229
t2_F2 = 40
t2_P2 = 535

# range of blue colors that will be detected based on blue image acquired before
img = cv2.imread("/Users/damianwilliams/Desktop/blue.jpg")
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
mu, sig = cv2.meanStdDev(hsv)

#--------------------------##------------Range of blue_values______________##

a = 15
#--------------------------##----------------------------------------------##

#--------------------------##------------threshold for rectangle size defined as liquid column size (ignores small
# area)______________##

minArea = 2000
#--------------------------##-------------------------------------------------------------------------------------------------------------##

change_resolution= urllib.request.urlopen("http://192.168.4.1/a?ql=6", timeout=5)
time.sleep(2)
change_resolution.close()

def CalculateTemp(temp1, temp2, pixel1, pixel2, yval):
    pix_per_deg = (pixel2 - pixel1) / (temp2 - temp1)
    pixels_diff = yval - pixel2
    delta_fah = pixels_diff / pix_per_deg
    fahrenheit = temp2 + delta_fah
    celcius = ((fahrenheit - 32) * (5 / 9))
    temps = [celcius,fahrenheit]
    return temps

idx = 0

while True:

    with open(url, "a") as f:
        writer = csv.writer(f, delimiter=",")

    try:

        imgResp = urllib.request.urlopen(url, timeout=10)
        print(imgResp)

        imgNp = np.array(bytearray(imgResp.read()), dtype=np.uint8)
        img = cv2.imdecode(imgNp, -1)

        #Crop image
        img = img[400:1000 + 50, 600:1500 + 100]
        #cv2.imshow("orig_thresh", img)

        Image_for_detection = img
        Thermometer_image = img

        height_image, width_image, channels_image = Image_for_detection.shape
        print(f'image dimensions (pixels):, height =  {height_image}, width = {width_image}')

        # Mask for blue color
        therm_hsv = cv2.cvtColor(Image_for_detection, cv2.COLOR_BGR2HSV)
        therm_hsv = cv2.inRange(therm_hsv, mu - a * sig, mu + a * sig)

# --------------------------##------------Parameters that will fill any small gaps in the color mask that
        # encompasses the thermometer-------##

        kernel = np.ones((3, 3), np.uint8)
        dilated_image = cv2.dilate(therm_hsv, kernel, iterations=1)

# --------------------------##----------------------------------------------##

        mask_image = cv2.cvtColor(dilated_image, cv2.COLOR_GRAY2RGB)
        contours, hierarchy = cv2.findContours(image=dilated_image, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_NONE)

        number_regions_found = 0

        for cnt in contours:
            x, y, w, h = cv2.boundingRect(cnt)
            area = cv2.contourArea(cnt)

            if area > minArea:

                #defines area where left thermometer would be found
                if x < 500:

                    ##########################################################################################
                    Thermometer_image = cv2.rectangle(Thermometer_image, (x, y), (x + w, y + h), (0, 255, 0), 1)
                    mask_image = cv2.rectangle(mask_image, (x, y), (x + w, y + h), (0, 255, 0), 1)
                    ##########################################################################################

                    therm_reading_1 = CalculateTemp(t1_F1, t1_F2, t1_P1, t1_P2, y)
                    therm_1_stringC = "%.1f" % round(therm_reading_1[0], 1)
                    print("Therm 1 Calculated_value: " + therm_1_stringC)
                    therm_1_stringF = "%.1f" % round(therm_reading_1[1], 1)
                    print("Therm 1 Calculated_value: " + therm_1_stringF)
                    number_regions_found = 1

                else:
                    #################################################################################################
                    Thermometer_image = cv2.rectangle(Thermometer_image, (x, y), (x + w, y + h), (0, 0, 255), 1)
                    mask_image = cv2.rectangle(mask_image, (x, y), (x + w, y + h), (0, 0, 255), 1)
                    #################################################################################################

                    therm_reading_2 = CalculateTemp(t2_F1, t2_F2, t2_P1, t2_P2, y)
                    therm_2_stringC = "%.1f" % round(therm_reading_2[0], 1)
                    print("Therm 2 Calculated_value C: " + therm_2_stringC)
                    therm_2_stringF = "%.1f" % round(therm_reading_2[1], 1)
                    print("Therm 2 Calculated_value C: " + therm_2_stringF)

                    number_regions_found = number_regions_found + 1

                if (number_regions_found == 2):

                    vis = np.concatenate((mask_image, Thermometer_image), axis=1)

                    #######################################################################
                    cv2.imshow("Image used for detection", vis)
                    #######################################################################

                    # Save data in csv file
                    now = datetime.now()
                    dt_string = now.strftime("%Y-%m-%d %H:%M:%S")
                    current_time = time.time()
                    for_log = [current_time, dt_string, therm_1_stringC, therm_2_stringC, therm_1_stringF,
                    therm_2_stringF]

                    #######################################################################
                    with open("/Users/damianwilliams/Desktop/test_data.csv", "a") as f:
                        writer = csv.writer(f, delimiter=",")
                        writer.writerow(for_log)
                    ######################################################################

                    # Create image with annotation
                    place = (f"{dt_string} \n"
                             f"{therm_1_stringF} F, {therm_1_stringC} C\n"
                             f"{therm_2_stringF} F, {therm_2_stringC} C\n")

                    position = (30, 30)
                    text = place
                    font_scale = 0.75
                    color = (255, 0, 0)
                    thickness = 2
                    font = cv2.FONT_HERSHEY_SIMPLEX
                    line_type = cv2.LINE_AA
                    text_size, _ = cv2.getTextSize(text, font, font_scale, thickness)
                    line_height = text_size[1] + 5
                    x, y0 = position
                    for i, line in enumerate(text.split("\n")):
                        y = y0 + i * line_height
                        img = cv2.putText(img,
                                    line,
                                    (x, y),
                                    font,
                                    font_scale,
                                    color,
                                    thickness,
                                    line_type)

                    #######################################################################################

                    #Save_name = "/Users/damianwilliams/Desktop/Image_set/thermometer_" + str(idx) + ".jpg"
                    #cv2.imwrite(Save_name, img)
                    #cv2.imshow('Thermometer_images', img)
                    ########################################################################################

                    print(idx)
                    idx = idx + 1

    except Exception as e:
        print(e)
        print("reset")
    pass

    cv2.waitKey(1000)
    cv2.destroyAllWindows()