import numpy as np
from csv import reader
import matplotlib.pyplot as plt
import math
import copy
import DecisionTree
import RFDecisionTree
import abDecisionTree
import bonusDecisionTree
import AdaBoost
import Bagging
import RandomForest
import LMS

# Load data
trainData = DecisionTree.loadCSV('data/bank-1/train.csv')
testData = DecisionTree.loadCSV('data/bank-1/test.csv')
features = {'age': {}, 'job': {}, 'marital': {}, 'education': {}, 'default': {}, 'balance': {}, 'housing': {},
            'loan': {}, 'contact': {}, 'day': {}, 'month': {}, 'duration': {}, 'campaign': {}, 'pdays': {},
            'previous': {}, 'poutcome': {}}

# Convert numerical attributes to binary based on median thresholds
numericalMedians = DecisionTree.setThreshold(trainData)
binaryTrainData = DecisionTree.setBinary(trainData, numericalMedians)
testData = DecisionTree.setBinary(testData, numericalMedians)

#============================================
# AdaBoost
#============================================
print('Running AdaBoost for 1 to 10 iterations...')
myAccuracy = []
maxAccuracy = 0
nt = range(1, 10, 1)
for n in nt:
    binaryTrainData1 = copy.deepcopy(binaryTrainData)
    binaryTrainData1 = AdaBoost.assignSampleWeights(binaryTrainData1)

    # Build stump
    stumps = []
    stumpWeights = []
    iterations = n
    newTrainData = binaryTrainData1
    weightLookup = None

    # Run adaBoost algorithm
    for run in range(iterations):
        eFeatures = copy.deepcopy(features)
        # Build dictionary of feature values
        c = 0
        for key in eFeatures.keys():
            for line in newTrainData:
                attr = line[c]
                clss = line[-2]
                if attr not in eFeatures[key].keys():
                    eFeatures[key][attr] = {clss: line[-1]}
                else:
                    if clss not in eFeatures[key][attr].keys():
                        eFeatures[key][attr][clss] = line[-1]
                    else:
                        eFeatures[key][attr][clss] += line[-1]
            c += 1
        newTrainData, stump, stumpWeight = AdaBoost.adaBoost(newTrainData, eFeatures, features)
        stumps.append(stump)
        stumpWeights.append(stumpWeight)

    #Test
    total = 0
    correct = 0
    incorrct = 0
    for sample in testData:
        total += 1
        votes = {'yes': 0, 'no': 0}
        # Run test sample through all trees and tally votes
        for i in range(len(stumps)):
            result = DecisionTree.test(sample, stumps[i], features)
            votes[result] += stumpWeights[i]
        predict = max(votes, key=votes.get)
        actual = sample[-1]
        if predict == actual:
            correct += 1
        else:
            incorrct += 1
    accuracy = correct / total
    if accuracy > maxAccuracy:
        maxAccuracy = accuracy
    myAccuracy.append(accuracy)

fig = plt.figure()
plt.plot(nt, myAccuracy)
plt.title('AdaBoost Test Accuracy')
plt.xlabel('Number of Stumps')
plt.ylabel('Accuracy')
plt.savefig('AdaBoostTest')
plt.show()

#============================================
# Bagging
#============================================
datLen = len(binaryTrainData)
nTrees = 10
nSamples = int(datLen*.6)
oFeatures = copy.deepcopy(features)
trees = []

print('Running Bagged Trees for 1 to 10 iterations...')
myAccuracy = []
nt = range(1, 11, 1)
for n in nt:
    nTrees = n
    trees = []
    for i in range(nTrees):
        trees.append(Bagging.baggedTrees(binaryTrainData, nSamples, oFeatures, datLen))

    #print('Testing Bagged Trees...\n')
    total = 0
    correct = 0
    incorrct = 0
    for line in testData:
        total += 1
        votes = {'yes': 0, 'no': 0}
        # Run test sample through all trees and tally votes
        for tree in trees:
            result = DecisionTree.test(line, tree, features)
            votes[result] += 1
        predict = max(votes, key=votes.get)
        actual = line[-1]
        if predict == actual:
            correct += 1
        else:
            incorrct += 1
    accuracy = correct / total
    myAccuracy.append(accuracy)
    #print('Accuracy: ', accuracy, '\n')

fig = plt.figure()
plt.plot(nt, myAccuracy)
plt.title('Bagged Trees Test Accuracy')
plt.xlabel('Number of trees')
plt.ylabel('Accuracy')
plt.savefig('BaggedTrees')
plt.show()

#============================================
# Random Forest
#============================================
myTrees = {}
myAccuracy = []
nt = range(1, 10, 1)
print('Running Random Forest for 1 to 10 iterations...')
for n in nt:
    nTrees = n
    nf = 2
    rTrees = []
    for i in range(nTrees):
        rTrees.append(RandomForest.randomForest(binaryTrainData, features, nf))
    #print('Random Forest Successfully Generated!\n')

    #print('Testing Random Forest...\n')
    total = 0
    correct = 0
    incorrct = 0
    for line in testData:
        total += 1
        votes = {'yes': 0, 'no': 0}
        # Run test sample through all trees and tally votes
        for tree in rTrees:
            result = RFDecisionTree.test(line, tree, features)
            votes[result] += 1
        predict = max(votes, key=votes.get)
        actual = line[-1]
        if predict == actual:
            correct += 1
        else:
            incorrct += 1
    accuracy = correct / total
    myAccuracy.append(accuracy)
    #print('Accuracy: ', accuracy, '\n')

fig = plt.figure()
plt.plot(nt, myAccuracy)
plt.title('Random Forest Test Accuracy')
plt.xlabel('Number of trees')
plt.ylabel('Accuracy')
plt.savefig('RandomForestT' + str())

#============================================
# Part 2 Problem 2c
#============================================
# predictors = []
# oFeatures = copy.deepcopy(features)
# datLen = len(binaryTrainData)
# nPredictors = 10
# for i in range(nPredictors):
#     trees = []
#     nTrees = 10
#     nSamples = 1000
#     # We will be sampling without replacement here, so we create a copy of our data
#     for i in range(nTrees):
#         dataCopy = binaryTrainData[:]
#         trees.append(Bagging.baggedTrees(dataCopy, nSamples, oFeatures, datLen, replace=False))
#     predictors.append(trees)
#
# myAccuracy = []
# total = 0
# correct = 0
# incorrct = 0
# for line in testData:
#     total += 1
#     votes = {'yes': 0, 'no': 0}
#
#     # Run test sample through FIRST TREE ONLY in all 100 bagged predictors
#     for p in predictors:
#         result = DecisionTree.test(line, p[0], features)
#         votes[result] += 1
#     print(votes)
#     #get ground truth label
#     actual = line[-1]
#
#     if actual == 'yes':
#         predict = votes[actual]/(votes[actual]+votes['no'])
#     elif actual == 'no':
#         predict = votes[actual] / (votes[actual] + votes['yes'])
#
#     bias = (predict-1)**2
#     print(bias)
#
#     #compute mean
#     mean = .5
#     print('mean', mean)
#========
#     if predict == actual:
#         correct += 1
#     else:
#         incorrct += 1
#
# accuracy = correct / total
# myAccuracy.append(accuracy)
# #print('Accuracy: ', accuracy, '\n')
# print(myAccuracy)

#============================================
# Part 3 - Bonus
#============================================
# data = abDecisionTree.loadCSV('data/credit_card.csv')
# features = {'LIMIT_BAL': {}, 'SEX': {}, 'MARRIAGE': {}, 'AGE': {}, 'PAY_0': {}, 'PAY_1': {}, 'PAY_2': {},
#             'PAY_3': {}, 'PAY_4': {}, 'PAY_5': {}, 'PAY_6': {}, 'BILL_AMT1': {}, 'BILL_AMT2': {}, 'BILL_AMT3': {},
#             'BILL_AMT4': {}, 'BILL_AMT5': {}, 'BILL_AMT6': {}, 'PAY_AMT1': {}, 'PAY_AMT2': {}, 'PAY_AMT3': {},
#             'PAY_AMT4': {}, 'PAY_AMT5': {}, 'PAY_AMT6': {}}
# trainData = []
# testData = []
# data.pop(0)
# data.pop(0)
# # Create training and testing sets from full dataset
# for i in range(24000):
#     rn = np.random.randint(0, len(data))
#     data[rn].pop(0)
#     trainData.append(data[rn])
#     data.pop(rn)
# for i in range(6000):
#     rn = np.random.randint(0, len(data))
#     data[rn].pop(0)
#     testData.append(data[rn])
#     data.pop(rn)
#
# # Convert numerical attributes to binary based on median thresholds
# numericalMedians = bonusDecisionTree.setThreshold(trainData)
# binaryTrainData = abDecisionTree.setBinary(trainData, numericalMedians)
# testData =abDecisionTree.setBinary(testData, numericalMedians)
#
# #==========
# # AdaBoost
# #==========
# print('BONUS\n')
# print('Running AdaBoost on credit card dataset for 1 to 10 iterations...')
# myAccuracy = []
# maxAccuracy = 0
# nt = range(1, 10, 1)
# for n in nt:
#     binaryTrainData1 = copy.deepcopy(binaryTrainData)
#     binaryTrainData1 = AdaBoost.assignSampleWeights(binaryTrainData1)
#
#     # Build stump
#     stumps = []
#     stumpWeights = []
#     iterations = n
#     newTrainData = binaryTrainData1
#     weightLookup = None
#
#     # Run adaBoost algorithm
#     for run in range(iterations):
#         eFeatures = copy.deepcopy(features)
#         # Build dictionary of feature values
#         c = 0
#         for key in eFeatures.keys():
#             for line in newTrainData:
#                 attr = line[c]
#                 clss = line[-2]
#                 if attr not in eFeatures[key].keys():
#                     eFeatures[key][attr] = {clss: line[-1]}
#                 else:
#                     if clss not in eFeatures[key][attr].keys():
#                         eFeatures[key][attr][clss] = line[-1]
#                     else:
#                         eFeatures[key][attr][clss] += line[-1]
#             c += 1
#         newTrainData, stump, stumpWeight = AdaBoost.adaBoost(newTrainData, eFeatures, features)
#         stumps.append(stump)
#         stumpWeights.append(stumpWeight)
#
#     #Test
#     total = 0
#     correct = 0
#     incorrct = 0
#     for sample in testData:
#         total += 1
#         votes = {'1': 0, '0': 0}
#         # Run test sample through all trees and tally votes
#         for i in range(len(stumps)):
#             result = bonusDecisionTree.test(sample, stumps[i], features)
#             votes[result] += stumpWeights[i]
#         predict = max(votes, key=votes.get)
#         actual = sample[-1]
#         if predict == actual:
#             correct += 1
#         else:
#             incorrct += 1
#     accuracy = correct / total
#     if accuracy > maxAccuracy:
#         maxAccuracy = accuracy
#     myAccuracy.append(accuracy)
#
# fig = plt.figure()
# plt.plot(nt, myAccuracy)
# plt.title('AdaBoost Test Accuracy')
# plt.xlabel('Number of Stumps')
# plt.ylabel('Accuracy')
# plt.savefig('AdaBoostTest')
# plt.show()

#====================
#====================

# datLen = len(binaryTrainData)
# nTrees = 10
# nSamples = int(datLen*.6)
# oFeatures = copy.deepcopy(features)
# trees = []
#
# myAccuracy = []
# nt = range(1, 11, 1)
# for n in nt:
#     nTrees = n
#     print(str(nTrees) + ' Trees...\n')
#     # print('Generating Random Forest With ' + str(nTrees) + ' Trees...\n')
#     trees = []
#     for i in range(nTrees):
#         trees.append(Bagging.baggedTrees(binaryTrainData, nSamples, oFeatures, datLen))
#     print('Bagged Trees Successfully Generated!\n')
#
#     print('Testing Bagged Trees...\n')
#     total = 0
#     correct = 0
#     incorrct = 0
#     for line in testData:
#         total += 1
#         votes = {'1': 0, '0': 0}
#         # Run test sample through all trees and tally votes
#         for tree in trees:
#             result = bonusDecisionTree.test(line, tree, features)
#             if result == 'no':
#                 print('j')
#             votes[result] += 1
#         predict = max(votes, key=votes.get)
#         actual = line[-1]
#         if predict == actual:
#             correct += 1
#         else:
#             incorrct += 1
#     accuracy = correct / total
#     myAccuracy.append(accuracy)
#     print('Accuracy: ', accuracy, '\n')
# print(myAccuracy)
#
# fig = plt.figure()
# plt.plot(nt, myAccuracy)
# plt.title('Bagged Trees Test Accuracy')
# plt.xlabel('Number of trees')
# plt.ylabel('Accuracy')
# plt.savefig('BaggedTrees')
# plt.show()

#============================================
# Least Mean Squares
#============================================
# Load data
trainData = DecisionTree.loadCSV('data/concrete/train.csv')
testData = DecisionTree.loadCSV('data/concrete/test.csv')

def plotFig(data, title, xlabel, ylabel):
    plt.figure()
    plt.plot(data)
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.show()

def test(coeff, predictFn):
    testCost = []
    for sample in testData:
        y0 = float(sample[-1])
        y = predictFn(sample, coeff)
        c = ((y - y0) ** 2) / 2
        #c = abs((y-y0)/y0)
        testCost.append(c)
    return testCost

X = []
Y = []
for sample in trainData:
    X.append([float(i) for i in sample[:7]])
    Y.append(float(sample[-1]))
X = np.array(X)
Y = np.array(Y)

B = np.zeros(len(X[1]))
lRate = .01
maxIter = 3000
convThresh = .000001
# Gradient Descent
print('Running Gradient Descent for LMS...\n')
coeff0, pastCost = LMS.BGD(X, Y, lRate, maxIter, convThresh)
print('The learned weight vector is: ', coeff0, '\n')
plotFig(pastCost, 'LMS with Gradient Descent', 'Iterations', 'Cost')
testCost = test(coeff0, LMS.predictG)
plotFig(testCost, 'Predictions with Gradient Descent', 'Sample', 'Cost')

# Stochastic Gradient Descent
print('Running Stochastic Gradient Descent for LMS...\n')
coeff1, pastCost = LMS.SGD(X, lRate, maxIter)
print('The learned weight vector is: ', coeff1, '\n')
plotFig(pastCost, 'LMS with Stochastic Gradient Descent', 'Iterations', 'Cost')
testCost = test(coeff1, LMS.predictG)
plotFig(testCost, 'Predictions with Stochastic Gradient Descent', 'Sample', 'Cost')

# Calculate optimal weight vector analytically
r = np.matmul(X.T, Y)
l = np.linalg.inv(np.matmul(X.T, X))
w = np.matmul(r, l)
print('Optimal weight vector solved analytically: ', w)
testCost = test(w, LMS.predictG)
plotFig(testCost, 'Predictions with Analytical Solution', 'Sample', 'Cost')


from __future__ import print_function

import matplotlib
matplotlib.use('Agg')
import cv2
import matplotlib.pyplot as plt
import numpy as np
from collections import defaultdict
import os
import sys
import tensorflow as tf
import random
import warnings
warnings.filterwarnings('ignore')

from keras import backend as K
from scipy.io import savemat
from skimage.io import imsave
from skimage.transform import resize

from data import load_data
from net import unet
from net import dice_coef

from model_dilation import get_frontend
from model_dilation import get_dilation_model_unet

import pdb

weights_path = '/media/alexshakouri/TOURO Mobile USB3.03/Research/Code/brain-segmentation-master/Rat_Brain_Sementation/results/weights_multiLabel_unet_dil_1_300.h5'
train_images_path = '/media/alexshakouri/TOURO Mobile USB3.03/Research/Code/brain-segmentation-master/data/dataAll_128/'
test_images_path = '/media/alexshakouri/TOURO Mobile USB3.03/Research/Code/brain-segmentation-master/data/dataAllVal_128_testIMG/'
predictions_path = '/media/alexshakouri/TOURO Mobile USB3.03/Research/Code/brain-segmentation-master/predictions/weights_singleLabel1_matlabtest/'

num_classes = 14

imSize = 128

output_rows = 280
output_cols = 200

gpu = '0'

random.seed(1)
class_colors = [ ( random.randint(0,255),random.randint(0,255),random.randint(0,255) ) for _ in range(num_classes) ]

def predict(mean=0.0, std=1.0):
    # load and normalize data
    if mean == 0.0 and std == 1.0:
        imgs_train, _, _ = load_data(train_images_path, num_classes)
        mean = np.mean(imgs_train)
        std = np.std(imgs_train)

    imgs_test, imgs_mask_test, names_test = load_data(test_images_path, num_classes)

    mean = np.mean(imgs_test)
    std = np.std(imgs_test)

    original_imgs_test = imgs_test.astype(np.uint8)

    imgs_test -= mean
    imgs_test /= std

    # load model with weights
    #model = unet(num_classes) #Unet model
    #model = get_frontend(imSize,imSize, num_classes) #Dilation model
    model = get_dilation_model_unet(imSize,imSize, num_classes) #combination model

    model.load_weights(weights_path)

    # make predictions
    imgs_mask_pred = model.predict(imgs_test, verbose=1)
    # save to mat file for further processing
    if not os.path.exists(predictions_path):
        os.mkdir(predictions_path)

    matdict = {
        'pred': imgs_mask_pred,
        'image': original_imgs_test,
        'mask': imgs_mask_test,
        'name': names_test
    }
    savemat(os.path.join(predictions_path, 'predictions.mat'), matdict)

    # save images with segmentation and ground truth mask overlay
    for i in range(len(imgs_test)):
        pred = imgs_mask_pred[i]
        #print(original_imgs_test.shape)
        image = original_imgs_test[i]
        mask = imgs_mask_test[i]

        # segmentation mask is for the middle slice
        image_rgb = gray2rgb(image[:, :, 0])

        # prediction contour image (add all the predictions)
        pred = (np.round(pred) * 255.0).astype(np.uint8)
        # ground truth contour image (add all the masks)
        mask = (np.round(mask) * 255.0).astype(np.uint8)

        # combine image with contours using red for pred and blue for mask
        pred_rgb = np.array(image_rgb)
        annotation = pred_rgb[:, :, 1]

        #Set all the pixels with the annotation to zero and fill it in with the color
        for c in range(num_classes):
            pred_temp = pred[:,:,c]
            mask_temp = mask[:,:,c]

            pred_temp, contours, _ = cv2.findContours(
                pred_temp.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
            pred_temp = np.zeros(pred_temp.shape)
            cv2.drawContours(pred_temp, contours, -1, (255, 0, 0), 1)

            mask_temp, contours, _ = cv2.findContours(
                mask_temp.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
            mask_temp = np.zeros(mask_temp.shape)
            cv2.drawContours(mask_temp, contours, -1, (255, 0, 0), 1)

            pred[:,:,c] = pred_temp
            mask[:,:,c] = mask_temp

            annotation[np.maximum(pred[:,:,c], mask[:,:,c]) == 255] = 0

        pred_rgb[:, :, 0] = pred_rgb[:, :, 1] = pred_rgb[:, :, 2] = annotation

        for c in range(num_classes):
            pred_rgb[:, :, 2] = np.maximum(pred_rgb[:, :, 2], mask[:,:,c])
            pred_rgb[: ,: ,1] = np.maximum(pred_rgb[: ,: ,1], (pred[:,:,c]/255)* class_colors[c][1])
            pred_rgb[:, :, 2] = np.maximum(pred_rgb[:, :, 2], (pred[:,:,c]/255)* class_colors[c][2])
            pred_rgb[:, :, 0] = np.maximum(pred_rgb[:, :, 0], (pred[:,:,c]/255)* class_colors[c][0])

        imsave(os.path.join(predictions_path,
                            names_test[i] + '.png'), pred_rgb)

    return imgs_mask_test, imgs_mask_pred, names_test

def evaluate(imgs_mask_test, imgs_mask_pred, names_test):
    test_pred = list(zip(imgs_mask_test, imgs_mask_pred))
    name_test_pred = list(zip(names_test, test_pred))
    name_test_pred.sort(key=lambda x: x[0])

    patient_ids = []
    dc_values = []

    i = 0  # start slice index
    for p in range(len(name_test_pred)):
        # get case id (names are in format <case_id>_<slice_number>)
        p_id = '_'.join(name_test_pred[p][0].split('_')[:-1])

        # if this is the last slice for the processed case
        if p + 1 >= len(name_test_pred) or p_id not in name_test_pred[p + 1][0]:
            # ground truth segmentation:
            p_slices_mask = np.array(
                [im_m[0] for im_id, im_m in name_test_pred[i:p + 1]])
            # predicted segmentation:
            p_slices_pred = np.array(
                [im_m[1] for im_id, im_m in name_test_pred[i:p + 1]])

            patient_ids.append(p_id)
            dc_values.append(dice_coefficient(p_slices_pred, p_slices_mask))
            print(p_id + ':\t' + str(dc_values[-1]))

            i = p + 1

    return dc_values, patient_ids

def dice_coefficient(prediction, ground_truth):
    prediction = np.squeeze(prediction)
    ground_truth = np.squeeze(ground_truth)
    prediction = np.round(prediction).astype(int)
    ground_truth = np.round(ground_truth).astype(int)

    return np.sum(prediction[ground_truth == 1]) * 2.0 / (np.sum(prediction) + np.sum(ground_truth))

def gray2rgb(im):
    w, h = im.shape
    ret = np.empty((w, h, 3), dtype=np.uint8)
    ret[:, :, 2] = ret[:, :, 1] = ret[:, :, 0] = im
    return ret

def plot_dc(labels, values):
    y_pos = np.arange(len(labels))

    fig = plt.figure(figsize=(12, 8))
    plt.barh(y_pos, values, align='center', alpha=0.5)
    plt.yticks(y_pos, labels)
    plt.xticks(np.arange(0.5, 1.0, 0.05))
    plt.xlabel('Dice coefficient', fontsize='x-large')
    plt.axes().xaxis.grid(color='black', linestyle='-', linewidth=0.5)
    axes = plt.gca()
    axes.set_xlim([0.5, 1.0])
    plt.tight_layout()
    axes.axvline(np.mean(values), color='green', linewidth=2)

    plt.savefig('DSC.png', bbox_inches='tight')
    plt.close(fig)

def post_process(imgs_mask, names_mask):
    #Step1: combine all the individual masks (128x128x14 to 128x128x1)
    #np.max as some of the pixel predictions overlap (this puts a higher weight on the later regions)
    imgs_mask_group = np.max(np.round(imgs_mask) * (np.arange(imgs_mask.shape[-1]) + 1),3)
    #step2: group the images into their own groups (total 12 groups)
    #assume the names are formatted: animalID_SliceNumber
    uniqNames = defaultdict(list)
    animalSliceNum = np.zeros(len(imgs_mask))
    for i in range(len(imgs_mask)):
        #seperate slice number from the animalID
        animalID = names_mask[i].split('_')[0]
        uniqNames[animalID].append(i)

        animalSliceNum[i] = names_mask[i].split('_')[1]

    #step3: sort the 44 images from 1-44 (from the names) and save the output
    for IDIter, imIter in uniqNames.items():
        namesID = names_mask[imIter]
        imgs_ID_mask = imgs_mask_group[imIter]
        namesSliceNum = animalSliceNum[imIter]

        sortSliceNum = np.argsort(namesSliceNum)

        sortNamesID = namesID[sortSliceNum]
        sort_imgs_mask = imgs_ID_mask[sortSliceNum, :, :]

        #Reverse the image processing
        #output is 280x200x44
        min_length = min(output_rows, output_cols)
        max_length = max(output_rows, output_cols)

        zeroPad = (np.ceil(((max_length*(imSize/min_length)) - imSize)/2)).astype(int)

        imgs_pad_mask = np.pad(sort_imgs_mask, ((0,0),(zeroPad,zeroPad),(0,0)), 'constant')
        imgs_post_mask = resize(imgs_pad_mask, (imgs_ID_mask.shape[0], output_rows, output_cols)).astype(np.int32)

        imsave(os.path.join(predictions_path, IDIter + '.tif'), imgs_post_mask)

    return 0

if __name__ == '__main__':

    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    sess = tf.Session(config=config)
    K.set_session(sess)

    if len(sys.argv) > 1:
        gpu = sys.argv[1]
    device = '/gpu:' + gpu

    #with tf.device(device):
    imgs_mask_test, imgs_mask_pred, names_test = predict()

    imgs_mask_post = post_process(imgs_mask_pred, names_test)

    values, labels = evaluate(imgs_mask_test, imgs_mask_pred, names_test)

    print('\nAverage DSC: ' + str(np.mean(values)))

    # plot results
    plot_dc(labels, values)


import sys
_module = sys.modules[__name__]
del sys
cnn_with_spp = _module
spp_layer = _module

from _paritybench_helpers import _mock_config, patch_functional
from unittest.mock import mock_open, MagicMock
from torch.autograd import Function
from torch.nn import Module
import abc, collections, copy, enum, functools, inspect, itertools, logging, math, matplotlib, numbers, numpy, pandas, queue, random, re, scipy, sklearn, string, tensorflow, time, torch, torchaudio, torchtext, torchvision, types, typing, uuid, warnings
import numpy as np
from torch import Tensor
patch_functional()
open = mock_open()
yaml = logging = sys = argparse = MagicMock()
ArgumentParser = argparse.ArgumentParser
_global_config = args = argv = cfg = config = params = _mock_config()
argparse.ArgumentParser.return_value.parse_args.return_value = _global_config
yaml.load.return_value = _global_config
sys.argv = _global_config
__version__ = '1.0.0'
xrange = range
wraps = functools.wraps

import torch

import torch.nn as nn

from torch.nn import init

import functools

from torch.autograd import Variable

import numpy as np

import torch.nn.functional as F

import math

def spatial_pyramid_pool(self, previous_conv, num_sample, previous_conv_size, out_pool_size):
    """
    previous_conv: a tensor vector of previous convolution layer
    num_sample: an int number of image in the batch
    previous_conv_size: an int vector [height, width] of the matrix features size of previous convolution layer
    out_pool_size: a int vector of expected output size of max pooling layer

    returns: a tensor vector with shape [1 x n] is the concentration of multi-level pooling
    """
    for i in range(len(out_pool_size)):
        h_wid = int(math.ceil(previous_conv_size[0] / out_pool_size[i]))
        w_wid = int(math.ceil(previous_conv_size[1] / out_pool_size[i]))
        h_pad = (h_wid * out_pool_size[i] - previous_conv_size[0] + 1) / 2
        w_pad = (w_wid * out_pool_size[i] - previous_conv_size[1] + 1) / 2
        maxpool = nn.MaxPool2d((h_wid, w_wid), stride=(h_wid, w_wid), padding=(h_pad, w_pad))
        x = maxpool(previous_conv)
        if i == 0:
            spp = x.view(num_sample, -1)
        else:
            spp = torch.cat((spp, x.view(num_sample, -1)), 1)
    return spp

class SPP_NET(nn.Module):
    """
    A CNN model which adds spp layer so that we can input multi-size tensor
    """

    def __init__(self, opt, input_nc, ndf=64, gpu_ids=[]):
        super(SPP_NET, self).__init__()
        self.gpu_ids = gpu_ids
        self.output_num = [4, 2, 1]
        self.conv1 = nn.Conv2d(input_nc, ndf, 4, 2, 1, bias=False)
        self.conv2 = nn.Conv2d(ndf, ndf * 2, 4, 1, 1, bias=False)
        self.BN1 = nn.BatchNorm2d(ndf * 2)
        self.conv3 = nn.Conv2d(ndf * 2, ndf * 4, 4, 1, 1, bias=False)
        self.BN2 = nn.BatchNorm2d(ndf * 4)
        self.conv4 = nn.Conv2d(ndf * 4, ndf * 8, 4, 1, 1, bias=False)
        self.BN3 = nn.BatchNorm2d(ndf * 8)
        self.conv5 = nn.Conv2d(ndf * 8, 64, 4, 1, 0, bias=False)
        self.fc1 = nn.Linear(10752, 4096)
        self.fc2 = nn.Linear(4096, 1000)

    def forward(self, x):
        x = self.conv1(x)
        x = self.LReLU1(x)
        x = self.conv2(x)
        x = F.leaky_relu(self.BN1(x))
        x = self.conv3(x)
        x = F.leaky_relu(self.BN2(x))
        x = self.conv4(x)
        spp = spatial_pyramid_pool(x, 1, [int(x.size(2)), int(x.size(3))], self.output_num)
        fc1 = self.fc1(spp)
        fc2 = self.fc2(fc1)
        s = nn.Sigmoid()
        output = s(fc2)
        return output


"""
The sensor module contains classes to define point or camera sensors
that can either be stationary and mobile.

.. rubric:: Contents

.. autosummary::

    Sensor
    Position
    Stationary
    Mobile
    Detector
    Point
    Camera
"""
from __future__ import print_function, division
import pandas as pd
import numpy as np
from scipy.interpolate import griddata
from scipy.spatial.distance import cdist
from scipy import integrate
from scipy import ndimage as sn

class Sensor(object):

    def __init__(self, position=None, detector=None):
        """
        Defines a sensor object and methods to calculate detection.

        Parameters
        ----------
        position : chama Position
            Sensor position
        detector : chama Detector
            Sensor detector, determines the method used to calculate detection
        """

        self.name = None

        if position is None or not isinstance(position, Position):
            raise ValueError('Must specify a Position object to create a '
                             'Sensor')
        self.position = position

        if detector is None or not isinstance(detector, Detector):
            raise ValueError('Must specify a Detector object to create a '
                             'Sensor')
        self.detector = detector

    def get_detected_signal(self, signal, interp_method=None,
                            min_distance=10.0):
        """
        Returns the detected signal.
        """
        return self.detector.get_detected_signal(signal, self.position,
                                                 interp_method, min_distance)

class Position(object):

    def __init__(self, location=None):
        """
        Defines a sensor's position.

        Parameters
        ----------
        location : (x,y,z) tuple or index
            The location of the Position object defined as an (x,y,z) tuple
            or location index, which can be a string or integer
        """
        self.location = location

    def __call__(self, time):
        """
        Returns the location at the specified time

        Parameters
        ----------
        time : int or float

        Returns
        -------
        Location tuple (x,y,z) or (j)

        """
        if isinstance(self.location, tuple):
            return tuple(self.location)
        else:
            return tuple([self.location])

class Stationary(Position):
    """
    Defines a stationary sensor's position.
    """
    pass

class Mobile(Position):
    """
    Defines a mobile sensor's position.
    A mobile position moves according to defined waypoints and speed. The
    mobile position is assumed to move in a straight line between waypoints
    and will repeat its path if needed.

    Parameters
    ----------
    locations : list of (x,y,z) tuples
        List of (x,y,z) tuples defining the waypoints of the mobile
        sensor's path.
    speed : int or float
        The speed of the mobile sensor in units consistent
        with the waypoints and sensor sample_times
    repeat : bool
        Boolean indicating if the path should repeat
    """
    def __init__(self, locations=None, speed=1, start_time=0, repeat=False):
        super(Mobile, self).__init__(locations)
        self.speed = speed
        self.start_time = start_time
        self.repeat = repeat
        self._d_btwn_locs = None

    def __call__(self, time):
        """
        Returns the position (x,y,z) at the specified time.

        Parameters
        ----------
        time : int or float

        Returns
        -------
        tuple
            The (x,y,z) location

        """
        # Calculate distance traveled at specified time
        delta_time = time - self.start_time
        if delta_time < 0:
            delta_time = 0
        distance = self.speed * delta_time

        temp_locs = [np.array(i) for i in self.location]
        if self.repeat:  # if path repeats
            temp_locs.append(temp_locs[0])

        if self._d_btwn_locs is None:
            # Distances between consecutive points
            self._d_btwn_locs = \
                [np.linalg.norm(temp_locs[i] - temp_locs[i + 1])
                 for i in range(len(temp_locs) - 1)]

        if self.repeat:  # if path repeats
            while distance > sum(self._d_btwn_locs):
                distance -= sum(self._d_btwn_locs)
        else:
            if distance > sum(self._d_btwn_locs):
                distance = sum(self._d_btwn_locs)

        i = 0
        # Figure out which line segment
        for i, _ in enumerate(self._d_btwn_locs):
            if sum(self._d_btwn_locs[:i + 1]) >= distance:
                distance -= sum(self._d_btwn_locs[:i])
                break

        # The two waypoints defining the line segment
        loc1 = temp_locs[i]
        loc2 = temp_locs[i + 1]

        location = loc1 + (loc2 - loc1) * (distance / self._d_btwn_locs[i])

        return tuple(location)

class Detector(object):
    """
    Defines a sensor's detector.

    Parameters
    ----------
    threshold : int
        The minimum signal that can be detected by the sensor
    sample_times : list of ints or floats
        List of the sensor's sample/measurement times
    """

    def __init__(self, threshold=None, sample_times=None):
        self.threshold = threshold
        self.sample_times = sample_times
        self.sample_points = None

    def get_sample_points(self, position):
        """
        Returns the sensor sample points in the form (t,x,y,z) or (t,j)

        Parameters
        ----------
        position : chama Position
            The position of the sensor

        Returns
        -------
        A list of sample points in the form (t,x,y,z) or (t,j)
        """

        if self.sample_points is None:
            self.sample_points = [(t,) + position(t) for t in
                                  self.sample_times]
        return self.sample_points

    def get_detected_signal(self, signal, position, interp_method,
                            min_distance):
        """
        Returns the signal detected by the sensor.

        Parameters
        ----------
        signal : pandas DataFrame
            DataFrame with the multi-index (T, X, Y, Z) or (T, Node) and columns
            containing the concentrations for different scenarios
        position : chama Position
            The position of the sensor
        interp_method : 'linear', 'nearest', or None
            Method used to interpolate the signal if needed.
            A value of 'linear' will use griddata to interpolate missing
            sample points. A value of 'nearest' will set the sample point to
            the nearest signal point within a minimum distance of min_distance.
            If there are no signal points within this distance then the
            signal will be set to zero at the sample point.
        min_distance : float
            The minimum distance when using the 'nearest' interp_method

        Returns
        -------
        A pandas Series with multi-index (T, Scenario) and signal values above
        the sensor threshold.

        """
        pts = self.get_sample_points(position)
        if len(pts) == 0:
            return pd.Series()

        signal_sample = self._get_signal_at_sample_points(signal, pts,
                                                          interp_method,
                                                          min_distance)
        if len(signal_sample) == 0:
            return pd.Series()

        # Reset the index
        signal_sample = signal_sample.reset_index()

        # At this point we don't need the X,Y,Z or Node columns
        if set(['X', 'Y', 'Z']) < set(list(signal_sample.columns)):
            signal_sample.drop(['X', 'Y', 'Z'], inplace=True, axis=1)
        elif set(['Node']) < set(list(signal_sample.columns)):
            signal_sample.drop(['Node'], inplace=True, axis=1)
        else:
            raise ValueError('Unrecognized signal format')
            return

        # Set T as the index
        signal_sample = signal_sample.set_index('T')

        # Apply threshold
        signal_sample = signal_sample[signal_sample >= self.threshold]
        if len(signal_sample) == 0:
            return pd.Series()

        # Name the columns so that the index is labeled after stacking
        signal_sample.columns.name = 'Scenario'

        # Drop Nan and stack by index
        return signal_sample.stack()

    def _get_signal_at_sample_points(self, signal, sample_points,
                                     interp_method, min_distance):
        raise NotImplementedError()

class Point(Detector):
    """
    Defines a point sensor.
    """

    def _get_signal_at_sample_points(self, signal, sample_points,
                                     interp_method, min_distance):
        """
        Returns the signal at the sensor sample points. If a sample point
        does not exist in the signal DataFrame then interpolate the signal.

        Parameters
        -----------
        signal : pandas DataFrame

        sample_points : list of tuples

        interp_method : 'linear', 'nearest', or None

        min_distance : float

        Returns
        ---------
        pandas DataFrame has a multi-index containing all of the
        sample_points and columns for each scenario with the
        concentration at each sample point

        """

        # Get subset of signal. If a sample point is not in signal then NaN
        # is inserted
        try:
            signal_subset = signal.loc[sample_points, :]
        except: # Some sample points are not in the signal
            if ['T', 'X', 'Y', 'Z'] == signal.index.names:
                signal_subset = pd.DataFrame(data=np.nan, columns=signal.columns,
                                         index=pd.MultiIndex.from_tuples(sample_points, names=['T', 'X', 'Y', 'Z']))
            elif ['T', 'Node'] == signal.index.names:
                signal_subset = pd.DataFrame(data=np.nan, columns=signal.columns,
                                         index=pd.MultiIndex.from_tuples(sample_points, names=['T', 'Node']))
            else:
                raise ValueError('Unrecognized signal format')

            sample_points_in_signal = list(set(sample_points).intersection(set(signal.index)))
            signal_subset.loc[sample_points_in_signal, :] = signal.loc[sample_points_in_signal, :]

        if interp_method is None:
            return signal_subset

        # Get the sample_points that need to be interpolated
        temp = signal_subset.isnull().any(axis=1)  # Get rows containing NaN
        interp_points = list(signal_subset[temp].index)  # Get their index

        if len(interp_points) == 0:
            return signal_subset

        # TODO: Revisit the distance calculation.
        # Scaling issue by including both time and xyz location in distance
        # calculation. Manually select the signal times bordering
        # interp_point times BEFORE calculating the distance? Or include a
        # time scaling parameter as an additional input?

        # get the distance between the signal points and the interp_points
        signal_points = list(signal.index)
        distdata = cdist(signal_points, interp_points)

        # Might not want to build this data frame when signal is very large
        dist = pd.DataFrame(data=distdata, index=signal.index)

        if interp_method == 'linear':

            # Loop over interp_points
            for i in range(len(dist.columns)):
                temp = dist.iloc[:, i]

                # Get the rows within dist_factor of the minimum distance
                dist_factor = 2
                temp2 = temp[temp < temp.min() * dist_factor]
                # Ensures that we get enough points to do the interpolation
                while len(temp2) < 100:
                    dist_factor += 1
                    temp2 = temp[temp < temp.min() * dist_factor]
                temp_signal = signal.loc[temp2.index, :]

                # Loop over scenarios
                for j in signal.columns:

                    interp_signal = griddata(list(temp_signal.index),
                                             list(temp_signal.loc[:, j]),
                                             interp_points[i],
                                             method=interp_method,
                                             rescale=True)
                    if np.isnan(interp_signal):
                        raise ValueError('Trying to interpolate a sample '
                                         'point outside of the signal grid. '
                                         'Make sure that all sensor '
                                         'locations are contained in the '
                                         'area spanned by the signal data.')
                    signal_subset.at[interp_points[i], j] = interp_signal

        elif interp_method == 'nearest':

            # Loop over interp_points
            for i in range(len(dist.columns)):
                temp = dist.iloc[:, i]

                if temp.min() > min_distance:
                    # Loop over scenarios
                    for j in signal.columns:
                        interp_signal = 0.0
                        signal_subset.at[interp_points[i], j] = interp_signal
                else:
                    temp2 = temp[temp < min_distance]
                    temp_signal = signal.loc[temp2.index, :]

                    # Loop over scenarios
                    for j in signal.columns:

                        interp_signal = griddata(list(temp_signal.index),
                                                 list(temp_signal.loc[:, j]),
                                                 interp_points[i],
                                                 method=interp_method,
                                                 rescale=True)

                        signal_subset.at[interp_points[i], j] = interp_signal
        else:
            raise ValueError('Unrecognized or unsupported interpolation method'
                             ' "%s" was specified. Only "linear" or "nearest" '
                             'interpolations are supported' % interp_method)

        return signal_subset

class Camera(Detector):
    """
    Defines a camera sensor.

    Parameters
    ----------
    threshold : int
        The minimum number of pixels that must detect something in order for
        the camera to detect.
    sample_times : list of ints or floats
        List of the sensor's sample/measurement times
    direction : (x, y, z) tuple
        Tuple representing the direction that the camera is pointing in (x,
        y, z) coordinates relative to the origin
    **kwds : dictionary
        Keyword arguments for setting parameter values in the camera model
    """

    def __init__(self, threshold=None, sample_times=None,
                 direction=(1, 1, 1), **kwds):

        super(Camera, self).__init__(threshold, sample_times)

        # Direction of the camera relative to the origin
        self.direction = direction

        # Set default camera properties

        # Transmission coefficient of air
        self.tau_air = kwds.pop('tau_air', 1)

        # Maximum distance that the camera can detect in (m)
        self.dist = kwds.pop('dist', 500.0)

        # TODO: Get descriptions of these from Arvind
        self.netd = kwds.pop('netd', 0.015)
        self.f_number = kwds.pop('f_number', 1.5)
        self.e_a = kwds.pop('e_a', 0.1)
        self.e_g = kwds.pop('e_g', 0.5)
        self.T_g = kwds.pop('T_g', 300)
        self.T_plume = kwds.pop('T_plume', 300)
        self.lambda1 = kwds.pop('lambda1', 3.2E-6)
        self.lambda2 = kwds.pop('lambda2', 3.4E-6)
        self.fov1 = kwds.pop('fov1', 24 * np.pi / 180)
        self.fov2 = kwds.pop('fov2', 18 * np.pi / 180)
        self.a_d = kwds.pop('a_d', 9.0E-10)
        self.Kav = kwds.pop('Kav', 2.191e-20)

        # Constants used in the camera model
        self.NA = 6.02E23  # Avogadro's number
        self.h = 6.626e-34  # Planck's constant [J-s]
        self.SIGMA = 5.67e-8  # Stefan-Boltzmann constant [W/m^2-K^4]
        self.c = 3.0e8  # Speed of light [m/s]
        self.k = 1.38e-23  # Boltzmann's constant [J/K]

    def _get_signal_at_sample_points(self, signal, sample_points,
                                     interp_method, min_distance):
        """
        Defines detection as seen by a camera sensor. Not just
        selecting/interpolating a subset of the signal DataFrame. We are using
        the CONCENTRATION signal DataFrame to calculate the PIXEL signal at the
        sample points.

        Parameters
        -----------
        signal : pandas DataFrame
            DataFrame has a multi-index with (T, X, Y, Z) points
            and each column in the frame contains concentration
            values at those points for one scenario

        sample_points : list of tuples (t,x,y,z)

        Returns
        ---------
        pandas DataFrame has a multi-index with the sensor's sample_points
        (T,X,Y,Z) and each column contains the number of pixels that
        detected something for one scenario
        """

        # TODO: Add option to specify a different camera direction at each
        # sample point
        CamDir = self.direction
        # Reset the index and set it to T
        allConc = signal.reset_index().set_index('T')

        # Create DataFrame to be returned
        newidx = pd.MultiIndex.from_tuples(sample_points,
                                           names=('T', 'X', 'Y', 'Z'))
        detected_pixels = pd.DataFrame(None, index=newidx,
                                       columns=signal.columns)

        # Check if all the sample times are time points included in the
        # signal dataframe
        signal_t = set(allConc.index)
        sample_t = set([p[0] for p in sample_points])
        if not sample_t.issubset(signal_t):
            raise ValueError('All sampling times for a camera sensor must be '
                             'contained in the signal data')

        # print('    Calculating camera signal detection')
        # TODO: Add interpolation for non-gridded or sparse signal data
        for point in sample_points:
            time = point[0]
            # print('        Time: ', time)
            CamLoc = point[1:]

            # We assume that every sample time is in the signal DataFrame
            # Extract the rows at the sample time
            Conc = allConc.loc[time, :]

            # Might want to move the below calculations to a new function to
            # avoid deeply nested for-loops. Any way to vectorize??

            # No longer need T column
            Conc = Conc.reset_index(drop=True)

            # Set and sort the index so that we can guarantee the order of
            # the rows and use numpy reshape to do the conversion to a 3D array
            Conc = Conc.set_index(['X', 'Y', 'Z'])
            Conc = Conc.sort_index()

            # Get all the unique X, Y, and Z grid points
            gridpoints = list(Conc.index)
            groupedpoints = list(zip(*gridpoints))
            X = np.unique(groupedpoints[0])
            Y = np.unique(groupedpoints[1])
            Z = np.unique(groupedpoints[2])

            # Check if signal is on a regular grid by looking at the number
            # of rows
            nx = len(X)
            ny = len(Y)
            nz = len(Z)

            if nx * ny * nz != Conc.shape[0]:
                raise ValueError('The camera sensor requires signal data '
                                 'to be on a regular grid')

            # Check to make sure X,Y,Z points are equally spaced
            xdiff = np.unique(X[1:] - X[:-1])
            ydiff = np.unique(Y[1:] - Y[:-1])
            zdiff = np.unique(Z[1:] - Z[:-1])
            if len(xdiff) > 1 or len(ydiff) > 1 or len(zdiff) > 1:
                raise ValueError('The camera sensor requires signal data to '
                                 'be equally spaced over a particular '
                                 'spatial axis (i.e. X, Y, and Z)')

            # Calculate angles (horizontal and vertical) associated with the
            # camera orientation. The vertical angle is complemented due to
            # spherical coordinate convention.
            dir1 = np.array(CamDir)
            dir2 = dir1 / (np.sqrt(dir1[0] ** 2 + dir1[1] ** 2 + dir1[2] ** 2))
            horiz = np.arccos(dir2[0])
            vert = np.arccos(dir2[2])

            # The camera has 320 X 240 pixels. To speed up computation, this
            # has been reduced proportionally to 80 X 60. The horizontal (vert)
            # field of view is divided equally among the 80 (60) horizontal
            # (vert) pixels
            # TODO: convert horiz/vert field of view degrees to parameters

            theta_h = np.linspace(horiz - np.pi / 15, horiz + np.pi / 15, 80)
            theta_v = np.linspace(vert - np.pi / 20, vert + np.pi / 20, 60)

            # factor_x, factor_y, factor_z are used later for
            # concentration-pathlength (CPL) calculations. Extrapolation to
            # calculate CPL happens in pixel-coordinates rather than real-life
            # coordinates. 'dist' is the maximum distance that the IR
            # camera can see
            Xstep, Ystep, Zstep = X[1] - X[0], Y[1] - Y[0], Z[1] - Z[0]
            factor_x = int(self.dist / Xstep)
            factor_y = int(self.dist / Ystep)
            factor_z = int((self.dist / 5) / Zstep)

            p, q = len(theta_h), len(theta_v)
            x_end = np.zeros((p, q))
            y_end = np.zeros((p, q))
            z_end = np.zeros((p, q))

            # Calculate the real-life coordinate of a point 'dist' m away for
            # each pixel orientation. This is used to calculate CPL. If 'dist'
            # goes outside the grid boundary the concentration is set to 0.
            for i in range(0, p):
                for j in range(0, q):
                    x_end[i, j] = factor_x * np.cos(theta_h[i]) * \
                                  np.sin(theta_v[j])
                    y_end[i, j] = factor_y * np.sin(theta_h[i]) * \
                                  np.sin(theta_v[j])
                    z_end[i, j] = factor_z * np.cos(theta_v[j])

            # Because calculations happen in pixel coordinates, the
            # location of the camera (start of calculation) and the
            # location of far-away point (end of calculation) is converted
            # to pixel coordinates.
            x_start = (CamLoc[0] - np.min(X)) / Xstep
            y_start = (CamLoc[1] - np.min(Y)) / Ystep
            z_start = (CamLoc[2] - np.min(Z)) / Zstep

            x_end += x_start
            y_end += y_start
            z_end += z_start

            # Calculate camera properties
            nep, tec = self._pixelprop()

            for scen in Conc.columns:
                # Extract the concentration values as a numpy array
                ppm = Conc.loc[:, scen].values
                # Reshape the concentration column as a 3D array
                ppm = ppm.reshape(nx, ny, nz)

                IntConc = np.zeros((p, q))
                CPL = np.zeros((p, q))

                # TODO: Convert this to vector operation to remove for-loops??
                for i in range(0, len(theta_h)):
                    for j in range(0, len(theta_v)):
                        # Calculate concentration pathlength (CPL)
                        IntConc[i, j] = self._pathlength(x_start, y_start,
                                                         z_start,
                                                         x_end[i, j],
                                                         y_end[i, j],
                                                         z_end[i, j], ppm)

                        CPL[i, j] = IntConc[i, j] * self.dist

                # Convert CPL to image contrast and compare it to nep
                # 1e-4 is conversion factor
                attn = CPL * self.Kav * self.NA * 1e-4
                temp = 1 - 10 ** (-attn)
                contrast = temp * np.abs(tec) * self.tau_air

                # Count the number of pixels with a contrast greater than nep
                pixels = sum(sum(contrast >= nep))

                # Camera pixels were truncated to 80 x 60 px, convert pixel
                # count back to the original scale
                pixel_final = 16 * pixels

                detected_pixels.at[point, scen] = pixel_final

        # print(detected_pixels)
        return detected_pixels

    def _pathlength(self, x0, y0, z0, x1, y1, z1, data):
        num = 201  # number of points in extrapolation
        x = np.linspace(x0, x1, num)
        y = np.linspace(y0, y1, num)
        z = np.linspace(z0, z1, num)
        concs = sn.map_coordinates(data, np.vstack((x, y, z)), order=1)
        # CPL as a fraction of total number of points in extrapolation
        avgconc = sum(concs) / num
        return avgconc

    def _pixelprop(self):
        """
        Calculates camera properties

        Returns
        -------
        nep : float
            Noise-equivalent power

        tec : float
            Temperature-emissivity contrast
        """

        T_a = self.T_g - 20

        w1g = self.h * self.c / (self.lambda2 * self.k * self.T_g)
        w2g = self.h * self.c / (self.lambda1 * self.k * self.T_g)
        n1 = 2 * np.pi * self.k ** 4 * self.T_g ** 3 / \
             (self.h ** 3 * self.c ** 2)
        temp_y1 = -np.exp(-w1g) * (720 + 720 * w1g + 360 * w1g ** 2 +
                                   120 * w1g ** 3 + 30 * w1g ** 4 +
                                   6 * w1g ** 5 + w1g ** 6)
        temp_y2 = -np.exp(-w2g) * (720 + 720 * w2g + 360 * w2g ** 2 +
                                   120 * w2g ** 3 + 30 * w2g ** 4 +
                                   6 * w2g ** 5 + w2g ** 6)
        y1 = temp_y2 - temp_y1
        y = y1 * n1
        nep = y * self.netd * self.a_d / (4 * self.f_number ** 2)

        ppixelg = self._pixel_power(self.T_g)
        ppixelp = self._pixel_power(self.T_plume)
        ppixela = self._pixel_power(T_a)

        tec = ppixelp - self.e_g * ppixelg \
              - self.e_a * (1 - self.e_g) * ppixela

        return nep, tec

    def _pixel_power(self, temp):
        """
        Calculates the the power incident on a pixel from an infinite blackbody
        emitter at a given temperature.

        Parameters
        -----------
        temp : float
            Temperature of the emitter (K)

        Returns
        ---------
        Power incident on the pixel (W)
        """

        # Calculate the nondimensional frequency limits of the sensor
        w1 = self.h * self.c / (self.lambda2 * self.k * temp)
        w2 = self.h * self.c / (self.lambda1 * self.k * temp)

        # Integrate the blackbody radiation over the frequency range
        temp_int = integrate.quad(lambda x: x ** 3 / (np.exp(x) - 1), w1, w2)

        # calculate the power incident on one camera pixel
        frac = temp_int[0] / (np.pi ** 4 / 15)
        sblaw = self.SIGMA * temp ** 4 * self.a_d
        power = (4 / np.pi) * sblaw * np.tan(self.fov1 / 2) * \
                np.tan(self.fov2 / 2)
        pixel_power = power * frac
        return pixel_power


import streamlit as st
import pickle
import pandas as pd
import numpy as np

# Memuat objek StandardScaler dan model MLPClassifier dari file pickle
with open('scaler.pkl', 'rb') as scaler_file:
    scaler = pickle.load(scaler_file)

with open('mlp_model.pkl', 'rb') as model_file:
    mlp = pickle.load(model_file)

# Fungsi untuk melakukan prediksi
def predict_verification_result(data):
    # Mengubah data masukan pengguna menjadi DataFrame
    input_data = pd.DataFrame([data], columns=['process.b1.capacity', 'process.b2.capacity', 'process.b3.capacity', 'process.b4.capacity', 'property.price', 'property.product', 'property.winner'])

    # Normalisasi data menggunakan objek StandardScaler yang telah di-load
    input_data = scaler.transform(input_data)

    # Melakukan prediksi dengan model MLPClassifier
    prediction = mlp.predict(input_data)

    return prediction[0]

# Judul aplikasi Streamlit
st.title('Klasifikasi Hasil Verifikasi')

# Input data dari pengguna
st.write('Masukkan data berikut untuk mengklasifikasikan hasil verifikasi:')
process_b1_capacity = st.number_input('process.b1.capacity', value=0.0)
process_b2_capacity = st.number_input('process.b2.capacity', value=0.0)
process_b3_capacity = st.number_input('process.b3.capacity', value=0.0)
process_b4_capacity = st.number_input('process.b4.capacity', value=0.0)
property_price = st.number_input('property.price', value=0.0)
property_product = st.number_input('property.product', value=0.0)
property_winner = st.number_input('property.winner', value=0.0)

# Tombol untuk melakukan prediksi
if st.button('Prediksi Hasil Verifikasi'):
    # Membuat data masukan dari input pengguna
    user_input = {
        'process.b1.capacity': process_b1_capacity,
        'process.b2.capacity': process_b2_capacity,
        'process.b3.capacity': process_b3_capacity,
        'process.b4.capacity': process_b4_capacity,
        'property.price': property_price,
        'property.product': property_product,
        'property.winner': property_winner
    }

    # Melakukan prediksi
    prediction = predict_verification_result(user_input)

    # Menampilkan hasil prediksi
    st.write(f'Hasil Verifikasi: {prediction}')


import numpy as np
from multiagent.core import World, Agent, Landmark
from multiagent.scenario import BaseScenario

class Scenario(BaseScenario):
    def make_world(self, task):
        world = World()
        # set any world properties first
        world.dim_c = 2
        num_agents, num_landmarks = task[1]

        world.collaborative = True
        # add agents
        world.agents = [Agent() for i in range(num_agents)]
        for i, agent in enumerate(world.agents):
            agent.name = 'agent %d' % i
            agent.collide = True
            agent.silent = True
            agent.size = 0.05
        # add landmarks
        world.landmarks = [Landmark() for i in range(num_landmarks)]
        for i, landmark in enumerate(world.landmarks):
            landmark.name = 'landmark %d' % i
            landmark.collide = False
            landmark.movable = False
        # make initial conditions
        self.reset_world(world)
        return world

    def reset_world(self, world):
        # random properties for agents
        for i, agent in enumerate(world.agents):
            agent.color = np.array([0.35, 0.35, 0.85])
        # random properties for landmarks
        for i, landmark in enumerate(world.landmarks):
            landmark.color = np.array([0.25, 0.25, 0.25])
        # set random initial states
        for agent in world.agents:
            agent.state.p_pos = np.random.uniform(-1, +1, world.dim_p)
            agent.state.p_vel = np.zeros(world.dim_p)
            agent.state.c = np.zeros(world.dim_c)
        for i, landmark in enumerate(world.landmarks):
            landmark.size = 0.05 * (i+1)
            landmark.state.p_pos = np.random.uniform(-1, +1, world.dim_p)
            landmark.state.p_vel = np.zeros(world.dim_p)

    def benchmark_data(self, agent, world):
        rew = 0
        collisions = 0
        occupied_landmarks = 0
        min_dists = 0
        for l in world.landmarks:
            dists = [np.sqrt(np.sum(np.square(a.state.p_pos - l.state.p_pos))) for a in world.agents]
            min_dists += min(dists)
            rew -= min(dists)
            if min(dists) < 0.1:
                occupied_landmarks += 1
        if agent.collide:
            for a in world.agents:
                if self.is_collision(a, agent):
                    rew -= 1
                    collisions += 1
        return (rew, collisions, min_dists, occupied_landmarks)

    def is_collision(self, agent1, agent2):
        delta_pos = agent1.state.p_pos - agent2.state.p_pos
        dist = np.sqrt(np.sum(np.square(delta_pos)))
        dist_min = agent1.size + agent2.size
        return True if dist < dist_min else False

    def reward(self, agent, world):
        rew = 0
        for l in world.landmarks:
            if self.is_collision(l, agent):
                numFeeders = 0
                for a in world.agents:
                    if self.is_collision(a, l):
                        numFeeders += 1
                rew += l.size / numFeeders * 10   # *5
        return rew

    def observation(self, agent, world):
        entity_pos = []
        for entity in world.landmarks:  # world.entities:
            entity_pos.append(entity.state.p_pos - agent.state.p_pos)
            entity_pos.append([entity.size])
        others = []
        for other in world.agents:
            if other is agent: continue
            o = []
            o.append(other.state.p_pos - agent.state.p_pos)
            o.append(other.state.p_vel)
            for en in world.landmarks:
                o.append(en.state.p_pos - other.state.p_pos)
                o.append([en.size])
            others += o
        return np.concatenate([agent.state.p_vel] + [agent.state.p_pos] + others+ entity_pos)


from dataclasses import dataclass
from datetime import datetime, timedelta
from decimal import Decimal
from typing import List

from django import forms
from django.db.models import QuerySet
from service_objects.fields import ModelField
from service_objects.services import Service

from finance_majordomo.stocks.models.accrual_models import AccrualsOfPortfolio
from finance_majordomo.stocks.services.transaction_services.transaction_calculation_services import \
    get_asset_quantity_for_portfolio
from finance_majordomo.users.models import Portfolio

@dataclass
class AccrualItem:
    """
    Instance to collect accruals data and indicators to show in view
    """
    asset_name: str
    asset_quantity: Decimal
    id: int
    amount: Decimal
    sum: Decimal
    date: datetime.date
    is_received: bool
    is_upcoming: bool

def execute_portfolio_accrual_view_context_service(
        portfolio: Portfolio, days_delta: int):
    """
    :param portfolio: Portfolio model object
    :param days_delta: shows the latest day on which accruals are paid
        (by default 90 days (1Q) is used). Used to limit future payments.
    :return: dictionary like:
        {'total_results': {'total_divs_payable': Decimal,
                           'total_divs_received': Decimal,
                           'total_divs_upcoming': Decimal
                           }
         'accrual_list': [AccrualItem]
         }
    """
    return PortfolioAccrualViewContextService.execute(
        {'portfolio': portfolio,
         'days_delta': days_delta
         }
    )

class PortfolioAccrualViewContextService(Service):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.portfolio_accrual_data = {
            'total_results': {
                'total_divs_payable': None,
                'total_divs_received': None,
                'total_divs_upcoming': None
            },
            'accrual_list': []
        }

    portfolio = ModelField(Portfolio)
    days_delta = forms.IntegerField()

    def process(self):

        self.portfolio = self.cleaned_data.get('portfolio')

        portfolio_accruals = self._get_portfolio_accruals()
        if not portfolio_accruals:
            print(self.portfolio_accrual_data)
            return self.portfolio_accrual_data

        self._fill_context_with_accrual_context_data(portfolio_accruals)
        return self.portfolio_accrual_data

    def _fill_context_with_accrual_context_data(self, portfolio_accruals):
        accrual_item_list = self._get_accrual_item_list(portfolio_accruals)
        self.portfolio_accrual_data['accrual_list'] = accrual_item_list
        self.portfolio_accrual_data['total_results']['total_divs_payable'] =\
            self._get_total_divs_payable(accrual_item_list)
        self.portfolio_accrual_data['total_results']['total_divs_received'] =\
            self._get_total_divs_received(accrual_item_list)
        self.portfolio_accrual_data['total_results']['total_divs_upcoming'] =\
            self._get_total_divs_upcoming(accrual_item_list)

    def _get_portfolio_accruals(self):

        days_delta = timedelta(days=self.cleaned_data.get('days_delta'))

        portfolio_accruals = AccrualsOfPortfolio.objects.filter(
            portfolio=self.portfolio,
            dividend__date__lte=self._get_today_date() + days_delta
        ).order_by('-dividend__date')

        return portfolio_accruals

    def _get_accrual_item_list(
            self, portfolio_accruals: QuerySet) -> List[AccrualItem]:

        portfolio_accruals_list = []

        for accrual in portfolio_accruals:

            accrual_date = accrual.dividend.date
            asset_quantity = get_asset_quantity_for_portfolio(
                self.portfolio.id, accrual.dividend.asset.id, accrual_date
            )

            if asset_quantity <= 0:
                continue

            accrual_amount = accrual.dividend.amount
            accrual_total = accrual_amount * asset_quantity
            accrual_id = accrual.dividend.id
            asset_name = accrual.dividend.asset.latname
            accrual_is_received = accrual.is_received
            accrual_is_upcoming = False if accrual_date <= \
                                           self._get_today_date() else True

            portfolio_accruals_list.append(
                AccrualItem(
                    id=accrual_id,
                    asset_name=asset_name,
                    asset_quantity=asset_quantity,
                    date=accrual_date,
                    amount=accrual_amount,
                    sum=accrual_total,
                    is_received=accrual_is_received,
                    is_upcoming=accrual_is_upcoming
                )
            )

        return portfolio_accruals_list

    @staticmethod
    def _get_total_divs_payable(portfolio_accruals_list):

        return sum(a.sum for a in portfolio_accruals_list if not a.is_upcoming)

    @staticmethod
    def _get_total_divs_received(portfolio_accruals_list):

        return sum(a.sum for a in portfolio_accruals_list
                   if not a.is_upcoming and a.is_received
                   )

    @staticmethod
    def _get_total_divs_upcoming(portfolio_accruals_list):
        return sum(a.sum for a in portfolio_accruals_list if a.is_upcoming)

    @staticmethod
    def _get_today_date():
        return datetime.today().date()


from django.db import models
from cloudinary.models import CloudinaryField

class Category(models.Model):
    class Meta(object):
        db_table = 'category'
        verbose_name_plural = "Categories"

    name = models.CharField(
        'Name', blank=False, null=False, max_length=200, db_index=True
    )
    image = CloudinaryField(
        "Category Image", blank=True, null=True
    )
    created_at = models.DateTimeField(
        'Creation Date', blank=True, auto_now_add=True
    )
    updated_at = models.DateTimeField(
        'Update Date', blank=True, auto_now=True
    )

    def __str__(self):
        return self.name


import os
import tensorflow as tf
import tensorflow_datasets as tfds
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
from tensorflow.keras.layers import Embedding, GlobalAvgPool1D, Dense
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import numpy as np
from history import plot_history, save_history

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

def retrieve_data():
    (ag_news_train, ag_news_test), info = tfds.load('ag_news_subset', split=['train','test'], with_info=True)
    return ag_news_train, ag_news_test, info

def split_feature_labels(dataset):
    features = []
    labels = []
    for ex in dataset:
        features.append(ex['title'].numpy())
        labels.append(ex['label'].numpy())
    features = np.array([x.decode('utf-8') for x in features])
    labels = np.array([float(x) for x in labels])
    return features, labels

def wrangle_data(tokenizer, features, seq_length):
    tokens = tokenizer.texts_to_sequences(features)
    features_padded = pad_sequences(tokens, maxlen=seq_length, padding='post', truncating='post')
    return np.array(features_padded), tokens

def dnn_model(word_dim, embedding_dim, seq_length):
    new_model = tf.keras.Sequential([
        Embedding(word_dim, embedding_dim, input_length=seq_length),
        GlobalAvgPool1D(),
        Dense(32, activation='relu'),
        Dense(4, activation='softmax')
    ])
    return compile_model(new_model)

def compile_model(new_model):
    new_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])
    print(new_model.summary())
    return new_model

def save_model(model, name, history, test_data, test_labels):
    test_loss, test_acc = model.evaluate(test_data, test_labels)

    # save model information
    save_name = f'../models/news/{name}-{len(history.epoch):02d}-{test_acc:0.4f}'
    model.save(f'{save_name}.h5')

    # Save history information
    save_history(history, save_name)

if __name__ == '__main__':

    train_ds, test_ds, ds_info = retrieve_data()

    train_titles, train_labels = split_feature_labels(train_ds)
    test_titles, test_labels = split_feature_labels(test_ds)

    word_dimension = 7000
    sequence_length =24

    tokenizer = Tokenizer(num_words=word_dimension, oov_token='~~~')
    tokenizer.fit_on_texts(train_titles)

    train_data, train_tokens = wrangle_data(tokenizer, train_titles, sequence_length)
    test_data, test_tokens = wrangle_data(tokenizer, test_titles, sequence_length)

    model_name = 'dnn'
    embedding_dimenstion = 9

    earlystop = EarlyStopping('val_loss', patience=3, restore_best_weights=True)
    checkpoint = ModelCheckpoint(filepath=f'../models/ckpts/news/{model_name}/'+'{epoch:02d}-{val_accuracy:.4f}')

    model = dnn_model(word_dimension, embedding_dimenstion, sequence_length)
    history = model.fit(train_data,train_labels, validation_split=0.1, batch_size=64, epochs=25, callbacks=[earlystop, checkpoint])

    plot_history(history)

    save_model(model, model_name, history, test_data, test_labels)


import random
import re
import sqlite3
import asyncio
import os
import datetime
from telebot.async_telebot import AsyncTeleBot
from telebot import types
from dotenv import load_dotenv, find_dotenv

admin_id = 1900666417

load_dotenv(find_dotenv())
bot = AsyncTeleBot(os.getenv('TOKEN_BOT'))

db = sqlite3.connect('db/fnaf.db', check_same_thread=False)
sql = db.cursor()

# Создание двух таблиц users и download
sql.execute("""CREATE TABLE IF NOT EXISTS download(
    id integer PRIMARY KEY AUTOINCREMENT,
    tg_id integer,
    link text,
    date date

)""")

sql.execute("""CREATE TABLE IF NOT EXISTS users(
    id integer PRIMARY KEY AUTOINCREMENT,
    tg_id integer,
    username text,
    utm text,
    date date
)""")

db_tiktok = sqlite3.connect('~TTsavee/db/ttsavee.db', check_same_thread=False)
db_insta = sqlite3.connect('~korzu/InstaSavee/db/instasavee.db', check_same_thread=False)

def checkSubscribe(tg_id, db):
    sql = db.cursor()

    sql.execute("SELECT * FROM users WHERE tg_id = $1;", tg_id)

    user = sql.fetchone()

    sql.close()

    if user:
        return True
    else:
        return False

async def all_bot(chat_id,message):
    markup = types.InlineKeyboardMarkup(row_width=1)
    button_1 = types.InlineKeyboardButton(text="Скачать видео из TikTok", url="https://t.me/saving_tt_video_bot?start=Fnaf")
    button_2 = types.InlineKeyboardButton(text="Скачать видео из Instagram",url="https://t.me/saving_insta_bot?start=Fnaf")

    markup.add(button_1,button_2)

    await bot.send_message(message.chat.id,
                           'Для того что бы узнать кто ты из FNAF, тебе так же нужно подписаться на наших ботов',reply_markup=markup)

# Команда для админа которая делает рассылку по всем пользователям из базы данных
@bot.message_handler(commands=['sendall'])
async def send_all_message(message: types.Message):
    sql.execute("SELECT tg_id FROM users;")
    users = sql.fetchall()
    if message.chat.id == admin_id:
        await bot.send_message(message.chat.id, '💌 Starting')
        for i in users:
            try:
                print("Send to: ", str(i[0]))
                await bot.send_message(i[0], message.text[message.text.find(' '):], parse_mode='html')
            except Exception as error:
                print("Blocked bot: ", str(i[0]))
            # await bot.send_message(i[0],message.text[message.text.find(' '):],parse_mode='html')
        await bot.send_message(message.chat.id, '✅ Successfully')
    else:
        await bot.send_message(message.chat.id, 'Вы не являетесь администратором!')

# Команда для админа которая скачивает с сервера базу данных
@bot.message_handler(commands=['download_db'])
async def command_download_db(message):
    if message.chat.id == admin_id:
        db = open('db/fnaf.db', 'rb')
        await bot.send_document(message.chat.id, db)
    else:
        await bot.send_message(message.chat.id, 'Вы не являетесь администратором!')

@bot.message_handler(commands=['users'])
async def all_users(message):
    if message.chat.id == admin_id:
        sql.execute("SELECT tg_id FROM users;")
        users = sql.fetchall()
        await bot.send_message(message.chat.id, f'👻 Общее количество пользователей: {len(users)}',
                               parse_mode='html')
    else:
        await bot.send_message(message.chat.id, f'🚫 Вы не является администратором')

name = ['😱 Оказывается ты Фредди! ', '😱 Оказывается ты Бонни!', '😱 Оказывается ты Чика!', '😱 Оказывается ты Фокси!',
        '😱 Оказывается ты Золотой Фредди!', '😱 Оказывается ты Марионетка!', '😱 Оказывается ты Балун Бой!',
        '😱 Оказывается ты Спрингтрап!', '😱 Оказывается ты Мангл!', '😱 Оказывается ты Циркус Бэби!',
        '😱 Оказывается ты Эннард!', '😱 Оказывается ты Баллора!', '😱 Оказывается ты Уильям Афтон!']

photo = ['img/freddy.jpg', 'img/bonny.jpg', 'img/chika.jpg', 'img/foxy.jpg', 'img/golden_freddy.jpg',
         'img/marionetka.jpg', 'img/balunboy.jpg', 'img/springtrap.jpg', 'img/mangl.jpg', 'img/cirk_baby.jpg',
         'img/ennar.jpg', 'img/ballora.jpg', 'img/aftin.jpg']

characters = dict(zip(name, photo))

@bot.message_handler(commands=['start'])
async def command_start(message):
    utm_label = (message.text).split(' ')

    print(len(utm_label))
    if len(utm_label) >= 2:
        utm = utm_label[1]
    else:
        utm = 'hull'

    date = datetime.datetime.now()
    tg_id = message.from_user.id
    sql.execute(f"SELECT tg_id FROM users WHERE tg_id={tg_id}")
    data = sql.fetchone()
    username = message.from_user.username

    sql.execute("SELECT tg_id FROM download;")
    quantity_download = sql.fetchall()
    if data is None:

        if message.from_user.username != None:
            await bot.send_message(admin_id, f'👤 New user : @{message.from_user.username} {message.chat.id}')
        else:
            await bot.send_message(admin_id, f'👤 New user : {message.chat.id}')

        if username != None:

            sql.execute("INSERT INTO users VALUES (?,?,?,?,?)", (None, tg_id, username, utm, date))
            db.commit()
        else:
            username_none = 'Stranger'
            sql.execute("INSERT INTO users VALUES (?,?,?,?,?)", (None, tg_id, username_none, utm, date))
            db.commit()

    with open('img/start.png', 'rb') as start_photo:
        await bot.send_photo(message.chat.id, start_photo,
                             'Привет! Я бот Кто ты из FNaF? 🐻🐰🦊\n\nДля этого мне понадобится твоя дата рождения. Пожалуйста, введи дату своего рождения в формате ДД.ММ.ГГГГ (например, 31.10.1990).\n\nПосле того как ты введешь свою дату рождения, я скажу, кто ты из персонажей FNaF.\n\nГотов начать? Введи свою дату рождения!',
                             parse_mode='html')

@bot.message_handler(content_types=['text'])
async def message_handler(message):
    checkSubscribe(message.chat.id, db_tiktok)
    if checkSubscribe == True:
        if re.match(r'\d{2}.\d{2}.\d{4}', message.text):
                if message.from_user.username != None:
                    await bot.send_message(admin_id,
                                               f'Username: @{message.from_user.username}\n👤 User id: {message.chat.id}\n⛓ text: {message.text}\n🟢 Status: 📩 Link accepted!',
                                               parse_mode='html')
                else:
                    await bot.send_message(admin_id,
                                               f'👤 User: {message.chat.id}\n⛓text: {message.text}\n🟢 Status: 📩 Link accepted!',
                                               parse_mode='html')

                random_name = random.choice(name)

                random_photo = characters[random_name]
                with open(f'{random_photo}', 'rb') as photo:

                    await bot.send_photo(message.chat.id, photo, f'{random_name}\n\nВаша связь с этим персонажем может быть ключом к раскрытию секретов игры и пониманию её сюжета. Приготовьтесь к захватывающему и опасному приключению, где вы будете играть важную роль.\n\n🌟Удачи в этой увлекательной игре!',
                                         parse_mode='html')
                    words = random_name.split()
                    last_word = words[-1]
                    inline_keyboard = types.InlineKeyboardMarkup()
                    share_button = types.InlineKeyboardButton("Поделиться", switch_inline_query=f'Помог мне узнать кто я из FNaF.\n\n😱 Как оказалось я {last_word}')

                    inline_keyboard.add(share_button)

                    await bot.send_message(message.chat.id, "Скорее делись результатом  со своими друзьями! 🐻🐰🦊",
                                           reply_markup=inline_keyboard)
                date = datetime.datetime.now()
                tg_id = message.from_user.id
                link = message.text
                sql.execute("INSERT INTO download VALUES (?,?,?,?)", (None, tg_id, link, date))
                db.commit()
        else:
            await bot.send_message(message.chat.id, 'Пожалуйста, введите дату в формате гггг-мм-дд.')
    else:
        await all_bot(message.chat.id,message)
asyncio.run(bot.polling(non_stop=True))


import logging
import sys
import os
import shutil
import sys
from datetime import datetime
from functools import partial
import tensorflow as tf
from absl import flags
from actorcritic.agent import ActorCriticAgent, ACMode
from actorcritic.runner import Runner, PPORunParams
from common.multienv import SubprocVecEnv, make_sc2env, SingleEnv

FLAGS = flags.FLAGS
flags.DEFINE_bool("visualize", False, "Whether to render with pygame.")
flags.DEFINE_integer("resolution", 32, "Resolution for screen and minimap feature layers.")
flags.DEFINE_integer("step_mul", 8, "Game steps per agent step.")
flags.DEFINE_integer("n_envs", 1, "Number of environments to run in parallel")
flags.DEFINE_integer("n_steps_per_batch", None,
    "Number of steps per batch, if None use 8 for a2c and 128 for ppo")
flags.DEFINE_integer("all_summary_freq", 50, "Record all summaries every n batch")
flags.DEFINE_integer("scalar_summary_freq", 5, "Record scalar summaries every n batch")
flags.DEFINE_string("checkpoint_path", "_files/models", "Path for agent checkpoints")
flags.DEFINE_string("summary_path", "_files/summaries", "Path for tensorboard summaries")
flags.DEFINE_string("model_name", "temp_testing", "Name for checkpoints and tensorboard summaries")
flags.DEFINE_integer("K_batches", -1,
    "Number of training batches to run in thousands, use -1 to run forever")
flags.DEFINE_string("map_name", "MoveToBeacon", "Name of a map to use.")
flags.DEFINE_float("discount", 0.95, "Reward-discount for the agent")
flags.DEFINE_boolean("training", True,
    "if should train the model, if false then save only episode score summaries"
)
flags.DEFINE_enum("if_output_exists", "fail", ["fail", "overwrite", "continue"],
    "What to do if summary and model output exists, only for training, is ignored if notraining")
flags.DEFINE_float("max_gradient_norm", 500.0, "good value might depend on the environment")
flags.DEFINE_float("loss_value_weight", 1.0, "good value might depend on the environment")
flags.DEFINE_float("entropy_weight_spatial", 1e-6,
    "entropy of spatial action distribution loss weight")
flags.DEFINE_float("entropy_weight_action", 1e-6, "entropy of action-id distribution loss weight")
flags.DEFINE_float("ppo_lambda", 0.95, "lambda parameter for ppo")
flags.DEFINE_integer("ppo_batch_size", None, "batch size for ppo, if None use n_steps_per_batch")
flags.DEFINE_integer("ppo_epochs", 3, "epochs per update")
flags.DEFINE_enum("agent_mode", ACMode.A2C, [ACMode.A2C, ACMode.PPO], "if should use A2C or PPO")

FLAGS(sys.argv)

#TODO this runner is maybe too long and too messy..
full_chekcpoint_path = os.path.join(FLAGS.checkpoint_path, FLAGS.model_name)

if FLAGS.training:
    full_summary_path = os.path.join(FLAGS.summary_path, FLAGS.model_name)
else:
    full_summary_path = os.path.join(FLAGS.summary_path, "no_training", FLAGS.model_name)

def check_and_handle_existing_folder(f):
    if os.path.exists(f):
        if FLAGS.if_output_exists == "overwrite":
            shutil.rmtree(f)
            print("removed old folder in %s" % f)
        elif FLAGS.if_output_exists == "fail":
            raise Exception("folder %s already exists" % f)

def _print(i):
    print(datetime.now())
    print("# batch %d" % i)
    sys.stdout.flush()

def _save_if_training(agent):
    if FLAGS.training:
        agent.save(full_chekcpoint_path)
        agent.flush_summaries()
        sys.stdout.flush()

def main():
    if FLAGS.training:
        check_and_handle_existing_folder(full_chekcpoint_path)
        check_and_handle_existing_folder(full_summary_path)

    env_args = dict(
        map_name=FLAGS.map_name,
        step_mul=FLAGS.step_mul,
        game_steps_per_episode=0,
        screen_size_px=(FLAGS.resolution,) * 2,
        minimap_size_px=(FLAGS.resolution,) * 2,
        visualize=FLAGS.visualize
    )

    envs = SubprocVecEnv((partial(make_sc2env, **env_args),) * FLAGS.n_envs)
    # envs = SingleEnv(make_sc2env(**env_args))

    tf.reset_default_graph()
    sess = tf.Session()

    agent = ActorCriticAgent(
        mode=FLAGS.agent_mode,
        sess=sess,
        spatial_dim=FLAGS.resolution,
        unit_type_emb_dim=5,
        loss_value_weight=FLAGS.loss_value_weight,
        entropy_weight_action_id=FLAGS.entropy_weight_action,
        entropy_weight_spatial=FLAGS.entropy_weight_spatial,
        scalar_summary_freq=FLAGS.scalar_summary_freq,
        all_summary_freq=FLAGS.all_summary_freq,
        summary_path=full_summary_path,
        max_gradient_norm=FLAGS.max_gradient_norm
    )

    agent.build_model()
    if os.path.exists(full_chekcpoint_path):
        agent.load(full_chekcpoint_path)
    else:
        agent.init()

    if FLAGS.n_steps_per_batch is None:
        n_steps_per_batch = 128 if FLAGS.agent_mode == ACMode.PPO else 8
    else:
        n_steps_per_batch = FLAGS.n_steps_per_batch

    if FLAGS.agent_mode == ACMode.PPO:
        ppo_par = PPORunParams(
            FLAGS.ppo_lambda,
            batch_size=FLAGS.ppo_batch_size or n_steps_per_batch,
            n_epochs=FLAGS.ppo_epochs
        )
    else:
        ppo_par = None

    runner = Runner(
        envs=envs,
        agent=agent,
        discount=FLAGS.discount,
        n_steps=n_steps_per_batch,
        do_training=FLAGS.training,
        ppo_par=ppo_par
    )

    runner.reset()

    if FLAGS.K_batches >= 0:
        n_batches = FLAGS.K_batches * 1000
    else:
        n_batches = -1

    i = 0

    try:
        while True:
            if i % 500 == 0:
                _print(i)
            if i % 4000 == 0:
                _save_if_training(agent)
            runner.run_batch()
            i += 1
            if 0 <= n_batches <= i:
                break
    except KeyboardInterrupt:
        pass

    print("Okay. Work is done")
    _print(i)
    _save_if_training(agent)

    envs.close()

if __name__ == "__main__":
    main()


import os
import tarfile
import argparse

import meta_extract.get_device_id as device
import download_data.download_videos as downloader
from meta_extract.get_highlight_flags import examine_mp4, sec2dtime
from shutil import move

ALL_METAS = [
    'ACCL', 'GYRO', 'SHUT', 'WBAL', 'WRGB',
    'ISOE', 'UNIF', 'FACE', 'CORI', 'MSKP',
    'IORI', 'GRAV', 'WNDM', 'MWET', 'AALP',
    'LSKP']

def extract_meta(video_folder, output_folder):
    videos = [f for f in os.listdir(video_folder) if f.endswith('.MP4')]
    for video in videos:
        video_path = os.path.join(video_folder, video)
        output_path = os.path.join(output_folder, video.split('.')[0])
        try:
            os.mkdir(output_path)
        except:
            print(f'{output_folder} already exists')
            pass

        for meta in ALL_METAS:
            meta_path = os.path.join(
                output_path, f'{meta}_meta.txt')
            print(video_path, meta_path)
            cmd = f'../gpmf-parser/gpmf-parser {video_path} -f{meta} -a | tee {meta_path}'
            os.system(cmd)

        get_highlight_and_device_id(video_path, output_path)

def compress_vid(video_path, output_folder):
    output_path = os.path.join(output_folder, os.path.basename(video_path))
    cmd = f"ffmpeg -i {video_path} -c:v libx265 -vtag hvc1 -strict -2 {output_path} -speed 1200"
    os.system(cmd)

def get_highlight_and_device_id(video_path, output_folder):
    def save_info(all_info, output_path, info_type):
        assert info_type in ['highlights', 'device_id'], \
            'info_type needs to be either device_id or highlights'
        str2insert = ""
        str2insert += fname + "\n"
        if info_type == 'highlights':
            for i, highl in enumerate(all_info):
                str2insert += "(" + str(i+1) + "): "
                str2insert += sec2dtime(highl) + "\n"
        elif info_type == 'device_id':
            str2insert += all_info
        str2insert += "\n"
        with open(output_path, "w") as f:
            f.write(str2insert)

    fname = os.path.basename(video_path).split('.')[0]
    # highlights = examine_mp4(video_path)
    # highlights.sort()
    # highlight_path = os.path.join(output_folder, f'GP-Highlights_{fname}.txt')
    # print(video_path)
    # print(highlight_path)
    # save_info(highlights, highlight_path, 'highlights')
    # device_id = device.examine_mp4(video_path)
    # device_id_path = os.path.join(output_folder, f'GP-Device_name_{fname}.txt')
    # save_info(device_id, device_id_path, 'device_id')
    # print(device_id_path)
    compress_vid(video_path, output_folder)

import json
import boto3
from zipfile import ZipFile
from botocore.exceptions import NoCredentialsError
#def aws_upload(output_folder, aws_access_key, aws_secret_key):
def aws_upload(output_folder, aws_key_path):
    aws_keys = json.load(open(aws_key_path, 'r'))
    aws_access_key = aws_keys['aws_access_key']
    aws_secret_key = aws_keys['aws_secret_key']
    s3 = boto3.client('s3', aws_access_key_id=aws_access_key,
                      aws_secret_access_key=aws_secret_key)
    bucket = 'babyview01'
    current_folder = os.getcwd()

    for vid_folder in os.listdir(output_folder):
        vid_folder = os.path.join(output_folder, vid_folder)
        if os.path.isdir(vid_folder):
            os.chdir(vid_folder)
            tar_file = f'{vid_folder}.tar'
            with tarfile.open(tar_file, "w") as tar_handle:
                for f in os.listdir(vid_folder):
                    tar_handle.add(f)

            s3_file = os.path.basename(tar_file)
            try:
                s3.upload_file(tar_file, bucket, s3_file)
                print(f'{s3_file} Upload successful!')
            except FileNotFoundError:
                print("The file was not found")
            except NoCredentialsError:
                print("Credentials not available")

            os.chdir(current_folder)
            s3.download_file(bucket, s3_file, f'./{s3_file}')

def main():
    parser = argparse.ArgumentParser(description="Data management pipeline for GoPro videos")
    # downloader args
    parser.add_argument(
        '--video_root', type=str,
        default='/data/ziyxiang/BabyView/raw',
        help='folder for downloaded videos'
        )
    parser.add_argument(
        '--cred_folder', type=str,
        default='/ccn2/u/ziyxiang/cloud_credentials/babyview',
        help='Google Drive API credentials'
        )
    # metadata extraction args
    parser.add_argument(
        '--output_folder', type=str,
        default='/data/ziyxiang/BabyView/processed',
        help='Save folder for processed videos'
        )
    # upload args
    parser.add_argument(
        '--aws_key_path', type=str,
        default='/ccn2/u/ziyxiang/cloud_credentials/babyview/aws_keys.json',
        help='JSON file path with AWS access key and secret key'
    )
    args = parser.parse_args()
    #gdrive_downloader = downloader.GoogleDriveDownloader(args)
    #gdrive_downloader.download_videos()
    extract_meta(args.video_root, args.output_folder)
    aws_upload(
        args.output_folder, args.aws_key_path)

if __name__ == '__main__':
    main()


from core.models import *
import math

X_N = 'latitude'
Y_N = 'longitude'

TEN_MINUTE = 0.05

def compute_score_for_measure(measure):
    return 1

def get_signal_map(operator, network, left_down_p, right_up_p,
                   resolution=100):
    left_down_p[X_N] = float(left_down_p[X_N])
    left_down_p[Y_N] = float(left_down_p[Y_N])
    right_up_p[Y_N] = float(right_up_p[Y_N])
    right_up_p[X_N] = float(right_up_p[X_N])
    relibility_range = max(
        abs(left_down_p[X_N] - right_up_p[X_N]),
        abs(left_down_p[Y_N] - right_up_p[Y_N])
                          )
    map = get_zeros_signal_map(left_down_p, right_up_p, resolution)
    coverages = get_coverages_intersected_with_rect(operator,
                                                    network,
                                                    left_down_p,
                                                    right_up_p
                                                    )
    n = len(coverages)
    print(f'coverages count: {n}')
    apply_coverages_on_map(map, coverages)
    measures = get_measures_in_rectangle(operator, network, left_down_p, right_up_p)
    n = len(measures)
    print(f'measures count = {n}; {left_down_p[X_N]}; {left_down_p[Y_N]} - {right_up_p[X_N]} ; {right_up_p[Y_N]}')
    apply_measures_on_map(map, measures, relibility_range)
    return map

def get_measures_in_rectangle(operator, network, left_down_p, right_up_p):
    return Measure.\
                objects.\
                filter(
                       latitude__gte=left_down_p[X_N],
                       latitude__lte=right_up_p[X_N],
                       longitude__gte=left_down_p[Y_N],
                       longitude__lte=right_up_p[Y_N],
                       operator_id=operator,
                       network_id=network
                       )

def get_coverages_intersected_with_rect(operator, network, left_down_p, right_up_p):
    return Coverage.\
            objects.\
            filter(
                   center_latitude__gte=left_down_p[X_N],
                   center_latitude__lte=right_up_p[X_N],
                   center_longitude__gte=left_down_p[Y_N],
                   center_longitude__lte=right_up_p[Y_N],
                   operator_id=operator,
                   network_id=network
                   )

def get_zeros_signal_map(left_down_p, right_up_p, resolution):
    map = []
    x = left_down_p[X_N]
    y = left_down_p[Y_N]
    x_step = (right_up_p[X_N] - left_down_p[X_N]) / resolution
    y_step = (right_up_p[Y_N] - left_down_p[Y_N]) / resolution
    while x <= right_up_p[X_N]:
        while y <= right_up_p[Y_N]:
            y += y_step
            map.append({'Points': [x, y],
                        'Reliability': 0,
                        'Signal': 0})
        y = left_down_p[Y_N]
        x += x_step
    return map

def apply_coverages_on_map(map, coverages):
    for cover in coverages:
        points = CoveragePoints.\
                    objects.\
                    filter(coverage_id=cover).\
                    order_by(X_N, Y_N)

        n = len(points)
        print(f'coverage points: {n}')
        if (n != 2):
            continue
        left_p = points[0]
        right_p = points[1]
        for map_point in map:
            if map_point['Reliability'] > cover.reliability:
                continue
            if is_point_inside(map_point, left_p, right_p):
                if map_point['Reliability'] < cover.reliability or\
                        map_point['Signal'] < cover.signal:
                    map_point['Signal'] = cover.signal

def apply_measures_on_map(map, measures, reliability_range):
    for measure in measures:
        map_len = len(map)
        print(f'maplen = {map_len}')
        for map_point in map:
            dist = dist_between_points(map_point['Points'],
                                       [measure.latitude,
                                        measure.longitude
                                        ]
                                       )
            reliability = 1 - dist / reliability_range
            if reliability > map_point['Reliability']:
                map_point['Signal'] = measure.signal
                map_point['Reliability'] = reliability
                # todo: update by time

def is_point_inside(map_point, left_p, right_p):
    return map_point['Points'][0] >= left_p.latitude and\
            map_point['Points'][0] <= right_p.latitude and\
            map_point['Points'][1] >= left_p.longitude and\
            map_point['Points'][1] <= right_p.longitude

def dist_between_points(point1, point2):
    cathet_x = point2[0] - point1[0]
    cathet_y = point2[1] - point1[1]
    return math.sqrt(cathet_x**2 + cathet_y**2)


import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

years = ['2019', '2020', '2021']
values = [100, 300, 600]
x = np.arange(len(years))

# bar차트 옵션
# width : 막대 너비 (default : 0.8)
# color : 막대 색상
# linewidth : 테두리 두께
# edgecolor : 테두리 색상
# align : tick 기준 막대 위치 조절 {'edge', 'center'} (default: center)
# tick_label : list 지정 시 순서대로 tick 표시
# log : y축 log scale로 표시

# 색상
color = ['red', 'yellow', 'blue', 'black']
bar_chart = plt.bar(x, values, width=0.4, color=color, linewidth=3, edgecolor="black", align='center', tick_label=years, log=True)

# 배경 색상 지정
plt.gca().set_facecolor('#E6F0F8')

# 축 색상 지정
plt.gca().spines['bottom'].set_color('red')

# 폰트 색상 지정
plt.xticks(color='#00517C', fontsize=10)

# 막대에 값 표기
for p in bar_chart.patches:
    left, bottom, width, height = p.get_bbox().bounds
    plt.annotate(f"{int(height)}", (left+width/2, height-8), ha='center', size=10, color='black')

# x축 tick
plt.xticks(x, years)
# y축 tick 제거
# plt.yticks(ticks= [])

# 테두리 제거
plt.gca().spines['right'].set_visible(False)
plt.gca().spines['top'].set_visible(False)

plt.show()

# 수평 막대 그래프
vertical_bar_chart = plt.barh(x, values, height=-0.4, color=color, linewidth=3, edgecolor="gray", align='center', tick_label=years, log=False)

plt.show()