# -*- coding: utf-8 -*-
"""
Created on Wed Apr 10 17:34:48 2019

@author: Arnold Yu
"""
# import libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# calculate accuracy
def calculateAcc(matrix):
    denom = matrix[0][0] + matrix[0][1] + matrix[1][0] + matrix[1][1]
    num = matrix[0][0] + matrix[1][1]
    return float(num)/float(denom)

# calculate percision
def calculatePer( matrix):
    denom = matrix[0][0]+ matrix[1][0]
    return float(matrix[0][0])/float(denom)

# calculate recall
def calculateRec( matrix):
    denom = matrix[0][0]+ matrix[0][1]
    return float(matrix[0][0])/float(denom)

# calculate F1
def calculateF1(percision, recall):
    return float(2.0 * percision * recall)/ float(percision + recall)

# calculate false positive rate
def FPR(matrix):
    denom = matrix[0][1]+ matrix[1][1]
    return float(matrix[0,1])/float(denom)

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# import dataset
train = pd.read_csv('data/train_replaceMissing.csv')
test = pd.read_csv('data/test_replaceMissing.csv')

from sklearn.preprocessing import StandardScaler, LabelEncoder

# turn categorical data into numerical
le = LabelEncoder()
columns_cat = ['Workclass', 'Education', 'Marital', 'Occupation', 'Relationship', 'Race', 'Sex', 'Country', 'Income']
for col in columns_cat:
    if train[col].dtype == object:
        train[col] = le.fit_transform(train[col])
        test[col] = le.fit_transform(test[col])

# standardlize
sc = StandardScaler()
all_data = pd.concat([train, test])
all_X = all_data.iloc[:, :-1]
all_y = all_data.iloc[:, -1]
all_X = sc.fit_transform(all_X)

# import all classifiers
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import AdaBoostClassifier
from sklearn.svm import SVC

models = {'Naïve Bayesian': GaussianNB(),
          'Decision tree classifier': DecisionTreeClassifier(criterion = 'entropy', random_state = 0),
          'k-NN': KNeighborsClassifier(n_neighbors = 5, metric = 'minkowski', p = 2),
          'Random Forest': RandomForestClassifier(n_estimators = 10, criterion = 'entropy', random_state = 0),
          'Logistic regression': LogisticRegression( solver='lbfgs', random_state = 0),
          'AdaBoost': AdaBoostClassifier(n_estimators = 50, learning_rate = 1. ,  random_state = 0),
          'SVM linear': SVC(kernel = 'linear', random_state = 0, probability = True)
          }
results = {'Naïve Bayesian': {},
          'Decision tree classifier':  {},
          'k-NN': {},
          'Random Forest':  {},
          'Logistic regression':  {},
          'AdaBoost':  {},
          'SVM linear':  {}
          }
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
methodP = ['predict', 'predict_proba']
target_names = ['Class 0', 'Class 1']

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# Naïve Bayesian classifier

y_prob_nb = cross_val_predict(estimator = models['Naïve Bayesian'],
                           X = all_X,
                           y= all_y,
                           groups=None,
                           cv= 10,
                           n_jobs= -1,
                           verbose=0,
                           fit_params=None,
                           pre_dispatch='2*n_jobs',
                           method= methodP[1])

y_pred_nb = np.zeros(len(y_prob_nb))
for i in range(0, len(y_prob_nb)):
    if y_prob_nb[i][0] > y_prob_nb[i][1]:
        y_pred_nb[i] = 0
    else:
        y_pred_nb[i] = 1
#import sklearn
#print('The scikit-learn version is {}.'.format(sklearn.__version__))
cm_nb = confusion_matrix(all_y, y_pred_nb)
dict_nb = classification_report(all_y, y_pred_nb, target_names = target_names,  output_dict= True)

results['Naïve Bayesian']['Confusion_matrix'] = cm_nb
results['Naïve Bayesian']['Classification_report'] = dict_nb
results['Naïve Bayesian']['Accuracy'] = calculateAcc(cm_nb)
results['Naïve Bayesian']['Percision'] = calculatePer(cm_nb)
results['Naïve Bayesian']['Recall'] = calculateRec(cm_nb)
results['Naïve Bayesian']['F1'] = calculateF1(results['Naïve Bayesian']['Percision'],results['Naïve Bayesian']['Recall'])

fpr, tpr, thresholds = roc_curve(all_y, y_prob_nb[:, 1])
auc = roc_auc_score(all_y, y_prob_nb[:, 1])
plt.plot(fpr, tpr, label='ROC curve:(area = %0.3f)' % (auc))
plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate or (Recall)')
plt.title('ROC : %s' % 'Naïve Bayesian')
plt.legend(loc="lower right")

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# Decision tree classifier
y_prob_dt = cross_val_predict(estimator = models['Decision tree classifier'],
                           X = all_X,
                           y= all_y,
                           groups=None,
                           cv= 10,
                           n_jobs=-1,
                           verbose=0,
                           fit_params=None,
                           pre_dispatch='2*n_jobs',
                           method= methodP[1])

y_pred_dt = np.zeros(len(y_prob_dt))
for i in range(0, len(y_prob_dt)):
    if y_prob_dt[i][0] > y_prob_dt[i][1]:
        y_pred_dt[i] = 0
    else:
        y_pred_dt[i] = 1

cm_dt = confusion_matrix(all_y, y_pred_dt)
dict_dt = classification_report(all_y, y_pred_dt, target_names = target_names,  output_dict= True)

results['Decision tree classifier']['Confusion_matrix'] = cm_dt
results['Decision tree classifier']['Classification_report'] = dict_dt
results['Decision tree classifier']['Accuracy'] = calculateAcc(cm_dt)
results['Decision tree classifier']['Percision'] = calculatePer(cm_dt)
results['Decision tree classifier']['Recall'] = calculateRec(cm_dt)
results['Decision tree classifier']['F1'] = calculateF1(results['Decision tree classifier']['Percision'],results['Decision tree classifier']['Recall'])

fpr, tpr, thresholds = roc_curve(all_y, y_prob_dt[:, 1])
auc = roc_auc_score(all_y, y_prob_dt[:, 1])
plt.plot(fpr, tpr, label='ROC curve:(area = %0.3f)' % (auc))
plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate or (Recall)')
plt.title('ROC : %s' % 'Decision tree classifier')
plt.legend(loc="lower right")

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# k-NN classifier
y_prob_knn = cross_val_predict(estimator = models['k-NN'],
                           X = all_X,
                           y= all_y,
                           groups=None,
                           cv= 10,
                           n_jobs=-1,
                           verbose=0,
                           fit_params=None,
                           pre_dispatch='2*n_jobs',
                           method= methodP[1])

y_pred_knn = np.zeros(len(y_prob_knn))
for i in range(0, len(y_prob_knn)):
    if y_prob_knn[i][0] > y_prob_knn[i][1]:
        y_pred_knn[i] = 0
    else:
        y_pred_knn[i] = 1

cm_knn = confusion_matrix(all_y, y_pred_knn)
dict_knn = classification_report(all_y, y_pred_knn, target_names = target_names,  output_dict= True)

results['k-NN']['Confusion_matrix'] = cm_knn
results['k-NN']['Classification_report'] = dict_knn
results['k-NN']['Accuracy'] = calculateAcc(cm_knn)
results['k-NN']['Percision'] = calculatePer(cm_knn)
results['k-NN']['Recall'] = calculateRec(cm_knn)
results['k-NN']['F1'] = calculateF1(results['k-NN']['Percision'],results['k-NN']['Recall'])

fpr, tpr, thresholds = roc_curve(all_y, y_prob_knn[:, 1])
auc = roc_auc_score(all_y, y_prob_knn[:, 1])
plt.plot(fpr, tpr, label='ROC curve:(area = %0.3f)' % (auc))
plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate or (Recall)')
plt.title('ROC : %s' % 'k-NN')
plt.legend(loc="lower right")

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# Random Forest classifier
y_prob_rf= cross_val_predict(estimator = models['Random Forest'],
                           X = all_X,
                           y= all_y,
                           groups=None,
                           cv= 10,
                           n_jobs= -1,
                           verbose=0,
                           fit_params=None,
                           pre_dispatch='2*n_jobs',
                           method= methodP[1])

y_pred_rf = np.zeros(len(y_prob_rf))
for i in range(0, len(y_prob_rf)):
    if y_prob_rf[i][0] > y_prob_rf[i][1]:
        y_pred_rf[i] = 0
    else:
        y_pred_rf[i] = 1

cm_rf = confusion_matrix(all_y, y_pred_rf)
dict_rf = classification_report(all_y, y_pred_rf, target_names = target_names,  output_dict= True)

results['Random Forest']['Confusion_matrix'] = cm_rf
results['Random Forest']['Classification_report'] = dict_rf
results['Random Forest']['Accuracy'] = calculateAcc(cm_rf)
results['Random Forest']['Percision'] = calculatePer(cm_rf)
results['Random Forest']['Recall'] = calculateRec(cm_rf)
results['Random Forest']['F1'] = calculateF1(results['Random Forest']['Percision'],results['Random Forest']['Recall'])

fpr, tpr, thresholds = roc_curve(all_y, y_prob_rf[:, 1])
auc = roc_auc_score(all_y, y_prob_rf[:, 1])
plt.plot(fpr, tpr, label='ROC curve:(area = %0.3f)' % (auc))
plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate or (Recall)')
plt.title('ROC : %s' % 'Random Forest')
plt.legend(loc="lower right")

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# Logistic regression classifier
y_prob_logis = cross_val_predict(estimator = models['Logistic regression'],
                           X = all_X,
                           y= all_y,
                           groups=None,
                           cv= 10,
                           n_jobs= -1,
                           verbose=0,
                           fit_params=None,
                           pre_dispatch='2*n_jobs',
                           method= methodP[1])

y_pred_logis = np.zeros(len(y_prob_logis))
for i in range(0, len(y_prob_logis)):
    if y_prob_logis[i][0] > y_prob_logis[i][1]:
        y_pred_logis[i] = 0
    else:
        y_pred_logis[i] = 1

cm_logis = confusion_matrix(all_y, y_pred_logis)
dict_logis = classification_report(all_y, y_pred_logis, target_names = target_names,  output_dict= True)

results['Logistic regression']['Confusion_matrix'] = cm_logis
results['Logistic regression']['Classification_report'] = dict_logis
results['Logistic regression']['Accuracy'] = calculateAcc(cm_logis)
results['Logistic regression']['Percision'] = calculatePer(cm_logis)
results['Logistic regression']['Recall'] = calculateRec(cm_logis)
results['Logistic regression']['F1'] = calculateF1(results['Logistic regression']['Percision'],results['Logistic regression']['Recall'])

fpr, tpr, thresholds = roc_curve(all_y, y_prob_logis[:, 1])
auc = roc_auc_score(all_y, y_prob_logis[:, 1])
plt.plot(fpr, tpr, label='ROC curve:(area = %0.3f)' % (auc))
plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate or (Recall)')
plt.title('ROC : %s' % 'Logistic regression')
plt.legend(loc="lower right")

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# AdaBoost classifier
y_prob_ada = cross_val_predict(estimator = models['AdaBoost'],
                           X = all_X,
                           y= all_y,
                           groups=None,
                           cv= 10,
                           n_jobs= -1,
                           verbose=0,
                           fit_params=None,
                           pre_dispatch='2*n_jobs',
                           method= methodP[1])

y_pred_ada = np.zeros(len(y_prob_ada))
for i in range(0, len(y_prob_ada)):
    if y_prob_ada[i][0] > y_prob_ada[i][1]:
        y_pred_ada[i] = 0
    else:
        y_pred_ada[i] = 1

cm_ada = confusion_matrix(all_y, y_pred_ada)
dict_ada = classification_report(all_y, y_pred_ada, target_names = target_names,  output_dict= True)

results['AdaBoost']['Confusion_matrix'] = cm_ada
results['AdaBoost']['Classification_report'] = dict_ada
results['AdaBoost']['Accuracy'] = calculateAcc(cm_ada)
results['AdaBoost']['Percision'] = calculatePer(cm_ada)
results['AdaBoost']['Recall'] = calculateRec(cm_ada)
results['AdaBoost']['F1'] = calculateF1(results['AdaBoost']['Percision'],results['AdaBoost']['Recall'])

fpr, tpr, thresholds = roc_curve(all_y, y_prob_ada[:, 1])
auc = roc_auc_score(all_y, y_prob_ada[:, 1])
plt.plot(fpr, tpr, label='ROC curve:(area = %0.3f)' % (auc))
plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate or (Recall)')
plt.title('ROC : %s' % 'AdaBoost')
plt.legend(loc="lower right")

#----------------------------------------------------------------------------------------------------------------------------------------------------------

# svm linear classifier
y_prob_svm = cross_val_predict(estimator = models['SVM linear'],
                           X = all_X,
                           y= all_y,
                           groups=None,
                           cv= 10,
                           n_jobs= -1,
                           verbose=0,
                           fit_params=None,
                           pre_dispatch='2*n_jobs',
                           method= methodP[1])

y_pred_svm = np.zeros(len(y_prob_svm))
for i in range(0, len(y_prob_svm)):
    if y_prob_svm[i][0] > y_prob_svm[i][1]:
        y_pred_svm[i] = 0
    else:
        y_pred_svm[i] = 1

cm_svm= confusion_matrix(all_y, y_pred_svm)
dict_svm = classification_report(all_y, y_pred_svm, target_names = target_names,  output_dict= True)

results['SVM linear']['Confusion_matrix'] = cm_svm
results['SVM linear']['Classification_report'] = dict_svm
results['SVM linear']['Accuracy'] = calculateAcc(cm_svm)
results['SVM linear']['Percision'] = calculatePer(cm_svm)
results['SVM linear']['Recall'] = calculateRec(cm_svm)
results['SVM linear']['F1'] = calculateF1(results['SVM linear']['Percision'],results['SVM linear']['Recall'])

fpr, tpr, thresholds = roc_curve(all_y, y_prob_svm[:, 1])
auc = roc_auc_score(all_y, y_prob_svm[:, 1])
plt.plot(fpr, tpr, label='ROC curve:(area = %0.3f)' % (auc))
plt.plot([0, 1], [0, 1], 'k--')  # random predictions curve
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate or (Recall)')
plt.title('ROC : %s' % 'SVM linear')
plt.legend(loc="lower right")

#----------------------------------------------------------------------------------------------------------------------------------------------------------
# graph learning curves of all classifiers

from sklearn.model_selection import learning_curve

train_sizes, train_scores, test_scores = learning_curve(estimator = models['Naïve Bayesian'],
                                                        X = all_X,
                                                        y = all_y,
                                                        # Number of folds in cross-validation
                                                        cv = 10,
                                                        # Evaluation metric
                                                        scoring='neg_log_loss', # this scoring can change to accuracy
                                                        # Use all computer cores
                                                        n_jobs = -1,
                                                        # 10 different sizes of the training set
                                                        train_sizes=np.linspace(0.01, 1.0, 10))

# Create means and standard deviations of training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)

# Create means and standard deviations of test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.grid()
# Draw lines
plt.plot(train_sizes, train_mean, '--', color="#111111",  label="Training score")
plt.plot(train_sizes, test_mean, color="#111111", label="Cross-validation score")

# Draw bands
plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color="#DDDDDD")
plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color="#DDDDDD")

# Create plot
plt.title("Learning Curve : Naïve Bayesian")
plt.xlabel("Training Set Size"), plt.ylabel("Neg Log Loss"), plt.legend(loc="best")
plt.tight_layout()
plt.show()

#----------------------------------------------------------------------------------------------------------------------------------------------------------

train_sizes, train_scores, test_scores = learning_curve(estimator = models['Decision tree classifier'],
                                                        X = all_X,
                                                        y = all_y,
                                                        # Number of folds in cross-validation
                                                        cv = 10,
                                                        # Evaluation metric
                                                        scoring='neg_log_loss', # this scoring can change to accuracy
                                                        # Use all computer cores
                                                        n_jobs = -1,
                                                        # 10 different sizes of the training set
                                                        train_sizes=np.linspace(0.01, 1.0, 10))

# Create means and standard deviations of training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)

# Create means and standard deviations of test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.grid()
# Draw lines
plt.plot(train_sizes, train_mean, '--', color="#111111",  label="Training score")
plt.plot(train_sizes, test_mean, color="#111111", label="Cross-validation score")

# Draw bands
plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color="#DDDDDD")
plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color="#DDDDDD")

# Create plot
plt.title("Learning Curve : Decision tree classifier")
plt.xlabel("Training Set Size"), plt.ylabel("Neg Log Loss"), plt.legend(loc="best")
plt.tight_layout()
plt.show()

#----------------------------------------------------------------------------------------------------------------------------------------------------------

train_sizes, train_scores, test_scores = learning_curve(estimator = models['k-NN'],
                                                        X = all_X,
                                                        y = all_y,
                                                        # Number of folds in cross-validation
                                                        cv = 10,
                                                        # Evaluation metric
                                                        scoring='neg_log_loss', # this scoring can change to accuracy
                                                        # Use all computer cores
                                                        n_jobs = -1,
                                                        # 10 different sizes of the training set
                                                        train_sizes=np.linspace(0.01, 1.0, 10))

# Create means and standard deviations of training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)

# Create means and standard deviations of test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.grid()
# Draw lines
plt.plot(train_sizes, train_mean, '--', color="#111111",  label="Training score")
plt.plot(train_sizes, test_mean, color="#111111", label="Cross-validation score")

# Draw bands
plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color="#DDDDDD")
plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color="#DDDDDD")

# Create plot
plt.title("Learning Curve : k-NN")
plt.xlabel("Training Set Size"), plt.ylabel("Neg Log Loss"), plt.legend(loc="best")
plt.tight_layout()
plt.show()

#----------------------------------------------------------------------------------------------------------------------------------------------------------

train_sizes, train_scores, test_scores = learning_curve(estimator = models['Random Forest'],
                                                        X = all_X,
                                                        y = all_y,
                                                        # Number of folds in cross-validation
                                                        cv = 10,
                                                        # Evaluation metric
                                                        scoring='neg_log_loss', # this scoring can change to accuracy
                                                        # Use all computer cores
                                                        n_jobs = -1,
                                                        # 10 different sizes of the training set
                                                        train_sizes=np.linspace(0.01, 1.0, 10))

# Create means and standard deviations of training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)

# Create means and standard deviations of test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.grid()
# Draw lines
plt.plot(train_sizes, train_mean, '--', color="#111111",  label="Training score")
plt.plot(train_sizes, test_mean, color="#111111", label="Cross-validation score")

# Draw bands
plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color="#DDDDDD")
plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color="#DDDDDD")

# Create plot
plt.title("Learning Curve : Random Forest")
plt.xlabel("Training Set Size"), plt.ylabel("Neg Log Loss"), plt.legend(loc="best")
plt.tight_layout()
plt.show()

#----------------------------------------------------------------------------------------------------------------------------------------------------------

train_sizes, train_scores, test_scores = learning_curve(estimator = models['Logistic regression'],
                                                        X = all_X,
                                                        y = all_y,
                                                        # Number of folds in cross-validation
                                                        cv = 10,
                                                        # Evaluation metric
                                                        scoring='neg_log_loss', # this scoring can change to accuracy
                                                        # Use all computer cores
                                                        n_jobs=-1,
                                                        # 10 different sizes of the training set
                                                        train_sizes=np.linspace(0.01, 1.0, 10))

# Create means and standard deviations of training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)

# Create means and standard deviations of test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.grid()
# Draw lines
plt.plot(train_sizes, train_mean, '--', color="#111111",  label="Training score")
plt.plot(train_sizes, test_mean, color="#111111", label="Cross-validation score")

# Draw bands
plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color="#DDDDDD")
plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color="#DDDDDD")

# Create plot
plt.title("Learning Curve : Logistic regression")
plt.xlabel("Training Set Size"), plt.ylabel("Neg Log Loss"), plt.legend(loc="best")
plt.tight_layout()
plt.show()

#----------------------------------------------------------------------------------------------------------------------------------------------------------

train_sizes, train_scores, test_scores = learning_curve(estimator = models['AdaBoost'],
                                                        X = all_X,
                                                        y = all_y,
                                                        # Number of folds in cross-validation
                                                        cv = 10,
                                                        # Evaluation metric
                                                        scoring='neg_log_loss', # this scoring can change to accuracy
                                                        # Use all computer cores
                                                        n_jobs=-1,
                                                        # 10 different sizes of the training set
                                                        train_sizes=np.linspace(0.01, 1.0, 10))

# Create means and standard deviations of training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)

# Create means and standard deviations of test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.grid()
# Draw lines
plt.plot(train_sizes, train_mean, '--', color="#111111",  label="Training score")
plt.plot(train_sizes, test_mean, color="#111111", label="Cross-validation score")

# Draw bands
plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color="#DDDDDD")
plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color="#DDDDDD")

# Create plot
plt.title("Learning Curve : AdaBoost")
plt.xlabel("Training Set Size"), plt.ylabel("Neg Log Loss"), plt.legend(loc="best")
plt.tight_layout()
plt.show()

#----------------------------------------------------------------------------------------------------------------------------------------------------------

train_sizes, train_scores, test_scores = learning_curve(estimator = models['SVM linear'],
                                                        X = all_X,
                                                        y = all_y,
                                                        # Number of folds in cross-validation
                                                        cv = 10,
                                                        # Evaluation metric
                                                        scoring='neg_log_loss', # this scoring can change to accuracy
                                                        # Use all computer cores
                                                        n_jobs = -1,
                                                        # 10 different sizes of the training set
                                                        train_sizes=np.linspace(0.01, 1.0, 10))

# Create means and standard deviations of training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)

# Create means and standard deviations of test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.grid()
# Draw lines
plt.plot(train_sizes, train_mean, '--', color="#111111",  label="Training score")
plt.plot(train_sizes, test_mean, color="#111111", label="Cross-validation score")

# Draw bands
plt.fill_between(train_sizes, train_mean - train_std, train_mean + train_std, color="#DDDDDD")
plt.fill_between(train_sizes, test_mean - test_std, test_mean + test_std, color="#DDDDDD")

# Create plot
plt.title("Learning Curve : SVM linear")
plt.xlabel("Training Set Size"), plt.ylabel("Neg Log Loss"), plt.legend(loc="best")
plt.tight_layout()
plt.show()


from collections import deque

def bfs():
    queue = deque()
    queue.append([home[0], home[1]])

    while queue:
        x, y = queue.popleft()

        if abs(x - destination[0]) + abs(y - destination[1]) <= 1000:
            print("happy")
            return
        for i in range(n):
            if not visited[i]:
                nx, ny = case[i]
                if abs(x - nx) + abs(y - ny) <= 1000:
                    queue.append([nx, ny])
                    visited[i] = 1
    print("sad")
    return

t = int(input())

for i in range(t):
    n = int(input())
    home = [int(x) for x in input().split()]

    case = []
    for i in range(n):
        x, y = map(int, input().split())
        case.append([x,y])

    destination = [int(x) for x in input().split()]

    visited = [0 for i in range(n+1)]
    bfs()


import streamlit as st
import numpy as np
from . import spearman_data as dt

def present_spearman():
    st.markdown(
        """
        # Análise de correlação de Spearman

        Para determinar a relação das diversas doenças com o suicídio nas cidades, foi realizada uma correlação de Spearman
        entre as taxas de internações correspondentes a cada uma delas e a taxa de suicídio por município, no período de 2008 a 2018.

        """
    )

    st.markdown(
        """
        ## Coeficiente de correlação de Spearman

        O coeficiente de **correlação de Spearman** é uma medida da intensidade e direção da associação monotônica entre duas variáveis.
        Em outras palavras, ele avalia o quão bem a relação entre duas variáveis pode ser descrita por meio de uma relação monotônica.

        O cálculo do coeficiente é feito através de uma correlação de Pearson entre os **rankings** das variáveis. O valor obtido é $r_s$, que é
        tal que $-1 \leq r_s \leq 1$:
        - $r_s = 1$ indica uma associação perfeita e positiva entre as váriáveis
        - $r_s = 0$ indica que não há nenhuma associação entre as variáveis
        - $r_s = -1$ indica uma associação perfeita e negativa entre as variávels

        """
    )

    st.markdown(
        """
        ## Tabela de correlações com suicídio

        A tabela abaixo contém os coeficientes de correlações de Spearman (bem como seus respectivos _p-values_) entre a média da taxa de internações referentes a diferentes doenças do DATASUS e a média da taxa de suicídios, no período de 2008-2018.

        A coluna "Qtd Municípios" indica a quantidade de municípios com dados não nulos presentes no conjunto de dados de cada doença.
        Foram excluídas as doenças cuja quantidade de municípios é inferior à metade do número total de municípios no Brasil, ou seja, $< 2785$.

        """
    )

    st.write(dt.get_top_correlations())

    st.markdown(
        """
        ## Gráfico de Doenças vs Suicídio

        """
    )

    disease_names = dt.get_diseases_select_names()
    if len(disease_names) > 0:
        options = np.append(['Selecione uma doença'], disease_names)
        selected_disease = st.selectbox('Selecione uma doença:', options)

        st.markdown(
            '
Para mais informações sobre uma doença, acesse o DATASUS.
', unsafe_allow_html=True
        )

        if (selected_disease != 'Selecione uma doença'):
            dt.plot_disease_vs_suicide(selected_disease)
    else:
        st.markdown(
            """
            ### **_Erro ao carregar nomes das doenças._**
            """
        )


from sys import stdout as out

import logging

class Logger(logging.Logger):
    """Logging handler"""
    def __init__(self, levl=logging.INFO, name='root',
                 **kwargs):
        super(Logger, self).__init__(name, **kwargs)
        self.setLevel(levl)
        if len(self.handlers) < 1:
            ch = logging.StreamHandler(out)
            ch.setLevel(levl)
            formatter = logging.Formatter('\r%(levelname)s '
                                          '- %(message)s')
            ch.setFormatter(formatter)
            self.addHandler(ch)
        self.last_wr = ''
        self.is_prog = False

    def progress(self, name, i, i_max, levl=logging.INFO):
        if self.level > levl:
            return
        if self.last_wr != name:
            if self.is_prog:
                out.write('\n')
            self.is_prog = True
            out.write('{} - {} - '.format(logging.getLevelName(levl), name))
            out.write(' '*7)
            self.last_wr = name
        out.write('\b'*7 + '{:7.2%}'.format(i/i_max))
        if i == i_max-1:
            out.write('\r{} - {} - Done   '
                      ''.format(logging.getLevelName(levl), name))
            out.write('\n')
            self.is_prog = False
            self.last_wr = ''
        out.flush()

    def set_mode(self, levl=logging.INFO):
        self.debug('Set mode: {}'.format(logging.getLevelName(levl)))
        self.setLevel(levl)
        for ch in self.handlers:
            ch.setLevel(levl)

    def debug(self, msg, **kwargs):
        if self.level > 10:
            return
        if self.is_prog:
            out.write('\n')
        self.is_prog = False
        self.last_wr = ''
        super(Logger, self).debug(msg, **kwargs)

    def info(self, msg, **kwargs):
        if self.level > 20:
            return
        if self.is_prog:
            out.write('\n')
        self.is_prog = False
        self.last_wr = ''
        super(Logger, self).info(msg, **kwargs)

    def warning(self, msg, **kwargs):
        if self.level > 30:
            return
        if self.is_prog:
            out.write('\n')
        self.is_prog = False
        self.last_wr = ''
        super(Logger, self).warning(msg, **kwargs)

    def error(self, msg, **kwargs):
        if self.level > 40:
            return
        if self.is_prog:
            out.write('\n')
        self.is_prog = False
        self.last_wr = ''
        super(Logger, self).error(msg, **kwargs)

    def critical(self, msg, **kwargs):
        if self.level > 50:
            return
        if self.is_prog:
            out.write('\n')
        self.is_prog = False
        self.last_wr = ''
        super(Logger, self).exception(msg, **kwargs)

if __name__ == '__main__':
    log = Logger(levl=logging.DEBUG)

    i_max = 1000000

    for i in range(i_max):
        log.progress('Counter', i, i_max)


import torch
import torch.nn as nn
class T(nn.Module):
    def __init__(self):
        super().__init__()
        self.alpha = nn.Linear(2, 2)
        self.alpha = nn.Linear(2, 2)
    def forward(self, x):
        return self.l(self.weight)

a = torch.ones(2, 2)
model = T()

optm = torch.optim.SGD(model.parameters(), 1e-3)
label = torch.tensor([1,0])
ct=torch.nn.CrossEntropyLoss()

result = model(a)
loss=ct(result[0], label)
opt.zero_grad()
loss.backward()


import shlex, subprocess
import os
from os import listdir
from random import sample
import numpy as np
import time

path_pcap_data = "./pcap"
dirs = os.listdir(path_pcap_data)

header_list = ['frame_number','date','time', 'time_second', 'src', 'dst', 'sport', 'dport', 'frame_len', 'columns_len', 'protocol', 'occurence']
df_15_mins = []
times = []

for i in range(len(dirs)):
    t0= time.perf_counter()
    if (i<9):
        index = "0" + str(i+1)

    else:
        index = str(i+1)

    x = "tshark -r 2020010113{}00.pcap -T fields -e frame.number -e frame.time -t e -e _ws.col.Time -e ip.src -e ip.dst -e tcp.srcport -e tcp.dstport -e frame.len -e _ws.col.Length -e _ws.col.Protocol -e frame.protocols -e _ws.col.Info -E separator=, -E occurrence=f > 2020010113{}00tshark1.csv".format(index, index)
    p = subprocess.call(x, shell=True)

    print(i+1, "Done!")
    t1 = time.perf_counter() - t0
    times.append(t1)
    print("Time elapsed: ", t1, 'second')


class Solution:
    # 深度优先遍历
    def isAdditiveNumber(self, num: str) -> bool:
        def dfs(previousNum,num):
            print("***",previousNum,num)
            # 如果还剩两个数字，则直接判断
            if len(num) == 2:
                if previousNum + int(num[0]) == int(num[1]):
                    return True
            for i in range(1,len(num)-1):
                if num[0] == "0" and int(num[:i]) != 0: continue# 防止有数字以0开头
                for j in range(i+1,len(num)+1):
                    print(previousNum,int(num[:i]),int(num[i:j]))
                    if num[i] == "0" and int(num[i:j]) != 0: continue# 防止有数字以0开头
                    if previousNum + int(num[:i]) == int(num[i:j]):
                        if j == len(num):return True # 如果剩下的数字也正好全符合，直接结束
                        if dfs(int(num[:i]),num[i:]):
                            return True
        # 少于3个数字直接False
        if len(num) < 3:return False
        for i in range(1,len(num)):
            print("----------------")
            previousNum = int(num[:i])
            if num[0] == "0" and previousNum != 0: continue# 防止有数字以0开头
            if dfs(previousNum,num[i:]):
                return True
        return False


# User function Template for python3
class Solution:

    def perfectSum(self, arr, n, sum):
        # code here
        t = [[1 if i == 0 and j == 0 else 0 for i in range(
            sum+1)] for j in range(n+1)]
        for i in range(1, n+1):
            for j in range(0, sum+1):
                if arr[i-1] > j:
                    t[i][j] = t[i-1][j]
                else:
                    t[i][j] = t[i-1][j-arr[i-1]] + t[i-1][j]
        return (t[n][sum] % (10**9 + 7))


from bridge import *
from three_particle_functions import *
from BasisVisualization import visbas
import operator

bastype = streukas[0]
bastype = boundstatekanal

angu = channels[bastype]

Jstreu = float(bastype.split('^')[0][-3:])
Jstreustring = '%s' % str(Jstreu)[:3]

os.chdir(v18uixpath)
print('(working dir) %s' % v18uixpath)

bas = np.loadtxt(litpath3He + 'basis_struct/LITbas_full_J%s_%s.dat' %
                 (Jstreustring, bastype)).astype(int)

basvs = {}
for bv in bas:
    try:
        basvs[bv[0]].append(bv[1])
    except:
        basvs[bv[0]] = [bv[1]]
sbas = sorted(basvs.items(), key=operator.itemgetter(0))
litbas = [bv for bv in np.unique(litbas, axis=0) if bv[1] != 0]

n3_inen_bdg(sbas, 1. / 2., costr, fn='INEN', pari=0, nzop=31, tni=11)

subprocess.run([home + '/kette_repo/ComptonLIT/src_nucl/TDR2END_AK.exe'])

os.system('cp OUTPUT OUTPUT_full')

red_mod_3(
    typ='',
    max_coeff=11000,
    min_coeff=150,
    target_size=20,
    nbr_cycles=20,
    max_diff=0.01,
    ord=0,
    tniii=31,
    delpred=2,
    dr2executable=home + '/kette_repo/ComptonLIT/src_nucl/TDR2END_AK.exe')

inen0 = [line for line in open('INEN')]

bas0 = []
nBV = int(inen0[7][4:8])
for bvn in range(1, 1 + nBV):
    bv = int(inen0[int(8 + 2 * (bvn - 1))][4:8])
    rel = np.argwhere(
        np.array(inen0[int(9 + 2 * (bvn - 1))].strip().split()).astype(
            int)).flatten() + 1
    bas0.append((bv, list(rel)))

with open(litpath3He + 'LITbas_red.dat', 'w') as f:
    np.savetxt(f, [[jj[0], kk] for jj in bas0 for kk in jj[1]], fmt='%d')
f.close()

os.system(
    home +
    '/kette_repo/ComptonLIT/src_nucl/TDR2END_I_2.exe && grep -A 3 \'EIGENWER\' OUTPUT'
)
os.system('cp OUTPUT OUTPUT_red')


# Tutor: Tran Tien Duc
# Gmail: trantienduc@gmail.com
# Created by Cao Le Cong Minh
# Gmail: caolecongminh1997@gmail.com
# Github: https://github.com/Minh-CaoLeCong

# USAGE
# python power.py --input imagePath --gamma floatValue
# example:  python power.py --input E:\\CVS\\Project\\Digital_Image_Processing\\python\\image\\power.tif --gamma 0.3
#           python power.py --input E:/CVS/Project/Digital_Image_Processing/python/image/power.tif --gamma 0.3

# import the necessary packages
import argparse
import cv2
import numpy as np
import ntpath
import time
import math

def power(imageInput, gamma):
    """
    NOTE: denote the values of pixels, before and after processing, by r and s, respectively
	'c' and 'gamma' is a positive constant.
	we can see:
		if 'gamma' < 1, make the image brighter
		if 'gamma' > 1, make the image darker

		if r = 0,		then s = 0
	and	if r = L - 1,	then s = L - 1
	so the form of Power-law transformations: s = c * pow(r, gamma)
	we can compute: c = (L - 1) / pow(L - 1, gamma) or c = pow(L - 1, 1 - gamma)
    """
    # the number of possible intensity levels in the image (256 for an 8-bit image)
    L = 256

    if (imageInput.shape[-1] == 3): # '-1' retrieves last item that is number of channels image, or could use '-2'
        imageInput = cv2.cvtColor(imageInput, cv2.COLOR_BGR2GRAY) #Convert RGB image to Gray image

    imageOutput = np.zeros((imageInput.shape[0], imageInput.shape[1]), np.uint8)

    M = imageInput.shape[0] # height of image
    N = imageInput.shape[1] # width of image

    c = math.pow(L - 1, 1 - gamma)

    # throughout each pixel of image
    for x in range(0, M):
        for y in range(0, N):
            # denote the values of pixels, before and after processing, by r and s, respectively
            r = imageInput[x, y]
            if (r==0):
                r = 1
            # power-law transformation function:
            s = c * math.pow(r, gamma)
            imageOutput[x, y] = int(s)

    return imageOutput

# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--input", required=True,\
    help="input image path to processing")
ap.add_argument("-g", "--gamma", required=True,\
    help="gamma constant")
args = vars(ap.parse_args())

# read original image from input image path
inputImagePath = args["input"]
imageFileName = ntpath.basename(inputImagePath)
imageOriginal = cv2.imread(inputImagePath)

# display original image
cv2.imshow('original', imageOriginal)

# convert RGB to grayscale
imageGrayscale = cv2.cvtColor(imageOriginal, cv2.COLOR_BGR2GRAY)

# display grayscale image
cv2.imshow('grayscale', imageGrayscale)

# power-law processing
gamma = float(args["gamma"])
startTime = time.time()
imagePower = power(imageGrayscale, gamma)
print("[INFOR]: Time execution: {}".format(time.time() - startTime))

# display power processing image
cv2.imshow('power', imagePower)

print("[INFOR]: Press 's' key to save result and quit")
print("[INFOR]: Press 'q' key to quit and NOT save")
key = cv2.waitKey(0) & 0xFF
if key == ord("s"):
    resultPath = "./output/" + "power" + imageFileName
    cv2.imwrite(resultPath, imagePower)


import csv
from datetime import datetime

#CSV source
user_logs_csv = 'data/raw/user_logs.csv'

# for transfering the type ID to text
log_action_type = ['LOGIN', 'LOGOFF', 'ACCESSCALENDAR', 'ACCESSGOALS', 'ACCESSADDACTIVITYPAGE', 'ADDEDACTIVITY', 'ACCESSGALLERY', 'ACCESSADDPICTUREPAGE',
    'VIEWACTIVITY', 'DELETEACTIVITY', 'DELETEPICTUREFROMACTIVITY', 'UNLINKPICTUREFROMACTIVITY', 'UPDATEACTIVITY',
    'DELETEPICTUREFROMGALLERY', 'SELECTPICTUREFROMGALLRERYPAGE', 'ACCESSADDPICTUREDIRECTLYPAGE', 'ADDEDPICTUREDIRECTLY',
    'LINKPICTURETOACTIVITY', 'UPDATECALENDARDOWN', 'UPDATECALENDARUP', 'UPDATECALENDARDIRECTLY', 'ACCESSSELECTMEASUREMENTPAGE',
    'ACCESSADDGLUCOSEPAGE', 'ADDEDGLUCOSE', 'VIEWMEASUREMENT', 'ADDEDPICTURE', 'REMOVEDMEASUREMENT', 'UPDATEGLUCOSE', 'UPDATEINSULIN',
    'UPDATEDGLUCOSE', 'UPDATEDINSULINE', 'UPDATEDCARBOHYDRATE', 'UPDATECARBOHYDRATE', 'ACCESSADDCARBOHYDRATEPAGE', 'ADDEDINSULIN',
    'ADDEDCARBOHYDRATE', 'UPLOADEDPICTURE', 'ACCESSGOALADDDAILYPAGE', 'ADDEDGOALDAILY', 'ACCESSGOALADDTOTALPAGE', 'ADDEDGOALTOTAL',
    'DELETEGOAL', 'ACCESSUPDATEACTIVITYPAGE', 'ACCESSADDACTIVITYTYPEPAGE', 'ACCESSADDINSULINPAGE', 'ACCESSUPDATEGLUCOSEPAGE',
    'ACCESSUPDATEINSULINPAGE', 'ADDEDACTIVITYTYPE', 'DELETEACTIVITYTYPE', 'DELETEGLUCOSE', 'DELETEINSULIN', 'ADDEDWITHGLUCONLINE',
    'TOGETHERORSELF', 'TOGETHER']

with open(user_logs_csv, 'rb') as csvFile:
    reader = csv.DictReader(csvFile)
    raw_user_logs = list(reader)

def isOnDelList(name):
    delList = ['mike.ligthart', 'hunter', 'elvira', 'viewer']
    return name in delList

def changeNameForNumber(name):
    numberForName = ['benja', 'sinne', 'daan', 'tom', 'rick', 'martijn', 'hunter', 'ilse', 'casper', 'safiye', 'yasmin', 'koen', 'maas', 'janwillem']
    return 1 + numberForName.index(name)

user_logs = []

for raw_user_log in raw_user_logs:
    if isOnDelList(raw_user_log['USER_USER_NAME']) == False:
        user_log = {}
        user_log['participant'] = changeNameForNumber(raw_user_log['USER_USER_NAME'])
        user_log['timestamp'] = datetime.strptime(raw_user_log['TIMESTAMP'].split('.')[0], '%Y-%m-%d %H:%M:%S')
        user_log['behavior'] = log_action_type[int(raw_user_log['TYPE'])]
        user_logs.append(user_log)

with open('data/cleaned/user_logs.csv', 'wb') as outputFile:
    writer = csv.DictWriter(outputFile, user_logs[0].keys())
    writer.writeheader()
    writer.writerows(user_logs)


import requests    # by Kirill Kasparov, 2023
from bs4 import BeautifulSoup
import pandas as pd
import time

# программа берет список ИНН из файла и проверяет наличие конкурсов на сайте zakupki.gov.

start = time.time()    # для таймера выполнения

head = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
       ' Chrome/86.0.4240.185 YaBrowser/20.11.2.78 Yowser/2.5 Safari/537.36', 'accept': '*/*'}

inn_list_df = pd.read_csv('inn_list.csv', sep=';', encoding='windows-1251', dtype='unicode', nrows=1000)
inn_list = list(inn_list_df['ИНН'])

total_results_df = []
last_purchase = []
count = 0

for inn in inn_list:
    url = 'https://zakupki.gov.ru/epz/order/extendedsearch/results.html?searchString=' + str(inn) + '&morphology=on&search-filter=%D0%94%D0%B0%D1%82%D0%B5+%D1%80%D0%B0%D0%B7%D0%BC%D0%B5%D1%89%D0%B5%D0%BD%D0%B8%D1%8F&pageNumber=1&sortDirection=false&recordsPerPage=_10&showLotsInfoHidden=false&sortBy=UPDATE_DATE&fz44=on&fz223=on&af=on&ca=on&pc=on&pa=on&currencyIdGeneral=-1'
    r = requests.get(url, headers=head)
    if r.status_code == 200:
        soup = BeautifulSoup(r.content, 'html.parser')
        total_results = soup.findAll('div', class_='search-results__total')
        for elem in total_results:
            total_results_df.append(elem.text.strip())
        data_value = soup.findAll('div', class_="data-block__value")
        if len(data_value) > 1:
            last_purchase.append(data_value[0].text.strip())
        else:
            last_purchase.append('нет')
    count += 1
    if count % 50 == 0:
        end = time.time()
        while True:  # проверка, если файл открыт
            try:
                inn_list_df.to_csv('inn_list_export.csv', sep=';', encoding='windows-1251', index=False, header=True,
                                   mode='a')
                print('Загружено строк:', count, "|  Время выполнения:", end-start, '|  Данные добавлены в файл inn_list_export.csv')
                break
            except IOError:  # PermissionError
                input('Необходимо закрыть файл inn_list.csv перед сохранением данных')

inn_list_df['Всего записей на zakupki.gov'] = total_results_df
inn_list_df['Последняя закупка'] = last_purchase

while True:  # проверка, если файл открыт
    try:
        inn_list_df.to_csv('inn_list_export.csv', sep=';', encoding='windows-1251', index=False, header=True)
        end = time.time()
        print('Загружено строк:', count, "|  Время выполнения:", end-start, '|  Итоговые данные сохранены в файл inn_list_export.csv')
        break
    except IOError:    # PermissionError
        input('Необходимо закрыть файл inn_list.csv перед сохранением данных')


"""
Additional propositional logic rules from Kaye, ch 6, defn. 6.3, p. 65.
Supplements rules in prop_common.
This module is an alternative to prop_derived or prop_constructive.
"""

from formula import FormulaPlaceholder
from prop_common import Not, Or

# Additional rules for propositional logic from Kaye
# Or-elim. rules here are allowed in both classic and constructive logics,
#  but both Bornat and H&R use a different form.
# Also Not-elim is here, only classical, not allowed in constructive.

# Define symbol for each rule, short for easy typing at Python interpreter
# Rule symbols are self-evaluating, used to write proof in save file format

oel, oer, ne = 'oel', 'oer', 'ne'

# Pretty-print names for rules.
# Each logic module merges its own rule_names with checker.rule_names
# Note _ prefix on _rule_names here makes it private

_rule_names = { oel: 'Or-Elimination (Left)',
                oer: 'Or-Elimination (Right)',
                ne: 'Not-Elimination',
              }

# Inference rules, dictionary of rule symbol and list of formulas:
#  list of premises, then conclusion last
# Subproofs in inference rules are nested lists

# Placeholders used in rules
m1, m2 = map(FormulaPlaceholder, ('m1','m2'))

_rules = { oel:    [ Or(m1,m2), Not(m1), m2 ],
           oer:    [ Or(m1,m2), Not(m2), m1 ],
           ne:     [ Not(Not(m1)), m1 ]
         }

# Import statement to write to save file, so it in turn can be imported

_imports = 'from flip.logic.prop_classic import *'


def checkNumber(string):
    string = string.replace('-', '').replace('(', '').replace(')', '')
    return string[-10:] \
        if len(string) > 7 \
        else '495' + string[-7:]

n = 4
notes = [input() for _ in range(n)]
for note in notes[1:]:
    print('YES' if checkNumber(notes[0]) == checkNumber(note) else 'NO')

Python/Телефонные номера.py

Телефонные номера.py


import cv2
import itertools
import os

import numpy as np
np.set_printoptions(precision=2)

import openface

class ImageParser(object):
    def __init__(self):
      fileDir = os.path.dirname(os.path.realpath(__file__))
      modelDir = os.path.join(fileDir, '/root/openface/', 'models')
      dlibModelDir = os.path.join(modelDir, 'dlib')
      openfaceModelDir = os.path.join(modelDir, 'openface')

      imgDim = 96
      self.align = openface.AlignDlib(os.path.join(dlibModelDir, "shape_predictor_68_face_landmarks.dat"))

      self.catHeadImg = cv2.imread('cats/cat_head.png', cv2.IMREAD_UNCHANGED)
      self.catMouthImg = cv2.imread('cats/cat_mouth.png', cv2.IMREAD_UNCHANGED)
      self.catWidth = self.catHeadImg.shape[0]
      self.catHeight = self.catHeadImg.shape[1]
      self.sf = 1.8
      self.yOffset = 1.2

    def process(self, img):
      bgrImg = img.copy()
      if bgrImg is None:
          raise Exception("Unable to load image: {}".format(imgPath))
      rgbImg = cv2.cvtColor(bgrImg, cv2.COLOR_BGR2RGB)

      bbs = self.align.getAllFaceBoundingBoxes(rgbImg)
      np_bbs = np.array([[b.left(), b.top(), b.right(), b.bottom()] for b in bbs])
      final_bbs = self.non_max_suppression_fast(np_bbs, 0.3)
      sorted_bbs = sorted(zip(final_bbs, bbs), key=lambda x: x[0][2]-x[0][0])
      for (bb, dlibBB) in sorted_bbs:
        landmarks = self.align.findLandmarks(rgbImg, dlibBB)

        bbWidth = (bb[2]-bb[0])
        resizeWidth = int(round(bbWidth * self.sf))
        resizeHeight = int(round(self.sf * self.catHeight * (bbWidth / float(self.catWidth))))
        resizedCatHeadImg = cv2.resize(self.catHeadImg, (resizeWidth, resizeHeight))
        resizedCatMouthImg = cv2.resize(self.catMouthImg, (resizeWidth, resizeHeight))
        drawX = int(round((bb[0] + bb[2])/2-(resizeWidth/2)))
        drawY = int(round((bb[1] + bb[3])/2-(resizeHeight/2) * self.yOffset))

        self.overlay_image_alpha(bgrImg,
                        resizedCatHeadImg[:, :, 0:3],
                        (drawX, drawY),
                        resizedCatHeadImg[:, :, 3] / 255.0)

        yParting = self.get_parting(landmarks)

        self.overlay_image_alpha(bgrImg,
                        resizedCatMouthImg[:, :, 0:3],
                        (drawX, drawY + yParting),
                        resizedCatMouthImg[:, :, 3] / 255.0)

      # cv2.rectangle(bgrImg, (bb.left(), bb.top()), (bb.right(), bb.bottom()), (0,255,0), 3)
      return bgrImg

    def get_parting(self, landmarks):
      diff1 = landmarks[67][1]-landmarks[61][1]
      diff2 = landmarks[66][1]-landmarks[62][1]
      diff3 = landmarks[65][1]-landmarks[63][1]
      parting = (diff1 + diff2 + diff3) / 3
      return parting

    def overlay_image_alpha(self, img, img_overlay, pos, alpha_mask):
        """Overlay img_overlay on top of img at the position specified by
        pos and blend using alpha_mask.

        Alpha mask must contain values within the range [0, 1] and be the
        same size as img_overlay.
        """

        x, y = pos

        # Image ranges
        y1, y2 = max(0, y), min(img.shape[0], y + img_overlay.shape[0])
        x1, x2 = max(0, x), min(img.shape[1], x + img_overlay.shape[1])

        # Overlay ranges
        y1o, y2o = max(0, -y), min(img_overlay.shape[0], img.shape[0] - y)
        x1o, x2o = max(0, -x), min(img_overlay.shape[1], img.shape[1] - x)

        # Exit if nothing to do
        if y1 >= y2 or x1 >= x2 or y1o >= y2o or x1o >= x2o:
            return

        channels = img.shape[2]

        alpha = alpha_mask[y1o:y2o, x1o:x2o]
        alpha_inv = 1.0 - alpha

        for c in range(channels):
            img[y1:y2, x1:x2, c] = (alpha * img_overlay[y1o:y2o, x1o:x2o, c] +
                                    alpha_inv * img[y1:y2, x1:x2, c])

    # Malisiewicz et al.
    def non_max_suppression_fast(self, boxes, overlapThresh):
      # if there are no boxes, return an empty list
      if len(boxes) == 0:
        return []

      # if the bounding boxes integers, convert them to floats --
      # this is important since we'll be doing a bunch of divisions
      if boxes.dtype.kind == "i":
        boxes = boxes.astype("float")

      # initialize the list of picked indexes
      pick = []

      # grab the coordinates of the bounding boxes
      x1 = boxes[:,0]
      y1 = boxes[:,1]
      x2 = boxes[:,2]
      y2 = boxes[:,3]

      # compute the area of the bounding boxes and sort the bounding
      # boxes by the bottom-right y-coordinate of the bounding box
      area = (x2 - x1 + 1) * (y2 - y1 + 1)
      idxs = np.argsort(y2)

      # keep looping while some indexes still remain in the indexes
      # list
      while len(idxs) > 0:
        # grab the last index in the indexes list and add the
        # index value to the list of picked indexes
        last = len(idxs) - 1
        i = idxs[last]
        pick.append(i)

        # find the largest (x, y) coordinates for the start of
        # the bounding box and the smallest (x, y) coordinates
        # for the end of the bounding box
        xx1 = np.maximum(x1[i], x1[idxs[:last]])
        yy1 = np.maximum(y1[i], y1[idxs[:last]])
        xx2 = np.minimum(x2[i], x2[idxs[:last]])
        yy2 = np.minimum(y2[i], y2[idxs[:last]])

        # compute the width and height of the bounding box
        w = np.maximum(0, xx2 - xx1 + 1)
        h = np.maximum(0, yy2 - yy1 + 1)

        # compute the ratio of overlap
        overlap = (w * h) / area[idxs[:last]]

        # delete all indexes from the index list that have
        idxs = np.delete(idxs, np.concatenate(([last],
          np.where(overlap > overlapThresh)[0])))

      # return only the bounding boxes that were picked using the
      # integer data type
      return boxes[pick].astype("int")


import angr

from ...storage.file import Flags
from .fstat import fstat

class stat(fstat):
    def run(self, p_addr, stat_buf):
        # open tempory fd
        strlen = angr.SIM_PROCEDURES["libc"]["strlen"]
        p_strlen = self.inline_call(strlen, p_addr)
        p_expr = self.state.memory.load(p_addr, p_strlen.max_null_index, endness="Iend_BE")
        file_path = self.state.solver.eval(p_expr, cast_to=bytes)
        fd = self.state.posix.open(file_path, Flags.O_RDONLY)

        # Use fstat to get the result and everything
        result = super().run(fd, stat_buf)

        # close tempory fd
        self.state.posix.close(fd)

        return result


'''Exercício Python 070: Crie um programa que leia o nome e o preço de vários produtos.
O programa deverá perguntar se o usuário vai continuar ou não. No final, mostre:
A) qual é o total gasto na compra.
B) quantos produtos custam mais de R$1000.
C) qual é o nome do produto mais barato. '''
total = totmil = menor = cont = 0
barato = ''
while True:
    nome = str(input('Qual o nome do produto? '))
    preco = float(input('Qual o preco? R$ '))
    cont += 1
    total += preco
    if preco > 1000:
        totmil += 1
    if cont == 1 or preco < menor:
        menor = preco
        barato = nome
    resp = ' '
    while resp not in 'SN':
        resp = str(input('quer continuar? [S/N]')).strip().upper()[0]
    if resp == 'N':
        break
print(f'o Total da compra foi: R${total:.2f}')
print(f'quantidade produtos que custam mais de R$1000: {totmil}')
print(f'O prod mais barato foi {barato} que custa R${menor:.2f}')


n_a = -1
NA = -1
def move_to_end(m_plus_n,size):
    j = size - 1
    for i in range(size - 1,-1,-1):
        if m_plus_n[i] != n_a:
            m_plus_n[j] = m_plus_n[i]
            j -= 1

# merge array N of size n into array of  m_plus_n of size m + n

def merge(m_plus_n,N,m,n):
    i = n # current index of input part of m_plus_n
    j = 0 # current index of N
    k = 0 # current index of output m_plus_n

    while k < m + n:
        """
        Take an element from m_plus_n if
            a -- value of the picked element is smaller and we have not reached at the end of it
            b -- we have reached the end of N[]
        """

        if (i < (m + n) and m_plus_n[i] <= N[j]) or (j == n):
            m_plus_n[k] = m_plus_n[i]
            k += 1
            i += 1
        else: # otherwise take the element form N[]

            m_plus_n[k] = N[j]
            k += 1
            j += 1

mPlusN = [2, 8, NA, NA, NA, 13, NA, 15, 20]
N = [5, 7, 9, 25]
n = len(N)

m = len(mPlusN) - n


import abc
import typing

from bytecodemanipulation.assembler.AbstractBase import AbstractSourceExpression
from bytecodemanipulation.assembler.AbstractBase import IAssemblyStructureVisitable
from bytecodemanipulation.assembler.AbstractBase import IIdentifierAccessor
from bytecodemanipulation.assembler.AbstractBase import ParsingScope
from bytecodemanipulation.assembler.AbstractBase import StaticIdentifier
from bytecodemanipulation.assembler.Lexer import SpecialToken
from bytecodemanipulation.assembler.Parser import Parser
from bytecodemanipulation.assembler.syntax_errors import PropagatingCompilerException
from bytecodemanipulation.assembler.util.parser import AbstractExpression
from bytecodemanipulation.data.shared.expressions.CompoundExpression import (
    CompoundExpression,
)
from bytecodemanipulation.data.shared.instructions.AbstractInstruction import (
    AbstractAssemblyInstruction,
)

class AbstractIFAssembly(AbstractAssemblyInstruction, abc.ABC):
    # IF <expression> ['\'' <label name> '\''] '{' <body> '}'
    NAME = "IF"

    @classmethod
    def consume(cls, parser: "Parser", scope: ParsingScope) -> "AbstractIFAssembly":
        source = parser.try_consume_access_to_value(allow_primitives=True, scope=scope)

        if source is None:
            raise PropagatingCompilerException(
                "expected <expression> after IF"
            ).add_trace_level(scope.get_trace_info().with_token(parser[0]))

        if parser.try_consume(SpecialToken("'")):
            label_name = parser.parse_jump_target(scope)

            if not parser.try_consume(SpecialToken("'")):
                raise PropagatingCompilerException(
                    "expected ' after label name declaration"
                ).add_trace_level(scope.get_trace_info().with_token(parser[0]))

        else:
            label_name = None

        body = parser.parse_body(scope=scope)

        return cls(
            source,
            body,
            label_name,
        )

    def __init__(
        self,
        source: AbstractSourceExpression,
        body: CompoundExpression,
        label_name: typing.List[IIdentifierAccessor] | str = None,
    ):
        self.source = source
        self.body = body
        self.label_name = (
            (
                [StaticIdentifier(e) for e in label_name.split(":")]
                if isinstance(label_name, str)
                else label_name
            )
            if label_name is not None
            else None
        )

    def copy(self):
        return type(self)(self.source.copy(), self.body.copy(), self.label_name.copy())

    def __eq__(self, other):
        return (
            type(self) == type(other)
            and self.source == other.source
            and self.body == other.body
            and self.label_name == other.label_name
        )

    def __repr__(self):
        c = '"'
        return f"IF({self.source}{'' if self.label_name is None else f', label={c}' + ':'.join(map(repr, self.label_name)) + c}) -> {{{self.body}}}"

    def visit_parts(
        self,
        visitor: typing.Callable[
            [IAssemblyStructureVisitable, tuple, typing.List[AbstractExpression]],
            typing.Any,
        ],
        parents: list,
    ):
        return visitor(
            self,
            (
                self.source.visit_parts(visitor, parents + [self]),
                self.body.visit_parts(visitor, parents + [self]),
            ),
            parents,
        )

    def visit_assembly_instructions(
        self, visitor: typing.Callable[[IAssemblyStructureVisitable, tuple], typing.Any]
    ):
        return visitor(self, (self.body.visit_assembly_instructions(visitor),))

    def get_labels(self, scope: ParsingScope) -> typing.Set[str]:
        label_name = (
            ":".join(e(scope) for e in self.label_name)
            if self.label_name is not None
            else None
        )

        return (
            set()
            if self.label_name is None
            else {label_name, f"{label_name}:END", f"{label_name}:HEAD"}
        ) | self.body.get_labels(scope)


from matplotlib import pyplot as plt
from matplotlib import dates as mplt_dates
from datetime import datetime, timedelta
import pandas as pd

# style
plt.style.use('seaborn')
dates = [
    datetime(2020, 8, 8),
    datetime(2020, 8, 9),
    datetime(2019, 8, 10),
    datetime(2020, 8, 11),
    datetime(2020, 8, 12),
    datetime(2020, 5, 22)]
y = [0, 1, 2, 3, 4, 5]
plt.plot_date(dates, y, linestyle='solid')
date_formater = mplt_dates.DateFormatter('%b,%d,%Y')
plt.gca().xaxis.set_major_formatter(date_formater)
plt.gcf().autofmt_xdate()
# show plot
plt.tight_layout()
plt.savefig('./plots/timeplot.png')
plt.show()


# train a model for text classification with tensorflow / keras
from __future__ import absolute_import, division, print_function

import re
import json
import pathlib

import string
import numpy as np
import pandas as pd  # pip install pandas

# nltk
import nltk  # pip install --user -U nltk
# nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer

# sklearn
from sklearn.model_selection import train_test_split
from sklearn.externals import joblib

def load_data(filename):
    data = []
    with open(str(pathlib.Path(__file__).parent.absolute()) + "/" + filename, 'r') as f:
        imported = json.load(f)
        data = pd.DataFrame(imported['data'])
        data.drop(['id', 'caseTitle', 'lbName', 'ltName', 'ltAlias'], axis=1, inplace=True)

    return data

### Text Normalizing function. Part of the following function was taken from this link.
### https://medium.com/@sabber/classifying-yelp-review-comments-using-lstm-and-word-embeddings-part-1-eb2275e4066b
def clean_text(text):
    """
        text: a string

        return: modified initial string
    """
    ## Remove puncuation
    text = text.translate(string.punctuation)

    ## Convert words to lower case and split them
    text = text.lower().split()

    ## Remove stop words
    stops = set(stopwords.words("german"))
    text = [w for w in text if not w in stops and len(w) >= 3]

    text = " ".join(text)
    ## Clean the text
    text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=äöüß]", " ", text)
    text = re.sub(r"\'s", " ", text)
    text = re.sub(r",", " ", text)
    text = re.sub(r"\.", " ", text)
    text = re.sub(r"!", " ! ", text)
    text = re.sub(r"\/", " ", text)
    text = re.sub(r"\^", " ^ ", text)
    text = re.sub(r"\+", " + ", text)
    text = re.sub(r"\-", " - ", text)
    text = re.sub(r"\=", " = ", text)
    text = re.sub(r"'", " ", text)
    text = re.sub(r"(\d+)(k)", r"\g<1>000", text)
    text = re.sub(r":", " : ", text)
    text = re.sub(r" e g ", " eg ", text)
    text = re.sub(r" b g ", " bg ", text)
    text = re.sub(r" u s ", " american ", text)
    text = re.sub(r"\0s", "0", text)
    text = re.sub(r" 9 11 ", "911", text)
    text = re.sub(r"e-mail", "email", text)
    text = re.sub(r"j k", "jk", text)
    text = re.sub(r"\s{2,}", " ", text)

    ## Stemming
    text = text.split()
    stemmer = SnowballStemmer('german')
    stemmed_words = [stemmer.stem(word) for word in text]
    text = " ".join(stemmed_words)

    return text

# load data, limit to TOP 10 categories and remove null-Values
df = load_data("data.json")
top10lb = list(df['lbAlias'].value_counts().head(10).index)
df = df[df['lbAlias'].isin(top10lb)]

# print(top10lb)
# print(filteredDf.head(20))

df = df.dropna()
df = df[df.caseDesc.apply(lambda x: x != "")]
df = df[df.lbAlias.apply(lambda x: x != "")]

# apply text cleaning function to df['text']
df['text'] = df['caseDesc'].map(lambda x: clean_text(x))
tags = pd.get_dummies(df['lbAlias']).columns.tolist()

# print(df['lbAlias'].value_counts())
# print(df.head(10))

X = df.text
y = df.lbAlias
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 42)

# naive bayes
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.metrics import classification_report
"""
nb = Pipeline([('vect', CountVectorizer()),
               ('tfidf', TfidfTransformer()),
               ('clf', MultinomialNB()),
              ])
nb.fit(X_train, y_train)

y_pred = nb.predict(X_test)

print('accuracy %s' % accuracy_score(y_pred, y_test))
print(classification_report(y_test, y_pred, labels=tags, zero_division=0)) """

# support vector machine (SVM)
""" from sklearn.linear_model import SGDClassifier

sgd = Pipeline([('vect', CountVectorizer()),
                ('tfidf', TfidfTransformer()),
                ('clf', SGDClassifier(loss='hinge', penalty='l2', alpha=1e-3, random_state=42, max_iter=5, tol=None)),
               ])
sgd.fit(X_train, y_train)

y_pred = sgd.predict(X_test)

print('accuracy %s' % accuracy_score(y_pred, y_test))
print(classification_report(y_test, y_pred, labels=tags, zero_division=0)) """

# logistic regression
# https://towardsdatascience.com/multi-class-text-classification-model-comparison-and-selection-5eb066197568
from sklearn.linear_model import LogisticRegression
from imblearn.pipeline import Pipeline as Pipeline_IMB   # pip install -U imbalanced-learn
from imblearn.over_sampling import SMOTE, SMOTENC
from imblearn.metrics import classification_report_imbalanced
'''
logreg = Pipeline([('vect', CountVectorizer()),
                ('tfidf', TfidfTransformer()),
                ('clf', LogisticRegression(n_jobs=1, C=1e5, max_iter=500)),
               ])
'''

count_vect = CountVectorizer()
X_train_counts = count_vect.fit_transform(X_train)

tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

print("Before oversampling: ", X_train_tfidf.shape)

sm = SMOTE(random_state=4)
train_text_res, train_y_res = sm.fit_sample(X_train_tfidf, y_train)

print("After oversampling: ", train_text_res.shape)

# logreg = LogisticRegression(n_jobs=1, C=1e5, max_iter=500)
logreg = LogisticRegression(solver='saga', multi_class='auto', max_iter=500, class_weight='balanced', n_jobs=-1, random_state=4)

# train model
logreg.fit(train_text_res, train_y_res)

# test model
X_new_counts = count_vect.transform(X_test)
X_new_tfidf = tfidf_transformer.transform(X_new_counts)
y_pred = logreg.predict(X_new_tfidf)

print('accuracy %s' % accuracy_score(y_pred, y_test))
# print(classification_report(y_test, y_pred, labels=tags, zero_division=0))
print(classification_report_imbalanced(y_test, y_pred))

# save model to file
joblib.dump(logreg, str(pathlib.Path(__file__).parent.absolute()) + '/legalcase_logreg_model.pkl')
print("Saved model to disk")

# make validation predictions here
validation_data = load_data("legalcase_validation.json")

# filter by known legal branches
validation_data = validation_data[validation_data['lbAlias'].isin(top10lb)]

# sanitize text
validation_data['caseDesc'] = validation_data['caseDesc'].map(lambda x: clean_text(x))

result = []
for index, row in validation_data.iterrows():
    X_new_counts = count_vect.transform([row['caseDesc']])
    X_new_tfidf = tfidf_transformer.transform(X_new_counts)
    predictions = logreg.predict(X_new_tfidf)
    result.append([predictions[0], row['lbAlias'], (predictions[0] == row['lbAlias'])])

rPd = pd.DataFrame(result, columns=['predicted', 'given', 'result'])
print("\n# of correct predictions: {}".format(len(rPd[(rPd.result == True)])))
print("# of incorrect predictions: {}\n".format(len(rPd[(rPd.result == False)])))
with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', None):  # more options can be specified also
    print(rPd)


import torch
import torch.nn.functional as F
import numpy as np
import math

from src import utils
from src.noise import GammaNetwork, PredefinedNoiseSchedule
from src.dynamics_gvp import DynamicsWithPockets
#from torch_scatter import scatter_mean

class AREDM(torch.nn.Module):
    def __init__(
            self,
            dynamics: DynamicsWithPockets,
            lig_nf: int, # number of lig features (10)
            pocket_nf: int, # number of pocket features (25)
            n_dims: int, # number of position features
            timesteps: int=500,
            n_hier_steps=8, #
            noise_schedule='learned',
            noise_precision=1e-5,
            loss_type='l2',
            norm_values=(1., 4., 1.),
            norm_biases=(None, 0., 0.),
            anchors_context=True, #
            center_of_mass='anchors',
    ):
        super().__init__()
        if noise_schedule == 'learned':
            assert loss_type == 'vlb', 'A noise schedule can only be learned with vlb objective'
            self.gamma = GammaNetwork()
        else:
            self.gamma = PredefinedNoiseSchedule(noise_schedule, timesteps=timesteps, precision=noise_precision)

        self.dynamics = dynamics
        self.lig_nf = lig_nf
        self.pocket_nf = pocket_nf
        self.n_dims = n_dims
        self.T = timesteps
        self.hier_steps = n_hier_steps
        self.norm_values = norm_values
        self.norm_biases = norm_biases
        self.anchors_context = anchors_context
        self.center_of_mass = center_of_mass

    def forward(self, x, h, pocket_x, pocket_h, extension_mask, scaffold_mask, anchors, pocket_anchors, pocket_mask):
        """
        inputs:
            x: [B, N_l, 3] coordinates of molecules
            h: [B, N_l, lig_nf] features of moleucles
            extension_masks: [B, N_l] mask on extension atoms
            scaffold_masks: [B, N_l] mask on scaffold atoms
            anchors: [B, N_l] mask on anchor atoms
            pocket_x: [B, N_p, 3] coordinates of pocket
            pocket_h: [B, N_p, pocket_nf] features of pocket atoms
            pocket_mask: [B, N_p] masking on all the pocket atoms
            pocket_anchors: [B, N_p] masking on anchor atoms
        """
        # NOTE: the molecule is already at the center of mass of the extension at the current step
        x, h = self.normalize(x, h)
        pocket_x, pocket_h = self.normalize(pocket_x, pocket_h)

        num_nodes = x.shape[1] + pocket_x.shape[1]
        # volume change loss term
        delta_log_px = self.delta_log_px(num_nodes)
        # sample t
        t_int = torch.randint(0, self.T+1, size=(x.size(0), 1), device=x.device).float() # [B, 1]
        s_int = t_int - 1
        t = t_int / self.T
        s = s_int / self.T

        # masks for t=0 and t>0
        t_is_zero = (t_int==0).squeeze().float()
        t_is_not_zero = 1 - t_is_zero

        # compute gamma_t and gamma_s according to noise schedule
        gamma_t = self.inflate_batch_array(self.gamma(t), x)
        gamma_s = self.inflate_batch_array(self.gamma(s), x)

        xh = torch.cat([x, h], dim=2) # [B, N_l, h_l+3]
        pocket_xh = torch.cat([pocket_x, pocket_h], dim=2) # [B, N_p, h_p+3]

        # compute alpha_t, sigma_t from gamma
        alpha_t = self.alpha(gamma_t, x)
        sigma_t = self.sigma(gamma_t, x)

        # sample noise (only for extension part)
        eps_t = self.sample_combined_position_features_noise(n_samples=x.size(0), n_nodes=x.size(1), mask=extension_mask.unsqueeze(-1))
        # keep the scaffold part unchanged
        z_t = alpha_t * xh + sigma_t * eps_t
        z_t = xh * scaffold_mask.unsqueeze(-1) + z_t * extension_mask.unsqueeze(-1)

        eps_t_hat = self.dynamics.forward(
            xh=z_t, # zt has random at extension masks and real pos/feat at scaffold mask
            t=t, # timestep
            pocket_xh=pocket_xh,
            extension_mask=extension_mask,
            scaffold_mask=scaffold_mask,
            anchors=anchors,
            pocket_anchors=pocket_anchors,
            pocket_mask=pocket_mask, # [B, N_p]
        ) # [B, N_l, lig_nf+3] # prediction only for the extension part of ligand

        eps_t_hat = eps_t_hat * extension_mask.unsqueeze(-1) # prediction only for the extension part # [B, N, nf+3]
        # computing basic error (further used for computing NLL and L2 loss)
        error_t = (eps_t - eps_t_hat) ** 2 # [B, N, nf+3]
        error_t = self.sum_except_batch(error_t)

        normalization = (self.n_dims + self.lig_nf) * self.numbers_of_nodes(extension_mask)
        l2_loss = error_t / normalization
        l2_loss = l2_loss.mean()

        # KL between q(z_t|x) and p(z_T) = Normal(0,1)
        kl_prior = self.kl_prior(xh, num_nodes).mean()

        # computing NLL middle term
        SNR_weight = (self.SNR(gamma_s - gamma_t)-1).squeeze(1).squeeze(1)
        loss_term_t = self.T * 0.5 * SNR_weight * error_t
        loss_term_t = (loss_term_t * t_is_not_zero).sum() / t_is_not_zero.sum()

        # computing noise returned by dynamics
        noise = torch.norm(eps_t_hat, dim=[1,2])
        noise_t = (noise * t_is_not_zero).sum() / t_is_not_zero.sum()

        if t_is_zero.sum() > 0:
            # the _constants depending on sigma_0 from the cross ent term E_q(z0|x) [log p(x|z0)]
            neg_log_constants = -self.log_constant_of_p_x_given_z0(x, num_nodes)
            # compute L0 term (even if gamma_t is not actually gamma_0)
            # and selected only relevant via masking
            loss_term_0 = -self.log_p_xh_given_z0_without_constants(h, z_t, gamma_t, eps_t, eps_t_hat, extension_mask.unsqueeze(-1))
            loss_term_0 = loss_term_0 + neg_log_constants
            loss_term_0 = (loss_term_0 * t_is_zero).sum() / t_is_zero.sum()

            # computing noise returned by dynamics
            noise_0 = (noise * t_is_zero).sum() / t_is_zero.sum()
        else:
            loss_term_0 = 0.
            noise_0 = 0.

        return delta_log_px, kl_prior, loss_term_t, loss_term_0, l2_loss, noise_t, noise_0

    @torch.no_grad()
    def sample_chain_single_fragment(self, x, h, extension_mask, scaffold_mask, anchors, pocket_x, pocket_h, pocket_mask, pocket_anchors, lj_guidance=False, prot_mol_lj_rm=None, all_H_coords=None, guidance_weights=None, keep_frames=None):

        n_samples = x.size(0)
        n_nodes = x.size(1)

        x, h = self.normalize(x, h)
        pocket_x, pocket_h = self.normalize(pocket_x, pocket_h)

        xh = torch.cat([x,h], dim=2)
        pocket_xh = torch.cat([pocket_x, pocket_h], dim=2)

        # sampling from N(0,1)
        z = self.sample_combined_position_features_noise(n_samples, n_nodes, mask=extension_mask.unsqueeze(-1))

        z = xh * scaffold_mask.unsqueeze(-1) + z * extension_mask.unsqueeze(-1)

        if keep_frames is None:
            keep_frames = self.T
        else:
            assert keep_frames <= self.T

        chain = torch.zeros((keep_frames,) + z.size(), device=z.device) # [num_frames, num_samples, n_nodes, nf+3]
        # sample p(zs|zt)
        for s in reversed(range(0, self.T)):
            s_array = torch.full((n_samples,1), fill_value=s, device=z.device)
            t_array = s_array + 1
            s_array = s_array / self.T
            t_array = t_array / self.T

            z = self.sample_p_zs_given_zt_only_extension(
                s=s_array,
                t=t_array,
                z_t=z,
                pocket_xh=pocket_xh,
                extension_mask=extension_mask,
                scaffold_mask=scaffold_mask,
                anchors=anchors,
                pocket_anchors=pocket_anchors,
                pocket_mask=pocket_mask,
            )

            # Move back to center of mass again
            #if self.center_of_mass == 'anchors':
            #    anchor_pos = torch.zeros((n_samples,3), device=x.device)
            #    row1, col1 = torch.where(pocket_anchors)
            #    anchor_pos[row1] = pocket_x[row1, col1]

            #    row2, col2 = torch.where(anchors)
            #    anchor_pos[row2] = x[row2, col2]
            #    z[:,:,:3] = z[:,:,:3] - anchor_pos.unsqueeze(1) * node_masks.unsqueeze(-1)
            #    pocket_xh[:,:,:3] = pocket_x[:,:,:3] - anchor_pos.unsqueeze(1) * pocket_mask.unsqueeze(-1)

            # NOTE: we are not doing LJ guidance in the last 20 steps
            if lj_guidance and s < 400:
                with torch.enable_grad():
                    lig_x, lig_h = z[:, :, :self.n_dims], z[:, :, self.n_dims:]
                    lig_x.requires_grad = True

                    #h_ext.requires_grad = True
                    if extension_mask.sum() != 0:
                        lj_prot_lig = self.compute_lj(lig_x, lig_h, extension_mask, pocket_xh, pocket_mask, prot_mol_lj_rm=prot_mol_lj_rm, all_H_coords=all_H_coords)
                        lj_grad = torch.autograd.grad(lj_prot_lig, lig_x, retain_graph=True)[0] #
                        energy_prot_mol_lj = 0.3 * lj_grad * extension_mask.unsqueeze(-1) #
                        energy_total = energy_prot_mol_lj * guidance_weights[s]
                        z[:, :, :self.n_dims] = z[:, :, :self.n_dims] - energy_total

            write_index = (s * keep_frames) // self.T
            chain[write_index] = self.unnormalize_z(z)

        # finally sample p(x, h | z0)
        x_out , h_out = self.sample_p_xh_given_z0_only_extension(
            z_0=z,
            pocket_xh=pocket_xh,
            extension_mask=extension_mask,
            scaffold_mask=scaffold_mask,
            anchors=anchors,
            pocket_anchors=pocket_anchors,
            pocket_mask=pocket_mask
        )

        chain[0] = torch.cat([x_out, h_out], dim=2)
        return x_out, h_out, chain

    def compute_lj(self, lig_x, lig_h, extension_mask, pocket_xh, pocket_mask, prot_mol_lj_rm, all_H_coords):
        """ compute the LJ between protein and ligand
        lig_x: [B, N, 3]
        lig_h: [B, N, hf]
        """

        #lig_x.requires_grad = True
        B = extension_mask.shape[0]
        lj_prot_lig = torch.tensor(0., device=lig_x.device)

        n = 0
        for j in range(B):
            num_atoms = extension_mask[j].sum()
            if num_atoms == 0:
                continue # no extension atoms
            n += 1

            x = lig_x[j][extension_mask[j].bool()]  # [N_l, 3] # onlly extension atoms
            h = lig_h[j][extension_mask[j].bool()]  # [N_l, hf] # only extension atoms

            pocket_x = pocket_xh[j][pocket_mask[j].bool()][:, :self.n_dims] # [N_p, 3]
            pocket_h = pocket_xh[j][pocket_mask[j].bool()][:, self.n_dims:][:,:4] # [N_p, hf]
            h_coords = all_H_coords[j] # [N_p, 3]

            # --------------- compute the LJ between protein and ligand --------------
            dists = torch.cdist(x, pocket_x, p=2)
            dists = torch.where(dists<0.5, 0.5, dists)
            inds_lig = torch.argmax(h, dim=1) # [N_l]
            inds_pocket = torch.argmax(pocket_h, dim=1).long() # [N_p]

            rm = prot_mol_lj_rm[inds_lig][:, inds_pocket] # [N_l, N_p]
            lj = ((rm / dists) ** 12 - (rm / dists) ** 6) # [N_l, N_p]

            lj[torch.isnan(lj)] = 0

            lj = torch.clamp(lj, min=0, max=1000) # [N_l, N_p]

            # -------------  compute the loss for h atoms ----------------
            dists_h = torch.cdist(x, h_coords, p=2)
            dists_h = torch.where(dists_h<0.5, 0.5, dists_h)
            inds_H = torch.ones(len(h_coords), device=x.device).long() * 10 # index of H is 10 in the table
            rm_h = prot_mol_lj_rm[inds_lig][:, inds_H]
            lj_h = ((rm_h / dists_h) ** 12 - (rm_h / dists_h) ** 6) # [N_l, N_p]

            lj_h[torch.isnan(lj_h)] = 0 # remove nan values
            lj_h = torch.clamp(lj_h, min=0, max=1000) # [N_l, N_p]

            lj = lj.mean()
            lj_h = lj_h.mean()

            lj_prot_lig = (lj_prot_lig + (lj + lj_h))

        lj_prot_lig = lj_prot_lig / n

        return lj_prot_lig

    @staticmethod
    def get_batch_mask(mask, device):
        n_nodes = mask.float().sum(dim=1).int()
        batch_size = mask.shape[0]
        batch_mask = torch.cat([torch.ones(n_nodes[i]) * i for i in range(batch_size)]).long().to(device)
        return batch_mask

    def sample_p_zs_given_zt_only_extension(self, s, t, z_t, pocket_xh, scaffold_mask, extension_mask, anchors, pocket_anchors, pocket_mask):
        """ samples zs ~ p(zs|zt) only used during sampling. Samples only the extension features and coords """
        gamma_s = self.gamma(s)
        gamma_t = self.gamma(t)

        sigma2_t_given_s, sigma_t_given_s, alpha_t_given_s = self.sigma_and_alpha_t_given_s(gamma_t, gamma_s, z_t)
        sigma_s = self.sigma(gamma_s, target_tensor=z_t)
        sigma_t = self.sigma(gamma_t, target_tensor=z_t)

        eps_hat = self.dynamics.forward(
            xh=z_t,
            pocket_xh=pocket_xh,
            t=t,
            extension_mask=extension_mask,
            scaffold_mask=scaffold_mask,
            anchors=anchors,
            pocket_anchors=pocket_anchors,
            pocket_mask=pocket_mask,
        )

        eps_hat = eps_hat * extension_mask.unsqueeze(-1) # only the extension part is unmasked
        # compute mu for p(z_s|z_t) (algorithm 2 EDM)
        mu = z_t / alpha_t_given_s - (sigma2_t_given_s / alpha_t_given_s / sigma_t) * eps_hat

        # compute sigma for p(z_s | z_t) (alg 2)
        sigma = sigma_t_given_s * sigma_s / sigma_t

        # sample z_s given the params for z_t
        z_s = self.sample_normal(mu, sigma, extension_mask.unsqueeze(-1))
        z_s = z_t * scaffold_mask.unsqueeze(-1) + z_s * extension_mask.unsqueeze(-1) # [B, N, 3]

        return z_s

    def sample_p_xh_given_z0_only_extension(self, z_0, pocket_xh, scaffold_mask, extension_mask, anchors, pocket_anchors, pocket_mask):

        zeros = torch.zeros(size=(z_0.size(0),1), device=z_0.device)
        gamma_0 = self.gamma(zeros)

        # compute sqrt(sigma_0^2/alpha_0^2)
        sigma_x = self.SNR(-0.5 * gamma_0).unsqueeze(1)

        eps_hat = self.dynamics.forward(
                                    t=zeros,
                                    xh=z_0,
                                    pocket_xh=pocket_xh,
                                    extension_mask=extension_mask,
                                    pocket_mask=pocket_mask,
                                    scaffold_mask=scaffold_mask,
                                    anchors=anchors,
                                    pocket_anchors= pocket_anchors,
                                    )

        eps_hat = eps_hat * extension_mask.unsqueeze(-1)
        mu_x = self.compute_x_pred(eps_t=eps_hat, z_t=z_0, gamma_t=gamma_0)
        xh = self.sample_normal(mu=mu_x, sigma=sigma_x, node_mask=extension_mask.unsqueeze(-1))
        xh = z_0 * scaffold_mask.unsqueeze(-1) + xh * extension_mask.unsqueeze(-1)

        x, h = xh[:, :, :self.n_dims], xh[:, :, self.n_dims:]
        x, h = self.unnormalize(x, h)

        node_mask = (scaffold_mask.bool() | extension_mask.bool()).unsqueeze(-1).float()
        h = F.one_hot(torch.argmax(h, dim=2), self.lig_nf) * node_mask
        return x, h

    def kl_prior(self, xh, num_nodes):
        """
        computes the KL between q(z1|x) and prior p(z1) = Normal(0,1)
        This is essentially a lot of work for something that is in practice neglibible in the loss.
        However, you compute it so that you see it when you've made a mistake in your noise schedule
        """
        # compute the last alpha value, alpha_T
        ones = torch.ones((xh.size(0),1), device=xh.device)
        gamma_T = self.gamma(ones) # the last step
        alpha_T = self.alpha(gamma_T, xh) # the last step

        # compute the means
        mu_T = alpha_T * xh
        mu_T_x, mu_T_h = mu_T[:, :, :self.n_dims], mu_T[:, :, self.n_dims:]

        # compute standard devs (only batch axis for x-part, inflated for h-part)
        sigma_T_x = self.sigma(gamma_T, mu_T_x).view(-1) # remove inflate, only keep batch dim for x-part
        sigma_T_h = self.sigma(gamma_T, mu_T_h)

        # compute KL for h-part
        zeros, ones = torch.zeros_like(mu_T_h), torch.ones_like(sigma_T_h)
        kl_distance_h = self.gaussian_kl(mu_T_h, sigma_T_h, zeros, ones)

        # compute KL for x-part
        zeros, ones = torch.zeros_like(mu_T_x), torch.ones_like(sigma_T_x)
        d = self.subspace_dimensionality(num_nodes)
        kl_distance_x = self.gaussian_kl_for_dimension(mu_T_x, sigma_T_x, zeros, ones, d=d)

        return kl_distance_x + kl_distance_h

    def log_constant_of_p_x_given_z0(self, x, num_nodes):
        batch_size = x.size(0)
        degrees_of_freedom = self.subspace_dimensionality(num_nodes)
        zeros = torch.zeros((batch_size, 1), device=x.device)
        gamma_0 = self.gamma(zeros)

        # Recall that sigma_x = sqrt(sigma_0^2 / alpha_0^2) = SNR(-0.5 gamma_0)
        log_sigma_x = 0.5 * gamma_0.view(batch_size)
        return degrees_of_freedom * (-log_sigma_x - 0.5 * np.log(2 * np.pi))

    def log_p_xh_given_z0_without_constants(self, h, z_0, gamma_0, eps, eps_hat, mask, epsilon=1e-10):
        """ compute log(p(x,h)|z0) without constants
        log p(x|z_0) (no const) = -0.5 ||eps-eps_hat||^2
        log p(h|z_0) (no const) = CDF(h + 0.5 - z_0)/sigma_0 - CDF(h - 0.5 - z_0)/sigma_0
            need to be normalized across categories -> log_sum_exp
        mask: must be the extension_mask for atoms to be generated
        """
        z_h = z_0[:, :, self.n_dims:]

        # take only part of over x
        eps_x = eps[:, :, :self.n_dims]
        eps_hat_x = eps_hat[:, :, :self.n_dims]

        # compute sigma_0 and rescale to integer scale of data
        sigma_0 = self.sigma(gamma_0, target_tensor=z_0) * self.norm_values[1]

        # compute teh error for distribution N(x | 1/alpha_0 z_0 - sigma_0/alpha_0, sigma_0/alpha_0)
        # the weighting of eps parameterization is exactly 1 (eq 19 edm)
        # NOTE: for the next prediction hte previous parts of eps_x and eps_hat_x must be the same
        log_p_x_given_z_without_constants = -0.5 * self.sum_except_batch((eps_x - eps_hat_x) ** 2)

        # categorical features
        # compute delta indicator mask
        h = h * self.norm_values[1] + self.norm_biases[1]
        estimated_h = z_h * self.norm_values[1] + self.norm_biases[1]

        # centered h_cat around 1, since onehot encoded
        centered_h = estimated_h - 1

        # compute integral from 0.5 to 1.5 of normal dist
        log_p_h_proportioal = torch.log(
            self.cdf_standard_gaussian(((centered_h + 0.5)/sigma_0)) -  # TODO: check if mask is actually needed here
            self.cdf_standard_gaussian(((centered_h - 0.5)/sigma_0)) +
            epsilon
        )

        # normalize the distribution over categories
        log_Z = torch.logsumexp(log_p_h_proportioal, dim=2, keepdim=True)
        log_probabilities = log_p_h_proportioal - log_Z

        # select the log_prob of current category using onehot repr
        # mask is [B, N] boolean matrix
        log_p_h_given_z = self.sum_except_batch(log_probabilities * h * mask)

        # combine log probs for x and h
        log_p_xh_given_z = log_p_x_given_z_without_constants + log_p_h_given_z
        return log_p_xh_given_z

    def compute_x_pred(self, eps_t, z_t, gamma_t): # NOTE: may need a mask
        """ computes x_pred, i.e. the most likely prediction of x"""
        sigma_t = self.sigma(gamma_t, target_tensor=eps_t)
        alpha_t = self.alpha(gamma_t, target_tensor=eps_t)
        x_pred = 1. / alpha_t * (z_t - sigma_t * eps_t)
        return x_pred

    def sample_combined_position_features_noise(self, n_samples, n_nodes, mask):
        """ samples gaussian noise for both positions and features and then concatenates them
        uses node_mask on the noises
        """
        z_x = utils.sample_gaussian_with_mask(
            size=(n_samples, n_nodes, self.n_dims),
            device=mask.device,
            node_mask=mask
        )
        z_h = utils.sample_gaussian_with_mask(
            size=(n_samples, n_nodes, self.lig_nf),
            device=mask.device,
            node_mask=mask
        )
        z = torch.cat([z_x, z_h], dim=2)
        return z

    def sample_normal(self, mu, sigma, node_mask):
        """ Samples from a normal dist """
        eps = self.sample_combined_position_features_noise(mu.size(0), mu.size(1), node_mask)
        return mu + sigma * eps

    def normalize(self, x, h):
        new_x = x / self.norm_values[0]
        new_h = (h.float() - self.norm_biases[1]) / self.norm_values[1]
        return new_x, new_h

    def unnormalize(self, x, h):
        new_x = x * self.norm_values[0]
        new_h = h * self.norm_values[1] + self.norm_biases[1]
        return new_x, new_h

    def unnormalize_z(self, z):
        assert z.size(2) == self.n_dims + self.lig_nf
        x, h  = z[:, :, :self.n_dims], z[:, :, self.n_dims:]
        x, h  = self.unnormalize(x, h)
        return torch.cat([x,h], dim=2)

    def delta_log_px(self, num_nodes):
        return -self.subspace_dimensionality(num_nodes) * np.log(self.norm_values[0])

    def dimensionality(self, mask):
        return self.numbers_of_nodes(mask) * self.n_dims

    def sigma(self, gamma, target_tensor):
        """Computes sigma given gamma."""
        return self.inflate_batch_array(torch.sqrt(torch.sigmoid(gamma)), target_tensor)

    def alpha(self, gamma, target_tensor):
        """Computes alpha given gamma."""
        return self.inflate_batch_array(torch.sqrt(torch.sigmoid(-gamma)), target_tensor)

    def SNR(self, gamma):
        """Computes signal to noise ratio (alpha^2/sigma^2) given gamma."""
        return torch.exp(-gamma)

    def sigma_and_alpha_t_given_s(self, gamma_t: torch.Tensor, gamma_s: torch.Tensor, target_tensor: torch.Tensor):
        """
        Computes sigma t given s, using gamma_t and gamma_s. Used during sampling.
        These are defined as:
            alpha t given s = alpha t / alpha s,
            sigma t given s = sqrt(1 - (alpha t given s) ^2 ).
        """
        sigma2_t_given_s = self.inflate_batch_array(
            -self.expm1(self.softplus(gamma_s) - self.softplus(gamma_t)),
            target_tensor
        )

        # alpha_t_given_s = alpha_t / alpha_s
        log_alpha2_t = F.logsigmoid(-gamma_t)
        log_alpha2_s = F.logsigmoid(-gamma_s)
        log_alpha2_t_given_s = log_alpha2_t - log_alpha2_s

        alpha_t_given_s = torch.exp(0.5 * log_alpha2_t_given_s)
        alpha_t_given_s = self.inflate_batch_array(alpha_t_given_s, target_tensor)
        sigma_t_given_s = torch.sqrt(sigma2_t_given_s)

        return sigma2_t_given_s, sigma_t_given_s, alpha_t_given_s

    def subspace_dimensionality(self, input_size):
        """Compute the dimensionality on translation-invariant linear subspace
        where distributions on x are defined."""
        return (input_size - 1) * self.n_dims

    @staticmethod
    def numbers_of_nodes(mask):
        if len(mask.squeeze().shape) > 1:
            return torch.sum(mask.squeeze(), dim=1)
        else:
            return torch.sum(mask.squeeze())

    @staticmethod
    def inflate_batch_array(array, target):
         """ inflates teh batch array with only a single axis (i.e. shape = (batch_size,)
         or possibly more empty axes (batch_size, 1, ..., 1) to match the target)
         """
         target_shape = (array.size(0),) + (1,) * (len(target.size())-1)
         return array.view(target_shape)

    @staticmethod
    def sum_except_batch(x):
        return x.view(x.size(0), -1).sum(-1) # sum everyting except batch dimension

    @staticmethod
    def expm1(x):
        return torch.expm1(x)

    @staticmethod
    def softplus(x):
        return F.softplus(x)

    @staticmethod
    def cdf_standard_gaussian(x):
        """ Returns the cumulative distribution function (CDF) of gaussian """
        return 0.5 * (1 + torch.erf(x/math.sqrt(2)))

    @staticmethod
    def gaussian_kl(q_mu, q_sigma, p_mu, p_sigma):
        """
        Computes KL between two normal dists
        Args:
            q_mu: Mean of q
            q_sigma: std of q
            p_mu: mean of p
            p_sigma: std of p
        Returns:
            KL distance summed over all dims except the batch dim
        """
        kl = torch.log(p_sigma / q_sigma) + 0.5 * (q_sigma ** 2 + (q_mu - p_mu) **2) / (p_sigma ** 2) - 0.5
        return AREDM.sum_except_batch(kl)

    def subspace_dimensionality(self, input_size):
        """Compute the dimensionality on translation-invariant linear subspace
        where distributions on x are defined."""
        return (input_size - 1) * self.n_dims

    @staticmethod
    def gaussian_kl_for_dimension(q_mu, q_sigma, p_mu, p_sigma, d):
        """
        computes the KL between two normal dists taking the dimension into account
        Returns:
            KL distance summed over all dims except the batch dim
        """
        mu_norm_2 = AREDM.sum_except_batch((q_mu - p_mu) ** 2)
        return d * torch.log(p_sigma / q_sigma) + 0.5 * (d * q_sigma ** 2 + mu_norm_2) / (p_sigma ** 2) - 0.5 * d


import boto3
import json
import string
import random

def lambda_handler(event, context):
    body = json.loads(event['body'])
    recipename = body.get('recipename')
    recipe = body.get('recipe')
    restaurantName = body.get('restaurantName')
    ingredients = body.get('ingredients')
    recipeid = body.get('recipeid')
    dynamodb = boto3.resource('dynamodb')
    table = dynamodb.Table('Recipe')
    print(body)

    #Author(s) name: Boto3 Docs
    #Date: 28 November 2022
    #Title of program/source code: 1.26.22 Documentations
    #Code version: v1
    #Type: Code
    #Web address: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/dynamodb.html#DynamoDB.Table.put_item

    table.put_item(
		Item = {
			'recipeid': recipeid,
			'restaurantName': restaurantName,
			'title': recipename,
			'recipe': recipe,
			'ingredients': ingredients
		}
	)

    return {
        'statusCode': 200,
        'body': json.dumps('Hello from Lambda!')
    }