from eflow.utils.sys_utils import *
from eflow.utils.pandas_utils import df_to_image
from eflow.utils.image_processing_utils import create_plt_png
from eflow._hidden.parent_objects import ModelAnalysis
from eflow._hidden.custom_exceptions import RequiresPredictionMethods, ProbasNotPossible, \
    UnsatisfiedRequirments
from eflow.data_analysis import FeatureAnalysis
from eflow.data_pipeline_segments import DataEncoder

from eflow._hidden.constants import GRAPH_DEFAULTS

from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
from sklearn.metrics import matthews_corrcoef
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import classification_report
import scikitplot as skplt
import numpy as np
import warnings
import copy
import pandas as pd
from IPython.display import display
import matplotlib.pyplot as plt
import seaborn as sns

__author__ = "Eric Cacciavillani"
__copyright__ = "Copyright 2019, eFlow"
__credits__ = ["Eric Cacciavillani"]
__license__ = "MIT"
__maintainer__ = "EricCacciavillani"
__email__ = "eric.cacciavillani@gmail.com"

class ClassificationAnalysis(ModelAnalysis):

    """
        Analyzes a classification model's result's based on the prediction
        function(s) passed to it. Creates graphs and tables to be saved in directory
        structure.
    """

    def __init__(self,
                 dataset_name,
                 model,
                 model_name,
                 feature_order,
                 target_feature,
                 pred_funcs_dict,
                 df_features,
                 sample_data,
                 project_sub_dir="Classification Analysis",
                 overwrite_full_path=None,
                 target_classes=None,
                 save_model=True,
                 notebook_mode=False):
        """
        Args:
            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            model:
                A fitted supervised machine learning model.

            model_name:
                The name of the model in string form.

            feature_order: collection object
                Features names in proper order to re-create the pandas dataframe.

            pred_funcs_dict:
                A dict of the name of the function and the function defintion for the
                model prediction methods.
                (Can handle either a return of probabilities or a singile value.)
                Init Example:
                pred_funcs = dict()
                pred_funcs["Predictions"] = model.predict
                pred_funcs["Probabilities"] = model.probas

            sample_data:
                Given data to then pass into our prediction functions to get a
                resultant to get the classification prediction 'type'.
                Can be a matrix or a vector.

            project_sub_dir:
                Creates a parent or "project" folder in which all sub-directories
                will be inner nested.

            overwrite_full_path:
                Overwrites the path to the parent folder.

            target_classes:
                Specfied list/np.array of targeted classes the model predicts. If set to
                none then it will attempt to pull from the sklearn's default attribute
                '.classes_'.

            df_features:
                DataFrameTypes object; organizes feature types into groups.
        """

        # Init parent object
        ModelAnalysis.__init__(self,
                               f'{dataset_name}/{project_sub_dir}/Target Feature: {target_feature}/{model_name}',
                               overwrite_full_path)

        # Init objects without pass by refrence

        # Remove target feature from feature order when trying to recreate dataframe
        self.__target_feature = copy.deepcopy(target_feature)
        self.__feature_order = copy.deepcopy(feature_order)

        if self.__target_feature in self.__feature_order:
            self.__feature_order.remove(self.__target_feature)

        del feature_order
        del target_feature

        self.__model = copy.deepcopy(model)

        self.__model_name = copy.deepcopy(model_name)
        self.__target_values = copy.deepcopy(target_classes)
        self.__df_features = copy.deepcopy(df_features)
        self.__pred_funcs_dict = copy.deepcopy(pred_funcs_dict)
        self.__pred_funcs_types = dict()
        self.__notebook_mode = copy.deepcopy(notebook_mode)

        # Determines if the perform was called
        self.__called_from_perform = False

        # Init on sklearns default target classes attribute
        if not self.__target_values:
            self.__target_values = copy.deepcopy(model.classes_)
        # ---
        if len(self.__target_values) != 2:
            self.__binary_classifcation = False
        else:
            self.__binary_classifcation = True

        # Attempt to save machine learning model
        try:
            if save_model:
                pickle_object_to_file(self.__model,
                                      self.folder_path,
                                      f'{self.__model_name}')
        except:
            pass
        # ---
        create_dir_structure(self.folder_path,
                             "_Extras")
        # Save predicted classes
        write_object_text_to_file(self.__target_values,
                                  self.folder_path + "_Extras",
                                  "_Classes")

        # Save feature order
        write_object_text_to_file(self.__feature_order,
                                  self.folder_path + "_Extras",
                                  "_Feature_Order")

        # Save features and or df_features object
        df_features.create_json_file_representation(self.folder_path + "_Extras",
                                                    "df_features.json")

        self.__sample_data = None

        # Extract sample data
        if len(sample_data.shape) == 2:
            self.__sample_data = np.reshape(sample_data[0],
                                            (-1, sample_data.shape[1]))
        elif len(sample_data.shape) == 1:
            self.__sample_data = [sample_data]
        else:
            raise UnsatisfiedRequirments("This program can only handle 1D and 2D matrices.")

        # Find the 'type' of each prediction. Probabilities or Predictions
        if self.__pred_funcs_dict:
            for pred_name, pred_func in self.__pred_funcs_dict.items():
                try:
                    model_output = pred_func(self.__sample_data)[0]
                except Exception as e:
                    model_output = pred_func(np.array(self.__sample_data))[0]

                # Confidence / Probability (Continuous output)
                if isinstance(model_output, list) or isinstance(model_output,
                                                                np.ndarray):
                    self.__pred_funcs_types[pred_name] = "Probabilities"

                # Classification (Discrete output)
                else:
                    self.__pred_funcs_types[pred_name] = "Predictions"
        else:
            raise RequiresPredictionMethods("This object requires you to pass the prediction methods in a dict with the name of the method as the key.")

        try:
            feature_importances = model.feature_importances_

            self.graph_model_importances(copy.deepcopy(self.__feature_order),
                                         feature_importances,
                                         display_visuals=False)

        except AttributeError:
            pass

    def get_predictions_names(self):
        return self.__pred_funcs_dict.keys()

    def perform_analysis(self,
                         X,
                         y,
                         dataset_name,
                         thresholds_matrix=None,
                         classification_error_analysis=False,
                         classification_correct_analysis=False,
                         ignore_metrics=[],
                         custom_metrics_dict=dict(),
                         average_scoring=["micro",
                                          "macro",
                                          "weighted"],
                         display_visuals=True):
        """

            Runs all available analysis functions on the models predicted data.

        Args:
            X:
                Feature matrix.

            y:
                Target data vector.

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds_matrix:
                List of list/matrix of thresholds.

                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            classification_error_analysis: bool
                Perform feature analysis on data that was incorrectly predicted.

            classification_correct_analysis: bool
                Perform feature analysis on data that was correctly predicted.

            figsize:
                All plot's dimension's.

            ignore_metrics:
                Specify the default metrics to not apply to the classification
                data_analysis.
                    * Precision
                    * MCC
                    * Recall
                    * F1-Score
                    * Accuracy

            custom_metrics_dict:
                Pass the name of metric(s) with the function definition(s) in a
                dictionary.

            average_scoring:
                Determines the type of averaging performed on the data.

            display_visuals:
                Controls visual display of error error data_analysis if it is able to run.

        Returns:
            Performs all classification functionality with the provided feature
            data and target data.
                * plot_precision_recall_curve
                * classification_evaluation
                * plot_confusion_matrix
        """
        try:
            # Convert to
            if isinstance(thresholds_matrix, np.ndarray):
                thresholds_matrix = thresholds_matrix.tolist()

            if not thresholds_matrix:
                thresholds_matrix = [[]]

            if isinstance(thresholds_matrix, list) and not isinstance(
                    thresholds_matrix[0], list):
                thresholds_matrix = list(thresholds_matrix)

            none_required = False
            for vector in thresholds_matrix:
                if not vector or len(vector) == 0:
                    none_required = True

            if not none_required:
                thresholds_matrix.append(None)

            self.__called_from_perform = True

            self.generate_matrix_meta_data(X,
                                           dataset_name + "/_Extras")

            print("\n\n" + "---" * 10 + f'{dataset_name}' + "---" * 10)
            first_iteration = True

            for pred_name, pred_type in self.__pred_funcs_types.items():

                # Nicer formating
                if not first_iteration:
                    print("\n\n\n")
                first_iteration = False

                for thresholds in thresholds_matrix:

                    print(f"Now running classification on {pred_name}", end='')
                    if pred_type == "Predictions":
                        print()
                        thresholds = None
                    else:
                        if thresholds:
                            print(f" on thresholds:")
                            for i,target_val in enumerate(self.__target_values):
                                try:
                                    print(f"\tTarget Value:{target_val}: Prediction weight: {thresholds[i]}")

                                except IndexError:
                                    raise IndexError("Thresholds must of the same length as the target values!")
                        else:
                            print(" on no thresholds.")

                    if display_visuals:
                        try:
                            print(f"\nShape of the data is {X.shape}")
                        except AttributeError:
                            pass

                    self.classification_metrics(X,
                                                y,
                                                pred_name=pred_name,
                                                dataset_name=dataset_name,
                                                thresholds=thresholds,
                                                ignore_metrics=ignore_metrics,
                                                custom_metrics_dict=custom_metrics_dict,
                                                average_scoring=average_scoring,
                                                display_visuals=display_visuals)

                    self.plot_confusion_matrix(X,
                                               y,
                                               pred_name=pred_name,
                                               dataset_name=dataset_name,
                                               thresholds=thresholds,
                                               normalize=True,
                                               display_visuals=display_visuals)

                    self.plot_confusion_matrix(X,
                                               y,
                                               pred_name=pred_name,
                                               dataset_name=dataset_name,
                                               thresholds=thresholds,
                                               normalize=False,
                                               display_visuals=display_visuals)

                    if pred_type == "Probabilities":
                        self.plot_precision_recall_curve(X,
                                                         y,
                                                         pred_name=pred_name,
                                                         dataset_name=dataset_name,
                                                         thresholds=thresholds,
                                                         display_visuals=display_visuals)
                        self.plot_roc_curve(X,
                                            y,
                                            pred_name=pred_name,
                                            dataset_name=dataset_name,
                                            thresholds=thresholds,
                                            display_visuals=display_visuals)

                        if self.__binary_classifcation:

                            self.plot_lift_curve(X,
                                                 y,
                                                 pred_name=pred_name,
                                                 dataset_name=dataset_name,
                                                 thresholds=thresholds,
                                                 display_visuals=display_visuals)

                            self.plot_ks_statistic(X,
                                                   y,
                                                   pred_name=pred_name,
                                                   dataset_name=dataset_name,
                                                   thresholds=thresholds,
                                                   display_visuals=display_visuals)

                            self.plot_cumulative_gain(X,
                                                      y,
                                                      pred_name=pred_name,
                                                      dataset_name=dataset_name,
                                                      thresholds=thresholds,
                                                      display_visuals=display_visuals)

                    if classification_error_analysis:
                        self.classification_error_analysis(X,
                                                           y,
                                                           pred_name,
                                                           dataset_name,
                                                           thresholds=thresholds,
                                                           display_visuals=False,
                                                           save_file=True,
                                                           aggerate_target=True,
                                                           display_print=False,
                                                           suppress_runtime_errors=True,
                                                           aggregate_target_feature=True,
                                                           selected_features=None,
                                                           extra_tables=True,
                                                           statistical_analysis_on_aggregates=True)

                    if classification_correct_analysis:
                        self.classification_correct_analysis(X,
                                                             y,
                                                             pred_name,
                                                             dataset_name,
                                                             thresholds=thresholds,
                                                             display_visuals=False,
                                                             save_file=True,
                                                             aggerate_target=True,
                                                             display_print=False,
                                                             suppress_runtime_errors=True,
                                                             aggregate_target_feature=True,
                                                             selected_features=None,
                                                             extra_tables=True,
                                                             statistical_analysis_on_aggregates=True)

                    print("-" * (len(dataset_name) + 60) + "\n")
                    if pred_type == "Predictions":
                        break
        finally:
            self.__called_from_perform = False

    def plot_ks_statistic(self,
                          X,
                          y,
                          pred_name,
                          dataset_name,
                          thresholds=None,
                          display_visuals=True,
                          save_file=True,
                          title=None,
                          ax=None,
                          figsize=GRAPH_DEFAULTS.FIGSIZE,
                          title_fontsize='large',
                          text_fontsize='medium'):

        """

            From scikit-plot documentation
            Link: http://tinyurl.com/y3ym5pyc
            Generates the KS Statistic plot from labels and scores/probabilities.

        Args:
            X:
                Feature matrix.

             y:
                Target data vector.

            pred_name:
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals:
                Visualize graph if needed.

            save_file:
                Boolean value to whether or not to save the file.
        """

        filename = f'KS Statistic on {dataset_name} on {self.__model_name}'
        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)
        if not title:
            title = filename

        skplt.metrics.plot_ks_statistic(y,
                                        self.__get_model_probas(pred_name,
                                                                X),
                                        title=title,
                                        ax=ax,
                                        figsize=figsize,
                                        title_fontsize=title_fontsize,
                                        text_fontsize=text_fontsize)
        legend = plt.legend(frameon=1)
        frame = legend.get_frame()
        frame.set_facecolor('white')
        frame.set_edgecolor('white')
        if save_file:
            self.save_plot(filename=filename,
                           sub_dir=sub_dir)

            if not self.__called_from_perform:
                self.generate_matrix_meta_data(X,
                                               dataset_name + "/_Extras")

        if display_visuals:
            plt.show()
        plt.close()

    def plot_roc_curve(self,
                       X,
                       y,
                       pred_name,
                       dataset_name,
                       thresholds=None,
                       display_visuals=True,
                       save_file=True,
                       title=None,
                       ax=None,
                       figsize=GRAPH_DEFAULTS.FIGSIZE,
                       title_fontsize='large',
                       text_fontsize='medium'):

        """

            From scikit-plot documentation
            Link: http://tinyurl.com/y3ym5pyc
            Creates ROC curves from labels and predicted probabilities.

        Args:
             X:
                Feature matrix.

             y:
                Target data vector.

            pred_name:
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals:
                Visualize graph if needed.

            save_file:
                Boolean value to whether or not to save the file.
        """
        filename = f'Roc Curve on {dataset_name} on {self.__model_name}'
        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)
        if not title:
            title = filename

        skplt.metrics.plot_roc(y,
                               self.__get_model_probas(pred_name,
                                                       X),
                               title=title,
                               ax=ax,
                               figsize=figsize,
                               title_fontsize=title_fontsize,
                               text_fontsize=text_fontsize)

        legend = plt.legend(frameon=1)
        frame = legend.get_frame()
        frame.set_facecolor('white')
        frame.set_edgecolor('white')

        if save_file:
            self.save_plot(filename=filename,
                           sub_dir=sub_dir)
            if not self.__called_from_perform:
                self.generate_matrix_meta_data(X,
                                               dataset_name + "/_Extras")

        if display_visuals:
            plt.show()
        plt.close()

    def plot_cumulative_gain(self,
                             X,
                             y,
                             pred_name,
                             dataset_name,
                             thresholds=None,
                             display_visuals=True,
                             save_file=True,
                             title=None,
                             ax=None,
                             figsize=GRAPH_DEFAULTS.FIGSIZE,
                             title_fontsize='large',
                             text_fontsize='medium'):

        """

            From scikit-plot documentation
            Link: http://tinyurl.com/y3ym5pyc
            Plots calibration curves for a set of classifier probability estimates.

        Args:
            X:
                Feature matrix.

             y:
                Target data vector.

            pred_name:
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals:
                Visualize graph if needed.

            save_file:
                Boolean value to whether or not to save the file.
        """
        filename = f'Cumulative Gain gain on {dataset_name} on {self.__model_name}'
        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)
        if not title:
            title = filename

        skplt.metrics.plot_cumulative_gain(y,
                                           self.__get_model_probas(pred_name,
                                                               X),
                                           title=title,
                                           ax=ax,
                                           figsize=figsize,
                                           title_fontsize=title_fontsize,
                                           text_fontsize=text_fontsize)

        legend = plt.legend(frameon=1)
        frame = legend.get_frame()
        frame.set_facecolor('white')
        frame.set_edgecolor('white')

        if save_file:
            self.save_plot(filename=filename,
                           sub_dir=sub_dir)

            if not self.__called_from_perform:
                self.generate_matrix_meta_data(X,
                                               dataset_name + "/_Extras")

        if display_visuals:
            plt.show()
        plt.close()

    def plot_precision_recall_curve(self,
                                    X,
                                    y,
                                    pred_name,
                                    dataset_name,
                                    thresholds=None,
                                    display_visuals=True,
                                    save_file=True,
                                    title=None,
                                    plot_micro=True,
                                    classes_to_plot=None,
                                    ax=None,
                                    figsize=GRAPH_DEFAULTS.FIGSIZE,
                                    cmap='nipy_spectral',
                                    title_fontsize='large',
                                    text_fontsize='medium'):
        """

            From scikit-plot documentation
            Link: http://tinyurl.com/y3ym5pyc
            Plots precision recall curve plot based on the models predictions.

        Args:
            X:
                Feature matrix.

            y:
                Target data vector.

            pred_name:
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals:
                Visualize graph if needed.

            save_file:
                Boolean value to whether or not to save the file.
        """

        filename = f'Precision Recall on {dataset_name} on {self.__model_name}'
        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)
        if not title:
            title = filename

        skplt.metrics.plot_precision_recall(y,
                                            self.__get_model_probas(pred_name,
                                                                    X),
                                            title=title,
                                            plot_micro=plot_micro,
                                            classes_to_plot=classes_to_plot,
                                            ax=ax,
                                            figsize=figsize,
                                            cmap=cmap,
                                            title_fontsize=title_fontsize,
                                            text_fontsize=text_fontsize)

        legend = plt.legend(frameon=1)
        frame = legend.get_frame()
        frame.set_facecolor('white')
        frame.set_edgecolor('white')

        if save_file:
            self.save_plot(filename=filename,
                           sub_dir=sub_dir)

            if not self.__called_from_perform:
                self.generate_matrix_meta_data(X,
                                               dataset_name + "/_Extras")

        if display_visuals:
            plt.show()
        plt.close()

    def plot_lift_curve(self,
                        X,
                        y,
                        pred_name,
                        dataset_name,
                        thresholds=None,
                        display_visuals=True,
                        save_file=True,
                        title=None,
                        ax=None,
                        figsize=GRAPH_DEFAULTS.FIGSIZE,
                        title_fontsize='large',
                        text_fontsize='medium'):
        """

            From scikit-plot documentation
            Link: http://tinyurl.com/y3ym5pyc
            The lift curve is used to determine the effectiveness of a binary classifier.
            A detailed explanation can be found at http://tinyurl.com/csegj9.
            The implementation here works only for binary classification.

        Args:
            X:
                Feature matrix.

             y:
                Target data vector.

            pred_name:
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals:
                Visualize graph if needed.

            save_file:
                Boolean value to whether or not to save the file.
        """

        filename = f'Lift Curve on {dataset_name} on {self.__model_name}'
        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)
        if not title:
            title = filename

        skplt.metrics.plot_lift_curve(y,
                                      self.__get_model_probas(pred_name,
                                                              X),
                                      title=title,
                                      ax=ax,
                                      figsize=figsize,
                                      title_fontsize=title_fontsize,
                                      text_fontsize=text_fontsize)

        legend = plt.legend(frameon=1)
        frame = legend.get_frame()
        frame.set_facecolor('white')
        frame.set_edgecolor('white')

        if save_file:
            self.save_plot(filename=filename,
                           sub_dir=sub_dir)

            if not self.__called_from_perform:
                self.generate_matrix_meta_data(X,
                                               dataset_name + "/_Extras")

        if display_visuals:
            plt.show()
        plt.close()

    def plot_confusion_matrix(self,
                              X,
                              y,
                              pred_name,
                              dataset_name,
                              thresholds=None,
                              display_visuals=True,
                              save_file=True,
                              title=None,
                              normalize=False,
                              hide_zeros=False,
                              hide_counts=False,
                              x_tick_rotation=0,
                              ax=None,
                              figsize=GRAPH_DEFAULTS.FIGSIZE,
                              cmap='Blues',
                              title_fontsize='large',
                              text_fontsize='medium'):
        """

            From scikit-plot documentation
            Link: http://tinyurl.com/y3ym5pyc
            Creates a confusion matrix plot based on the models predictions.

        Args:
            X:
                Feature matrix.

             y:
                Target data vector.

            pred_name:
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals:
                Visualize graph if needed.

            save_file:
                Boolean value to whether or not to save the file.
        """
        filename = f'Confusion Matrix: {dataset_name} on {self.__model_name} Normalized: {normalize}'
        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)
        if not title:
            title = filename

        warnings.filterwarnings('ignore')
        ax = skplt.metrics.plot_confusion_matrix(
             self.__get_model_prediction(pred_name,
                                         X,
                                         thresholds),
             y,
             title=title,
             normalize=normalize,
             hide_zeros=hide_zeros,
             hide_counts=hide_counts,
             x_tick_rotation=x_tick_rotation,
             ax=ax,
             figsize=figsize,
             cmap=cmap,
             title_fontsize=title_fontsize,
             text_fontsize=text_fontsize)
        warnings.filterwarnings('default')

        if save_file:
            self.save_plot(filename=filename,
                           sub_dir=sub_dir)
            if not self.__called_from_perform:
                self.generate_matrix_meta_data(X,
                                               dataset_name + "/_Extras")

        if display_visuals:
            plt.show()
        plt.close()

    def classification_metrics(self,
                               X,
                               y,
                               pred_name,
                               dataset_name,
                               thresholds=None,
                               display_visuals=True,
                               save_file=True,
                               title="",
                               custom_metrics_dict=dict(),
                               ignore_metrics=[],
                               average_scoring=["micro",
                                                "macro",
                                                "weighted"]):
        """

            Creates a dataframe based on the prediction metrics
            of the feature matrix and target vector.

        Args:
            X:
                Feature matrix.

            y: list or np.array
                Target data vector.

            pred_name:
                The name of the prediction function in questioned stored
                in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals:
                Display tables.

            save_file:
                Determines whether or not to save the generated document.

            title:
                Adds to the column 'Metric Score'.

            custom_metrics_dict:
                Pass the name of metric(s) and the function definition(s) in a
                dictionary.

            ignore_metrics:
                Specify the default metrics to not apply to the classification
                data_analysis.
                    * Precision
                    * MCC
                    * Recall
                    * F1-Score
                    * Accuracy

            average_scoring:
                Determines the type of averaging performed on the data.
                    * micro
                    * macro
                    * weighted

        Returns:
            Return a dataframe object of the metrics value.
        """
        filename = f'Metric Evaluation on {dataset_name} on {self.__model_name}'
        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)

        if not isinstance(average_scoring, list):
            average_scoring = [average_scoring]

        # Default metric name's and their function
        metric_functions = dict()
        metric_functions["Precision"] = precision_score
        metric_functions["MCC"] = matthews_corrcoef
        metric_functions["Recall"] = recall_score
        metric_functions["F1-Score"] = f1_score
        metric_functions["Accuracy"] = accuracy_score

        warnings.filterwarnings('ignore')

        # Ignore default metrics if needed
        for remove_metric in ignore_metrics:
            if remove_metric in metric_functions:
                del metric_functions[remove_metric]

        # Add in custom metrics
        if len(custom_metrics_dict.keys()):
            metric_functions.update(custom_metrics_dict)

        # Evaluate model on metrics
        evaluation_report = dict()
        for metric_name in metric_functions.keys():
            for average_score in average_scoring:

                model_predictions = self.__get_model_prediction(pred_name,
                                                                X,
                                                                thresholds)
                try:
                    evaluation_report[f'{metric_name}({average_score})'] = \
                        metric_functions[metric_name](y_true=y,
                                                      y_pred=model_predictions,
                                                      average=average_score)
                except TypeError:

                    if metric_name not in evaluation_report.keys():
                        evaluation_report[metric_name] = metric_functions[
                            metric_name](y,
                                         model_predictions)
                    continue

        warnings.filterwarnings('default')

        if title and len(title) > 0:
            index_name = f"Metric Scores ({title})"
        else:
            index_name = "Metric Scores"

        # ---
        evaluation_report = pd.DataFrame({index_name:
                                              [f'{metric_score:.4f}'
                                               for metric_score
                                               in evaluation_report.values()]},
                                         index=list(evaluation_report.keys()))

        if display_visuals:
            if self.__notebook_mode:
                display(evaluation_report)
            else:
                print(evaluation_report)

        if save_file:
            df_to_image(evaluation_report,
                        self.folder_path,
                        sub_dir,
                        convert_to_filename(filename),
                        col_width=20,
                        show_index=True,
                        format_float_pos=4)

            if not self.__called_from_perform:
                self.generate_matrix_meta_data(X,
                                               dataset_name + "/_Extras")

    def classification_correct_analysis(self,
                                        X,
                                        y,
                                        pred_name,
                                        dataset_name,
                                        thresholds=None,
                                        display_visuals=True,
                                        save_file=True,
                                        aggerate_target=False,
                                        display_print=True,
                                        suppress_runtime_errors=True,
                                        aggregate_target_feature=True,
                                        selected_features=None,
                                        extra_tables=True,
                                        statistical_analysis_on_aggregates=True):
        """

            Compares the actual target value to the predicted value and performs
            analysis of all the data.

        Args:
            X: np.matrix or lists of lists
                Feature matrix.

            y: list or np.array
                Target data vector.

            pred_name: str
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name: str
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            feature_order: collection object
                Features names in proper order to re-create the pandas dataframe.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals: bool
                Boolean value to whether or not to display visualizations.

            display_print: bool
                Determines whether or not to print function's embedded print
                statements.

            save_file: bool
                Boolean value to whether or not to save the file.

            dataframe_snapshot: bool
                Boolean value to determine whether or not generate and compare a
                snapshot of the dataframe in the dataset's directory structure.
                Helps ensure that data generated in that directory is correctly
                associated to a dataframe.

            suppress_runtime_errors: bool
                If set to true; when generating any graphs will suppress any runtime
                errors so the program can keep running.

            extra_tables: bool
                When handling two types of features if set to true this will
                    generate any extra tables that might be helpful.
                    Note -
                        These graphics may create duplicates if you already applied
                        an aggregation in 'perform_analysis'

            aggregate_target_feature: bool
                Aggregate the data of the target feature if the data is
                non-continuous data.

                Note
                    In the future I will have this also working with continuous
                    data.

            selected_features: collection object of features
                Will only focus on these selected feature's and will ignore
                the other given features.

            statistical_analysis_on_aggregates: bool
                If set to true then the function 'statistical_analysis_on_aggregates'
                will run; which aggregates the data of the target feature either
                by discrete values or by binning/labeling continuous data.
        """

        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)

        model_predictions = self.__get_model_prediction(pred_name,
                                                        X,
                                                        thresholds=thresholds)

        if sum(model_predictions != y) == len(y):
            print("Your model predicted everything correctly for this dataset! No correct analysis needed!")
            print("Also sorry for your model...zero correct? Dam...")
        else:
            print("\n\n" + "*" * 10 +
                  "Generating graphs for when the model predicted correctly" +
                  "*" * 10 + "\n")

            # Generate error dataframe
            correct_df = pd.DataFrame.from_records(X[model_predictions == y])
            correct_df.columns = self.__feature_order
            correct_df[self.__target_feature] = y[model_predictions == y]

            # Directory path
            create_dir_structure(self.folder_path,
                                 sub_dir + "/Correctly Predicted Data/All Correct Data")
            output_path = f"{self.folder_path}/{sub_dir}/Correctly Predicted Data"

            tmp_df_features = copy.deepcopy(self.__df_features)

            data_encoder = DataEncoder(create_file=False)

            data_encoder.revert_dummies(correct_df,
                                        tmp_df_features,
                                        qualitative_features=list(self.__df_features.get_dummy_encoded_features().keys()))

            data_encoder.decode_data(correct_df,
                                     tmp_df_features,
                                     apply_value_representation=False)

            data_encoder.apply_value_representation(correct_df,
                                                    tmp_df_features)
            del data_encoder

            # Create feature analysis
            feature_analysis = FeatureAnalysis(tmp_df_features,
                                               overwrite_full_path=output_path + "/All Correct Data")
            feature_analysis.perform_analysis(correct_df,
                                              dataset_name=dataset_name,
                                              target_features=[self.__target_feature],
                                              save_file=save_file,
                                              selected_features=selected_features,
                                              suppress_runtime_errors=suppress_runtime_errors,
                                              display_print=display_print,
                                              display_visuals=display_visuals,
                                              dataframe_snapshot=False,
                                              aggregate_target_feature=aggregate_target_feature,
                                              extra_tables=extra_tables,
                                              statistical_analysis_on_aggregates=statistical_analysis_on_aggregates)

            # Aggregate target by predicted and actual
            if aggerate_target:
                targets = set(y)
                for pred_target in targets:
                    if pred_target != pred_target:
                        create_dir_structure(output_path,
                                             f"/Actual and Predicted:{pred_target}")

                        # Create predicted vs actual dataframe
                        tmp_df = copy.deepcopy(correct_df[correct_df[
                                                              self.__target_feature] == pred_target])

                        if tmp_df.shape[0]:
                            # Create feature analysis directory structure with given graphics
                            feature_analysis = FeatureAnalysis(
                                tmp_df_features,
                                overwrite_full_path=f"/Actual and Predicted:{pred_target}")
                            feature_analysis.perform_analysis(tmp_df,
                                                              dataset_name=dataset_name,
                                                              target_features=[
                                                                  self.__target_feature],
                                                              save_file=save_file,
                                                              selected_features=selected_features,
                                                              suppress_runtime_errors=suppress_runtime_errors,
                                                              display_print=display_print,
                                                              display_visuals=display_visuals,
                                                              dataframe_snapshot=False,
                                                              extra_tables=extra_tables,
                                                              statistical_analysis_on_aggregates=statistical_analysis_on_aggregates)

    def classification_error_analysis(self,
                                      X,
                                      y,
                                      pred_name,
                                      dataset_name,
                                      thresholds=None,
                                      display_visuals=True,
                                      save_file=True,
                                      aggerate_target=False,
                                      display_print=True,
                                      suppress_runtime_errors=True,
                                      aggregate_target_feature=True,
                                      selected_features=None,
                                      extra_tables=True,
                                      statistical_analysis_on_aggregates=True):
        """

            Compares the actual target value to the predicted value and performs
            analysis of all the data.

        Args:
            X: np.matrix or lists of lists
                Feature matrix.

            y: list or np.array
                Target data vector.

            pred_name: str
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name: str
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            feature_order: collection object
                Features names in proper order to re-create the pandas dataframe.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals: bool
                Boolean value to whether or not to display visualizations.

            display_print: bool
                Determines whether or not to print function's embedded print
                statements.

            save_file: bool
                Boolean value to whether or not to save the file.

            dataframe_snapshot: bool
                Boolean value to determine whether or not generate and compare a
                snapshot of the dataframe in the dataset's directory structure.
                Helps ensure that data generated in that directory is correctly
                associated to a dataframe.

            suppress_runtime_errors: bool
                If set to true; when generating any graphs will suppress any runtime
                errors so the program can keep running.

            extra_tables: bool
                When handling two types of features if set to true this will
                    generate any extra tables that might be helpful.
                    Note -
                        These graphics may create duplicates if you already applied
                        an aggregation in 'perform_analysis'

            aggregate_target_feature: bool
                Aggregate the data of the target feature if the data is
                non-continuous data.

                Note
                    In the future I will have this also working with continuous
                    data.

            selected_features: collection object of features
                Will only focus on these selected feature's and will ignore
                the other given features.

            statistical_analysis_on_aggregates: bool
                If set to true then the function 'statistical_analysis_on_aggregates'
                will run; which aggregates the data of the target feature either
                by discrete values or by binning/labeling continuous data.
        """

        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)

        model_predictions = self.__get_model_prediction(pred_name,
                                                        X,
                                                        thresholds=thresholds)

        if sum(model_predictions == y) == len(y):
            print("Your model predicted everything correctly for this dataset! No error analysis needed!")
        else:
            print("\n\n" + "*" * 10 +
                  "Generating graphs for when the model predicted incorrectly" +
                  "*" * 10 + "\n")

            # Generate error dataframe
            error_df = pd.DataFrame.from_records(X[model_predictions != y])
            error_df.columns = self.__feature_order
            error_df[self.__target_feature] = y[model_predictions != y]
            error_df.reset_index(drop=True,
                                 inplace=True)

            # Directory path
            create_dir_structure(self.folder_path,
                                 sub_dir + "/Incorrectly Predicted Data/All Incorrect Data")
            output_path = f"{self.folder_path}/{sub_dir}/Incorrectly Predicted Data"

            tmp_df_features = copy.deepcopy(self.__df_features)

            data_encoder = DataEncoder(create_file=False)

            data_encoder.revert_dummies(error_df,
                                        tmp_df_features,
                                        qualitative_features=list(self.__df_features.get_dummy_encoded_features().keys()))

            data_encoder.decode_data(error_df,
                                     tmp_df_features,
                                     apply_value_representation=False)

            data_encoder.apply_value_representation(error_df,
                                                    tmp_df_features)
            del data_encoder

            # Create feature analysis
            feature_analysis = FeatureAnalysis(tmp_df_features,
                                               overwrite_full_path=output_path + "/All Incorrect Data")
            feature_analysis.perform_analysis(error_df,
                                              dataset_name=dataset_name,
                                              target_features=[self.__target_feature],
                                              save_file=save_file,
                                              selected_features=selected_features,
                                              suppress_runtime_errors=suppress_runtime_errors,
                                              display_print=display_print,
                                              display_visuals=display_visuals,
                                              dataframe_snapshot=False,
                                              aggregate_target_feature=aggregate_target_feature,
                                              extra_tables=extra_tables,
                                              statistical_analysis_on_aggregates=statistical_analysis_on_aggregates)

            # Aggregate target by predicted and actual
            if aggerate_target:
                targets = set(y)
                prediction_feature = self.__target_feature + "_MODEL_PREDICTIONS_"
                error_df[prediction_feature] = model_predictions[model_predictions != y]
                for actual_target in targets:
                    for pred_target in targets:
                        if pred_target != actual_target:
                            create_dir_structure(output_path,
                                                 f"/Predicted:{pred_target} Actual: {actual_target}")

                            # Create predicted vs actual dataframe
                            tmp_df = copy.deepcopy(error_df[error_df[
                                                                self.__target_feature] == actual_target][
                                                       error_df[
                                                           prediction_feature] == pred_target])

                            tmp_df.drop(columns=[prediction_feature],
                                        inplace=True)
                            if tmp_df.shape[0]:
                                # Create feature analysis directory structure with given graphics
                                feature_analysis = FeatureAnalysis(tmp_df_features,
                                                                   overwrite_full_path=f"{output_path}/Predicted:{pred_target} Actual: {actual_target}")
                                feature_analysis.perform_analysis(tmp_df,
                                                                  dataset_name=dataset_name,
                                                                  target_features=[
                                                                      self.__target_feature],
                                                                  save_file=save_file,
                                                                  selected_features=selected_features,
                                                                  suppress_runtime_errors=suppress_runtime_errors,
                                                                  display_print=display_print,
                                                                  display_visuals=display_visuals,
                                                                  dataframe_snapshot=False,
                                                                  extra_tables=extra_tables,
                                                                  statistical_analysis_on_aggregates=statistical_analysis_on_aggregates)

    def classification_report(self,
                              X,
                              y,
                              pred_name,
                              dataset_name,
                              thresholds=None,
                              display_visuals=True,
                              save_file=True):
        """
            Creates a report of all target's metric evaluations
            based on the model's prediction output from the classification report
            from the sklearn.

        Args:
            X:
                Feature matrix.

            y:
                Target data vector.

            pred_name:
                The name of the prediction function in questioned
                stored in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

            display_visuals:
                Visualize graph if needed.

            save_file:
                Boolean value to whether or not to save the file.
        """
        filename = f'Classification Report {dataset_name} on {self.__model_name}'
        sub_dir = self.__create_sub_dir_with_thresholds(pred_name,
                                                        dataset_name,
                                                        thresholds)

        # Create dataframe report
        report_df = pd.DataFrame(classification_report(y,
                                                       self.__get_model_prediction(
                                                           pred_name,
                                                           X,
                                                           thresholds),
                                                       output_dict=True))

        # ---
        if display_visuals:
            if self.__notebook_mode:
                display(report_df)
            else:
                print(report_df)

        if save_file:
            # Output dataframe as png
            df_to_image(report_df,
                        self.folder_path,
                        sub_dir,
                        filename,
                        col_width=20,
                        show_index=True,
                        format_float_pos=4)

            if not self.__called_from_perform:
                self.generate_matrix_meta_data(X,
                                               dataset_name + "/_Extras")

    def graph_model_importances(self,
                                feature_order,
                                feature_importances,
                                display_visuals=True):
        """
            Graph given models's feature importances

        Args:
            feature_order: list
                Features names in proper order to re-create the pandas dataframe.

            feature_importances: list
                List of floats that represent each corresponding features importance

            display_visuals: bool
                Visualize graph if needed.
        """

        feature_importances, feature_order = zip(
            *sorted(zip(feature_importances, feature_order), reverse=True))
        feature_order = list(feature_order)

        plt.figure(figsize=(20, 10))

        palette = "PuBu"

        # Color ranking
        rank_list = np.argsort(-np.array(feature_importances)).argsort()
        pal = sns.color_palette(palette, len(feature_importances))
        palette = np.array(pal[::-1])[rank_list]

        plt.clf()

        plt.title("Feature Importances")

        ax = sns.barplot(x=feature_importances,
                         y=feature_order,
                         palette=palette,
                         order=feature_order)

        self.save_plot("Feature Importances",
                       "_Extras")

        pickle_object_to_file(dict(zip(feature_order, feature_importances)),
                              self.folder_path + "/_Extras",
                              "Feature Importances")

        if self.__notebook_mode and display_visuals:
            plt.show()

        plt.close("all")

    def __get_model_prediction(self,
                               pred_name,
                               X,
                               thresholds=None):
        """

            Finds the model's predicted labels.

        Args:
            X: np.matrix or np.array
                Feature matrix.

            pred_name: str
                The name of the prediction function in questioned stored in 'self.__pred_funcs_dict'

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

        Returns:
            Returns back a predicted value based for a given matrix.
            Handles prediction function 'types' Predictions and Probabilities.
            Helps streamline the entire process of evaluating classes.
        """

        # Check if function name exists in the dictionary
        if pred_name not in self.__pred_funcs_types:
            raise KeyError("The function name has not be recorded!")

        # Must be a prediction function
        if self.__pred_funcs_types[pred_name] == "Predictions":
            return self.__pred_funcs_dict[pred_name](X)

        # Output must be continuous; Probabilities
        elif self.__pred_funcs_types[pred_name] == "Probabilities":

            # Validate probabilities
            if thresholds:
                if isinstance(thresholds, list) or \
                        isinstance(thresholds, np.ndarray):
                    if len(thresholds) != len(self.__target_values):
                        raise UnsatisfiedRequirments("Length of thresholds must match the same length as the associated classes.")
                else:
                    raise UnsatisfiedRequirments("Threshold object is not a list or numpy array!")

            # Get model probability output
            model_output = self.__get_model_probas(pred_name,
                                                   X)

            # No thresholds found
            if not thresholds:
                return np.asarray([self.__target_values[np.argmax(proba)]
                                   for proba in model_output])

            # Apply thresholds to model's probability output
            else:
                model_output = model_output - thresholds

                return np.asarray([self.__target_values[np.argmax(proba)]
                                   for proba in model_output])
        else:
            raise ValueError(f"Unknown type '{self.__pred_funcs_types[pred_name]}' was found!")

    def __get_model_probas(self,
                           pred_name,
                           X):
        """

            Attempts to get the probabilities from the prediction function.

        Args:
            X:
                Feature matrix.

            pred_name: str
                The name of the prediction function in questioned stored in 'self.__pred_funcs_dict'

        Raises:
             If probabilities isn't possible with the given function that it will invoke an error.

        Returns:
            Returns back a series of values between 0-1 to represent it's confidence.
        """

        if self.__pred_funcs_types[pred_name] == "Probabilities":
            model_output = self.__pred_funcs_dict[pred_name](X)

            # ---
            if isinstance(model_output, list):
                model_output = np.asarray(model_output)

            return model_output
        else:
            raise ProbasNotPossible

    def __create_sub_dir_with_thresholds(self,
                                         pred_name,
                                         dataset_name,
                                         thresholds):
        """

            Iterates through directory structure looking at each text file
            containing a string representation of the given threshold;
            keeps comparing the passed value of 'thresholds' to the text file.

        Args:
            pred_name:
                The name of the prediction function in questioned stored in 'self.__pred_funcs_dict'

            dataset_name:
                The dataset's name; this will create a sub-directory in which your
                generated graph will be inner-nested in.

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

        Returns:
            Looking at the root of the starting directory and looking at each
            '_Thresholds.txt' file to determine if the files can be outputed
            to that directory. The content of the file must match the content
            of the list/numpy array 'thresholds'.
        """

        sub_dir = f'{dataset_name}/{pred_name}'

        # Only generate extra folder structure if function type is Probabilities
        if self.__pred_funcs_types[pred_name] == "Probabilities":

            # ------
            if not thresholds:
                sub_dir = f'{sub_dir}/No Thresholds'
            else:
                i = 0
                sub_dir = f'{sub_dir}/Thresholds'
                tmp_sub_dir = copy.deepcopy(sub_dir)
                while True:
                    threshold_dir = self.folder_path
                    if i > 0:
                        tmp_sub_dir = (sub_dir + f' {i}')
                    threshold_dir += tmp_sub_dir

                    # If file exists with the same thresholds; than use this directory
                    if os.path.exists(threshold_dir):
                        if self.__compare_thresholds_to_saved_thresholds(
                                threshold_dir,
                                thresholds):
                            sub_dir = tmp_sub_dir
                            break

                    # Create new directory
                    else:
                        os.makedirs(threshold_dir)
                        write_object_text_to_file(thresholds,
                                                  threshold_dir,
                                                  "_Thresholds")
                        sub_dir = tmp_sub_dir
                        break

                    # Iterate for directory name change
                    i += 1

        return sub_dir

    def __compare_thresholds_to_saved_thresholds(self,
                                                 directory_path,
                                                 thresholds):
        """

            Compare the thresholds object to a threshold text file found in
            the directory; returns true if the file exists and the object's
            value matches up.

        Args:
            directory_path:
                Path to the given folder where the "_Thresholds.txt"

            thresholds:
                If the model outputs a probability list/numpy array then we apply
                thresholds to the ouput of the model.
                For classification only; will not affect the direct output of
                the probabilities.

        Returns:
            Compare the thresholds object to the text file; returns true if
            the file exists and the object's value matches up.
        """

        file_directory = correct_directory_path(directory_path)

        if os.path.exists(file_directory):

            # Extract file contents and convert to a list object
            file = open(file_directory + "_Thresholds.txt", "r")
            line = file.read()
            converted_list = line.split("=")[-1].strip().strip('][').split(
                ', ')
            converted_list = [float(val) for val in converted_list]
            file.close()

            if thresholds == converted_list:
                return True
            else:
                return False
        else:
            return False


import pytest
import pyspark
#import spark
from pyspark.sql import SparkSession
import pyspark.sql.functions as f
from pyspark import SparkContext
import pyspark.sql
from pyspark.sql import SQLContext
from pyspark.sql.functions import col
from pyspark.sql.types import *
import urllib.request
from io import StringIO

# location and name of test file
filename = 'C:\\Users\\asuhajda\\JupyterRoot\\practice_scripts\\Sparkreader\\test_files\\spark_reader_test.csv'

'''
Function 1 - creates a  spark session for this demo
'''

def create_session():
    #initiate sparkcontext if not exist to read into
    spark = SparkSession \
    .builder \
   .appName("read_files") \
    .getOrCreate()

    return(spark)

'''
Test 1 - Checks if spark session is created with spark session object type & if what's been created is the  active session

'''

def test_1():
	x = create_session()
	lst = []
	lst.append(type(x))
	type(x) == lst[0]
	s = SparkSession.getActiveSession()

	assert lst[0] == pyspark.sql.session.SparkSession
	assert s == x

'''
Function 2 -  reads csv or json file defined by 'filename' into a spark dataframe
   - Arg 1 'filename': filename and location if not in workdir
   - Arg 2 'header': does it have a header, takes true or false. Defaults to 'true' if argument not provided
   - Arg 3 'multiline': for json files only, true or false. Defaults to false. Indicates if json file is in multiline format

  Returns a spark dataframe from a csv or json file
'''

def read_file(filename, header = 'true', multiline = 'false'):

    # initiate sparkcontext if doesn't exists to read into
    spark = create_session()

    # checks if extension is csv, if yes reads csv into spark
    if(filename[-4:] == '.csv'):
        return(spark.read.load(filename, format = 'csv', sep = ',', header = header))

    # checks if extension is json, reads into spark
    elif(filename[-4:] == 'json'):
        return(spark.read.load(filename, format = 'json', sep = "true", header = header, multiline = multiline))

    # in case extension is not csv or json, returns error
    else:
        print("##ERROR: file is not in csv or json format #################")

'''
Test 2 - checks if file was read  into a spark dataframe
'''

def test_2():

	x = read_file(filename)
	lst = []
	lst.append(type(x))

	assert type(x) == pyspark.sql.dataframe.DataFrame

'''
Function 3 -  takes a dataframe and a list with 2 elements (column name and StringType).

    - Arg 1 'dataframe': spark dataframe
    - Arg 2 'columns': list with 2 elements - 1st item is column name, 2nd item column type.

List of data types: https://spark.apache.org/docs/latest/sql-ref-datatypes.html
Example input for function:
            columns_to_cast = [ ('column1', 'StringType()'),
                               ('column2','StringType()'),
                               ('column3','IntegerType()'),
                              ('column4','StringType()')]

Note: Casting a string column to Float can result in loss of data. In case column had strings, that'll default to nulls.

Function casts column types to provided types or returns error.
'''

def column_caster(dataframe, columns):

    try:

        for i,j in columns:
            dataframe = dataframe.withColumn(i,col(i).cast(eval(j)))

        return(dataframe)

    except:
        print('Column value couldnt be casted, please check data types!')

'''
Test 3 - asserts if column types have changed, samples if Interger type was casetd
'''

def test_3():

	columns_to_cast = [ ('user', 'StringType()'),
                   ('motorcycle','StringType()'),
                   ('km','IntegerType()'),
                  ('serial','StringType()')]

	x = read_file(filename)
	types_orig = x.dtypes

	x = column_caster(x, columns_to_cast)
	types_new = x.dtypes

	assert types_orig != types_new
	assert types_new[2] == ('km', 'int')

'''
Function 4 - NULL_CHECK function takes 2 arguments:
    - Arg 1 'dataframe': a spark dataframe
    - Arg 2 'columns': a list containing the column names to be checked, f.e: columns = ('user','motorcycle','km')

Function returns columns with NULL values
'''

def null_check(dataframe, columns):

    # creates empty list to append column names with null values to
    null_error = list()

    # loops through each columns
    for i in columns:

        # checks if column contains null values. larger than 1 if yes
        check = dataframe.select("*").where(col(i).isNull())
        check = len(check.head(1))

        # if column contains null values, appends them to null_error list
        if(check >= 1):
             null_error.append(i)

    # returns columns with null values
    return(null_error)

'''
Test 4 - only 'user' column has null value, function should therefore only return name of that column
'''

def test_4():

	x = read_file(filename)

	l = x.columns
	check = null_check(x, l)

	assert check == ['user']

'''
Function 5 - takes a dataframe, a list of columns and a regular expression. Looks for regex in provided list of columns,
and returns which columns contains the regex or warns if none
    - Arg 1 'dataframe': a spark dataframe
    - Arg 2 'columns': a list containing the column names
    - Arg 3 'regex': expression to look for

Function returns columns where not every field contains regex provided

'''

def regex_check(dataframe, columns, regex):

    # empty list to append to
    regex_list = list()

    # loops through each column
    for i in columns:

        # checks if column contains regex provided, larger than 1 if yes
        check = dataframe.filter(dataframe[i].rlike(regex))
        check = check.count()

        # if column contains regex, stores it in regex_list
        if(check != dataframe.count()):
            regex_list.append(i)

        elif(check == 0):
            pass

    return(regex_list)

'''
Test 5 - only serial column has strin 'bsn' in all fields. Function should return every column except for Serial
'''

def test_5():
	x = read_file(filename)

	l = x.columns
	check = regex_check(x, l, 'bsn')

	assert check == ['user', 'motorcycle', 'km']

'''
Function 6 - Unique_check function checks if provided dataframe column contains null values
    -Arg 1 'dataframe': takes a spark dataframe
    - Arg 2 'columns': takes a list containing column names to check

Retruns list of columns with non-unique values
'''

def unique_check(dataframe, columns):

    # empty list to append non-unique cols to
    nonunique_cols = list()

    # loops through each column
    for i in columns:

        # case if column only has unique values
        if( (dataframe.select(i).distinct().count() == dataframe.count()) == False ):
            nonunique_cols.append(i)

        # case if column has non-unique values
        elif( (dataframe.select(i).distinct().count() == dataframe.count()) == True ):
            pass

    # returns list of columns with non-unique values
    return(nonunique_cols)

'''
Test 6 - only serial and motorcycle columns in test dataframe are'uniques. should return 'user' and 'km' columns, those being non-unique
'''

def test_6():
	x = read_file(filename)

	l = x.columns
	check = unique_check(x, l)

	assert check == ['user',  'km']

'''
FUNCTION 7 - TAKES 4 ARGUMENTS:
    - Arg 1 'filename': path and name of csv or json file to be checked
    - Arg 2: 'header': indicates if file to read has header
    - Arg 3 'multiline': only for json files, defaults to false. Indicates if json is multiline
    - Arg 2 'columns_to_cast': column names for which type needs to be casted. If not provided, function doesn't case types
    - Arg 3 'columns':  list of columns to perform the checks on. If not provided, all columns will be checked
    - Arg 4 'regex': provide value if you need to make sure column contain a given expression

Function:
    1) reads cvs / json file provided
    2) casts column types if provided
    3) checks if column(s) contain null values
    4) checks if column(s) contain specified expression in all fields
    5) checks if column(s) has non-unique values
    6) if any of the below is true, returns error
    7) otherwise returns dataframe

'''

def check_my_data(filename, columns_to_cast = '', columns = '', regex = '', header = 'true', multiline = 'false'):

    # get dataframe
    df = read_file(filename, header = header, multiline = multiline)

    # cast columns if provided
    if(len(columns_to_cast)>0):
        df = column_caster(df, columns_to_cast)
    else:
        pass

    # get columns if not provided
    if(len(columns)==0):
        columns=df.columns
    else:
        pass

    # returns cols with NULL values
    nc = null_check(df, columns)

    # returns cols where not all field contains REGEX
    reg_check = regex_check(df, columns, 'bsn')

    # returns list of columns with non-unique values
    u_check = unique_check(df, columns)

    # logical checks
    if(len(nc + reg_check + u_check)>0):
        print('DATA QUALITY ERROR, VALIDATION FAILED AT: ')

    if(len(nc)>0):
        print('FIELD HAS NULLS: '), print(nc)

    if(len(reg_check)>0):
        print('REGEX IS NOT IN ALL FIELDS IN: '), print(reg_check)

    if(len(u_check)>0):
        print('NON-UNIQUE FIELDS IN: '), print(u_check)

    if(len(nc + reg_check + u_check)==0):
        print('DATA SUCCESSFULLY LOADED!'), print(df.show(5))

        # if data is ready, return dataframe
        return(df)

'''
Test 7 -

	x runs through every column and fails to load the dataframe if one column fails any checks. since only column that passes all checks is the 'serial',
	x will fail therefore not returning a spark dataframe obejct but a null type

	y only runs the checks for the 'serial column, therefore returning a spark dataframe
	y should have columns ['user', 'motorcycle', 'km', 'serial']

'''

def test_7():

	x = check_my_data(filename)

	cols = []
	cols.append('serial')
	y = check_my_data(filename, columns = cols)

	check = []
	check.append(type(x))

	assert type(x) == check[0]
	assert type(y) == pyspark.sql.dataframe.DataFrame
	assert y.columns == ['user', 'motorcycle', 'km', 'serial']


import sys
import os
import shutil
import numpy as np
from datetime import date, datetime
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib import colors as mcolors
from CHECLabPy.core.io import DL1Reader
import pandas as pd
import argparse
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib import gridspec

def draw_camera_outline(ax, **plot_opt):
    """Draw camera outline

    Parameters
    ----------
    ax : matplotlib.pylab.axis
        axis on which draw the camera outline
    **plot_opt : variuos
        options for plot
    """
    ms = (51.4 + 1.93) / 1000.0
    ax.plot([-3 * ms, -3 * ms], [-2 * ms, +2 * ms], **plot_opt)
    ax.plot([+3 * ms, +3 * ms], [-2 * ms, +2 * ms], **plot_opt)
    ax.plot([-2 * ms, -2 * ms], [-3 * ms, -2 * ms], **plot_opt)
    ax.plot([+2 * ms, +2 * ms], [-3 * ms, -2 * ms], **plot_opt)
    ax.plot([-2 * ms, -2 * ms], [+2 * ms, +3 * ms], **plot_opt)
    ax.plot([+2 * ms, +2 * ms], [+2 * ms, +3 * ms], **plot_opt)
    ax.plot([-2 * ms, +2 * ms], [-3 * ms, -3 * ms], **plot_opt)
    ax.plot([-2 * ms, +2 * ms], [+3 * ms, +3 * ms], **plot_opt)
    ax.plot([-3 * ms, -2 * ms], [+2 * ms, +2 * ms], **plot_opt)
    ax.plot([-3 * ms, -2 * ms], [-2 * ms, -2 * ms], **plot_opt)
    ax.plot([+2 * ms, +3 * ms], [+2 * ms, +2 * ms], **plot_opt)
    ax.plot([+2 * ms, +3 * ms], [-2 * ms, -2 * ms], **plot_opt)

def inside_camera_mask(x, y):
    ms = (51.4 + 1.93) / 1000.0
    m = (x >= -3 * ms) & (x <= 3 * ms) & (y >= -3 * ms) & (y <= 3 * ms)
    m = m & ~((x >= -3 * ms) & (x <= -2 * ms) & (y >= -3 * ms) & (y <= -2 * ms))
    m = m & ~((x >= -3 * ms) & (x <= -2 * ms) & (y >= 2 * ms) & (y <= 3 * ms))
    m = m & ~((x >= 2 * ms) & (x <= 3 * ms) & (y >= -3 * ms) & (y <= -2 * ms))
    m = m & ~((x >= 2 * ms) & (x <= 3 * ms) & (y >= 2 * ms) & (y <= 3 * ms))
    return m

parser = argparse.ArgumentParser(description="")
parser.add_argument(
    "-i", "--files", type=str, help=".h5 (dl1) filename (with path)", required=True
)

args = parser.parse_args()
filename = args.files
#Create a Reader
r = DL1Reader(filename)
#Get camera mapping
m = r.mapping
#Load data as pandas dataframe
d = r.load_entire_table()
#Group data by pixel number
df = d[d.duplicated('pixel', keep=False)].groupby('pixel')['charge_cc'].apply(list).reset_index()
#Evaluate the mean value on all events per pixel
a = []
for i in range(len(df["charge_cc"])):
    a.append(np.mean(df["charge_cc"][i]))
a = np.array(a)
#Draw histo2D
fig, ax = plt.subplots()
fig.set_size_inches(8.5,7.)
x = np.array(m["xpix"])
y = np.array(m["ypix"])
ma = inside_camera_mask(x,y)
x_min = np.amin(x)
x_max = np.amax(x)
y_max =np.amax(y)
y_min = np.amin(y)
bins_x = np.linspace(x_min - 0.006/2, x_max + 0.006/2, 49)
bins_y = np.linspace(y_min - 0.006/2, y_max + 0.006/2, 49)
h, bin_x, bin_y, image = ax.hist2d(
        x[ma],
        y[ma],
        bins = [bins_x, bins_y],
        weights=a[ma],
    )

plt.colorbar(image, ax=ax, label="Charge")
draw_camera_outline(ax, color="k", lw=2)
ax.set_xlabel("x [m]")
ax.set_ylabel("y [m]")
plt.title(filename[-15:-7])
fig.savefig(filename[-15:-7]+"cameraimage.png")
plt.show()


import pickle

# global variable
global model, scaler

#load model dan scaler
def load():
    global model, scaler
    model = pickle.load(open('model/model_knn.pkl', 'rb'))
    scaler = pickle.load(open('model/scaler_churn.pkl', 'rb'))

#fungsi prediksi data dan mereturn hasil
def prediksi_data(data):
    data = scaler.transform(data)
    prediksi = int(model.predict(data))
    if prediksi == 0:
        hasil_prediksi = "No"
    else:
        hasil_prediksi = "Yes"
    return hasil_prediksi


'''Annotating title and abstract for biological terms with BECAS'''

import becas
import sys
import pandas as pd
import csv
import time
from preprocessing_data import printProgressBar

def becas_annotation(title,abstract,email):

    '''
    Annotating title and abstract for biological terms with BECAS
    Input: text from title and abstract from pubmed_scrap.py output
    '''
    becas.email=email
    try:
        title_annotate=becas.annotate_text(title)
    except AttributeError:
        title_annotate={'entities':['nothingggggggg']}
    try:
        abstract_annotate=becas.annotate_text(abstract)
    except AttributeError:
        abstract_annotate={'entities':['nothingggggggg']}
    if not title_annotate['entities']:
        title_annotation='nothingggggggg'
    else:
        title_annotation=','.join([str.split(i,'|')[0] for i in title_annotate['entities']])
    if not abstract_annotate['entities']:
        abstract_annotation='nothingggggggg'
    else:
        abstract_annotation=','.join([str.split(i,'|')[0] for i in abstract_annotate['entities']])
    combined_annotation=','.join([title_annotation,abstract_annotation])
    return combined_annotation

def annotating_df_to_file(df,email,filename):

    '''
    Make sure the columns of the dataframe is defined as follows:
    'Title', 'MeshHeadIndex', 'Abstract', 'Date', 'Type','PMID'

    Take input the csv file. Note to remove the header.
    Output: A new csv file with annotated biological terms.
    '''

    chunksize=10
    for chunk_i in range(0,df.shape[0],chunksize):
        chunk=df.iloc[chunk_i:chunk_i+chunksize,:]
        with open(filename,'a+',newline='') as data_file:
            file_writer=csv.writer(data_file,delimiter=',',quoting=csv.QUOTE_MINIMAL)
            for row in range(chunk.shape[0]):
                annotation=becas_annotation(chunk['Title'].iloc[row],chunk['Abstract'].iloc[row],email)
                file_writer.writerow([chunk['Title'].iloc[row],annotation,
                                      chunk['PMID'].iloc[row]])
        printProgressBar(iteration=chunk_i+chunksize,total=df.shape[0])#shows a progress bar on the console.
        time.sleep(3.0)#to not overload the system

if __name__ == "__main__":

    email=input('Email: ')
    df=pd.read_csv(sys.argv[1],header=None)
    df.columns=['Title','MeshHeadIndex','Abstract','Date','Type','PMID']
    filename=input('Output csv filename: ')
    annotating_df_to_file(df,email,filename)


import torch, torch.nn as nn
import torch.nn.functional as F
import numpy as np
from torch.autograd import Variable

"""================================================================================================="""
class Adversarial(torch.nn.Module):
    def __init__(self,hidden_adversarial_size,direction_dict,decorrnet_lr=0.00001):
        """Decorrelation sub embeddings .

        Args:
            hidden_adversarial_size (int): the size for hidden layer
            direction_dict (dict): {"discriminative-shared":{'dim':'96-120', 'weight':150}}, for example,
            the name of soure embedding is 'discriminative' with size 96, target embedding is 'shared' with size 120,
            the weight for decorrelation is 150
            decorrnet_lr (float, optional): Defaults to 0.00001.
        """
        super(Adversarial,self).__init__()

        self.directions = direction_dict # {direction:{dim: str ,weight: float}}
        self.proj_dim   = hidden_adversarial_size
        self.lr   =  decorrnet_lr
        #Projection network
        self.regressors = nn.ModuleDict()
        for key in self.directions.keys():
            source_dim, target_dim =  self.directions[key]['dim'].split('-')
            self.regressors[key] = torch.nn.Sequential(torch.nn.Linear(int(source_dim), self.proj_dim),
                                                                torch.nn.ReLU(),
                                                                torch.nn.Linear(self.proj_dim, int(target_dim))
                                                            ).to(torch.float).cuda()
    def forward(self, feature_dict):
        """
        Args:
            feature_dict (dict): eg.{'discriminative': sub_embeddings1, 'shared': sub_embeddings2},
                           sub_embeddings1 [batch_size, dim1], sub_embeddings2 [batch_size, dim2]
        """
        feature_names = np.unique(np.array([key.split('-') for key in self.directions.keys()]).reshape(-1))
        assert len(feature_names) == len([key for key in feature_dict.keys()])

        #Apply gradient reversal on input embeddings.
        adj_feature_dict = {key:torch.nn.functional.normalize(grad_reverse(features),dim=-1) for key, features in feature_dict.items()}
        #Project one embedding to the space of the other (with normalization), then compute the correlation.
        sim_loss = 0
        for key in self.directions.keys():
            source, target = key.split('-')
            source_data, target_data = adj_feature_dict[source], adj_feature_dict[target]
            weight = self.directions[key]['weight']
            regressor = self.regressors[key]
            sim_loss += -1.*weight*torch.mean(torch.mean((target_data*torch.nn.functional.normalize(regressor(source_data),dim=-1))**2,dim=-1))
        return sim_loss

### Gradient Reversal Layer
class GradRev(torch.autograd.Function):
    """
    Implements an autograd class to flip gradients during backward pass.
    """
    @staticmethod
    def forward(self, x):
        """
        Container which applies a simple identity function.

        Input:
            x: any torch tensor input.
        """
        return x.view_as(x)
    @staticmethod
    def backward(self, grad_output):
        """
        Container to reverse gradient signal during backward pass.

        Input:
            grad_output: any computed gradient.
        """
        return (grad_output * -1.)

### Gradient reverse function
def grad_reverse(x):
    """
    Applies gradient reversal on input.

    Input:
        x: any torch tensor input.
    """
    #return GradRev()(x)
    return GradRev.apply(x)


"""tests related to helper functions."""
import os
import struct
from typing import Union, Optional
import pytest
from circuitpython_nrf24l01.rf24 import address_repr
from circuitpython_nrf24l01.network.mixins import _lvl_2_addr
from circuitpython_nrf24l01.fake_ble import (
    whitener,
    crc24_ble,
    reverse_bits,
    swap_bits,
    BLE_FREQ,
    FakeBLE,
    ServiceData,
    BATTERY_UUID,
)

# pylint: disable=redefined-outer-name

@pytest.mark.parametrize(
    "addr,expected", [(b"1Node", "65646F4E31"), (b"\0\xFF\1", "01FF00")]
)
def test_addr_repr(addr: Union[bytes, bytearray], expected: str):
    """test address_repr()"""
    assert expected == address_repr(addr)

@pytest.mark.parametrize(
    "lvl,expected", [(0, 0), (1, 0o1), (2, 0o10), (3, 0o100), (4, 0o1000)]
)
def test_lvl_mask(lvl: int, expected: int):
    """test _lvl_2_addr()"""
    assert expected == _lvl_2_addr(lvl)

def test_whitener(ble_obj: FakeBLE):
    """test whitening and dewhitening an arbitrary buffer."""
    buf = os.urandom(24)
    for freq in range(3):
        coef = (freq + 37) | 0x40
        whitened = ble_obj.whiten(buf)
        assert buf == whitener(whitened, coef)
        ble_obj.hop_channel()

def test_crc24():
    """test CRC 24 bit checksum."""
    buf = bytes(range(24))
    checksum = crc24_ble(buf)
    assert len(checksum) == 3
    assert checksum == bytearray(b"\x1d`\xeb")

def test_rev_bits():
    """test reverse_bits()"""
    buf = bytes(range(21))
    result = reverse_bits(buf)
    assert len(result) == len(buf)
    for x, y in zip(result, buf):
        assert x == swap_bits(y)

@pytest.mark.parametrize("addr", [None, 0xAABBCCDDEEFF, b"\xAA\xBB\xCC"])
def test_ble_mac(ble_obj: FakeBLE, addr: Optional[Union[int, bytes]]):
    """test the FakeBLE mac attribute."""
    assert isinstance(ble_obj.mac, (bytes, bytearray))
    ble_obj.mac = addr
    assert ble_obj.mac is not None and len(ble_obj.mac) == 6
    if addr is not None:
        if isinstance(addr, int):
            mac = (addr).to_bytes(6, byteorder="little", signed=False)
            assert mac == ble_obj.mac
        else:
            assert addr in ble_obj.mac

@pytest.mark.parametrize(
    "name",
    [
        "nRF24L01",
        b"nRF24L01",
        pytest.param(0, marks=pytest.mark.xfail),
        pytest.param("A really long name!", marks=pytest.mark.xfail),
    ],
)
def test_name(ble_obj: FakeBLE, name: Optional[Union[str, bytes, int]]):
    """test FakeBLE name attribute."""
    ble_obj.name = name
    if isinstance(name, str):
        assert name.encode(encoding="utf-8") == ble_obj.name
    else:
        assert ble_obj.name == name

@pytest.mark.parametrize(
    "name",
    [
        "nRF24L01",
        pytest.param("a really long name", marks=pytest.mark.xfail),
    ],
)
def test_show_pa_level(ble_obj: FakeBLE, name: Optional[Union[str, bytes, int]]):
    """test FakeBLE show_pa_level attribute."""
    assert not ble_obj.show_pa_level
    ble_obj.name = name
    ble_obj.show_pa_level = True
    assert ble_obj.show_pa_level

def test_hop_channel(ble_obj: FakeBLE):
    """test FakeBLE hop_channel()."""
    for freq in BLE_FREQ:
        assert ble_obj.channel == freq
        ble_obj.hop_channel()
    assert ble_obj.channel == BLE_FREQ[0]

@pytest.mark.parametrize(
    "attr",
    [
        "dynamic_payloads",
        "data_rate",
        "address_length",
        "auto_ack",
        "ack",
        "crc",
    ],
)
@pytest.mark.xfail
def test_not_implemented(ble_obj: FakeBLE, attr: str):
    """assert some FakeBLE attributes cannot be set."""
    setattr(ble_obj, attr, None)  # the value doesn't matter

@pytest.mark.parametrize(
    "func_name,args",
    [
        ("open_rx_pipe", [0, b"AnyA"]),
        ("open_tx_pipe", ["AnyA"]),
    ],
)
@pytest.mark.xfail
def test_pipe_restricted(ble_obj: FakeBLE, func_name: str, args: list):
    """assert FakeBLE pipe addresses cannot be altered."""
    getattr(ble_obj, func_name)(*args)

def test_print_details(ble_obj: FakeBLE, capsys: pytest.CaptureFixture):
    """test print_details()"""
    ble_obj.print_details(dump_pipes=True)
    out, _ = capsys.readouterr()
    assert "BLE device name" in out
    assert "Broadcasting PA Level" in out

@pytest.fixture
def inject_rx_payload(ble_obj: FakeBLE, monkeypatch: pytest.MonkeyPatch):
    """Inject a payload into the state machine's RX FIFO."""
    service_data = ServiceData(BATTERY_UUID)
    service_data.data = bytes([42])
    assert isinstance(service_data.data, bytes)
    with ble_obj as ble:
        ble.name = "test"
        ble.show_pa_level = True
        monkeypatch.setattr(ble, "send", ble._spi._spi.state.rx_fifo.append)
        ble.advertise(service_data.buffer, data_type=0x16)
    ble_obj._spi._spi.state.registers[7][0] &= 0xF1
    return repr(service_data)

def test_rx_payload(ble_obj: FakeBLE, inject_rx_payload: pytest.fixture):
    """test available() and read()"""
    buf_repr = inject_rx_payload
    assert ble_obj.available()
    payload = ble_obj.read()
    assert payload.mac == ble_obj.mac
    assert payload.name == "test"
    assert not payload.pa_level
    for data in payload.data:
        print("data type", type(data), data)
        if isinstance(data, ServiceData):
            assert address_repr(data.buffer, False) == buf_repr
            assert data.data == 42
            assert data.uuid == struct.pack("<H", BATTERY_UUID)
            break
    else:  # pragma: no cover
        raise ValueError("ServiceData item not found in decrypted payload")
    assert ble_obj.read() is None