# Tensorflow/Keras related imports
import tensorflow as tf
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.applications import ResNet50
from tensorflow.keras.applications import VGG19
from tensorflow.keras.layers import Dense
from tensorflow.keras import Sequential
from keras_lr_finder import LRFinder
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Flatten, Dropout, AveragePooling2D

class ImageClassificationTF:
    def __init__(self, path, image_size):
        self.path = path
        self.image_size = image_size

    def visualization(self):
        train_ds = tf.keras.preprocessing.image_dataset_from_directory(
            self.path + '/train',
            validation_split=0.2,
            subset="training",
            seed=1337,
            image_size=self.image_size,
            batch_size=32,
        )
        plt.figure(figsize=(10, 10))
        for images, labels in train_ds.take(1):
            for i in range(9):
                ax = plt.subplot(3, 3, i + 1)
                plt.imshow(images[i].numpy().astype("uint8"))
                plt.title(int(labels[i]))
                plt.axis("off")
                plt.suptitle('Dataset visualization with labels')

        plt.savefig('results/keras/dataset_visualization.jpg')

        train_ds = train_ds.prefetch(buffer_size=32)
        return train_ds

    def test_train_val(self):
        train_datagen = ImageDataGenerator(
            rescale=1. / 255,
            shear_range=0.2,
            zoom_range=0.2,
            horizontal_flip=True,
            height_shift_range=0.1,
            validation_split=0.2,
            rotation_range=30,
            brightness_range=(0.5, 1.1),
            fill_mode='nearest')

        self.train_generator = train_datagen.flow_from_directory(
            self.path + '/train',
            target_size=self.image_size,
            batch_size=24,
            class_mode='binary',
            subset='training')

        self.validation_generator = train_datagen.flow_from_directory(
            self.path + '/train',
            target_size=self.image_size,
            batch_size=16,
            class_mode='binary',
            subset='validation')

        test_datagen = ImageDataGenerator(rescale=1. / 255)
        self.test_pictures = test_datagen.flow_from_directory(self.path + '/test/', target_size=self.image_size,
                                                              batch_size=32,
                                                              class_mode='binary')
        return self.train_generator, self.validation_generator, self.test_pictures

    def augumented_visualization(self):
        imgs, labels = next(self.train_generator)
        fig, axes = plt.subplots(1, 10, figsize=(20, 20))
        axes = axes.flatten()
        for img, ax in zip(imgs, axes):
            ax.imshow(img)
            ax.axis('off')
            plt.suptitle('Examples of data augmentation')
        plt.tight_layout()
        plt.savefig('results/keras/augmentation_examples.png')
        plt.show()

    def customized_model(self, image_size, num_classes):
        inputs = keras.Input(shape=image_size + (3,))
        # Entry block
        x = layers.Rescaling(1.0 / 255)(inputs)
        x = layers.Conv2D(32, 3, strides=2, padding="same")(x)
        x = layers.BatchNormalization()(x)
        x = layers.Activation("relu")(x)
        x = layers.Conv2D(64, 3, padding="same")(x)
        x = layers.BatchNormalization()(x)
        x = layers.Activation("relu")(x)
        previous_block_activation = x  # Set aside residual
        for self.size in [128, 256, 512, 728]:
            x = layers.Activation("relu")(x)
            x = layers.SeparableConv2D(self.size, 3, padding="same")(x)
            x = layers.BatchNormalization()(x)
            x = layers.Activation("relu")(x)
            x = layers.SeparableConv2D(self.size, 3, padding="same")(x)
            x = layers.BatchNormalization()(x)
            x = layers.MaxPooling2D(3, strides=2, padding="same")(x)
            # Project residual
            residual = layers.Conv2D(self.size, 1, strides=2, padding="same")(
                previous_block_activation
            )
            x = layers.add([x, residual])  # Add back residual
            previous_block_activation = x  # Set aside next residual
        x = layers.SeparableConv2D(1024, 3, padding="same")(x)
        x = layers.BatchNormalization()(x)
        x = layers.Activation("relu")(x)
        x = layers.GlobalAveragePooling2D()(x)
        if num_classes == 2:
            activation = "sigmoid"
            units = 1
        else:
            activation = "softmax"
            units = num_classes
        x = layers.Dropout(0.2)(x)
        outputs = layers.Dense(units, activation=activation)(x)
        customized_model = keras.Model(inputs, outputs)

        return customized_model

    def pretrained_model(self, model, unfreeze):
        if model == 'resnet':
            baseModel = ResNet50(include_top=False, input_shape=self.image_size + (3,), weights='imagenet')
            for layer in baseModel.layers[:]:
                if unfreeze == 1:
                    layer.trainable = True
                elif unfreeze == 0:
                    layer.trainable = False
            headModel = baseModel.output
            headModel = AveragePooling2D(pool_size=(7, 7))(headModel)
            headModel = Flatten(name="flatten")(headModel)
            headModel = Dense(256, activation="relu")(headModel)
            headModel = Dropout(0.5)(headModel)
            headModel = Dense(1, activation="sigmoid")(headModel)
            customized_model = Model(inputs=baseModel.input, outputs=headModel)

        elif model == 'VGG19':
            model = VGG19(include_top=False, input_shape=self.image_size + (3,), weights='imagenet')
            customized_model = Sequential()

        # Add layers of base model
            for layer in model.layers[:]:
                if unfreeze == 1:
                    layer.trainable = True
                    customized_model.add(layer)
                elif unfreeze == 0:
                    layer.trainable = False
                    customized_model.add(layer)

                else:
                    raise ValueError('Please choose either 0 - frozen layers or 1 - trainable layers')

            customized_model.add(layers.Flatten())
            customized_model.add(tf.keras.layers.Dropout(0.5))

        # Add last layer with sigmoid activation function since we have 2 classes
            customized_model.add(Dense(units=1, activation='sigmoid'))

        return customized_model

    def training_process(self, model, epochs, train_ds, val_ds, test_set):
        model.summary()  # model layers+info

        # Compile model
        model.compile(
            optimizer=keras.optimizers.Adam(),  # most popular optimizer at the moment
            loss="binary_crossentropy",  # loss for classification problem
            metrics=["accuracy"],# required metric
        )

        lr_finder = LRFinder(model)

        # Train a model with batches
        # with learning rate growing exponentially from 0.0001 to 1
        lr_finder.find_generator(train_ds, start_lr=0.000001, end_lr=1, epochs=50)
        # Plot the loss, ignore 20 batches in the beginning and 5 in the end
        lr_finder.plot_loss(n_skip_beginning=20, n_skip_end=5)
        plt.savefig('results/keras/plot_loss.png')
        plt.show()

        # Plot rate of change of the loss
        # Ignore 20 batches in the beginning and 5 in the end
        # Smooth the curve using simple moving average of 20 batches
        # Limit the range for y axis to (-0.02, 0.01)
        lr_finder.plot_loss_change(sma=20, n_skip_beginning=20, n_skip_end=5, y_lim=(-0.01, 0.01))
        plt.savefig('results/keras/plot_loss_change.png')
        plt.show()

        # Now we are changing learning rate per visualization and recompile the model - fastest decrease on the graph
        learning_rate = float(input('Please type in learning rate per visualization '))
        # Compile model
        model.compile(
            optimizer=keras.optimizers.Adam(learning_rate),  # most popular optimizer at the moment
            loss="binary_crossentropy",  # loss for classification problem
            metrics=["accuracy"],  # required metric
        )
        model.summary()

        # Fit model with data
        history = model.fit_generator(train_ds, epochs=epochs,
                                      validation_data=val_ds)
        # Model results visualization

        # summarize history for accuracy
        plt.plot(history.history['accuracy'])
        plt.plot(history.history['val_accuracy'])
        plt.title('model accuracy')
        plt.ylabel('accuracy')
        plt.xlabel('epoch')
        plt.legend(['train', 'validation'], loc='upper left')
        plt.savefig('results/keras/accuracy.png')
        plt.show()

        # summarize history for loss
        plt.plot(history.history['loss'])
        plt.plot(history.history['val_loss'])
        plt.title('model loss')
        plt.ylabel('loss')
        plt.xlabel('epoch')
        plt.legend(['train', 'validation'], loc='upper left')
        plt.savefig('results/keras/loss.png')
        plt.show()
        score = model.evaluate(test_set, verbose=0)
        print(f'Test loss: {score[0]} / Test accuracy: {score[1]}')


import numpy as np
import pandas as pd
import hashlib
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
from sklearn.feature_selection import SelectKBest, f_classif
from xgboost import XGBClassifier
from sklearn.utils.class_weight import compute_class_weight
from imblearn.under_sampling import CondensedNearestNeighbour
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn import metrics, random
from sklearn.ensemble import RandomForestClassifier, StackingClassifier
from imblearn.over_sampling import SMOTE
from sklearn.metrics import roc_curve, roc_auc_score, auc
from sklearn.preprocessing import label_binarize
from sklearn.metrics import f1_score
from sklearn.metrics import matthews_corrcoef
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_curve, auc, balanced_accuracy_score
from sklearn.preprocessing import label_binarize
from scipy import interp
from itertools import cycle
from sklearn.metrics import precision_recall_fscore_support
import shap

def make_seed(seedStr):
    return int(hashlib.md5(seedStr.encode("utf-8")).hexdigest()[24:], 16)

def get_actual_class(tumor_code, actual_classes):
    return actual_classes.get(tumor_code)
def optimize_thresholds(y_test, y_pred_proba):
    best_mcc = -1
    best_thresholds = (0, 0)

    for t_low in np.linspace(0, 1, num=100):
        for t_high in np.linspace(t_low, 1, num=100):
            y_pred = np.zeros_like(y_test)
            y_pred[y_pred_proba[:, 0] > t_low] = 0
            y_pred[y_pred_proba[:, 1] > t_high] = 1

            mcc = matthews_corrcoef(y_test, y_pred)

            if mcc > best_mcc:
                best_mcc = mcc
                best_thresholds = (t_low, t_high)
                print (best_mcc)
    return best_thresholds

dataDel = pd.read_csv("Delayed1.csv")
dataPre = pd.read_csv("Precontrast1.csv")
dataArt = pd.read_csv("Arterial1.csv")
dataVen = pd.read_csv("Venous1.csv")

data = pd.concat([dataDel, dataPre, dataArt, dataVen], axis=1)

labelclassnum2 = data['Grade']
code_data = data[['tumor and ID']].copy()
data = data.drop(['RPCID', 'tumorKey', 'patientId', 'Grade', 'tumor and ID'], axis=1)

datafeatures = np.array(data)

accuracy = np.zeros([100])
output_file = "all_reports_stack.txt"
with open(output_file, "w") as f:
    f.write("")
print ('Starting the Code ...')
misclassified_count = {}
actual_classes = {}
for i in range(100):

    seed1 = make_seed(f"growth1{i}")
    seed2 = make_seed(f"growth2{i}")
    seed3 = make_seed(f"growth3{i}")
    seed4 = make_seed(f"growth4{i}")

    np.random.seed(seed3)
    random.seed(seed4)

    x_train, x_test, y_train, y_test, train_index, test_index = train_test_split(datafeatures, labelclassnum2, data.index,
                                                                                test_size=0.15, random_state=seed1,
                                                                                stratify=labelclassnum2)

    cnn = CondensedNearestNeighbour(random_state=42)
    x_train, y_train = cnn.fit_resample(x_train, y_train)

    smote = SMOTE(random_state=42)
    x_train, y_train = smote.fit_resample(x_train, y_train)

    class_weights = compute_class_weight(class_weight='balanced', classes=np.unique(y_train), y=y_train)
    weights = dict(zip(np.unique(y_train), class_weights))
    scale_pos_weight = weights[1] / weights[0]

    scaler = StandardScaler()
    x_train = scaler.fit_transform(x_train)
    x_test = scaler.transform(x_test)

    k_best = SelectKBest(score_func=f_classif, k=20)
    x_train = k_best.fit_transform(x_train, y_train)
    x_test = k_best.transform(x_test)
    print ('DONE')
    # Grid search for XGBoost
    xgb_model = XGBClassifier(tree_method='gpu_hist', gpu_id=0)
    xgb_param_grid = {
        'scale_pos_weight': [scale_pos_weight],
        'max_depth': [3, 4, 5, 6],
        'min_child_weight': [1, 2, 3],
        'gamma': [0, 0.1, 0.2],
        'colsample_bytree': [0.6, 0.8, 1.0],
        'subsample': [0.6, 0.8, 1.0],
    }

    xgb_grid_search = GridSearchCV(xgb_model, xgb_param_grid, scoring='f1', cv=5, n_jobs=-1)
    xgb_grid_search.fit(x_train, y_train)
    print ('DONE XGB')
    # Grid search for Random Forest
    rf_model = RandomForestClassifier()
    rf_param_grid = {
        'n_estimators': [100, 200, 300],
        'max_depth': [None, 3, 4, 5],
        'min_samples_split': [2, 5, 10],
        'min_samples_leaf': [1, 2, 4],
        'class_weight': ['balanced', 'balanced_subsample'],
    }
    print ('DONE RF')
    rf_grid_search = GridSearchCV(rf_model, rf_param_grid, scoring='f1', cv=5, n_jobs=-1)
    rf_grid_search.fit(x_train, y_train)

    best_xgb_model = xgb_grid_search.best_estimator_
    best_xgb_model.fit(x_train, y_train)

    best_rf_model = rf_grid_search.best_estimator_
    best_rf_model.fit(x_train, y_train)
    '''
    xgb_y_pred_proba = best_xgb_model.predict_proba(x_test)
    rf_y_pred_proba = best_rf_model.predict_proba(x_test)
    avg_y_pred_proba = (xgb_y_pred_proba + rf_y_pred_proba) / 2
    avg_y_pred = np.argmax(avg_y_pred_proba, axis=1)

    # ENSEMBLING: Evaluate the performance of the ensemble model
    ensemble_accuracy = accuracy_score(y_test, avg_y_pred)
    print("Ensemble Accuracy: %.2f%%" % (ensemble_accuracy * 100.0))
    '''
    stacking_clf = StackingClassifier(
        estimators=[('xgb', best_xgb_model), ('rf', best_rf_model)],
        final_estimator=LogisticRegression(),
        cv=5
    )
    stacking_clf.fit(x_train, y_train)

    stacked_y_pred = stacking_clf.predict(x_test)
    stacked_y_pred_proba = stacking_clf.predict_proba(x_test)
    stacked_accuracy = accuracy_score(y_test, stacked_y_pred)
    print("Stacking Classifier Accuracy: %.2f%%" % (stacked_accuracy * 100.0))

    test_code_data = code_data.iloc[test_index]
    pred_data = list(zip(test_code_data['tumor and ID'], stacked_y_pred, y_test))

    for code, pred, actual in pred_data:
        if code not in misclassified_count:
            misclassified_count[code] = 0
            actual_classes[code] = actual
        if pred != actual:
            misclassified_count[code] += 1

    ensemble_report = classification_report(y_test, stacked_y_pred)
    print(ensemble_report)

    y_pred_proba_low_grade = stacked_y_pred_proba[:, 0]
    y_pred_proba_high_grade = stacked_y_pred_proba[:, 1]

    optimal_threshold_low, optimal_threshold_high = optimize_thresholds(y_test, stacked_y_pred_proba)

    y_pred_optimal = np.zeros_like(y_test)
    y_pred_optimal = np.where(stacked_y_pred_proba[:, 1] > optimal_threshold_high, 1, 0)
    optimal_classification_report = classification_report(y_test, y_pred_optimal)
    print(optimal_classification_report)

    optimal_y_pred_proba_pos = stacked_y_pred_proba[:, 1]

    optimal_fpr, optimal_tpr, _ = roc_curve(y_test, optimal_y_pred_proba_pos)
    optimal_roc_auc = auc(optimal_fpr, optimal_tpr)

    lw = 2
    plt.figure()
    plt.plot(optimal_fpr, optimal_tpr, color='darkorange', lw=lw,
             label=f'ROC curve (area = {optimal_roc_auc:.2f})')
    plt.plot([0, 1], [0, 1], 'k--', lw=lw)
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic (ROC)')
    plt.legend(loc="lower right")
    plt.savefig(f'roc_curve_optimal_stack{i}.png')
    plt.close()

    optimal_conf_mat = confusion_matrix(y_test, y_pred_optimal)
    disp = ConfusionMatrixDisplay(confusion_matrix=optimal_conf_mat, display_labels=['LowGrade', 'HighGrade'])
    disp.plot(cmap='Blues')
    plt.savefig(f'optimal_confusion_matrix_stack{i}.png')
    '''
    explainer = shap.Explainer(stacking_clf.named_estimators_['xgb'], x_train)

    # Calculate SHAP values for the x_test dataset
    shap_values = explainer(x_test)

    # Create a SHAP force plot and save it to a file
    test_sample_idx = 0  # Choose the index of the test sample you want to visualize
    shap.plots.force(shap_values[test_sample_idx], matplotlib=True, show=False)
    plt.savefig(f'shap_force_plot_optimal_iter_{i+1}.png', bbox_inches='tight')
    plt.close()
    '''
    with open(output_file, "a") as f:
        f.write(f"Iteration {i+1} - Optimal Thresholds\n")
        f.write(f"Low Grade: {optimal_threshold_low}, High Grade: {optimal_threshold_high}\n")
        f.write(optimal_classification_report)
        f.write("\n")

threshold = max(20, 0.25 * 100)
repeatedly_wrong_codes = {code: count for code, count in misclassified_count.items() if count >= threshold}

with open("repeatedly_wrong_codes_stack.csv", "w") as f:
    f.write("Code,Count,Actual\n")
    for code, count in repeatedly_wrong_codes.items():
        actual_class = get_actual_class(code, actual_classes)
        f.write(f"{code},{count},{actual_class}\n")

average_accuracy = np.mean(accuracy)
print("Average accuracy: %.2f%%" % (average_accuracy * 100.0))


import abc
from tempfile import TemporaryDirectory
from typing import Any, Callable, Dict, List, Tuple, Type, Union

from nebullvm.config import ACTIVATION_METRIC_DROP_THS
from nebullvm.core.models import (
    OptimizedModel,
    OptimizationTime,
    ModelParams,
    ModelCompiler,
    ModelCompressor,
    DeepLearningFramework,
    DeviceType,
    QuantizationType,
)
from nebullvm.operations.base import Operation
from nebullvm.operations.inference_learners.base import (
    BuildInferenceLearner,
)
from nebullvm.operations.inference_learners.builders import (
    DeepSparseBuildInferenceLearner,
    FasterTransformerBuildInferenceLearner,
    IntelNeuralCompressorBuildInferenceLearner,
    ONNXApacheTVMBuildInferenceLearner,
    ONNXBuildInferenceLearner,
    ONNXTensorRTBuildInferenceLearner,
    OpenVINOBuildInferenceLearner,
    PyTorchApacheTVMBuildInferenceLearner,
    PyTorchTensorRTBuildInferenceLearner,
    TensorflowBuildInferenceLearner,
    TFLiteBuildInferenceLearner,
    TorchNeuronBuildInferenceLearner,
    TorchXLABuildInferenceLearner,
    TorchDynamoBuildInferenceLearner,
    TorchScriptBuildInferenceLearner,
)
from nebullvm.operations.measures.measures import MetricDropMeasure
from nebullvm.operations.measures.utils import (
    compute_optimized_running_time,
    compute_relative_difference,
)
from nebullvm.operations.optimizations.compilers.base import Compiler
from nebullvm.operations.optimizations.compilers.deepsparse import (
    DeepSparseCompiler,
)
from nebullvm.operations.optimizations.compilers.faster_transformer import (
    FasterTransformerCompiler,
)
from nebullvm.operations.optimizations.compilers.intel_neural_compressor import (  # noqa: E501
    IntelNeuralCompressorCompiler,
)
from nebullvm.operations.optimizations.compilers.onnxruntime import (
    ONNXCompiler,
)
from nebullvm.operations.optimizations.compilers.openvino import (
    OpenVINOCompiler,
)
from nebullvm.operations.optimizations.compilers.tensor_rt import (
    ONNXTensorRTCompiler,
    PyTorchTensorRTCompiler,
)
from nebullvm.operations.optimizations.compilers.tensorflow import (
    TensorflowBackendCompiler,
    TFLiteBackendCompiler,
)
from nebullvm.operations.optimizations.compilers.torch_dynamo import (
    TorchDynamoCompiler,
)
from nebullvm.operations.optimizations.compilers.torch_neuron import (
    TorchNeuronCompiler,
)
from nebullvm.operations.optimizations.compilers.torch_xla import (
    TorchXLACompiler,
)
from nebullvm.operations.optimizations.compilers.torchscript import (
    TorchScriptCompiler,
)
from nebullvm.operations.optimizations.compilers.tvm import (
    ONNXApacheTVMCompiler,
    PyTorchApacheTVMCompiler,
)
from nebullvm.optional_modules.tensorflow import tensorflow as tf
from nebullvm.optional_modules.torch import torch
from nebullvm.tools.data import DataManager
from nebullvm.tools.transformations import MultiStageTransformation
from nebullvm.tools.utils import get_throughput

class Optimizer(Operation, abc.ABC):
    def __init__(self):
        super().__init__()
        self.optimized_models = []
        self.source_dl_framework = None
        self.pipeline_dl_framework = None
        self.compiler_ops = {}
        self.build_inference_learner_ops = {}
        self.validity_check_op = MetricDropMeasure()

    def execute(
        self,
        model: Any,
        input_data: DataManager,
        optimization_time: OptimizationTime,
        metric_drop_ths: float,
        metric: Callable,
        model_params: ModelParams,
        model_outputs: List[Tuple[Any, ...]],
        ignore_compilers: List[ModelCompiler],
        ignore_compressors: List[ModelCompressor],
        source_dl_framework: DeepLearningFramework,
        is_diffusion: bool = False,
    ) -> List[OptimizedModel]:
        self.source_dl_framework = source_dl_framework

        # TODO: implement and select compressors from hardware

        compilers = self._select_compilers_from_hardware()

        remove_compiler_list = []
        add_compiler_list = []
        for compiler in ignore_compilers:
            if compiler in MULTI_FRAMEWORK_COMPILERS:
                add_compiler_list += MULTI_FRAMEWORK_COMPILERS[compiler]
                remove_compiler_list.append(compiler)

        for c in remove_compiler_list:
            ignore_compilers.remove(c)

        ignore_compilers += add_compiler_list

        (
            self.compiler_ops,
            self.build_inference_learner_ops,
        ) = self._load_compilers(
            ignore_compilers=ignore_compilers,
            compilers=compilers,
        )
        self._optimize(
            model=model,
            input_data=input_data,
            optimization_time=optimization_time,
            metric_drop_ths=metric_drop_ths,
            metric=metric,
            model_params=model_params,
            model_outputs=model_outputs,
            ignore_compilers=ignore_compilers,
            is_diffusion=is_diffusion,
        )

        return self.optimized_models

    @abc.abstractmethod
    def _select_compilers_from_hardware(self):
        raise NotImplementedError()

    @staticmethod
    def _load_compilers(
        ignore_compilers: List[ModelCompiler],
        compilers: List[ModelCompiler],
    ):
        compiler_ops = {
            compiler: COMPILER_TO_OPTIMIZER_MAP[compiler]()
            for compiler in compilers
            if compiler not in ignore_compilers
            and compiler in COMPILER_TO_OPTIMIZER_MAP
        }
        build_inference_learner_ops = {
            compiler: COMPILER_TO_INFERENCE_LEARNER_MAP[compiler]()
            for compiler in compilers
            if compiler not in ignore_compilers
            and compiler in COMPILER_TO_OPTIMIZER_MAP
        }

        return compiler_ops, build_inference_learner_ops

    def free_model_gpu(self, model: Any):
        # Free gpu memory
        if self.device.type is DeviceType.GPU:
            try:
                model.cpu()
            except Exception:
                pass
            try:
                with torch.cuda.device(self.device.to_torch_format()):
                    torch.cuda.empty_cache()
            except Exception:
                pass

    def _optimize(
        self,
        model: Union[torch.nn.Module, tf.Module, str],
        input_data: DataManager,
        optimization_time: OptimizationTime,
        metric_drop_ths: float,
        metric: Callable,
        model_params: ModelParams,
        model_outputs: List[Tuple[Any, ...]],
        ignore_compilers: List[ModelCompiler],
        is_diffusion: bool = False,
    ):

        if metric_drop_ths is not None:
            q_types = [
                None,
            ]
            if metric_drop_ths > 0:
                q_types.append(QuantizationType.HALF)
            if metric_drop_ths > ACTIVATION_METRIC_DROP_THS:
                q_types.append(QuantizationType.DYNAMIC)
                if input_data is not None:
                    q_types.append(QuantizationType.STATIC)
        else:
            q_types = [None]

        optimization_info = []
        for compiler, compiler_op, build_inference_learner_op in zip(
            self.compiler_ops.keys(),
            self.compiler_ops.values(),
            self.build_inference_learner_ops.values(),
        ):
            for q_type in q_types:
                input_tfms = MultiStageTransformation([])

                self.free_model_gpu(model)

                with TemporaryDirectory() as tmp_dir:
                    try:
                        compiler_op.to(self.device).execute(
                            model=model,
                            input_data=input_data,
                            model_params=model_params,
                            metric_drop_ths=metric_drop_ths
                            if q_type is not None
                            else None,
                            quantization_type=q_type,
                            input_tfms=input_tfms,
                            onnx_output_path=tmp_dir,
                            is_diffusion=is_diffusion,
                        )

                        compiled_model = compiler_op.get_result()
                        if compiled_model is not None:
                            build_inference_learner_op.to(self.device).execute(
                                model=compiled_model,
                                model_orig=compiler_op.model_orig
                                if hasattr(compiler_op, "model_orig")
                                else None,
                                model_params=model_params,
                                input_tfms=input_tfms,
                                source_dl_framework=self.source_dl_framework,
                                quantization_type=q_type,
                            )
                            inference_learner = (
                                build_inference_learner_op.get_result()
                            )

                            if inference_learner is not None:
                                test_input_data, ys = input_data.get_split(
                                    "test"
                                ).get_list(with_ys=True)

                                self.validity_check_op.execute(
                                    inference_learner,
                                    test_input_data,
                                    model_outputs,
                                    metric_drop_ths,
                                    metric_func=metric
                                    if q_type is not None
                                    else compute_relative_difference,
                                    ys=ys,
                                )

                                if self.validity_check_op.valid:
                                    latency = compute_optimized_running_time(
                                        inference_learner, input_data
                                    )
                                    self.logger.info(
                                        f"Optimized model latency: {latency} "
                                        f"sec/iter"
                                    )

                                    if (
                                        compiler not in ignore_compilers
                                        and optimization_time
                                        is OptimizationTime.CONSTRAINED
                                    ):
                                        ignore_compilers.append(compiler)

                                    self.optimized_models.append(
                                        OptimizedModel(
                                            inference_learner=inference_learner,  # noqa: E501
                                            metric_drop=self.validity_check_op.measure_result,  # noqa: E501
                                            compiler=compiler,
                                            technique=q_type.name
                                            if q_type is not None
                                            else "None",
                                            latency_seconds=latency,
                                            throughput=get_throughput(
                                                latency,
                                                # Normal models have batch
                                                # size B, diffusion models
                                                # have batch size 2B
                                                model_params.batch_size
                                                if not is_diffusion
                                                else model_params.batch_size
                                                / 2,
                                            ),
                                            size_mb=inference_learner.get_size()  # noqa: E501
                                            / 1e6,
                                        )
                                    )

                                    opt_info_dict = {
                                        "compiler": f"{self.pipeline_dl_framework.value}_{compiler.value}",  # noqa: E501
                                        "technique": q_type.value
                                        if q_type
                                        else "none",
                                        "latency": latency,
                                    }
                                    if (
                                        metric_drop_ths is not None
                                        and q_type is not None
                                    ):
                                        opt_info_dict[
                                            "metric_loss"
                                        ] = (
                                            self.validity_check_op.measure_result  # noqa: E501
                                        )
                                        opt_info_dict[
                                            "metric"
                                        ] = metric.__name__
                                    optimization_info.append(opt_info_dict)
                                else:
                                    self.logger.warning(
                                        "The optimized model will be "
                                        "discarded due to poor results "
                                        "obtained with the given metric."
                                    )

                                if self.device.type in [
                                    DeviceType.GPU,
                                    DeviceType.TPU,
                                ]:
                                    inference_learner.free_gpu_memory()
                    except Exception as ex:
                        self.logger.warning(
                            f"Optimization failed with "
                            f"{self.pipeline_dl_framework} "
                            f"interface of {compiler}. Got error {ex}. "
                            f"If possible the compilation will be re-scheduled"
                            f" with another interface. Please consult the "
                            f"documentation for further info or open an issue "
                            f"on GitHub for receiving assistance."
                        )
                        optimization_info.append(
                            {
                                "compiler": compiler.value,
                                "technique": q_type.value
                                if q_type
                                else "none",
                                "latency": -1,
                            }
                        )
        if self.feedback_collector is not None:
            self.feedback_collector.store_info(
                key="optimizations",
                value=optimization_info,
            )

MULTI_FRAMEWORK_COMPILERS = {
    ModelCompiler.TENSOR_RT: [
        ModelCompiler.TENSOR_RT_TORCH,
        ModelCompiler.TENSOR_RT_ONNX,
    ],
    ModelCompiler.APACHE_TVM: [
        ModelCompiler.APACHE_TVM_TORCH,
        ModelCompiler.APACHE_TVM_ONNX,
    ],
}

COMPILER_TO_OPTIMIZER_MAP: Dict[ModelCompiler, Type[Compiler]] = {
    ModelCompiler.TORCHSCRIPT: TorchScriptCompiler,
    ModelCompiler.DEEPSPARSE: DeepSparseCompiler,
    ModelCompiler.INTEL_NEURAL_COMPRESSOR: IntelNeuralCompressorCompiler,
    ModelCompiler.TENSOR_RT_TORCH: PyTorchTensorRTCompiler,
    ModelCompiler.TENSOR_RT_ONNX: ONNXTensorRTCompiler,
    ModelCompiler.APACHE_TVM_TORCH: PyTorchApacheTVMCompiler,
    ModelCompiler.APACHE_TVM_ONNX: ONNXApacheTVMCompiler,
    ModelCompiler.ONNX_RUNTIME: ONNXCompiler,
    ModelCompiler.OPENVINO: OpenVINOCompiler,
    ModelCompiler.TFLITE: TFLiteBackendCompiler,
    ModelCompiler.XLA: TensorflowBackendCompiler,
    ModelCompiler.TORCH_NEURON: TorchNeuronCompiler,
    ModelCompiler.TORCH_XLA: TorchXLACompiler,
    ModelCompiler.TORCH_DYNAMO: TorchDynamoCompiler,
    ModelCompiler.FASTER_TRANSFORMER: FasterTransformerCompiler,
}

COMPILER_TO_INFERENCE_LEARNER_MAP: Dict[
    ModelCompiler, Type[BuildInferenceLearner]
] = {
    ModelCompiler.TORCHSCRIPT: TorchScriptBuildInferenceLearner,
    ModelCompiler.DEEPSPARSE: DeepSparseBuildInferenceLearner,
    ModelCompiler.INTEL_NEURAL_COMPRESSOR: IntelNeuralCompressorBuildInferenceLearner,  # noqa: E501
    ModelCompiler.TENSOR_RT_TORCH: PyTorchTensorRTBuildInferenceLearner,
    ModelCompiler.TENSOR_RT_ONNX: ONNXTensorRTBuildInferenceLearner,
    ModelCompiler.APACHE_TVM_TORCH: PyTorchApacheTVMBuildInferenceLearner,
    ModelCompiler.APACHE_TVM_ONNX: ONNXApacheTVMBuildInferenceLearner,
    ModelCompiler.ONNX_RUNTIME: ONNXBuildInferenceLearner,
    ModelCompiler.OPENVINO: OpenVINOBuildInferenceLearner,
    ModelCompiler.TFLITE: TFLiteBuildInferenceLearner,
    ModelCompiler.XLA: TensorflowBuildInferenceLearner,
    ModelCompiler.TORCH_NEURON: TorchNeuronBuildInferenceLearner,
    ModelCompiler.TORCH_XLA: TorchXLABuildInferenceLearner,
    ModelCompiler.TORCH_DYNAMO: TorchDynamoBuildInferenceLearner,
    ModelCompiler.FASTER_TRANSFORMER: FasterTransformerBuildInferenceLearner,
}


import itertools

import torch
from tqdm import tqdm

from src.metrics import get_au_fa_fr

class BaseTrainer:
    def __init__(self, opt, train_loader, val_loader=None,
                 device=torch.device('cpu'), max_grad_norm=5.0):
        self.opt = opt
        self.train_loader = train_loader
        self.val_loader = val_loader
        self.device = device
        self.max_grad_norm = max_grad_norm

        self.best_score = 1.0
        self.best_state = None

    def clip_grad_norm(self):
        parameters = itertools.chain.from_iterable(
            (param_group['params'] for param_group in self.opt.param_groups)
        )
        torch.nn.utils.clip_grad_norm_(parameters, self.max_grad_norm)

    def eval_mode(self):
        '''
        this function is expected to switch trainer to eval mode
        '''
        raise NotImplementedError

    def save_best_state(self):
        '''
        this function is expected to update self.best_state
        '''
        raise NotImplementedError

    def train(self, n_epochs, make_plots=True):
        if make_plots:
            from IPython.display import clear_output
            import matplotlib.pyplot as plt
        val_history = []
        for n in range(1, n_epochs + 1):
            self.train_epoch()
            au_fa_fr = self.validation()
            val_history.append(au_fa_fr)
            if au_fa_fr < self.best_score:
                self.best_score = au_fa_fr
                self.save_best_state()
            if make_plots:
                clear_output()
                plt.plot(val_history)
                plt.ylabel('Metric')
                plt.xlabel('Epoch')
                plt.grid()
                plt.show()
            print(f'END OF EPOCH {n}, val metric: {au_fa_fr}')
        return val_history

    def train_mode(self):
        '''
        this function is expected to switch trainer to train mode
        '''
        raise NotImplementedError

    def train_batch(self, batch, labels):
        '''
        this function is expected to return batch loss
        '''
        raise NotImplementedError

    @torch.no_grad()
    def validate_batch(self, batch, labels):
        '''
        this function is expected to return probs, i.e.,
        probs[i] = P[label(batch[i]) == 1]
        '''
        raise NotImplementedError

    def train_epoch(self):
        self.train_mode()
        for batch, labels in tqdm(self.train_loader,
                                  total=len(self.train_loader)):
            batch, labels = batch.to(self.device), labels.to(self.device)
            loss = self.train_batch(batch, labels)
            self.opt.zero_grad()
            loss.backward()
            self.clip_grad_norm()
            self.opt.step()

    @torch.no_grad()
    def validation(self):
        self.eval_mode()
        all_probs, all_labels = [], []
        for batch, labels in tqdm(self.val_loader,
                                  total=len(self.val_loader)):
            batch, labels = batch.to(self.device), labels.to(self.device)
            probs = self.validate_batch(batch, labels)
            all_probs.append(probs.cpu())
            all_labels.append(labels.cpu())
        return get_au_fa_fr(torch.cat(all_probs, dim=0), all_labels)


# This program will analyze the financial statements and stock
# quote of any publically traded company listed in Yahoo Finance
# using financial ratio analysis.
#
# The program output a report file in HTML or PDF format of the
# results of the analysis.
#
# The comments in this program will be written in Javadoc style. This ensures
# compatibility with Doxygen documentation tool.

import requests
import pandas as pd
import numpy as np

# Constants

ROUNDING_PRECISION = 3

#
# Stores financial data from financial statements in one
# convenient place.
class FinancialData (object):
    # Initializes the financial data class by collecting data
    # from specified financial statements.
    def __init__ (self, balSht, incStmt, cfStmt, stkQte, dates):
        # Balance Sheet Values
        self.__cce = pd.to_numeric(balSht["Cash And Cash Equivalents"])
        self.__currLia = pd.to_numeric(balSht["Total Current Liabilities"])
        self.__currAssets = pd.to_numeric(balSht["Total Current Assets"])
        self.__shortTermInv = pd.to_numeric(balSht["Short Term Investments"])
        self.__NR = pd.to_numeric(balSht["Net Receivables"])
        self.__inventory = pd.to_numeric(balSht["Inventory"])
        self.__totalAssets = pd.to_numeric(balSht["Total Assets"])
        self.__totalLia = pd.to_numeric(balSht["Total Liabilities"])
        self.__totalSE = pd.to_numeric(balSht["Total Stockholder Equity"])

        # Income Statement Values
        self.__sales = pd.to_numeric(incStmt["Total Revenue"])
        self.__gp = pd.to_numeric(incStmt["Gross Profit"])
        self.__noil = pd.to_numeric(incStmt["Operating Income or Loss"])
        self.__ie = pd.to_numeric(incStmt["Interest Expense"])
        self.__ebit = pd.to_numeric(\
                        incStmt["Earnings Before Interest and Taxes"])
        self.__cor = pd.to_numeric(incStmt["Cost of Revenue"])

        # Net Income Value
        temp = incStmt["Net Income"]
        temp.columns = ["NanCol", "Net Income"]
        self.__ni = pd.to_numeric(temp["Net Income"])

        # Cash Flow Statement Values
        self.__ocf = pd.to_numeric(cfStmt\
                       ["Total Cash Flow From Operating Activities"])

    def getCashAndCashEquivalents (self):
        return self.__cce

    def getCurrentLiabilities (self):
        return self.__currLia

    def getCurrentAssets (self):
        return self.__currAssets

    def getShortTermInvestments(self):
        return self.__shortTermInv

    def getNetReceivables(self):
        return self.__NR

    def getInventory(self):
        return self.__inventory

    def getTotalAssets(self):
        return self.__totalAssets

    def getTotalLiabilities(self):
        return self.__totalLia

    def getTotalStockholderEquity(self):
        return self.__totalSE

    def getTotalShareholderEquity(self):
        return self.getTotalStockholderEquity();

    def getSales(self):
        return self.__sales

    def getTotalRevenue(self):
        return self.getSales();

    def getGrossProfit(self):
        return self.__gp

    def getOperatingIncomeOrLoss(self):
        return self.__noil

    def getNetOperatingIncomeOrLoss(self):
        return self.getOperatingIncomeOrLoss()

    def getInterestExpense(self):
        return self.__ie

    def getOperatingCashFlows(self):
        return self.__ocf

    def getEBIT(self):
        return self.__ebit

    def getCostOfRevenue(self):
        return self.__cor

    def getCOR(self):
        return self.getCostOfRevenue()

    def getNetIncome(self):
        return self.__ni

# Functions

# Retrieves financial statements and stock quote of a company
# from Yahoo's website.
def getFinancialStatementsFromYahoo (ticker):
    # URLs of various documents.
    url_cf_stmt = \
        "https://finance.yahoo.com/quote/%s/cash-flow?p=%s" %(ticker,ticker)
    url_income_stmt = \
        "https://finance.yahoo.com/quote/%s/financials?p=%s" %(ticker,ticker)
    url_balance_sheet = \
    "https://finance.yahoo.com/quote/%s/balance-sheet?p=%s" %(ticker,ticker)
    url_stock_quote = "https://finance.yahoo.com/quote/%s/" %(ticker)

    # To be on the safe side, we disguise this request as a Web Browser
    # request by adding this header.
    #
    # The request MIGHT successfully work without this header.

    myHeader = {
      "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.75 Safari/537.36",
      "X-Requested-With": "XMLHttpRequest"
    }

    # Make a request for EACH document
    requestCfStmt = requests.get(url_cf_stmt, headers=myHeader)
    print("Retrieved Cash Flow Statement.")
    requestBalSht = requests.get(url_balance_sheet, headers=myHeader)
    print("Retrieved Balance Sheet.")
    requestIncStmt = requests.get(url_income_stmt, headers=myHeader)
    print("Retrieved Income Statement.")
    requestStkQte = requests.get(url_stock_quote,headers=myHeader)
    print("Retrieved Stock Quote Statement.")

    # Format the tables into Pandas Dataflows
    # Note: It is best to have repetitve code, so that
    # the code can easily be changed if Yahoo Inc changes
    # its website's formatting.
    tablesCfStmt = pd.read_html(requestCfStmt.text)
    tablesBalSht = pd.read_html(requestBalSht.text)
    tablesIncStmt = pd.read_html(requestIncStmt.text)
    tablesStkQte = pd.read_html(requestStkQte.text)

    cfStmt = tablesCfStmt[0].transpose()
    cfStmt.columns = cfStmt.iloc[0]
    cfStmt = cfStmt[1:]

    balSht = tablesBalSht[0].transpose()
    balSht.columns = balSht.iloc[0]
    balSht = balSht[1:]

    incStmt = tablesIncStmt[0].transpose()
    incStmt.columns = incStmt.iloc[0]
    incStmt = incStmt[1:]

    stkQte0 = tablesStkQte[0].transpose()
    stkQte0.columns = stkQte0.iloc[0]
    stkQte1 = tablesStkQte[1].transpose()
    stkQte1.columns = stkQte1.iloc[0]
    stkQte = stkQte0.join(stkQte1, how='left')[1:]

    dates = incStmt[incStmt.columns[0]].values

    # Return stock information.
    return balSht, incStmt, cfStmt, stkQte, dates

# Calculates the Weighted Average Cost of Capital (WACC) of the firm.
# Parameters are retrieved from getFinancialStatementsFromYahoo().
def calculateWACC(balSht, incStmt, stkQte):
    # Calculate the cost of debt for the most recent year.
    interestExpense = np.array(list(map(int,\
                        incStmt["Interest Expense"].values[1:])))
    totalDebt = np.array(list(map(int,\
                        balSht["Total Liabilities"].values[1:])))

    costOfDebt4Yrs = np.abs(interestExpense)/totalDebt

    costOfDebt = pd.DataFrame({"K_d":costOfDebt4Yrs})[:1]\
        ["K_d"].values[0]

    # Calculate the Cost of Equity for the most recent year,
    # using the current beta value.
    beta = float(stkQte["Beta (3Y Monthly)"].values)

    # These rates are educated guesses.
    riskFreeRate = 0.02 # 2% Assumed
    marketRate = 0.098 # Average historical rate of return for S&P 500.

    costOfEquity = riskFreeRate + beta*(marketRate - riskFreeRate)

    totalLiabilities = float(balSht["Total Liabilities"][1])
    totalEquity = float(balSht["Total Stockholder Equity"][1])
    totalLiaEquity = totalLiabilities + totalEquity

    # Calculate WACC
    WACC = (totalLiabilities/totalLiaEquity)*costOfDebt +\
        (totalEquity/totalLiaEquity)*costOfEquity

    return WACC

# Liquidity Ratios
def calculateCashRatio (dates):
    global financialData
    cce = financialData.getCashAndCashEquivalents()
    currLia = financialData.getCurrentLiabilities()

    ratioAsSeries = cce/currLia

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Cash Ratio"])

def calculateQuickRatio (dates):
    global financialData
    cce = financialData.getCashAndCashEquivalents()
    si = financialData.getShortTermInvestments()
    nr = financialData.getNetReceivables()
    qa = cce + si + nr
    currLia = financialData.getCurrentLiabilities()

    ratioAsSeries = qa/currLia

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Quick Ratio"])

def calculateCurrentRatio (dates):
    global financialData
    currAssets = financialData.getCurrentAssets()
    currLia = financialData.getCurrentLiabilities()

    ratioAsSeries = currAssets/currLia

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Current Ratio"])

def calculateWorkingCapital(dates):
    global financialData
    currAssets = financialData.getCurrentAssets()
    currLia = financialData.getCurrentLiabilities()

    # Since all numbers are in thousands, we multiply
    # current assets and liabilities each by 1000.
    ratioAsSeries = 1000*currAssets - 1000*currLia

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Working Capital (WC)"])

def calculateCashToWorkingCapitalRatio(dates):
    global financialData
    cce = financialData.getCashAndCashEquivalents()
    wc = calculateWorkingCapital(dates)
    wc_series = wc["Working Capital (WC)"]
    wc_series.index = cce.index

    # CCE in thousands, working capital in exact amount.
    ratioAsSeries = 1000*cce/wc_series

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Cash To WC Ratio"])

def calculateInventoryToWorkingCapitalRatio(dates):
    global financialData
    inventory = financialData.getInventory()
    wc = calculateWorkingCapital(dates)
    wc_series = wc["Working Capital (WC)"]
    wc_series.index = inventory.index

    # Inventory in thousands, working capital in exact amount.
    ratioAsSeries = 1000*inventory/wc_series

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Inventory To WC Ratio"])

def salesToWorkingCapitalRatio(dates):
    global financialData
    sales = financialData.getSales()
    wc = calculateWorkingCapital(dates)
    wc_series = wc["Working Capital (WC)"]
    wc_series.index = sales.index

    # Sales in thousands, working capital in exact amount.
    ratioAsSeries = 1000*sales/wc_series

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Sales To WC Ratio"])

def salesToCurrentAssetsRatio(dates):
    global financialData
    sales = financialData.getSales()
    ca = financialData.getCurrentAssets()

    ratioAsSeries = sales/ca

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Sales To Current Assets Ratio"])

# Solvency Ratios

def calculateDebtRatio(dates):
    global financialData
    tl = financialData.getTotalLiabilities()
    ta = financialData.getTotalAssets()

    ratioAsSeries = tl/ta

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Debt Ratio"])

def calculateEquityRatio(dates):
    global financialData
    te = financialData.getTotalStockholderEquity()
    ta = financialData.getTotalAssets()

    ratioAsSeries = te/ta

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Equity Ratio"])

def calculateDebtToEquityRatio(dates):
    global financialData
    te = financialData.getTotalStockholderEquity()
    tl = financialData.getTotalLiabilities()
    ratioAsSeries = tl/te

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Debt to Equity Ratio"])

def calculateDebtToIncomeRatio(dates):
    global financialData
    tl = financialData.getTotalLiabilities()
    gp = financialData.getGrossProfit()

    ratioAsSeries = tl/gp

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Debt to Income Ratio"])

def calculateDebtServiceCoverageRatio(dates):
    global financialData
    noi = financialData.getOperatingIncomeOrLoss()
    ie = financialData.getInterestExpense()

    ratioAsSeries = noi/ie

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Debt Service Coverage Ratio"])

def calculateCashFlowToDebtRatio(dates):
    global financialData
    ocf = financialData.getOperatingCashFlows()
    tl = financialData.getTotalLiabilities()

    ratioAsSeries = ocf/tl

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Cash Flow to Debt Ratio"])

def calculateWCToDebtRatio(dates):
    global financialData
    tl = financialData.getTotalLiabilities()
    wc = calculateWorkingCapital(dates)
    wc_series = wc["Working Capital (WC)"]
    wc_series.index = tl.index

    ratioAsSeries = wc_series/tl

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["WC to Debt Ratio"])

def calculateTimesInterestEarned(dates):
    global financialData
    ebit = financialData.getEBIT()
    ie = np.abs(financialData.getInterestExpense())

    ratioAsSeries = ebit/ie

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Times Interest Earned"])

# Efficiency Ratios

def calculateAssetTurnoverRatio(dates):
    global financialData
    sales = financialData.getSales()
    totalAssets = financialData.getTotalAssets().values
    avgAssets = np.array([None]*len(totalAssets))

    for index in range(len(totalAssets)-1):
        avgAssets[index] = 0.5*(totalAssets[index]+totalAssets[index+1])

    avgAssets = pd.Series(avgAssets, index=range(1,len(avgAssets)+1))

    ratioAsSeries = sales/avgAssets

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Asset Turnover Ratio"])

def calculateInventoryTurnoverRatio(dates):
    global financialData
    cor = financialData.getCOR()
    totalInventory = financialData.getInventory().values
    avgInventory = np.array([None]*len(totalInventory))

    for index in range(len(totalInventory)-1):
        avgInventory[index] = 0.5*(totalInventory[index]+totalInventory[index+1])

    avgInventory = pd.Series(avgInventory, index=range(1,len(avgInventory)+1))

    ratioAsSeries = cor/avgInventory

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Inventory Turnover Ratio"])

def calculateAccountsReceivableTurnoverRatio(dates):
    global financialData
    sales = financialData.getSales()
    totalAR = financialData.getNetReceivables().values
    avgAR = np.array([None]*len(totalAR))

    for index in range(len(totalAR)-1):
        avgAR[index] = 0.5*(totalAR[index]+totalAR[index+1])

    avgAR = pd.Series(avgAR, index=range(1,len(avgAR)+1))

    ratioAsSeries = sales/avgAR

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Accounts Receivable Turnover Ratio"])

# Profitability Ratios

def calculateROARatio(dates):
    global financialData
    ni = financialData.getNetIncome()
    totalAssets = financialData.getTotalAssets().values
    avgAssets = np.array([None]*len(totalAssets))

    for index in range(len(totalAssets)-1):
        avgAssets[index] = 0.5*(totalAssets[index]+totalAssets[index+1])

    avgAssets = pd.Series(avgAssets, index=range(1,len(avgAssets)+1))

    ratioAsSeries = ni/avgAssets

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Return on Assets"])

def calculateROERatio(dates):
    global financialData
    ni = financialData.getNetIncome()
    totalSE = financialData.getTotalStockholderEquity()

    ratioAsSeries = ni / totalSE

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Return on Equity"])

def calculateROSRatio(dates):
    global financialData
    ebit = financialData.getEBIT()
    sales = financialData.getSales()

    ratioAsSeries = ebit / sales

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Return on Sales"])

def calculateNetProfitMarginRatio(dates):
    global financialData
    ni = financialData.getNetIncome()
    sales = financialData.getSales()

    ratioAsSeries = ni/sales

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Net Profit Margin Ratio"])

def calculateGrossProfitMarginRatio(dates):
    global financialData
    gp = financialData.getGrossProfit()
    sales = financialData.getSales()

    ratioAsSeries = gp/sales

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Gross Profit Margin Ratio"])

def calculateOperatingProfitMarginRatio(dates):
    global financialData
    op = financialData.getOperatingIncomeOrLoss()
    sales = financialData.getSales()

    ratioAsSeries = op/sales

    return pd.DataFrame(ratioAsSeries.values, index=dates,\
                        columns=["Operating Profit Margin Ratio"])

# Report-generating Functions
def getLiquidityRatios(dates):
    listOfRatioTables = [calculateCashRatio(dates),
    calculateQuickRatio(dates),calculateCurrentRatio(dates),
    calculateWorkingCapital(dates),
    calculateCashToWorkingCapitalRatio(dates),
    calculateInventoryToWorkingCapitalRatio(dates),
    salesToWorkingCapitalRatio(dates),
    salesToCurrentAssetsRatio(dates)]

    table = listOfRatioTables[0]

    for index in range(1,len(listOfRatioTables)):
        table = table.join(listOfRatioTables[index], how="left")

    return table.round(ROUNDING_PRECISION).astype(object).transpose()

def getSolvencyRatios(dates):
    listOfRatioTables = \
    [calculateDebtRatio(dates),
     calculateEquityRatio(dates),
     calculateDebtToEquityRatio(dates),
     calculateDebtToIncomeRatio(dates),
     calculateDebtServiceCoverageRatio(dates),
     calculateCashFlowToDebtRatio(dates),
     calculateWCToDebtRatio(dates),
     calculateTimesInterestEarned(dates)]

    table = listOfRatioTables[0]

    for index in range(1,len(listOfRatioTables)):
        table = table.join(listOfRatioTables[index], how="left")

    return table.round(ROUNDING_PRECISION).astype(object).transpose()

def getEfficiencyRatios(dates):
    listOfRatioTables = \
    [calculateAssetTurnoverRatio(dates),
     calculateInventoryTurnoverRatio(dates),
     calculateAccountsReceivableTurnoverRatio(dates)]

    table = listOfRatioTables[0]

    for index in range(1,len(listOfRatioTables)):
        table = table.join(listOfRatioTables[index], how="left")

    return table.round(ROUNDING_PRECISION).astype(object).transpose()

def getProfitabilityRatios(dates):
    listOfRatioTables = \
    [calculateROARatio(dates),\
     calculateROERatio(dates),\
     calculateROSRatio(dates),\
     calculateNetProfitMarginRatio(dates),\
     calculateGrossProfitMarginRatio(dates),\
     calculateOperatingProfitMarginRatio(dates)]

    table = listOfRatioTables[0]

    for index in range(1,len(listOfRatioTables)):
        table = table.join(listOfRatioTables[index], how="left")

    return table.round(ROUNDING_PRECISION).astype(object).transpose()

def createMasterReport(dates):
    liqTable = getLiquidityRatios(dates)
    solvTable = getSolvencyRatios(dates)
    effTable = getEfficiencyRatios(dates)
    profitTable = getProfitabilityRatios(dates)

    liqTableHTML = liqTable.to_html() # "liqRatios.html"
    solvTableHTML = solvTable.to_html() # "solvRatios.html"
    effTableHTML = effTable.to_html() # "effRatios.html"
    profitTableHTML = profitTable.to_html()

    header = \
    """
    <html>
    <head>
    <title> Financial Ratios </title>
    <body>
    """

    liqTableHTML = \
    """
    <h1> <u> Liquidity Ratios </u> </h1>
    """ + liqTableHTML

    solvTableHTML = \
    """
    <h1> <u> Solvency Ratios </u> </h1>
    """ + solvTableHTML

    effTableHTML = \
    """
    <h1> <u> Efficiency Ratios </u> </h1>
    """ + effTableHTML

    profitTableHTML = \
    """
    <h1> <u> Profitability Ratios </u> </h1>
    """ + profitTableHTML

    footer = \
    """
    </body>
    </html>
    """

    src = header + liqTableHTML + solvTableHTML + effTableHTML + \
        profitTableHTML + footer

    myFile = open("FinancialRatios.html", "w")
    myFile.write(src)
    myFile.close()

# Main Program
if __name__ == "__main__":
    balSht, incStmt, cfStmt, stkQte, dates = \
        getFinancialStatementsFromYahoo("KO")
    financialData = FinancialData(balSht, incStmt, cfStmt, stkQte, dates)
    createMasterReport(dates)


import asyncio
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

from hummingbot.connector.utilities.oms_connector import oms_connector_constants as CONSTANTS
from hummingbot.connector.utilities.oms_connector.oms_connector_auth import OMSConnectorAuth
from hummingbot.connector.utilities.oms_connector.oms_connector_web_utils import (
    OMSConnectorURLCreatorBase,
    OMSConnectorWebAssistantsFactory,
)
from hummingbot.core.data_type.order_book_message import OrderBookMessage, OrderBookMessageType
from hummingbot.core.data_type.order_book_tracker_data_source import OrderBookTrackerDataSource
from hummingbot.core.utils.tracking_nonce import NonceCreator
from hummingbot.core.web_assistant.connections.data_types import RESTMethod, WSJSONRequest
from hummingbot.core.web_assistant.rest_assistant import RESTAssistant
from hummingbot.core.web_assistant.ws_assistant import WSAssistant

if TYPE_CHECKING:
    from hummingbot.connector.utilities.oms_connector.oms_connector_exchange import OMSExchange

class OMSConnectorAPIOrderBookDataSource(OrderBookTrackerDataSource):
    def __init__(
        self,
        trading_pairs: List[str],
        connector: 'OMSExchange',
        api_factory: OMSConnectorWebAssistantsFactory,
        url_provider: OMSConnectorURLCreatorBase,
        oms_id: int,
    ):
        super().__init__(trading_pairs)
        self._connector = connector
        self._api_factory = api_factory
        self._rest_assistant: Optional[RESTAssistant] = None
        self._ws_assistant: Optional[WSAssistant] = None
        self._auth: OMSConnectorAuth = api_factory.auth
        self._url_provider = url_provider
        self._oms_id = oms_id
        self._nonce_provider = NonceCreator.for_milliseconds()

    async def get_last_traded_prices(
        self, trading_pairs: List[str], domain: Optional[str] = None
    ) -> Dict[str, float]:
        return await self._connector.get_last_traded_prices(trading_pairs=trading_pairs)

    async def _parse_trade_message(self, raw_message: List[Dict[int, Union[int, float]]], message_queue: asyncio.Queue):
        raise NotImplementedError  # OMS connectors do not provide a public trades endpoint

    async def _parse_order_book_diff_message(
        self, raw_message: List[List[Union[int, float]]], message_queue: asyncio.Queue
    ):
        msg_data = raw_message[CONSTANTS.MSG_DATA_FIELD]
        first_row = msg_data[0]
        ts_ms = first_row[CONSTANTS.DIFF_UPDATE_TS_FIELD]
        update_id = self._nonce_provider.get_tracking_nonce(timestamp=ts_ms * 1e-3)
        instrument_id = first_row[CONSTANTS.DIFF_UPDATE_INSTRUMENT_ID_FIELD]
        trading_pair = await self._connector.trading_pair_associated_to_exchange_symbol(symbol=str(instrument_id))
        bids, asks = self._get_bids_and_asks_from_snapshot(msg_data)

        order_book_message_content = {
            "trading_pair": trading_pair,
            "update_id": update_id,
            "bids": bids,
            "asks": asks,
        }
        diff_message: OrderBookMessage = OrderBookMessage(
            OrderBookMessageType.DIFF,
            order_book_message_content,
            ts_ms * 1e-3,
        )

        message_queue.put_nowait(diff_message)

    async def _order_book_snapshot(self, trading_pair: str) -> OrderBookMessage:
        snapshot_response = await self._request_order_book_snapshot(trading_pair)
        first_row = snapshot_response[0]
        ts_ms = first_row[CONSTANTS.DIFF_UPDATE_TS_FIELD]
        update_id = self._nonce_provider.get_tracking_nonce(timestamp=ts_ms * 1e-3)
        bids, asks = self._get_bids_and_asks_from_snapshot(snapshot_response)

        order_book_message_content = {
            "trading_pair": trading_pair,
            "update_id": update_id,
            "bids": bids,
            "asks": asks,
        }
        snapshot_msg = OrderBookMessage(OrderBookMessageType.SNAPSHOT, order_book_message_content, ts_ms)
        return snapshot_msg

    @staticmethod
    def _get_bids_and_asks_from_snapshot(
        snapshot: List[List[Union[int, float]]]
    ) -> Tuple[List[Tuple[float, float]], List[Tuple[float, float]]]:
        """OMS connectors do not guarantee that the data is sorted in any way."""
        asks = []
        bids = []
        for row in snapshot:
            update = (row[CONSTANTS.DIFF_UPDATE_PRICE_FIELD], row[CONSTANTS.DIFF_UPDATE_AMOUNT_FIELD])
            if row[CONSTANTS.DIFF_UPDATE_SIDE_FIELD] == CONSTANTS.BUY_ACTION:
                bids.append(update)
            else:
                asks.append(update)
        return bids, asks

    async def _request_order_book_snapshot(self, trading_pair: str) -> List[List[Union[int, float]]]:
        instrument_id = await self._connector.exchange_symbol_associated_to_pair(trading_pair=trading_pair)
        params = {
            CONSTANTS.OMS_ID_FIELD: self._oms_id,
            CONSTANTS.INSTRUMENT_ID_FIELD: int(instrument_id),
            CONSTANTS.DEPTH_FIELD: CONSTANTS.MAX_L2_SNAPSHOT_DEPTH,
        }
        rest_assistant = await self._get_rest_assistant()
        url = self._url_provider.get_rest_url(path_url=CONSTANTS.REST_GET_L2_SNAPSHOT_ENDPOINT)
        resp = await rest_assistant.execute_request(
            url=url,
            throttler_limit_id=CONSTANTS.REST_REQ_LIMIT_ID,
            params=params,
            method=RESTMethod.GET,
        )
        return resp

    async def _get_rest_assistant(self) -> RESTAssistant:
        if self._rest_assistant is None:
            self._ensure_authenticated()
            self._rest_assistant = await self._api_factory.get_rest_assistant()
        return self._rest_assistant

    async def _connected_websocket_assistant(self) -> WSAssistant:
        ws: WSAssistant = await self._get_ws_assistant()
        url = self._url_provider.get_ws_url()
        await ws.connect(ws_url=url, message_timeout=CONSTANTS.WS_MESSAGE_TIMEOUT)
        return ws

    async def _get_ws_assistant(self) -> WSAssistant:
        if self._ws_assistant is None:
            self._ensure_authenticated()
            self._ws_assistant = await self._api_factory.get_ws_assistant()
        return self._ws_assistant

    def _ensure_authenticated(self):
        if not self._auth.initialized:
            raise RuntimeError("The authenticator is not initialized.")

    async def _subscribe_channels(self, ws: WSAssistant):
        try:
            for trading_pair in self._trading_pairs:
                instrument_id = await self._connector.exchange_symbol_associated_to_pair(trading_pair)
                req_params = {
                    CONSTANTS.OMS_ID_FIELD: self._oms_id,
                    CONSTANTS.INSTRUMENT_ID_FIELD: int(instrument_id),
                    CONSTANTS.DEPTH_FIELD: CONSTANTS.MAX_L2_SNAPSHOT_DEPTH,
                }
                payload = {
                    CONSTANTS.MSG_ENDPOINT_FIELD: CONSTANTS.WS_L2_SUB_ENDPOINT,
                    CONSTANTS.MSG_DATA_FIELD: req_params,
                }
                subscribe_orderbook_request = WSJSONRequest(payload=payload)

                async with self._api_factory.throttler.execute_task(limit_id=CONSTANTS.WS_REQ_LIMIT_ID):
                    await ws.send(subscribe_orderbook_request)

            self.logger().info("Subscribed to public order book and trade channels...")
        except asyncio.CancelledError:
            raise
        except Exception:
            self.logger().exception("Unexpected error occurred subscribing to order book trading and delta streams...")
            raise

    def _channel_originating_message(self, event_message: Dict[str, Any]) -> str:
        channel = ""
        if event_message[CONSTANTS.MSG_TYPE_FIELD] != CONSTANTS.ERROR_MSG_TYPE:
            event_channel = event_message[CONSTANTS.MSG_ENDPOINT_FIELD]
            if event_channel == CONSTANTS.WS_L2_EVENT:
                channel = self._diff_messages_queue_key
        return channel

    async def _process_websocket_messages(self, websocket_assistant: WSAssistant):
        while True:
            try:
                async for ws_response in websocket_assistant.iter_messages():
                    data: Dict[str, Any] = ws_response.data
                    channel: str = self._channel_originating_message(event_message=data)
                    if channel in [self._diff_messages_queue_key, self._trade_messages_queue_key]:
                        self._message_queue[channel].put_nowait(data)
            except asyncio.TimeoutError:
                ping_payload = {
                    CONSTANTS.MSG_ENDPOINT_FIELD: CONSTANTS.WS_PING_REQUEST,
                    CONSTANTS.MSG_DATA_FIELD: {},
                }
                ping_request = WSJSONRequest(payload=ping_payload)
                async with self._api_factory.throttler.execute_task(limit_id=CONSTANTS.WS_PING_REQUEST):
                    await websocket_assistant.send(request=ping_request)


x = -5
if x > 0:
    raise Exception('x should be positive')

# assert(x >= 0), 'x is not positive'

# try:
#     a = 5 / 0
# # except:
# #     print('error')
# except Exception as e:
#     print(e) #division by zero

# try:
#     a = 5 / 1
#     b = a + '10'
# except ZeroDivisionError as e:
#     print(e)
# except TypeError as e:
#     print(e) #unsupported operand type(s) for +: 'float' and 'str'
# else:
#     print('everything is fine')
# finally:
#     print("cleaning up...")

class ValueTooHighError(Exception):
    def __init__(self,message,value):
        self.message = message
        self.value = value

def test_value(x):
    if x > 100:
        raise ValueTooHighError('Value is too high',x)

try:
    test_value(200)
except ValueTooHighError as e:
    print(f"The error message is : {e.message}, the input is : {e.value}.")


from tkinter.messagebox import NO
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from config import Machine_State, config
import time

def heatmap(data, row_labels, col_labels, ax=None,
            cbar_kw={}, cbarlabel="", **kwargs):
    """
    Create a heatmap from a numpy array and two lists of labels.

    Parameters
    ----------
    data
        A 2D numpy array of shape (M, N).
    row_labels
        A list or array of length M with the labels for the rows.
    col_labels
        A list or array of length N with the labels for the columns.
    ax
        A `matplotlib.axes.Axes` instance to which the heatmap is plotted.  If
        not provided, use current axes or create a new one.  Optional.
    cbar_kw
        A dictionary with arguments to `matplotlib.Figure.colorbar`.  Optional.
    cbarlabel
        The label for the colorbar.  Optional.
    **kwargs
        All other arguments are forwarded to `imshow`.
    """

    if not ax:
        ax = plt.gca()

    # Plot the heatmap
    im = ax.imshow(data, **kwargs)

    # Create colorbar
    # cbar = ax.figure.colorbar(im, ax=ax, **cbar_kw)
    # cbar.ax.set_ylabel(cbarlabel, rotation=-90, va="bottom")

    # Show all ticks and label them with the respective list entries.
    ax.set_xticks(np.arange(data.shape[1]), labels=col_labels)
    ax.set_yticks(np.arange(data.shape[0]), labels=row_labels)

    # Let the horizontal axes labeling appear on top.
    ax.tick_params(top=True, bottom=False,
                   labeltop=True, labelbottom=False)

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=-30, ha="right",
             rotation_mode="anchor")

    # Turn spines off and create white grid.
    ax.spines[:].set_visible(False)

    ax.set_xticks(np.arange(data.shape[1]+1)-.5, minor=True)
    ax.set_yticks(np.arange(data.shape[0]+1)-.5, minor=True)
    ax.grid(which="minor", color="w", linestyle='-', linewidth=3)
    ax.tick_params(which="minor", bottom=False, left=False)

    return im

def annotate_heatmap(im, data=None, valfmt="{x:.2f}",
                     textcolors=("black", "white"),
                     threshold=None, **textkw):
    """
    A function to annotate a heatmap.

    Parameters
    ----------
    im
        The AxesImage to be labeled.
    data
        Data used to annotate.  If None, the image's data is used.  Optional.
    valfmt
        The format of the annotations inside the heatmap.  This should either
        use the string format method, e.g. "$ {x:.2f}", or be a
        `matplotlib.ticker.Formatter`.  Optional.
    textcolors
        A pair of colors.  The first is used for values below a threshold,
        the second for those above.  Optional.
    threshold
        Value in data units according to which the colors from textcolors are
        applied.  If None (the default) uses the middle of the colormap as
        separation.  Optional.
    **kwargs
        All other arguments are forwarded to each call to `text` used to create
        the text labels.
    """

    if not isinstance(data, (list, np.ndarray)):
        data = im.get_array()

    # Normalize the threshold to the images color range.
    if threshold is not None:
        threshold = im.norm(threshold)
    else:
        threshold = im.norm(data.max())/2.

    # Set default alignment to center, but allow it to be
    # overwritten by textkw.
    kw = dict(horizontalalignment="center",
              verticalalignment="center")
    kw.update(textkw)

    # Get the formatter in case a string is supplied
    if isinstance(valfmt, str):
        valfmt = matplotlib.ticker.StrMethodFormatter(valfmt)

    # Loop over the data and create a `Text` for each "pixel".
    # Change the text's color depending on the data.
    texts = []
    for i in range(data.shape[0]):
        for j in range(data.shape[1]):
            kw.update(color=textcolors[int(im.norm(data[i, j]) > threshold)])
            text = im.axes.text(j, i, valfmt(data[i, j], None), **kw)
            texts.append(text)

    return texts

def plot_cluster(plt:plt, cluster):
    fig, axs = plt.subplots(config.z_unit_num, config.x_unit_num, figsize=
    (config.x_node_num_each_unit, config.y_unit_num))  # axs array shape [x, z]
    # plt.rcParams['figure.figsize']=(1000, 500)
    fig.set_figheight(8)
    fig.set_figwidth(15)
    for z in range(config.z_unit_num):
        for x in range(config.x_unit_num):
            ax = axs[z, x]
            id_data = np.zeros((config.y_unit_num, config.x_node_num_each_unit))
            color_data = np.zeros((config.y_unit_num, config.x_node_num_each_unit))
            if cluster:
                for y in range(config.y_unit_num):
                    for x1 in range(config.x_node_num_each_unit):
                        key = ','.join([str(x*config.x_node_num_each_unit+x1),str(y),str(z)])
                        node_id = cluster.xyz_to_id(key)
                        id_data[y, x1] = node_id
                        if cluster.machines[node_id].state == Machine_State.run_state:
                            color_data[y, x1] = 1
                        # if node_id % 2 == 0:
                        #     color_data[y, x1] = 1

            container_id = ["y {}".format(i) for i in range(config.y_unit_num)]
            node_index_in_one_container = ["x {}".format(i) for i in range(config.x_node_num_each_unit)]
            im= heatmap(data = color_data, row_labels=container_id, col_labels=node_index_in_one_container,
            ax=ax, cmap="Wistia", cbarlabel="taken by job")
            annotate_heatmap(im, data = id_data, valfmt="{x:.0f}", size=9)

    plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0.1, hspace=0.1)
    plt.pause(0.5)
    plt.close()

def plot_cluster_with_scheduled_jobs(plt:plt, cluster, scheduled_jobs):
    fig, axs = plt.subplots(config.z_unit_num, config.x_unit_num, figsize=
    (config.x_node_num_each_unit, config.y_unit_num))
    fig.set_figheight(8)
    fig.set_figwidth(15)
    axs_iddata = {}
    axs_colordata = {}
    axs_num = axs.shape[0] * axs.shape[1] if config.z_unit_num > 1 or config.x_unit_num > 1 else 1
    axs = axs.reshape((1, axs_num))[0].tolist() if config.z_unit_num > 1 or config.x_unit_num > 1 else axs

    container_id = ["y {}".format(i) for i in range(config.y_unit_num)]
    node_index_in_one_container = ["x {}".format(i) for i in range(config.x_node_num_each_unit)]
    for axid in range(axs_num):
        axs_iddata[axid] = np.zeros((config.y_unit_num, config.x_node_num_each_unit))
        axs_colordata[axid] = np.zeros((config.y_unit_num, config.x_node_num_each_unit))

    jobid_to_color_index = {}
    color_index = 0
    for jobid in scheduled_jobs.keys():
        assert not jobid_to_color_index.__contains__(jobid)
        jobid_to_color_index[jobid] = color_index
        color_index += 1

    for jobid in scheduled_jobs.keys():
        for nodeid in scheduled_jobs[jobid]:
            axz, axx, y, x1 = node_id_to_point_in_plot(cluster, nodeid)
            axid = axz * config.x_unit_num + axx
            axs_iddata[axid][y, x1] = jobid
            axs_colordata[axid][y, x1] = jobid_to_color_index[jobid]

    for axid in range(axs_num):
        im = heatmap(data = axs_colordata[axid],
        row_labels=container_id, col_labels=node_index_in_one_container,
        ax = axs[axid] if config.z_unit_num > 1 or config.x_unit_num > 1 else axs, cmap = "magma_r")
        annotate_heatmap(im, data = axs_iddata[axid], valfmt="{x:.0f}", size=7, textcolors=("red", "green"))

    plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0.1, hspace=0.1)
    plt.pause(0.5)
    plt.close()

def node_id_to_point_in_plot(cluster, node_id):
    rack_id = int(node_id/cluster.nb_machines_one_rack)
    axz = int(rack_id/config.x_unit_num)
    axx = rack_id - axz * config.x_unit_num
    node_index_in_one_rack = node_id - cluster.nb_machines_one_rack * rack_id
    y = int(node_index_in_one_rack/config.x_node_num_each_unit)
    x1 = node_index_in_one_rack - y * config.x_node_num_each_unit
    return axz, axx, y, x1

if __name__ == '__main__':
    from hpc.cluster import ClusterWithTopology
    cluster = ClusterWithTopology()
    plot_cluster(plt, cluster)


num1 = int(input())
for i in range(num1):
    count=0
    flag =0
    n = int(input())
    a = list(map(int, input().split()))
    for i in range(n):
        if i%2!=a[i]%2:
            if (i%2==0):
                count+=1
            else:
                flag+=1
    if count == flag:
        print(count)
    else:
        print("-1")


import requests

# Define the Riot API endpoint.
endpoint = "https://na.api.riotgames.com/val/account/v1/accounts/by-riot-id/"

# Define the Valorant API endpoint for retrieving player stats.
stats_endpoint = "https://na.api.riotgames.com/val/stats/v1/player-stats/"

# Define the API headers, including your Riot API key.
headers = {
    "X-Riot-Token": "Your Riot API-KEY should go here"
}

# Request the player's Riot ID.
riot_id = input("Enter player Riot ID (e.g. player1#1234): ")
region = input("Enter player region (e.g. na): ")
url = endpoint + riot_id + "?game=val&region=" + region
response = requests.get(url, headers=headers)

# Retrieve the player's performance metrics if the Riot ID was found.
if response.status_code == 200:
    player_id = response.json()["puuid"]
    stats_url = stats_endpoint + player_id + "/competitive"
    stats_response = requests.get(stats_url, headers=headers)

    # Parse the performance metrics from the response and display them.
    if stats_response.status_code == 200:
        stats_json = stats_response.json()
        print(f"\nPerformance metrics for {riot_id}:")
        print(f"Kills per round: {stats_json['killStats']['killsPerRound']:.2f}")
        print(f"Deaths per round: {stats_json['killStats']['deathsPerRound']:.2f}")
        print(f"Assists per round: {stats_json['assistsPerRound']:.2f}")
        print(f"Headshot percentage: {stats_json['headshotPercentage']:.2f}%")
        print(f"Win percentage: {stats_json['winRate']:.2f}%")
    else:
        print("Unable to retrieve player stats.")
        print(f"Response code: {stats_response.status_code}")
        print(f"Response message: {stats_response.json()['message']}")
else:
    print("Player not found.")
    print(f"Response code: {response.status_code}")
    print(f"Response message: {response.json()['message']}")


from database.tvshow_db import TVShowDb
from model.tvshow import TVShow
from model.tvshow_episodes import TVShowEpisodes
import unittest
import os

# RUN - python -m unittest -v
# RUN - python -m unittest -v test.test_db

class MyTestDb(unittest.TestCase):
    db_name = 'database_test.db'
    # data for testing
    tvs1_id = 123
    tvs1 = (tvs1_id, 'Teste 1', 1, True, False)
    tvs2_id = 456
    tvs2 = (tvs2_id, 'Teste 2', 2, False, True)

    @classmethod
    def setUpClass(cls) -> None:
        cls.db = TVShowDb(db_name=cls.db_name)
        cls.main_table_name = 'tvshows'
        cls.episode_table_name = 'episodes'

    @classmethod
    def tearDownClass(cls) -> None:
        if os.path.isfile(cls.db_name):
            os.remove(cls.db_name)

    def check_episode_seen(self, line, value):
        self.assertIsInstance(line, tuple)
        ep = TVShowEpisodes(from_db=line)
        self.assertIsInstance(ep.watched, bool)
        self.assertIsInstance(line[4], int)
        if value:  # seen
            self.assertTrue(ep.watched)
            self.assertEqual(ep.watched_table, '✅')
            self.assertTrue(line[4])
            self.assertEqual(line[4], 1)
        else:  # not seen
            self.assertFalse(ep.watched)
            self.assertEqual(ep.watched_table, '')
            self.assertFalse(line[4])
            self.assertEqual(line[4], 0)

    def check_season_seen(self, lines, value):
        self.assertIsInstance(lines, list)
        for line in lines:
            self.check_episode_seen(line=line, value=value)

    def test_db_00_create_db_file(self):
        if os.path.isfile(self.db_name):
            os.remove(self.db_name)
        self.db.create_db_file()
        self.assertTrue(os.path.isfile(self.db_name))

    def test_db_01_01_create_main_table(self):
        cols = ['id', 'name', 'total_seasons', 'eu', 'pai']
        self.db.create_main_table()
        cols_ret = self.db.list_columns_from_table(table_name=self.main_table_name)
        self.assertEqual(cols_ret, cols)

    def test_db_01_02_create_episodes_table(self):
        cols = ['id', 'season', 'episode', 'air_date', 'watched']
        self.db.create_episode_table()
        cols_ret = self.db.list_columns_from_table(table_name=self.episode_table_name)
        self.assertEqual(cols_ret, cols)

    def test_db_01_03_all_tables(self):
        all_tables = [(self.episode_table_name,), (self.main_table_name,)]
        all_tables_ret = self.db.list_all_tables()
        self.assertEqual(all_tables_ret, all_tables)

    def test_db_02_01_insert_tvshow(self):
        self.db.insert_tvshow(*self.tvs1)
        self.db.insert_tvshow(*self.tvs2)
        lines = self.db.select_all_tvshows()
        self.assertEqual(len(lines), 2)
        self.assertEqual(lines[0], self.tvs1)
        self.assertEqual(lines[1], self.tvs2)

    def test_db_02_02_insert_tvshow_duplicate(self):
        ret1 = self.db.select_tvshow_exist(id=self.tvs1_id)
        self.assertEqual(ret1, 1)
        ret2 = self.db.select_tvshow_exist(id=self.tvs2_id)
        self.assertEqual(ret2, 1)
        ret3 = self.db.select_tvshow_exist(id=55323324332532523)
        self.assertEqual(ret3, 0)

    def test_db_02_03_insert_episode(self):
        self.db.insert_episode_mock(id=self.tvs1_id, season=1, episode_max=3, air_date=None, watched=False)
        self.db.insert_episode_mock(id=self.tvs2_id, season=1, episode_max=2, air_date=None, watched=False)
        lines = self.db.select_all_episodes()
        self.assertEqual(len(lines), 5)

    def test_db_03_01_select_all_tvshows(self):
        lines = self.db.select_all_tvshows()
        self.assertEqual(len(lines), 2)
        # eu
        tvs_eu = TVShow(tuple_from_db=lines[0])
        self.assertEqual(tvs_eu.id, self.tvs1[0])
        self.assertEqual(tvs_eu.name, self.tvs1[1])
        self.assertEqual(tvs_eu.number_of_seasons, self.tvs1[2])
        self.assertEqual(tvs_eu.eu, self.tvs1[3])
        self.assertEqual(tvs_eu.eu_desc, 'Sim')
        self.assertEqual(tvs_eu.pai, self.tvs1[4])
        self.assertEqual(tvs_eu.pai_desc, '')
        # pai
        tvs_pai = TVShow(tuple_from_db=lines[1])
        self.assertEqual(tvs_pai.id, self.tvs2[0])
        self.assertEqual(tvs_pai.name, self.tvs2[1])
        self.assertEqual(tvs_pai.number_of_seasons, self.tvs2[2])
        self.assertEqual(tvs_pai.eu, self.tvs2[3])
        self.assertEqual(tvs_pai.eu_desc, '')
        self.assertEqual(tvs_pai.pai, self.tvs2[4])
        self.assertEqual(tvs_pai.pai_desc, 'Sim')

    def test_db_03_02_select_all_tvshows(self):
        lines = self.db.select_all_episodes()
        self.assertEqual(len(lines), 5)

    def test_db_03_03_select_tvshow_tuple(self):
        tvs_tuple = (self.tvs1_id, 'Teste 1', 1, 3, 'Sim', '')
        lines = self.db.select_all_tvshows()
        self.db.select_all_episodes_by_tvshow(id=self.tvs1_id, debug=False)
        tvs = TVShow(tuple_from_db=lines[0], db_name=self.db_name)
        self.assertEqual(tvs.to_tuple(), tvs_tuple)

    def test_db_03_04_select_episode_by_tvshow(self):
        lines = self.db.select_all_episodes_by_tvshow(id=self.tvs1_id)
        self.assertEqual(len(lines), 3)
        ep_cls = TVShowEpisodes(from_db=lines[0])
        self.assertEqual(ep_cls.id, self.tvs1_id)
        self.assertEqual(ep_cls.season, 1)
        self.assertEqual(ep_cls.episode, 1)
        self.assertEqual(ep_cls.air_date, '')
        self.assertFalse(ep_cls.watched)

    def test_db_04_01_mark_episode_seen(self):
        line = self.db.select_episode(id=self.tvs1_id, season=1, episode=1)
        self.check_episode_seen(line=line, value=False)
        self.db.mark_reset_episode_seen(id=self.tvs1_id, season=1, episode=1, value=True)
        self.assertEqual(self.db.get_row_count(), 1)
        line2 = self.db.select_episode(id=self.tvs1_id, season=1, episode=1)
        self.check_episode_seen(line=line2, value=True)

    def test_db_04_02_reset_episode_seen(self):
        line = self.db.select_episode(id=self.tvs1_id, season=1, episode=1)
        self.check_episode_seen(line=line, value=True)
        self.db.mark_reset_episode_seen(id=self.tvs1_id, season=1, episode=1, value=False)
        self.assertEqual(self.db.get_row_count(), 1)
        line2 = self.db.select_episode(id=self.tvs1_id, season=1, episode=1)
        self.check_episode_seen(line=line2, value=False)

    def test_db_04_03_mark_season_seen(self):
        lines = self.db.select_episodes_from_season(id=self.tvs2_id, season=1)
        self.check_season_seen(lines=lines, value=False)
        self.db.mark_reset_season_seen(id=self.tvs2_id, season=1, value=True)
        self.assertEqual(self.db.get_row_count(), 2)
        lines2 = self.db.select_episodes_from_season(id=self.tvs2_id, season=1)
        self.check_season_seen(lines=lines2, value=True)

    def test_db_04_04_reset_season_seen(self):
        lines = self.db.select_episodes_from_season(id=self.tvs2_id, season=1)
        self.check_season_seen(lines=lines, value=True)
        self.db.mark_reset_season_seen(id=self.tvs2_id, season=1, value=False)
        self.assertEqual(self.db.get_row_count(), 2)
        lines2 = self.db.select_episodes_from_season(id=self.tvs2_id, season=1)
        self.check_season_seen(lines=lines2, value=False)

    def test_db_05_01_delete_episode_by_tvshow(self):
        lines = self.db.select_all_episodes_by_tvshow(id=self.tvs2_id)
        self.assertEqual(len(lines), 2)
        self.db.delete_all_episodes_from_tvshow(id=self.tvs2_id)
        lines = self.db.select_all_episodes_by_tvshow(id=self.tvs2_id)
        self.assertEqual(len(lines), 0)

    def test_db_05_02_delete_all_episodes(self):
        lines = self.db.select_all_episodes()
        self.assertEqual(len(lines), 3)
        self.db.delete_all_episodes()
        lines = self.db.select_all_episodes()
        self.assertEqual(len(lines), 0)

    def test_db_05_03_delete_all_tvshow(self):
        lines = self.db.select_all_tvshows()
        self.assertEqual(len(lines), 2)
        self.db.delete_all_tvshows()
        lines = self.db.select_all_tvshows()
        self.assertEqual(len(lines), 0)

if __name__ == '__main__':
    # unittest.main(failfast=True, exit=True)
    unittest.main()


import unittest
import doctest
import zope.component
from zope.component.testing import setUp, tearDown, PlacelessSetup
from zope.site.testing import siteSetUp
from zope.app.testing.setup import placefulSetUp
from zope.configuration.xmlconfig import XMLConfig
from ZODB.tests.util import DB
import transaction
import zope.intid

from zope.keyreference.testing import SimpleKeyReference

import quotationtool.quotation

_flags = doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS

def setUpRoot(test):
    db = DB()
    conn = db.open()
    dbroot = conn.root()
    dbroot['root'] = root = siteSetUp(True)
    transaction.commit()
    test.globs['root'] = root
    test.globs['db'] = db
    test.globs['conn'] = conn

    zope.component.provideAdapter(SimpleKeyReference)

    sm = root.getSiteManager()
    intids = sm['default']['intids'] = zope.intid.IntIds()
    sm.registerUtility(intids, zope.intid.interfaces.IIntIds)

def tearDownConnection(test):
    test.globs['conn'].close()
    tearDown(test)

def setUpZCML(test):
    setUp(test)
    XMLConfig('configure.zcml', quotationtool.quotation)()

def setUpOnlySome(test):
    """ Register only some components."""
    from quotationtool.quotation.source import plainTextQuotationFactory, restQuotationFactory
    from quotationtool.quotation.source import QuotationSourceTypesVocabulary
    from quotationtool.quotation.interfaces import IQuotationSourceFactory
    from zope.schema.interfaces import IVocabularyFactory
    import zope.component
    zope.component.provideUtility(
        plainTextQuotationFactory, IQuotationSourceFactory, 'plaintext')
    zope.component.provideUtility(
        restQuotationFactory, IQuotationSourceFactory, 'rest')
    zope.component.provideUtility(
        QuotationSourceTypesVocabulary,
        IVocabularyFactory,
        'quotationtool.quotation.SourceTypes')
    import zope
    import quotationtool
    XMLConfig('meta.zcml', zope.component)()
    XMLConfig('meta.zcml', zope.security)()
    XMLConfig('configure.zcml', zope.security)()
    XMLConfig('configure.zcml', zope.app.schema)()
    XMLConfig('configure.zcml', zope.component)()
    XMLConfig('configure.zcml', zope.security)()
    XMLConfig('configure.zcml', zope.site)()
    XMLConfig('configure.zcml', zope.annotation)()
    XMLConfig('configure.zcml', zope.dublincore)()
    XMLConfig('configure.zcml', quotationtool.site)()
    # subscribers
    from quotationtool.site.interfaces import INewQuotationtoolSiteEvent
    import quotationtool.relation
    zope.component.provideHandler(
        quotationtool.relation.createRelationCatalog,
        adapts=[INewQuotationtoolSiteEvent])
    zope.component.provideHandler(
        quotationtool.quotation.quotation.createRelationIndex,
        adapts=[INewQuotationtoolSiteEvent])
    # container object annotation
    from zope.annotation.interfaces import IAttributeAnnotatable

class SourceTests(PlacelessSetup, unittest.TestCase):

    def setUp(self):
        super(SourceTests, self).setUp()
        setUpZCML(self)

    def test_Types(self):
        import zope.schema
        field = zope.schema.Choice(
            title = u"source type",
            vocabulary = 'quotationtool.quotation.SourceTypes')
        self.assertTrue(field.validate(u'plaintext') is None)
        self.assertTrue(field.validate(u'rest') is None)
        self.assertRaises(zope.schema.interfaces.ConstraintNotSatisfied,
                          field.validate, (u'fails'))
        # if there is an other type registered in the renderer
        # package that is not known here, it should fail
        from quotationtool.renderer.plaintext import plainTextFactory
        from quotationtool.renderer.interfaces import ISourceFactory
        zope.component.provideUtility(
            plainTextFactory,
            provides = ISourceFactory,
            name = 'other_text_syntax')
        self.assertRaises(zope.schema.interfaces.ConstraintNotSatisfied,
                          field.validate, (u'other_text_syntax'))
        # but it should be OK for quotationtool.renderer.SourceTypes
        rfield = zope.schema.Choice(
            title = u"source type",
            vocabulary = 'quotationtool.renderer.SourceTypes')
        self.assertTrue(rfield.validate('other_text_syntax') is None)

    def test_PlainText(self):
        """ Test if quotationtool.renderer still works."""
        quotation = zope.component.createObject('plaintext', u"Hello World!")
        from quotationtool.renderer.plaintext import PlainText
        self.assertTrue(isinstance(quotation, PlainText))

class SiteCreationTests(PlacelessSetup, unittest.TestCase):

    def setUp(self):
        super(SiteCreationTests, self).setUp()
        setUpOnlySome(self)
        import quotationtool.site
        XMLConfig('configure.zcml', quotationtool.site)

    def test_RelationIndex(self):
        """ Test if a relation index is created on a new site event."""
        from quotationtool.site.site import QuotationtoolSite
        from zope.container.btree import BTreeContainer
        root = BTreeContainer()
        root['quotationtool'] = site = QuotationtoolSite()
        from zc.relation.interfaces import ICatalog
        cat = zope.component.getUtility(
            ICatalog, context = site)
        #self.assertTrue('ifigure-reference' in list(cat.iterSearchIndexes()))

def test_suite():
    return unittest.TestSuite((
            doctest.DocTestSuite('quotationtool.quotation.interfaces',
                                 setUp = setUpZCML,
                                 tearDown = tearDown,
                                 optionflags=_flags),
            doctest.DocTestSuite('quotationtool.quotation.quotation',
                                 setUp = setUpZCML,
                                 tearDown = tearDown,
                                 optionflags=_flags),
            doctest.DocFileSuite('README.txt',
                                 setUp = setUpZCML,
                                 tearDown = tearDown,
                                 optionflags=_flags),
            unittest.makeSuite(SourceTests),
            unittest.makeSuite(SiteCreationTests),
            ))


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from mozperftest.metrics.notebook.utilities import flat

class SingleJsonRetriever:
    """Transforms perfherder data into the standardized data format."""

    entry_number = 0

    def transform(self, data):
        self.entry_number += 1

        # flat(data, ()) returns a dict that have one key per dictionary path
        # in the original data.
        return [
            {
                "data": [{"value": i, "xaxis": self.entry_number} for i in v],
                "subtest": k,
            }
            for k, v in flat(data, ()).items()
        ]

    def merge(self, sde):
        grouped_data = {}
        for entry in sde:
            subtest = entry["subtest"]
            data = grouped_data.get(subtest, [])
            data.extend(entry["data"])
            grouped_data.update({subtest: data})

        merged_data = [{"data": v, "subtest": k} for k, v in grouped_data.items()]

        self.entry_number = 0
        return merged_data

    def summary(self, suite):
        """Summarize a suite of perfherder data into a single value.

        Returning None means that there's no summary. Otherwise, an integer
        or float must be returned.

        Only available in the Perfherder layer.
        """
        return None

    def subtest_summary(self, subtest):
        """Summarize a set of replicates for a given subtest.

        By default, it returns a None so we fall back to using the
        average of the replicates which is the default.

        Only available in the Perfherder layer.
        """
        return None


class CustomMetadataSchema:
    def __init__(
        self,
        type=None,
        select_options=None,
        default_value=None,
        is_value_required=None,
        min_value=None,
        max_value=None,
        min_length=None,
        max_length=None,
        **kwargs

    ):
        self.type = type
        self.select_options = select_options
        self.default_value = default_value
        self.is_value_required = is_value_required
        self.min_value = min_value
        self.max_value = max_value
        self.min_length = min_length
        self.max_length = max_length
        for key in kwargs.keys():
                self.__setattr__(key,kwargs[key])
    def __getattr__(self,key):
        return None