import sys
import getopt

from datetime import datetime
from typing import Optional

import wandb

import datasets
import torch
from torch.utils.data import DataLoader
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
)
from pytorch_lightning import LightningDataModule, LightningModule, Trainer, seed_everything
from pytorch_lightning.loggers import WandbLogger

class GLUEDataModule(LightningDataModule):
    task_text_field_map = {
        "cola": ["sentence"],
        "sst2": ["sentence"],
        "mrpc": ["sentence1", "sentence2"],
        "qqp": ["question1", "question2"],
        "stsb": ["sentence1", "sentence2"],
        "mnli": ["premise", "hypothesis"],
        "qnli": ["question", "sentence"],
        "rte": ["sentence1", "sentence2"],
        "wnli": ["sentence1", "sentence2"],
        "ax": ["premise", "hypothesis"],
    }

    glue_task_num_labels = {
        "cola": 2,
        "sst2": 2,
        "mrpc": 2,
        "qqp": 2,
        "stsb": 1,
        "mnli": 3,
        "qnli": 2,
        "rte": 2,
        "wnli": 2,
        "ax": 3,
    }

    loader_columns = [
        "datasets_idx",
        "input_ids",
        "token_type_ids",
        "attention_mask",
        "start_positions",
        "end_positions",
        "labels",
    ]

    def __init__(
        self,
        model_name_or_path: str,
        task_name: str = "mrpc",
        max_seq_length: int = 128,
        train_batch_size: int = 32,
        eval_batch_size: int = 32,
        **kwargs,
    ):
        super().__init__()
        self.model_name_or_path = model_name_or_path
        self.task_name = task_name
        self.max_seq_length = max_seq_length
        self.train_batch_size = train_batch_size
        self.eval_batch_size = eval_batch_size

        self.text_fields = self.task_text_field_map[task_name]
        self.num_labels = self.glue_task_num_labels[task_name]
        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path, use_fast=True)

    def setup(self, stage: str):
        self.dataset = datasets.load_dataset("glue", self.task_name)

        for split in self.dataset.keys():
            self.dataset[split] = self.dataset[split].map(
                self.convert_to_features,
                batched=True,
                remove_columns=["label"],
            )
            self.columns = [c for c in self.dataset[split].column_names if c in self.loader_columns]
            self.dataset[split].set_format(type="torch", columns=self.columns)

        self.eval_splits = [x for x in self.dataset.keys() if "validation" in x]

    def prepare_data(self):
        datasets.load_dataset("glue", self.task_name)
        AutoTokenizer.from_pretrained(self.model_name_or_path, use_fast=True)

    def train_dataloader(self):
        return DataLoader(self.dataset["train"], batch_size=self.train_batch_size, shuffle=True)

    def val_dataloader(self):
        if len(self.eval_splits) == 1:
            return DataLoader(self.dataset["validation"], batch_size=self.eval_batch_size)
        elif len(self.eval_splits) > 1:
            return [DataLoader(self.dataset[x], batch_size=self.eval_batch_size) for x in self.eval_splits]

    def test_dataloader(self):
        if len(self.eval_splits) == 1:
            return DataLoader(self.dataset["test"], batch_size=self.eval_batch_size)
        elif len(self.eval_splits) > 1:
            return [DataLoader(self.dataset[x], batch_size=self.eval_batch_size) for x in self.eval_splits]

    def convert_to_features(self, example_batch, indices=None):
        # Either encode single sentence or sentence pairs
        if len(self.text_fields) > 1:
            texts_or_text_pairs = list(zip(example_batch[self.text_fields[0]], example_batch[self.text_fields[1]]))
        else:
            texts_or_text_pairs = example_batch[self.text_fields[0]]

        # Tokenize the text/text pairs
        features = self.tokenizer.batch_encode_plus(
            texts_or_text_pairs, max_length=self.max_seq_length, pad_to_max_length=True, truncation=True
        )

        # Rename label to labels to make it easier to pass to model forward
        features["labels"] = example_batch["label"]

        return features

class GLUETransformer(LightningModule):
    def __init__(
        self,
        model_name_or_path: str,
        num_labels: int,
        task_name: str,
        learning_rate: float = 2e-5,
        adam_epsilon: float = 1e-8,
        warmup_steps: int = 0,
        weight_decay: float = 0.0,
        train_batch_size: int = 32,
        eval_batch_size: int = 32,
        eval_splits: Optional[list] = None,
        **kwargs,
    ):
        super().__init__()

        self.save_hyperparameters()

        self.config = AutoConfig.from_pretrained(model_name_or_path, num_labels=num_labels)
        self.model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, config=self.config)
        self.metric = datasets.load_metric(
            "glue", self.hparams.task_name, experiment_id=datetime.now().strftime("%d-%m-%Y_%H-%M-%S")
        )
        self.save_hyperparameters()

    def forward(self, **inputs):
        return self.model(**inputs)

    def training_step(self, batch, batch_idx):
        outputs = self(**batch)
        loss = outputs[0]
        self.log("train_loss", loss)
        return loss

    def validation_step(self, batch, batch_idx, dataloader_idx=0):
        outputs = self(**batch)
        val_loss, logits = outputs[:2]

        if self.hparams.num_labels > 1:
            preds = torch.argmax(logits, axis=1)
        elif self.hparams.num_labels == 1:
            preds = logits.squeeze()

        labels = batch["labels"]

        return {"loss": val_loss, "preds": preds, "labels": labels}

    def validation_epoch_end(self, outputs):
        if self.hparams.task_name == "mnli":
            for i, output in enumerate(outputs):
                # matched or mismatched
                split = self.hparams.eval_splits[i].split("_")[-1]
                preds = torch.cat([x["preds"] for x in output]).detach().cpu().numpy()
                labels = torch.cat([x["labels"] for x in output]).detach().cpu().numpy()
                loss = torch.stack([x["loss"] for x in output]).mean()
                self.log(f"val_loss_{split}", loss, prog_bar=True)
                split_metrics = {
                    f"{k}_{split}": v for k, v in self.metric.compute(predictions=preds, references=labels).items()
                }
                self.log_dict(split_metrics, prog_bar=True)
            return loss

        preds = torch.cat([x["preds"] for x in outputs]).detach().cpu().numpy()
        labels = torch.cat([x["labels"] for x in outputs]).detach().cpu().numpy()
        loss = torch.stack([x["loss"] for x in outputs]).mean()
        self.log("val_loss", loss, prog_bar=True)
        self.log_dict(self.metric.compute(predictions=preds, references=labels), prog_bar=True)

    def configure_optimizers(self):
        """Prepare optimizer and schedule (linear warmup and decay)"""
        model = self.model
        no_decay = ["bias", "LayerNorm.weight"]
        optimizer_grouped_parameters = [
            {
                "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
                "weight_decay": self.hparams.weight_decay,
            },
            {
                "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
                "weight_decay": 0.0,
            },
        ]
        optimizer = torch.optim.AdamW(optimizer_grouped_parameters, lr=self.hparams.learning_rate, eps=self.hparams.adam_epsilon)

        scheduler = get_linear_schedule_with_warmup(
            optimizer,
            num_warmup_steps=self.hparams.warmup_steps,
            num_training_steps=self.trainer.estimated_stepping_batches,
        )
        scheduler = {"scheduler": scheduler, "interval": "step", "frequency": 1}
        return [optimizer], [scheduler]

def load_arguments(argv, file_path, learning_rate, epsilon, warmup_steps, entity, project_name):
  arg_help = "{0} -f <file_path> -l <learning_rate> -e <epsilon> -w <warmup_steps> -y <entity> -p <project_name>".format(argv[0])

  try:
    opts, args = getopt.getopt(argv[1:], "hf:l:e:w:y:p:", ["help", "file_path=",
      "learning_rate=", "epsilon=", "warmup_steps=", "entity=", "project_name="])
  except:
      print(arg_help)
      sys.exit(2)

  for opt, arg in opts:
    if opt in ("-h", "--help"):
      print(arg_help)  # print the help message
      sys.exit(2)
    elif opt in ("-f", "--file_path"):
      file_path = arg
    elif opt in ("-l", "--learning_rate"):
      print(f'l: {arg}')
      learning_rate = float(arg)
    elif opt in ("-e", "--epsilon"):
      print(f'e: {arg}')
      epsilon = float(arg)
    elif opt in ("-w", "--warmup_steps"):
      print(f'w: {arg}')
      warmup_steps = int(arg)
    elif opt in ("-y", "--entity"):
        entity = arg
    elif opt in ("-p", "--project_name"):
        project_name = arg

  return file_path, learning_rate, epsilon, warmup_steps, entity, project_name

if __name__ == "__main__":
  seed_everything(42)

  learning_rate: float = 2e-5
  epsilon: float = 1e-8
  warmup_steps: int = 0
  file_path = "model.pth"
  entity = "mlops"
  project_name = "project-2"

  file_path, learning_rate, epsilon, warmup_steps, entity, project_name = load_arguments(sys.argv, file_path, learning_rate, epsilon, warmup_steps, entity, project_name)

  wandb.init(entity=entity, project=project_name)

  wandb_logger = WandbLogger(project=project_name, log_model="all")

  dm = GLUEDataModule(
      model_name_or_path="distilbert-base-uncased",
      task_name="mrpc",
  )
  dm.setup("fit")
  model = GLUETransformer(
      model_name_or_path="distilbert-base-uncased",
      num_labels=dm.num_labels,
      eval_splits=dm.eval_splits,
      task_name=dm.task_name,
      learning_rate=learning_rate,
      adam_epsilon=epsilon,
      warmup_steps=warmup_steps
  )

  trainer = Trainer(
      max_epochs=3,
      accelerator="auto",
      devices=1 if torch.cuda.is_available() else None,
      logger=wandb_logger
  )

  trainer.fit(model, datamodule=dm)

  torch.save(model.state_dict(), file_path)
  wandb.finish()


# inter not loppable, as well as float and bool, only string can loop
# for g in range(10):
#     print("**********")

# for i in '101':   COMMON ERROR
#     print("**********")
            # start, stop, step
# for q in range(10, -11, -2):
#     print(f"x = {q}")
# for q in range(1, 11, 1):
#     print(f"{q}", end = " ,")
# f = format

result = 0
for w in range(1, 11):
    old_result = result
    result = result + w
    print(f"{old_result}+{w}={result}")


from pandas import read_csv, DataFrame
from sklearn.linear_model import LinearRegression
from numpy import ndarray
import numpy as np
from numpy import ndarray
from pandas import Series
from sklearn.metrics import r2_score, mean_absolute_error
from regressors import stats
from scipy.stats import f

class Correlation:

    def __init__(self, data_frame: DataFrame, relevance_min_value: float):
        self.instance = data_frame.corr()
        self.relevance_min_value = relevance_min_value
        self.correlated_params = self.get_correlated_params()

    def get_correlated_params(self) -> list:
        fields = []
        params = []
        for i in self.instance:
            for j in self.instance.index[abs(self.instance[i]) > self.relevance_min_value]:
                if i != j and j not in fields and i not in fields:
                    fields.append(j)
                    params.append((i, j, self.instance[i][self.instance.index==j].values[0]))

        return params

class Regression:
    def __init__(self, data_frame: DataFrame, drop_params: list, correlation: Correlation):
        self.model = LinearRegression()
        self.correlation = correlation
        self.result_df = self._build_result_df(data_frame)
        self.result_nd = self.result_df.values.ravel()
        self.params_df = self._build_params_df(data_frame, drop_params)
        self.fit()
        self.predicted = self.predict()
        self.params_names = self.params_df.columns.values
        self.deviation = None
        self.std_errors = None
        self.t_values = None
        self.p_values = None
        self.f_stat = None
        self.dfn = self.params_df.shape[1]
        self.dfd = self.params_df.shape[0] - self.dfn - 1
        self.common_p_value = None
        self.f_crit = None
        self.r_squared = None
        self.adj_r_squared = None

    def calculate_metrics(self):
        self.deviation = self.calculate_deviation()
        self.std_errors = self.calculate_std_errors()
        self.t_values = self.calculate_t_values()
        self.p_values = self.calculate_p_values()
        self.f_stat = self.calculate_f_stat()
        self.common_p_value = self.calculate_common_p_value(self.f_stat)
        self.f_crit = self.calculate_f_crit()
        self.adj_r_squared = self.calculate_adj_r_squared()
        self.r_squared = self.calculate_r_squared()

    def calculate_r_squared(self):
        return r2_score(self.result_nd, self.predicted)

    def calculate_adj_r_squared(self):
        return stats.adj_r2_score(self.model, self.params_df, self.result_nd)

    def calculate_f_crit(self):
        return f.ppf(1-0.05, dfn=self.dfn, dfd=self.dfd)

    def calculate_common_p_value(self, f_stat):
        return f.sf(f_stat, self.dfn, self.dfd)

    def calculate_f_stat(self):
        return stats.f_stat(self.model, self.params_df, self.result_nd)

    def calculate_p_values(self):
        return stats.coef_pval(self.model, self.params_df, self.result_nd)

    def calculate_t_values(self):
        return stats.coef_tval(self.model, self.params_df, self.result_nd)

    def calculate_std_errors(self):
        return stats.coef_se(self.model, self.params_df, self.result_nd)

    def calculate_deviation(self):
        return mean_absolute_error(self.result_nd, self.predicted)

    @staticmethod
    def _build_params_df(data_frame: DataFrame, drop_params: list = None):
        if drop_params is None:
            drop_params = ['y']
        else:
            drop_params.append('y')

        return data_frame.drop(drop_params, axis=1)

    @staticmethod
    def _build_result_df(data_frame: DataFrame) -> DataFrame:
        return data_frame[['y']]

    def fit(self, x=None, y=None, weight=None):
        if x is not None and y is not None:
            self.model.fit(x, y, weight)

        self.model.fit(self.params_df, self.result_df.values.ravel())

    def predict(self, x=None) -> ndarray:
        if x is not None:
            values = self.model.predict(x)
        else:
            values = self.model.predict(self.params_df)

        return values

class RegressionAnalyzer:
    def __init__(self, file_path):
        self.file_path = file_path
        self.data_frame = self.read_file()
        self.correlation = None

    def read_file(self) -> DataFrame:
        return read_csv(self.file_path, ';')

    def print_data_frame(self):
        print(self.data_frame)

    def run(self) -> Regression:
        print("Загруженные данные:")
        self.print_data_frame()
        print("Построение кореляции:")
        relevance_value = float(input("Считать коллинеарными параметры больше:"))
        self.correlation = Correlation(self.data_frame, relevance_value)
        print("Построение регрессионной модели")
        drop_params = self.read_drop_params()
        if len(drop_params) > 0:
            print("Параметры {} будут исключены из модели".format(','.join(drop_params)))

        return Regression(self.data_frame, drop_params, self.correlation)

    def read_drop_params(self) -> list:
        params = [x for x in self.data_frame.columns.values if x != 'y']
        print("Какие параметры исключить из модели? (перечислить через запятую, доступны {})".format(','.join(params)))
        drop = []
        for param in input().split(','):
            if param not in params:
                continue
            drop.append(param)

        return drop


# Problem Statement - Write a Python program to flatten a shallow list.

# If list contains sublist
from itertools import chain

lst = [[10, 20, 30, 40], [50, 60, 70, 80], [90, 100, 110, 120]]

# # Make it one flat list i.e. [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]
# # Just take each individual sublist from the overall list,
# # and then take every item in each sublist and append to the flat list
# flatlist = [item for sublist in lst for item in sublist]

# Method 2 is to use chain from itertools but need to add asterisks on the iterable (the list)
# to ensure the list unpacked before iteration
# Else it will become a list within a list
flatlist = list(chain(*lst))

print(flatlist)


import tensorflow as tf

def batch_tensor(tensor, batch_size):
    """
        Batches a tensor
        Args:
            tensor,     a tensor of shape *
            batch_size, the batch_size
        Returns:
            a tensor of shape (batch_size, *) witht the tensor repeated in each
            batch entry
    """
    batch = tf.expand_dims(tensor, axis=0)
    return tf.repeat(batch, batch_size, axis=0)

def get_random_bool():
    """
        Generates a random bool scalar tensor
    """
    return tf.greater(tf.random.uniform((), dtype=tf.float32), 0.5)

def horizontal_flip(image, gt_boxes):
    """
        Flips the image horizontally
        Args:
            image,    the image tensor
            gt_boxes, the normalized ground truth boxes tensor
        Returns:
            image,    flipped image
            gt_boxes, flipped gt_boxes
    """
    flipped_image = tf.image.flip_left_right(image)
    flipped_gt_boxes = tf.stack(
        [
            1.0 - gt_boxes[..., 2],
            gt_boxes[..., 1],
            1.0 - gt_boxes[..., 0],
            gt_boxes[..., 3]
        ], axis=-1
    )
    return flipped_image, flipped_gt_boxes

def gaussian_noise(image, gt_boxes):
    """
        Applies color modifications to the images
        Args:
             image, image tensor
             gt_boxes, normalized gt_boxes
        Returns:
             noised_image, color modified image tensor
             gt_boxes, normalized gt_boxes
    """
    color_noise = tf.random.normal(tf.shape(image), 0.0,  0.1, tf.float32)
    noised_image = tf.add(image, color_noise)
    return tf.clip_by_value(noised_image, 0.0, 1.0), gt_boxes

def process_data(image, gt_boxes, height, width, apply_augmentation=False):
    """
        Data processing operation
        Args:
            image,              the image
            gt_boxes,           the ground truth boxes
            height,             the height to resize the image data to
            width,              the width to resize the image data to
            apply_augmentation, whether to apply data augmentation
        Returns:
            image, a tensor containing the image of shape (height, width, channels)
            gt_boxes, a tensor containing normalized ground truth boxes of
            shape (gt_boxes_count, 4 [x1, y1, x2, y2])
    """
    image = tf.image.convert_image_dtype(image, tf.float32)
    image = tf.image.resize(image, (height, width))
    if apply_augmentation:
        image, gt_boxes = tf.cond(get_random_bool(), lambda: horizontal_flip(image, gt_boxes), lambda: (image, gt_boxes))
        image, gt_boxes = tf.cond(get_random_bool(), lambda: gaussian_noise(image, gt_boxes), lambda: (image, gt_boxes))
    return image, gt_boxes


import numpy

from shapely.geometry import MultiLineString, LineString
from shapely.geometry import MultiPoint
from shapely.ops import split
from gistools.utils.collection import MemCollection
from gistools.utils.geometry import TLine, TMultiLineString

def representative_length(line_col, profile_col, id_field):
    """ creates a MemCollection with a line geometry, attributes copied from the input profile file,
        and three added fields for the representative length: pre length, post length and total
        representative length.

        receives two line input files:
        - line shape representing the waterways
        - line shape representing the profiles
        - id-field representing unique profile codes

        returns MemCollection profile_col with the representative length data added, and a MemCollection rep_lines_col
        with the lines based on the representative lengths
        """

    # Initalize new line collection for the split lines
    rep_lines_col = MemCollection(geometry_type='LineString')

    # Loop through each line and select the profiles that intersect the line
    for feature in line_col:
        # Check if line is LineString or MultiLineString and create the appropriate line object
        if type(feature['geometry']['coordinates'][0][0]) != tuple:
            line = TLine(feature['geometry']['coordinates'])
        else:
            line = TMultiLineString(feature['geometry']['coordinates'])

        # Initialize the list that collects the points that intersect with the line
        points = []

        # Loop through the profiles within the bounding box of the line
        for profile in profile_col.filter(bbox=line.bounds, precision=10**-6):
            if type(profile['geometry']['coordinates'][0][0]) != tuple:
                prof = LineString(profile['geometry']['coordinates'])
            else:
                prof = MultiLineString(profile['geometry']['coordinates'])

            # Making an intersection of the profile with the line creates a point
            x = line.intersection(prof)

            if isinstance(x, MultiPoint):
                raise ValueError('Profiel {0} heeft meerdere intersecties met de onderliggende watergang.'.format(
                    profile['properties'][id_field]
                ))

            # If the profile has an intersect with the line, and thus a point is created,
            # distance from the beginning of the line to the point is calculated, and it is added to
            # the points collection. Also, references to the original profile and line are added.
            if not x.is_empty:
                points.append({
                    'coords': x.coords[0],
                    'distance': line.project(x),
                    'profile': profile,
                    'line': line
                })

        # No profiles on the line
        if not points:
            continue

        # After all profiles on a line are defined, sort them by distance so that they are in the correct order
        sorted_points = sorted(points, key=lambda x: x['distance'])

        # Initialize list of points on which the lines will be split
        list_points = []

        # Loops again through the points to calculate the representative length variables. Four different situations
        # are treated: only one point on the line, the first point on the line, the last point on the line, and
        # points in between other points. At the same time split points are being calculated and added to the list.
        for i, point in enumerate(sorted_points):
            distance = point['distance']
            if len(sorted_points) == 1:
                point['na_lengte'] = line.length - distance
                point['voor_lengte'] = distance
            elif i != len(sorted_points) - 1:
                point['na_lengte'] = (sorted_points[i + 1]['distance'] - distance) / 2
                if i == 0:
                    point['voor_lengte'] = distance
                else:
                    point['voor_lengte'] = (distance - sorted_points[i - 1]['distance']) / 2
                    split_distance = distance - ((distance - sorted_points[i - 1]['distance']) / 2)
                    split_point = line.interpolate(split_distance)
                    list_points.append(split_point)
            else:
                point['voor_lengte'] = (distance - sorted_points[i - 1]['distance']) / 2
                split_distance = distance - ((distance - sorted_points[i - 1]['distance']) / 2)
                split_point = line.interpolate(split_distance)
                list_points.append(split_point)
                point['na_lengte'] = line.length - distance

        # If split points are present, lines will be split. Otherwise the whole line will be added to the rep_lines_col
        if list_points:

            # Define the line used for splitting
            current_line = line

            # Loop through the split points and define perpendicular lines to ensure splitting
            for j, point in enumerate(list_points):
                haaks = LineString(line.get_haakselijn_point(point, 0.001))
                split_line = split(current_line, haaks)

                # Initialize line parts (used to store the part of the line that belongs to the profile); also
                # initialize current parts, a list used to store the remaining parts of the line (that potentially need
                # to be split further. Also define the representative length per profile (distance_point). The total
                # distance of the line parts should correspond to this length.
                line_parts = []
                current_parts = []
                total_distance = 0
                distance_point = sorted_points[j]['voor_lengte'] + sorted_points[j]['na_lengte']

                # Loop through the line parts: in case of multilinestrings the split function causes all parts of the
                # line to be converted into linestrings, also when they were not split. These need to be grouped back
                # and converted back into a multilinestring. To see which lines belong together, the total
                # representative length per profile is used.
                for k, part in enumerate(split_line):
                    total_distance += part.length
                    if total_distance < distance_point:
                        line_parts.append(part)
                    elif numpy.isclose(total_distance, distance_point, 0.0000005):
                        line_parts.append(part)
                    elif total_distance > distance_point:
                        current_parts.append(part)

                # Convert line parts to multilinestring, and redefine the current line
                segment = MultiLineString(line_parts)
                current_line = MultiLineString(current_parts)

                # Copy properties to avoid aliasing and set the profile id to which this line part belongs.
                # Also add total length and write this part to the rep_lines_col
                props = feature['properties'].copy()
                props[id_field] = sorted_points[j]['profile']['properties'][id_field]
                props['tot_leng'] = segment.length
                # This can cause an error.
                if segment._is_empty:
                    continue

                try:
                    rep_lines_col.writerecords([
                    {'geometry': {'type': 'MultiLineString',
                                  'coordinates': [[p for p in l.coords] for l in segment]},
                     'properties': props}])
                except:
                    #'Fout in coordinaten bij profiel. Profiel wordt niet meegenomen: '+ str(props[id_field]))
                    continue

                # If the split point was the last to be treated, also add the remaining part of the line to the
                # last profile.
                if j == (len(list_points) - 1):
                    props = feature['properties'].copy()
                    props[id_field] = sorted_points[j + 1]['profile']['properties'][id_field]
                    props['tot_leng'] = current_line.length

                    rep_lines_col.writerecords([
                        {'geometry': {'type': 'MultiLineString',
                                      'coordinates': [[p for p in l.coords] for l in current_line]},
                         'properties': props}])

        else:
            props = feature['properties'].copy()
            props[id_field] = sorted_points[0]['profile']['properties'][id_field]
            props['tot_leng'] = line.length
            rep_lines_col.writerecords([
                    {'geometry': {'type': 'MultiLineString',
                                  'coordinates': [[p for p in l.coords] for l in line]},
                     'properties': props}])

        # The representative length data is written in the properties of the
        # profile collection, after which this updated profile collection and the rep_lines_collection with the split
        # lines is returned.
        for point in sorted_points:
            profile = point['profile']
            profile['properties']['voor_leng'] = int(round(float(point['voor_lengte']), 0))
            profile['properties']['na_leng'] = int(round(float(point['na_lengte']), 0))
            profile['properties']['tot_leng'] = profile['properties']['voor_leng'] + profile['properties']['na_leng']

    return profile_col, rep_lines_col


import csv
from tqdm import tqdm
from typing import Dict, List
from collections import defaultdict

from torch.utils.data import DataLoader

from molecular_ml.dataset.loader import MoleculeDataset, MoleculeDatapoint, MoleculeBitMapDataset
from molecular_ml.models.metrics import get_metric_func

def get_attributes(path: str, smiles_columns, target_columns):
    # Load data
    with open(path) as f:
        reader = csv.DictReader(f)
        all_smiles, all_targets = [], []
        for i, row in enumerate(tqdm(reader)):
            smiles = [row[c] for c in smiles_columns]
            targets = []
            for column in target_columns:
                value = row[column]
                targets.append(float(value))

            all_smiles.append(smiles)
            all_targets.append(targets)

    return all_smiles, all_targets

def get_data_from_smile(smile: str) -> MoleculeDataset:
    all_smiles = [[smile]]
    all_targets = [[0.0]]
    data = MoleculeDataset([
        MoleculeDatapoint(
            smiles=smiles,
            targets=targets,
            ) for i, (smiles, targets) in tqdm(enumerate(zip(all_smiles, all_targets)),
                                            total=len(all_smiles))
        ])
    return data

def get_data(path: str, cfg, skip_invalid_smiles = True) -> MoleculeDataset:
    """
    Gets SMILES and target values from a CSV file.
    :param path: Path to a CSV file.
    :param cfg: config data.
    :param skip_invalid_smiles: Whether to skip and filter out invalid smiles.
    :return: A :class:`dataset.loader.MoleculeDataset` containing SMILES and target values along
             with other info such as additional features when desired.
    """

    smiles_columns = [cfg.DATA_DIGEST.smiles_name]
    target_columns = cfg.DATA_DIGEST.labels_name

    all_smiles, all_targets = get_attributes(path, smiles_columns, target_columns)
    data = MoleculeDataset([
        MoleculeDatapoint(
            smiles=smiles,
            targets=targets,
            ) for i, (smiles, targets) in tqdm(enumerate(zip(all_smiles, all_targets)),
                                            total=len(all_smiles))
        ])

    # Filter out invalid SMILES
    if skip_invalid_smiles:
        original_data_len = len(data)
        data = filter_invalid_smiles(data)

        if len(data) < original_data_len:
            print(f'Warning: {original_data_len - len(data)} SMILES are invalid.')

    return data

def get_bit_map_loader_from_smile(smile: str) -> MoleculeDataset:
    all_smiles = [[smile]]
    all_targets = [[0.0]]
    molecule_bit_map_dataset = MoleculeBitMapDataset(all_smiles, all_targets)

    return DataLoader(
        molecule_bit_map_dataset,
        batch_size=1,
        shuffle=False,
        num_workers=1)

def get_molecule_bit_map_loader(path: str, cfg, max_len: int = 64):

    smiles_columns = [cfg.DATA_DIGEST.smiles_name]
    target_columns = cfg.DATA_DIGEST.labels_name

    all_smiles, all_targets = get_attributes(path, smiles_columns, target_columns)
    molecule_bit_map_dataset = MoleculeBitMapDataset(all_smiles, all_targets)

    return DataLoader(
        molecule_bit_map_dataset,
        batch_size=cfg.MODEL.TRAINING.batch_size,
        shuffle=False,
        num_workers=cfg.MODEL.num_workers,
    )

def filter_invalid_smiles(data: MoleculeDataset) -> MoleculeDataset:
    """
    Filters out invalid SMILES.
    :param data: A :class:`dataset.loader.MoleculeDataset`.
    :return: A :class:`dataset.loader.MoleculeDataset` with only the valid molecules.
    """
    return MoleculeDataset([datapoint for datapoint in tqdm(data)
                            if all(s != '' for s in datapoint.smiles) and all(m is not None for m in datapoint.mol)
                            and all(m.GetNumHeavyAtoms() > 0 for m in datapoint.mol if not isinstance(m, tuple))
                            and all(m[0].GetNumHeavyAtoms() + m[1].GetNumHeavyAtoms() > 0 for m in datapoint.mol if isinstance(m, tuple))])

def evaluate_predictions(preds: List[List[float]],
                         targets: List[List[float]],
                         metrics: List[str],
                         dataset_type: str) -> Dict[str, List[float]]:
    """
    Evaluates predictions using a metric function after filtering out invalid targets.

    :param preds: A list of lists of shape :code:`(data_size, num_tasks)` with model predictions.
    :param targets: A list of lists of shape :code:`(data_size, num_tasks)` with targets.
    :param num_tasks: Number of tasks.
    :return: A dictionary mapping each metric in :code:`metrics` to a list of values for each task.
    """

    metric_to_func = {metric: get_metric_func(metric) for metric in metrics}

    if len(preds) == 0:
        return {metric: [float('nan')] for metric in metrics}

    valid_preds = []
    valid_targets = []
    for j in range(len(preds)):
        if targets[j][0] is not None:  # Skip those without targets
            valid_preds.append(preds[j][0])
            valid_targets.append(targets[j][0])

    results = defaultdict(list)

    if dataset_type == 'classification':
        nan = False
        if all(target == 0 for target in valid_targets) or all(target == 1 for target in valid_targets):
            nan = True
        if all(pred == 0 for pred in valid_preds) or all(pred == 1 for pred in valid_preds):
            nan = True

        if nan:
            for metric in metrics:
                results[metric].append(float('nan'))

    for metric, metric_func in metric_to_func.items():
        if dataset_type == 'multiclass' and metric == 'cross_entropy':
            results[metric].append(metric_func(valid_targets, valid_preds,
                                            labels=list(range(len(valid_preds[0])))))
        else:
            results[metric].append(metric_func(valid_targets, valid_preds))

    results = dict(results)

    return results


# Team Project
# Bennett Miller
# Griffin Lumb
# Logan Courtney
# Terryl Dodson

import warnings
import numpy as np
import pandas as pd
import sys
import sklearn as sk
from sklearn import tree
import matplotlib.pyplot as plt
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix

warnings.simplefilter(action='ignore', category=FutureWarning)

data = pd.read_csv("clean_data.csv", usecols=[i for i in range(3,11)])

training_data, test_data = np.split(data, [int(.9 * len(data))])

training_y = training_data['death_yn']
test_y = test_data['death_yn']

# Encode target variables to boolean
pd.set_option("display.max_columns", 100)
for i in range(0, len(training_y)):
    if training_y[i] == "No":
        training_y[i] = 0
    else:
        training_y[i] = 1

for i in range(221781, len(test_y)+221781):
    if test_y[i] == "No":
        test_y[i] = 0
    else:
        test_y[i] = 1

del training_data['death_yn']
del test_data['death_yn']

x_training = training_data
x_test = test_data
x_test['age_group'] = x_test['age_group'].replace('Unknown', "20 - 29 Years")
x_test.loc[x_test.index[-1], 'age_group'] = "70 - 79 Years"

# Encode training data as boolean/splits up nominal multivariate categorical variables as multiples
x_training = pd.get_dummies(x_training, drop_first=True)
x_test = pd.get_dummies(x_test, drop_first=True)
features = list(x_training.columns)
print(features)

# as ints
training_y = training_y.astype('int')
test_y = test_y.astype('int')

# train decision tree on training data
clf = tree.DecisionTreeClassifier(max_depth=4)
fit = clf.fit(x_training, training_y)
tree.export_graphviz(fit, out_file="tree.dot", feature_names=features, class_names=True)
print(clf.get_depth())

# Predict test data with DT classifier
y_predict = clf.predict(x_test)
print("Accuracy = " + str(accuracy_score(test_y, y_predict)))

# Confusion Matrix
print(pd.DataFrame(
    confusion_matrix(test_y, y_predict),
    columns = ['Predicted Dead', 'Predicted Survival'],
    index = ['True Dead', 'True Survival']
))


import os

folder_name = "your-py-project"

baseCode = """# !/usr/bin/env python3

print("Hello World!")
"""

def create_a_folder():
    os.mkdir(folder_name)

def main():
    os.mkdir(f"{folder_name}/src")
    main_file = open(f'{folder_name}/src/main.py', 'w')
    main_file.write(baseCode)
    main_file.close()

def readme():
    readme_git = open(f'{folder_name}/README.md', 'w')
    readme_git.write('### My First Python App!')
    readme_git.close()

def license_project():
    license_git = open(f'{folder_name}/LICENSE', 'w')
    license_git.write('License goes here.')
    license_git.close()

def start():
    create_a_folder()
    main()
    license_project()
    readme()


import asyncio

import akinator
from akinator.async_aki import Akinator
from discord.ext.commands import BucketType, cooldown

aki = Akinator()

import datetime

import discord
import requests
from discord.ext import commands

class GuessName(commands.Cog):
    def __init__(self, client):
        self.client = client

    @commands.command()
    @cooldown(1, 60, BucketType.user)
    async def guess(self, ctx):
        q = await aki.start_game()
        question_num = 1
        while aki.progression <= 80:
            question = q + "\n\t"
            embed = discord.Embed(color=0xFF0000)
            embed.add_field(
                name=f"Q-{question_num}\n{question}",
                value="Reply with yes/y/no/n/i/idk/i don't know/p/probably/proably not/pn",
            )
            await ctx.send(embed=embed)

            def check(m):
                replies = [
                    "yes",
                    "y",
                    "no",
                    "n",
                    "i",
                    "idk",
                    "i don't know",
                    "probably",
                    "p",
                    "probably not",
                    "pn",
                ]
                return (
                    m.content in replies
                    and m.channel == ctx.channel
                    and m.author == ctx.author
                )

            msg = await self.client.wait_for("message", check=check)
            if msg.content == "b":
                try:
                    q = await aki.back()
                except akinator.CantGoBackAnyFurther:
                    pass
            else:
                q = await aki.answer(msg.content)
            question_num += 1
        await aki.win()

        embed = discord.Embed(
            title=f"It's {aki.first_guess['name']} ({aki.first_guess['description']})! Was I correct?\n\t",
            color=0xFF0000,
        )
        embed.add_field(name="Reply with yes/y/no/n", value="\u200b")
        await ctx.send(embed=embed)

        def check(m):
            return (
                m.content == "yes"
                and m.channel == ctx.channel
                and m.author == ctx.author
            )

        correct = await self.client.wait_for("message", check=check)
        if correct.content.lower() == "yes" or correct.content.lower() == "y":
            embed = discord.Embed(title="Yay! I guessed it right", color=0xFF0000)
            await ctx.send(embed=embed)
        else:
            embed = discord.Embed(title="Yay! I guessed it right", color=0xFF0000)
            await ctx.send(embed=embed)

        loop = asyncio.get_event_loop()
        loop.run_until_complete(GuessName.guess())
        loop.close()

def setup(client):
    client.add_cog(GuessName(client))


# -*- coding: utf-8 -*-
# @Time    : 30/12/2018 03:00
# @Author  : weiziyang
# @FileName: solution.py
# @Software: PyCharm
import GMK
from backend. MiniMax import MiniMax, BLACK_AGENT, WHITE_AGENT
import logging
import json

def MiniMaxSearch_solve(pos, agent, state, depth):
    new_agent = WHITE_AGENT if agent == BLACK_AGENT else BLACK_AGENT
    winner = GMK.check_win(state)
    if winner:
        return pos, state, agent, winner
    minimax = MiniMax(new_agent, state, pos, depth)
    solution = minimax.solve()
    logging.basicConfig(level=logging.DEBUG, format='%(asctime)s : %(message)s', filename='test')
    info = {
        'pos': pos,
        'depth': minimax.depth,
        'heuristic': minimax.heuristic,
        'nodes_num': minimax.nodes_num,
        'searched_nodes': minimax.searched_nodes,
        'cut_time': minimax.cut_time,
        'computing_time': minimax.evaluate_time,
        'mark': solution[2]
    }
    logging.info(json.dumps(info))
    new_pos = solution[0]
    new_board = solution[1]
    winner = GMK.check_win(new_board)
    return new_pos, new_board, new_agent, winner


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
The identities module is part of the nmeta suite

It provides an abstraction for participants (identities), using
a MongoDB database for storage and data retention maintenance.

Identities are identified via  TBD....

There are methods (see class docstring) that provide harvesting
of identity metadata and various retrieval searches
"""
import sys
import struct

#*** For packet methods:
import socket

#*** Import dpkt for packet parsing:
import dpkt

#*** mongodb Database Import:
import pymongo
from pymongo import MongoClient

#*** For timestamps:
import datetime

#*** For logging configuration:
from baseclass import BaseClass

#*** For Regular Expression searches:
import re

#*** For hashing of identities:
import hashlib

#*** How long in seconds to cache ARP responses for (in seconds):
ARP_CACHE_TIME = 14400
#*** DHCP lease time to use if none present (in seconds):
DHCP_DEFAULT_LEASE_TIME = 3600

class Identities(BaseClass):
    """
    An object that represents identity metadata

    Main function used to harvest identity metadata:
    (assumes class instantiated as an object called 'ident')

        ident.harvest(pkt, flow.packet)
            Passed a raw packet and packet metadata from flow object.
            Check a packet_in event and harvest any relevant identity
            indicators to metadata

    Functions available for Classifiers:
    (assumes class instantiated as an object called 'ident')

        ident.findbymac(mac_address)
            Look up identity object for a MAC address

        ident.findbynode(host_name)
            Look up identity object by host name (aka node)
            Additionally, can set:
                regex=True       Treat service_name as a regular expression
                harvest_type=    Specify what type of harvest (i.e. DHCP)

        ident.findbyservice(service_name)
            Look up identity object by service name
            Additionally, can set:
                regex=True        Treat service_name as a regular expression
                harvest_type=     Specify what type of harvest (i.e. DNS_A)
                ip_address=       Look for specific IP address

    See function docstrings for more information
    """

    def __init__(self, config, policy):
        """
        Initialise an instance of the Identities class
        """
        self.policy = policy
        #*** Required for BaseClass:
        self.config = config
        #*** Set up Logging with inherited base class method:
        self.configure_logging(__name__, "identities_logging_level_s",
                                       "identities_logging_level_c")
        #*** Get parameters from config:
        mongo_addr = config.get_value("mongo_addr")
        mongo_port = config.get_value("mongo_port")
        mongo_dbname = self.config.get_value("mongo_dbname")
        #*** Max bytes of the identities capped collection:
        identities_max_bytes = config.get_value("identities_max_bytes")
        #*** How far back in time to go back looking for an identity:
        self.identity_time_limit = datetime.timedelta \
                              (seconds=config.get_value("identity_time_limit"))
        #*** Max bytes of the dhcp_messages capped collection:
        dhcp_messages_max_bytes = config.get_value("dhcp_messages_max_bytes")
        #*** How far back in time to go back looking for an dhcp message:
        self.dhcp_messages_time_limit = datetime.timedelta \
                         (seconds=config.get_value("dhcp_messages_time_limit"))

        #*** Start mongodb:
        self.logger.info("Connecting to MongoDB database...")
        mongo_client = MongoClient(mongo_addr, mongo_port)

        #*** Connect to MongoDB nmeta database:
        db_nmeta = mongo_client[mongo_dbname]

        #*** Delete (drop) previous identities collection if it exists:
        self.logger.debug("Deleting previous identities MongoDB collection...")
        db_nmeta.identities.drop()

        #*** Create the identities collection, specifying capped option
        #*** with max size in bytes, so MongoDB handles data retention:
        self.identities = db_nmeta.create_collection('identities', capped=True,
                                            size=identities_max_bytes)

        #*** Index to improve look-up performance:
        self.identities.create_index([('valid_from', pymongo.DESCENDING),
                                        ('valid_to', pymongo.DESCENDING),
                                        ('ip_address', pymongo.ASCENDING),
                                        ('harvest_type', pymongo.ASCENDING)
                                        ],
                                        unique=False)

        #*** Index to improve MAC address look-up performance:
        self.identities.create_index([('mac_address', pymongo.ASCENDING),
                             ('valid_from', pymongo.DESCENDING)], unique=False)

        #*** Index to improve Node (host_name) look-up performance:
        self.identities.create_index([('host_name', pymongo.ASCENDING),
                             ('valid_from', pymongo.DESCENDING)], unique=False)

        #*** Index to improve Service look-up performance:
        self.identities.create_index([('service_name', pymongo.ASCENDING),
                             ('valid_from', pymongo.DESCENDING)], unique=False)

        #*** Delete (drop) previous dhcp_messages collection if it exists:
        self.logger.debug("Deleting previous dhcp_messages MongoDB "
                                                               "collection...")
        db_nmeta.dhcp_messages.drop()

        #*** Create the dhcp_messages collection, specifying capped option
        #*** with max size in bytes, so MongoDB handles data retention:
        self.dhcp_messages = db_nmeta.create_collection('dhcp_messages',
                                     capped=True, size=dhcp_messages_max_bytes)

        #*** Index dhcp_messages to improve look-up performance:
        self.dhcp_messages.create_index([('ingest_time', pymongo.DESCENDING),
                                        ('transaction_id', pymongo.ASCENDING),
                                        ('message_type', pymongo.ASCENDING)
                                        ],
                                        unique=False)

    class Identity(object):
        """
        An object that represents an individual Identity Indicator
        """
        def __init__(self):
            #*** Initialise identity variables:
            self.dpid = 0
            self.in_port = 0
            self.mac_address = ""
            self.ip_address = ""
            self.harvest_type = 0
            self.harvest_time = 0
            self.host_name = ""
            self.host_type = ""
            self.host_os = ""
            self.host_desc = ""
            self.service_name = ""
            self.service_alias = ""
            self.user_id = ""
            self.valid_from = ""
            self.valid_to = ""
            self.id_hash = ""
            self.location_logical = ""
            self.location_physical = ""

        def dbdict(self):
            """
            Return a dictionary object of identity metadata
            parameters for storing in the database
            """
            return self.__dict__

    class DHCPMessage(object):
        """
        An object that represents an individual DHCP message.
        Used for storing DHCP state by recording DHCP events
        """
        def __init__(self):
            #*** Initialise identity variables:
            self.dpid = 0
            self.in_port = 0
            self.ingest_time = 0
            self.eth_src = 0
            self.eth_dst = 0
            self.ip_src = 0
            self.ip_dst = 0
            self.tp_src = 0
            self.tp_dst = 0
            self.transaction_id = 0
            self.message_type = 0
            self.host_name = ""
            self.ip_assigned = 0
            self.ip_dhcp_server = 0
            self.lease_time = 0

        def dbdict(self):
            """
            Return a dictionary object of dhcp message
            parameters for storing in the database
            """
            return self.__dict__

    def harvest(self, pkt, flow_pkt):
        """
        Passed a raw packet and packet metadata from flow object.
        Check a packet_in event and harvest any relevant identity
        indicators to metadata
        """
        #*** ARP:
        if flow_pkt.eth_type == 2054:
            self.harvest_arp(pkt, flow_pkt)

        #*** DHCP:
        elif flow_pkt.eth_type == 2048 and flow_pkt.proto == 17 and \
                              (flow_pkt.tp_dst == 67 or flow_pkt.tp_dst == 68):
            self.harvest_dhcp(flow_pkt)

        #*** LLDP:
        elif flow_pkt.eth_type == 35020:
            self.harvest_lldp(flow_pkt)

        #*** DNS:
        elif (flow_pkt.proto == 6 or flow_pkt.proto == 17) and \
                        (flow_pkt.tp_src == 53 or flow_pkt.tp_src == 53):
            self.harvest_dns(flow_pkt)

        else:
            #*** Not an identity indicator
            return 0

    def harvest_arp(self, pkt, flow_pkt):
        """
        Harvest ARP identity metadata into database.
        Passed packet-in metadata from flow object.
        Check ARP reply and harvest identity
        indicators to metadata
        """
        self.logger.debug("Harvesting metadata from ARP request")

        eth = dpkt.ethernet.Ethernet(pkt)
        pkt_arp = eth.arp
        if pkt_arp:
            #*** It's an ARP, but is it a reply (opcode 2) for IPv4?:
            if pkt_arp.op == 2 and pkt_arp.pro == 2048:
                #*** Instantiate an instance of Indentity class:
                ident = self.Identity()
                ident.dpid = flow_pkt.dpid
                ident.in_port = flow_pkt.in_port
                ident.mac_address = mac_addr(pkt_arp.sha)
                ident.ip_address = socket.inet_ntoa(pkt_arp.spa)
                ident.harvest_type = 'ARP'
                ident.harvest_time = flow_pkt.timestamp
                ident.valid_from = flow_pkt.timestamp
                ident.valid_to = flow_pkt.timestamp + \
                                    datetime.timedelta(0, ARP_CACHE_TIME)
                ident.id_hash = self._hash_identity(ident)
                ident.location_logical = self.policy.locations.get_location \
                                                    (ident.dpid, ident.in_port)
                db_dict = ident.dbdict()
                #*** Write ARP identity metadata to database collection:
                self.logger.debug("writing db_dict=%s", db_dict)
                self.identities.insert_one(db_dict)
        return 1

    def harvest_dhcp(self, flow_pkt):
        """
        Harvest DHCP identity metadata into database.
        Passed packet-in metadata from flow object.
        Check LLDP TLV fields and harvest any relevant identity
        indicators to metadata
        """
        self.logger.debug("Harvesting metadata from DHCP request")
        dhcp_hostname = ""
        dhcp_leasetime = 0
        #*** Use dpkt to parse UDP DHCP data:
        try:
            pkt_dhcp = dpkt.dhcp.DHCP(flow_pkt.payload)
        except:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            self.logger.error("DHCP extraction failed "
                        "Exception %s, %s, %s",
                         exc_type, exc_value, exc_traceback)
            return 0
        #*** Turn DHCP options list of tuples into a dictionary:
        dhcp_opts = dict(pkt_dhcp.opts)
        self.logger.debug("dhcp_opts=%s", dhcp_opts)
        #*** Get the type of the DHCP message:
        try:
            dhcp_type = ord(dhcp_opts[53])
        except:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            self.logger.error("DHCP type extraction failed "
                        "Exception %s, %s, %s",
                         exc_type, exc_value, exc_traceback)
            return 0
        #*** Do stuff based on the DHCP message type:
        if dhcp_type == dpkt.dhcp.DHCPDISCOVER:
            self.logger.debug("Matched DHCPDISCOVER, TBD - not handled")
        elif dhcp_type == dpkt.dhcp.DHCPOFFER:
            self.logger.debug("Matched DHCPOFFER, TBD - not handled")
        elif dhcp_type == dpkt.dhcp.DHCPREQUEST:
            self.logger.debug("Matched DHCPREQUEST")
            if dpkt.dhcp.DHCP_OPT_HOSTNAME in dhcp_opts:
                #*** Instantiate an instance of DHCP class:
                self.dhcp_msg = self.DHCPMessage()
                self.dhcp_msg.dpid = flow_pkt.dpid
                self.dhcp_msg.in_port = flow_pkt.in_port
                self.dhcp_msg.ingest_time = flow_pkt.timestamp
                self.dhcp_msg.eth_src = flow_pkt.eth_src
                self.dhcp_msg.eth_dst = flow_pkt.eth_dst
                self.dhcp_msg.ip_src = flow_pkt.ip_src
                self.dhcp_msg.ip_dst = flow_pkt.ip_dst
                self.dhcp_msg.tp_src = flow_pkt.tp_src
                self.dhcp_msg.tp_dst = flow_pkt.tp_dst
                self.dhcp_msg.transaction_id = hex(pkt_dhcp.xid)
                self.dhcp_msg.host_name = str(dhcp_opts
                                                 [dpkt.dhcp.DHCP_OPT_HOSTNAME])
                self.dhcp_msg.message_type = 'DHCPREQUEST'
                #*** Record DHCP event to db collection:
                db_dict = self.dhcp_msg.dbdict()
                #*** Write DHCP message to db collection:
                self.logger.debug("writing dhcp_messages db_dict=%s", db_dict)
                self.dhcp_messages.insert_one(db_dict)
                return 1
        elif dhcp_type == dpkt.dhcp.DHCPDECLINE:
            self.logger.debug("Matched DHCPDECLINE, TBD - not handled")
        elif dhcp_type == dpkt.dhcp.DHCPACK:
            self.logger.debug("Matched DHCPACK")
            xid = hex(pkt_dhcp.xid)
            #*** Look up dhcp db collection for DHCPREQUEST:
            db_data = {'transaction_id': xid,
                        'message_type': 'DHCPREQUEST'}
            #*** Filter by documents that are still within 'best before' time:
            db_data['ingest_time'] = {'$gte': datetime.datetime.now() -
                                                 self.dhcp_messages_time_limit}
            #*** Run db search:
            result = self.dhcp_messages.find(db_data).sort('ingest_time', -1) \
                                                                      .limit(1)
            if result.count():
                result0 = list(result)[0]
                self.logger.debug("Found DHCPREQUEST for DHCPACK")
                #*** Found a DHCP Request for the ACK, record results:
                #*** Instantiate an instance of DHCP class:
                self.dhcp_msg = self.DHCPMessage()
                self.dhcp_msg.dpid = flow_pkt.dpid
                self.dhcp_msg.in_port = flow_pkt.in_port
                self.dhcp_msg.ingest_time = flow_pkt.timestamp
                self.dhcp_msg.eth_src = flow_pkt.eth_src
                self.dhcp_msg.eth_dst = flow_pkt.eth_dst
                self.dhcp_msg.ip_src = flow_pkt.ip_src
                self.dhcp_msg.ip_dst = flow_pkt.ip_dst
                self.dhcp_msg.tp_src = flow_pkt.tp_src
                self.dhcp_msg.tp_dst = flow_pkt.tp_dst
                self.dhcp_msg.transaction_id = hex(pkt_dhcp.xid)
                self.dhcp_msg.ip_assigned = \
                           socket.inet_ntoa(struct.pack(">L", pkt_dhcp.yiaddr))
                if dpkt.dhcp.DHCP_OPT_LEASE_SEC in dhcp_opts:
                    self.dhcp_msg.lease_time = struct.unpack('>L', dhcp_opts
                                            [dpkt.dhcp.DHCP_OPT_LEASE_SEC])[0]
                    self.logger.debug("Found dhcp_leasetime=%s",
                                                      self.dhcp_msg.lease_time)
                else:
                    self.dhcp_msg.lease_time = DHCP_DEFAULT_LEASE_TIME
                    self.logger.debug("Using default dhcp_leasetime=%s",
                                                      self.dhcp_msg.lease_time)
                self.dhcp_msg.message_type = 'DHCPACK'
                #*** Record DHCP event to db collection:
                db_dict = self.dhcp_msg.dbdict()
                #*** Write DHCP message to db collection:
                self.logger.debug("writing dhcp_messages db_dict=%s", db_dict)
                self.dhcp_messages.insert_one(db_dict)
                #*** Instantiate an instance of Identity class:
                ident = self.Identity()
                ident.dpid = flow_pkt.dpid
                ident.in_port = flow_pkt.in_port
                ident.mac_address = flow_pkt.eth_dst
                ident.ip_address = self.dhcp_msg.ip_assigned
                ident.harvest_type = 'DHCP'
                ident.host_name = result0['host_name']
                ident.harvest_time = flow_pkt.timestamp
                ident.valid_from = flow_pkt.timestamp
                #*** Calculate validity:
                ident.valid_to = flow_pkt.timestamp + \
                                datetime.timedelta(0, self.dhcp_msg.lease_time)
                ident.id_hash = self._hash_identity(ident)
                ident.location_logical = self.policy.locations.get_location \
                                                    (ident.dpid, ident.in_port)
                db_dict = ident.dbdict()
                #*** Write DHCP identity metadata to db collection:
                self.logger.debug("writing db_dict=%s", db_dict)
                self.identities.insert_one(db_dict)
                return 1
            else:
                self.logger.debug("Prev DHCP host_name not found")
                return 0
        elif dhcp_type == dpkt.dhcp.DHCPNAK:
            self.logger.debug("Matched DHCPNAK, TBD - not handled")
        elif dhcp_type == dpkt.dhcp.DHCPRELEASE:
            self.logger.debug("Matched DHCPRELEASE, TBD - not handled")
        elif dhcp_type == dpkt.dhcp.DHCPINFORM:
            self.logger.debug("Matched DHCPINFORM, TBD - not handled")
        else:
            self.logger.debug("Unknown DHCP option 53 value: %s", dhcp_type)
            return 0

    def harvest_lldp(self, flow_pkt):
        """
        Harvest LLDP identity metadata into database.
        Passed packet-in metadata from flow object.
        Check LLDP TLV fields and harvest any relevant identity
        indicators to metadata
        """
        self.logger.debug("Checking LLDP for metadata")
        payload = flow_pkt.payload
        lldp_dict = self._parse_lldp_detail(payload)
        if not len(lldp_dict):
            self.logger.warning("Failed to parse LLDP")
            return 0
        self.logger.debug("LLDP parsed %s", lldp_dict)
        #*** Instantiate an instance of Indentity class:
        ident = self.Identity()
        if 'system_name' in lldp_dict:
            ident.host_name = lldp_dict['system_name']
        if 'system_desc' in lldp_dict:
            ident.host_desc = lldp_dict['system_desc']
        if 'TTL' in lldp_dict:
            ttl = lldp_dict['TTL']
        else:
            #*** TBD, handle this better:
            ttl = 60
        ident.dpid = flow_pkt.dpid
        ident.in_port = flow_pkt.in_port
        ident.mac_address = flow_pkt.eth_src
        ident.harvest_type = 'LLDP'
        ident.harvest_time = flow_pkt.timestamp
        ident.valid_from = flow_pkt.timestamp
        #*** valid to based on LLDP TTL:
        ident.valid_to = flow_pkt.timestamp + \
                                    datetime.timedelta(0, ttl)
        #*** Try looking up an IP for the LLDP source MAC:
        ident2 = self.findbymac(ident.mac_address)
        if 'ip_address' in ident2:
            ident.ip_address = ident2['ip_address']
            self.logger.debug("Found ip=%s for LLDP flow_hash=%s",
                                    ident.ip_address, flow_pkt.flow_hash)
        else:
            self.logger.debug("Could not find IP for LLDP flow_hash=%s",
                                    flow_pkt.flow_hash)
        ident.id_hash = self._hash_identity(ident)
        ident.location_logical = self.policy.locations.get_location \
                                                    (ident.dpid, ident.in_port)
        #*** Write LLDP identity metadata to db collection:
        db_dict = ident.dbdict()
        self.logger.debug("writing db_dict=%s", db_dict)
        self.identities.insert_one(db_dict)
        return 1

    def harvest_dns(self, flow_pkt):
        """
        Harvest DNS identity metadata into database.
        Passed packet-in metadata from flow object.
        Check DNS answer(s) and harvest any relevant identity
        indicators to metadata
        """
        self.logger.debug("Checking DNS for metadata")
        #*** Use dpkt to parse DNS:
        try:
            pkt_dns = dpkt.dns.DNS(flow_pkt.payload)
        except:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            self.logger.error("DNS extraction failed "
                        "Exception %s, %s, %s",
                         exc_type, exc_value, exc_traceback)
            return 0
        answers = pkt_dns.an
        for answer in answers:
            if answer.type == 1:
                #*** DNS A Record:
                ident = self.Identity()
                ident.dpid = flow_pkt.dpid
                ident.in_port = flow_pkt.in_port
                ident.harvest_type = 'DNS_A'
                ident.ip_address = socket.inet_ntoa(answer.rdata)
                ident.service_name = answer.name
                ident.harvest_time = flow_pkt.timestamp
                ident.valid_from = flow_pkt.timestamp
                ident.valid_to = flow_pkt.timestamp + \
                                    datetime.timedelta(0, answer.ttl)
                ident.id_hash = self._hash_identity(ident)
                ident.location_logical = self.policy.locations.get_location \
                                                    (ident.dpid, ident.in_port)
                db_dict = ident.dbdict()
                #*** Write DNS identity metadata to database collection:
                self.logger.debug("writing db_dict=%s", db_dict)
                self.identities.insert_one(db_dict)
            elif answer.type == 5:
                #*** DNS CNAME Record:
                ident = self.Identity()
                ident.dpid = flow_pkt.dpid
                ident.in_port = flow_pkt.in_port
                ident.harvest_type = 'DNS_CNAME'
                ident.service_name = answer.name
                ident.service_alias = answer.cname
                ident.harvest_time = flow_pkt.timestamp
                ident.valid_from = flow_pkt.timestamp
                ident.valid_to = flow_pkt.timestamp + \
                                    datetime.timedelta(0, answer.ttl)
                ident.id_hash = self._hash_identity(ident)
                ident.location_logical = self.policy.locations.get_location \
                                                    (ident.dpid, ident.in_port)
                db_dict = ident.dbdict()
                #*** Write DNS identity metadata to database collection:
                self.logger.debug("writing db_dict=%s", db_dict)
                self.identities.insert_one(db_dict)
            else:
                #*** Not a type that we handle yet
                self.logger.debug("Unhandled DNS answer type=%s", answer.type)

    def findbymac(self, mac_addr, test=0):
        """
        Passed a MAC address and reverse search identities collection
        returning first match as a dictionary version of
        an Identity class, or empty dictionary if not found

        Setting test=1 returns database query execution statistics
        """
        db_data = {'mac_address': mac_addr}
        if not test:
            result = self.identities.find(db_data).sort('valid_from', -1).limit(1)
        else:
            return self.identities.find(db_data).sort('valid_from', -1).limit(1).explain()
        if result.count():
            result0 = list(result)[0]
            self.logger.debug("found result=%s len=%s", result0, len(result0))
            return result0
        else:
            self.logger.debug("mac_addr=%s not found", mac_addr)
            return {}

    def findbynode(self, host_name, harvest_type='any', regex=False, test=0):
        """
        Find by node name
        Pass it the name of the node to search for. Additionally,
        can set:
          regex=True       Treat service_name as a regular expression
          harvest_type=    Specify what type of harvest (i.e. DHCP)
        Returns a dictionary version of an Identity class, or 0 if not found

        Setting test=1 returns database query execution statistics
        """
        db_data = {'host_name': host_name}
        if harvest_type != 'any':
            #*** Filter by harvest type:
            db_data['harvest_type'] = harvest_type
        if regex:
            #*** Regular expression search on service name:
            regx = re.compile(host_name)
            db_data['host_name'] = regx
        #*** Filter by documents that are still within 'best before' time:
        db_data['valid_to'] = {'$gte': datetime.datetime.now()}
        #*** Run db search:
        if not test:
            result = self.identities.find(db_data).sort('valid_from', -1).limit(1)
        else:
            return self.identities.find(db_data).sort('valid_from', -1).limit(1).explain()
        if result.count():
            result0 = list(result)[0]
            self.logger.debug("found result=%s len=%s", result0, len(result0))
            return result0
        else:
            self.logger.debug("host_name=%s not found", host_name)
            return 0

    def findbyservice(self, service_name, harvest_type='any', regex=False,
                        ip_address='any', test=0):
        """
        Find by service name
        Pass it the name of the service to search for. Additionally,
        can set:
          regex=True        Treat service_name as a regular expression
          harvest_type=     Specify what type of harvest (i.e. DNS_A)
          ip_address=       Look for specific IP address
        Returns boolean

        Setting test=1 returns database query execution statistics
        """
        db_data = {'service_name': service_name}
        if harvest_type != 'any':
            #*** Filter by harvest type:
            db_data['harvest_type'] = harvest_type
        if ip_address != 'any':
            #*** Filter by IP address:
            db_data['ip_address'] = ip_address
        if regex:
            #*** Regular expression search on service name:
            regx = re.compile(service_name)
            db_data['service_name'] = regx
        #*** Filter by documents that are still within 'best before' time:
        db_data['valid_to'] = {'$gte': datetime.datetime.now()}
        #*** Run db search:
        if not test:
            result = self.identities.find(db_data).sort('valid_from', -1).limit(1)
        else:
            return self.identities.find(db_data).sort('valid_from', -1).limit(1).explain()
        if result.count():
            result0 = list(result)[0]
            self.logger.debug("found result=%s len=%s", result0, len(result0))
            return result0
        else:
            self.logger.debug("service_name=%s not found", service_name)
            return 0

    def _hash_identity(self, ident):
        """
        Generate a hash of the current identity used for deduplication
        where the same identity is received periodically, or from multiple
        sources.
        """
        hash_result = hashlib.md5()
        id_tuple = (ident.harvest_type,
                    ident.host_name,
                    ident.service_name,
                    ident.user_id)
        id_tuple_as_string = str(id_tuple)
        hash_result.update(id_tuple_as_string)
        return hash_result.hexdigest()

    #=================== PRIVATE ==============================================
    def _parse_lldp_detail(self, lldpPayload):
        """
        Parse basic LLDP parameters from an LLDP packet payload
        """
        result = {}

        while lldpPayload:
            tlv_header = struct.unpack("!H", lldpPayload[:2])[0]
            tlv_type = tlv_header >> 9
            tlv_len = (tlv_header & 0x01ff)
            lldpDU = lldpPayload[2:tlv_len + 2]
            if tlv_type == 0:
                #*** TLV type 0 is end of TLVs so break the while loop:
                break
            else:
                tlv_subtype = struct.unpack("!B", lldpDU[0:1]) \
                                                    if tlv_type is 2 else ""
                startbyte = 1 if tlv_type is 2 else 0
                tlv_datafield = lldpDU[startbyte:tlv_len]
            #*** Pull out values from specific TLVs:
            if tlv_type == 3:
                result['TTL'] = struct.unpack("!h", tlv_datafield)[0]
            elif tlv_type == 4:
                result['port_desc'] = tlv_datafield
            elif tlv_type == 5:
                result['system_name'] = tlv_datafield
            elif tlv_type == 6:
                result['system_desc'] = tlv_datafield
            else:
                pass
            lldpPayload = lldpPayload[2 + tlv_len:]
        return result

def mac_addr(address):
    """
    Convert a MAC address to a readable/printable string
    """
    return ':'.join('%02x' % ord(b) for b in address)


# -*- coding: utf-8 -*-
"""
@author: Cédric Berteletti
Calculs à partir des données de la centrale inertielle
"""

import logging

from dataclasses import dataclass

FILTRE_VITESSE_ANGULAIRE_MIN = 0.2 # en ° par seconde
FILTRE_ACCELERATION_MIN = 0.001 # en g (gravité terrestre 9.8 m/s²)

class CentraleInertielle():
    "Classe gérant toutes les données inertielles"

    def __init__(self):
        self.courant = DonneesInertielles()
        logging.debug(self.courant)
        self.data_liste = []
        self.offset_ax = 0.0
        self.offset_ay = 0.0
        self.offset_az = 0.0
        self.offset_valpha = 0.0
        self.offset_vbeta = 0.0
        self.offset_vgamma = 0.0

    def ajouter_telemetrie(self, log):
        if (log):
            data = DonneesInertielles()
            tokens = log.split("\", \"")
            if(len(tokens[1]) > 1):
                values = tokens[1][:-1].split(", ")

                # Récupération des valeurs de la télémétrie, en tenant compte de la calibration
                data.t = int(values[0])
                data.ax = float(values[1]) + self.offset_ax
                data.ay = float(values[2]) + self.offset_ay
                data.az = float(values[3]) + self.offset_az
                data.valpha = float(values[4]) + self.offset_valpha
                data.vbeta = float(values[5]) + self.offset_vbeta
                data.vgamma = float(values[6]) + self.offset_vgamma

                # Pour éviter la dérive des gyroscopes,
                # on filtre les trop petites vitesses angulaires
                if (abs(data.valpha) < FILTRE_VITESSE_ANGULAIRE_MIN):
                    data.valpha = 0.0
                if (abs(data.vbeta) < FILTRE_VITESSE_ANGULAIRE_MIN):
                    data.vbeta = 0.0
                if (abs(data.vgamma) < FILTRE_VITESSE_ANGULAIRE_MIN):
                    data.vgamma = 0.0

                # De même pour les accélérations
                if (abs(data.ax) < FILTRE_ACCELERATION_MIN):
                    data.ax = 0.0
                if (abs(data.ay) < FILTRE_ACCELERATION_MIN):
                    data.ay = 0.0
                if (abs(data.az) < FILTRE_ACCELERATION_MIN):
                    data.az = 0.0

                if (self.courant.t > 0):
                    # Il ne s'agit pas de la première donnée reçue, on peut donc calculer
                    # l'orientation et la position
                    temps_ecoule = (data.t - self.courant.t) / 1000
                    data.alpha = self.courant.alpha + data.valpha * temps_ecoule
                    logging.debug(f"Calcul du nouvel angle à deltat={temps_ecoule} s : a'={data.alpha} ; a={self.courant.alpha} ; va={data.valpha}")
                    data.beta = self.courant.beta + data.vbeta * temps_ecoule
                    data.gamma = self.courant.gamma + data.vgamma * temps_ecoule
                    # TODO

                # Mise à jour de la liste
                self.courant = data
                self.data_liste.append(data)

    def effacer_donnees(self):
        "Efface toutes les données stockées, mais ne modifie pas la calibration"
        self.data_liste = []
        self.courant = DonneesInertielles()

    def calibrer(self):
        sum_ax = 0.0
        sum_ay = 0.0
        sum_az = 0.0
        sum_valpha = 0.0
        sum_vbeta = 0.0
        sum_vgamma = 0.0
        n = 0

        # Calcul de la déviation sur tous les échantillons
        for data in self.data_liste:
            sum_ax = sum_ax + data.ax
            sum_ay = sum_ay + data.ay
            sum_az = sum_az + data.az
            sum_valpha = sum_valpha + data.valpha
            sum_vbeta = sum_vbeta + data.vbeta
            sum_vgamma = sum_vgamma + data.vgamma
            n = n + 1

        # Déviation moyenne en fonction du nombre n d'échantillons
        if(n>0):
            self.offset_ax = - sum_ax / n
            self.offset_ay = - sum_ay / n
            self.offset_az = - sum_az / n
            self.offset_valpha = - sum_valpha / n
            self.offset_vbeta = - sum_vbeta / n
            self.offset_vgamma = - sum_vgamma / n
        else:
            # Pas de donnée : on efface la calibration
            self.offset_ax = 0.0
            self.offset_ay = 0.0
            self.offset_az = 0.0
            self.offset_valpha = 0.0
            self.offset_vbeta = 0.0
            self.offset_vgamma = 0.0

        logging.info(f"CALIBRATION {self.offset_ax} ; {self.offset_ay} ; {self.offset_az} ; {self.offset_valpha} ; {self.offset_vbeta} ; {self.offset_vgamma}")

@dataclass
class DonneesInertielles:
    "Classe représentant les données de la centrale inertielle et les valeurs calculées à un instant t donné"
    t : int = 0             # en ms
    ax : float = 0.0        # en g (gravité terrestre 9.8 m/s²)
    ay : float = 0.0        # en g (gravité terrestre 9.8 m/s²)
    az : float = 0.0        # en g (gravité terrestre 9.8 m/s²)
    vx : float = 0.0        # en m/s
    vy : float = 0.0        # en m/s
    vz : float = 0.0        # en m/s
    x : float = 0.0         # en m
    y : float = 0.0         # en m
    z : float = 0.0         # en m
    valpha : float = 0.0    # en ° par seconde
    vbeta : float = 0.0     # en ° par seconde
    vgamma : float = 0.0    # en ° par seconde
    alpha : float = 0.0     # en °
    beta : float = 0.0      # en °
    gamma : float = 0.0     # en °


import warnings
warnings.filterwarnings("ignore", category = UserWarning, module = "matplotlib")

import matplotlib.pyplot as pl
import numpy as np
import sklearn.learning_curve as curves
from sklearn.tree import DecisionTreeRegressor
from sklearn.cross_validation import ShuffleSplit, train_test_split

def ModelLearning(X, y):
    cv = ShuffleSplit(X.shape[0], n_iter = 10, test_size = 0.2, random_state = 0)
    train_sizes = np.rint(np.linspace(1, X.shape[0]*0.8 - 1, 9)).astype(int)
    fig = pl.figure(figsize=(10,7))
    for k, depth in enumerate([1,3,6,10]):
        regressor = DecisionTreeRegressor(max_depth = depth)
        sizes, train_scores, test_scores = curves.learning_curve(regressor, X, y, \
            cv = cv, train_sizes = train_sizes, scoring = 'r2')
        train_std = np.std(train_scores, axis = 1)
        train_mean = np.mean(train_scores, axis = 1)
        test_std = np.std(test_scores, axis = 1)
        test_mean = np.mean(test_scores, axis = 1)
        ax = fig.add_subplot(2, 2, k+1)
        ax.plot(sizes, train_mean, 'o-', color = 'r', label = 'Training Score')
        ax.plot(sizes, test_mean, 'o-', color = 'g', label = 'Testing Score')
        ax.fill_between(sizes, train_mean - train_std, \
            train_mean + train_std, alpha = 0.15, color = 'r')
        ax.fill_between(sizes, test_mean - test_std, \
            test_mean + test_std, alpha = 0.15, color = 'g')

        ax.set_title('max_depth = %s'%(depth))
        ax.set_xlabel('Number of Training Points')
        ax.set_ylabel('Score')
        ax.set_xlim([0, X.shape[0]*0.8])
        ax.set_ylim([-0.05, 1.05])

    ax.legend(bbox_to_anchor=(1.05, 2.05), loc='lower left', borderaxespad = 0.)
    fig.suptitle('Learning Performances', fontsize = 16, y = 1.03)
    fig.tight_layout()
    fig.show()

def ModelComplexity(X, y):
    cv = ShuffleSplit(X.shape[0], n_iter = 10, test_size = 0.2, random_state = 0)
    max_depth = np.arange(1,11)
    train_scores, test_scores = curves.validation_curve(DecisionTreeRegressor(), X, y, \
        param_name = "max_depth", param_range = max_depth, cv = cv, scoring = 'r2')
    train_mean = np.mean(train_scores, axis=1)
    train_std = np.std(train_scores, axis=1)
    test_mean = np.mean(test_scores, axis=1)
    test_std = np.std(test_scores, axis=1)
    pl.figure(figsize=(7, 5))
    pl.title('Complexity Performance')
    pl.plot(max_depth, train_mean, 'o-', color = 'r', label = 'Training Score')
    pl.plot(max_depth, test_mean, 'o-', color = 'g', label = 'Validation Score')
    pl.fill_between(max_depth, train_mean - train_std, \
        train_mean + train_std, alpha = 0.15, color = 'r')
    pl.fill_between(max_depth, test_mean - test_std, \
        test_mean + test_std, alpha = 0.15, color = 'g')

    pl.legend(loc = 'lower right')
    pl.xlabel('Maximum Depth')
    pl.ylabel('Score')
    pl.ylim([-0.05,1.05])
    pl.show()


from common.mails.BaseMails import BaseMails
from partner.models import InitialContactDetails
from django.core.mail import BadHeaderError, EmailMessage
from django.http import HttpResponse
from django.template.loader import get_template
from django.template import Context
from common.receivers import choose_template

class PartnerRegistration(BaseMails):
    def __init__(self, partner_detail):
        self.send_partner_registration_mail(partner_detail)
        super(PartnerRegistration, self).__init__(partner_detail['partner_detail'], 'PartnerRegistration')

    def send_partner_registration_mail(self, partner_detail):
        detail = partner_detail['partner_detail']
        partner = InitialContactDetails.objects.filter(partner=detail['id'], type='P').first()
        subject = 'Customer registration' if detail['customer'] else 'Partner registration'
        t = 'Customer' if detail['customer'] else 'Partner'
        template = choose_template('PartnerRegistration')
        from_email = 'cloudsupport@redington.co.in'
        html = get_template(template['html'])
        assigning_values = Context({'username': partner.name, 'url': partner_detail['url'], 'type': t})
        html_content = html.render(assigning_values)

        try:
            mail = EmailMessage(subject, html_content, to=[partner.email], from_email=from_email)
            mail.content_subtype = 'html'
            mail.send(fail_silently=True)
        except BadHeaderError:
            return HttpResponse('Invalid header found.')
        return HttpResponse(True)


from multiprocessing import Process, freeze_support
from concurrent.futures import ProcessPoolExecutor
import time
def show ( name ) :
    print ( f'starting { name } ... ' )
    time.sleep ( 3 )
    print ( f'Ending {name } ... ' )
def main ( ) :
    with ProcessPoolExecutor (max_workers=2) as executor :
        names = [ ' one ' , ' Two ' , ' Three ' , " Four " ]
        executor.map ( show , names )

if __name__ == '__main__':
    freeze_support()
    main ( )


import os.path as op
from typing import List
from utils.iotools import read_json
from .bases import BaseDataset

class F30K(BaseDataset):
    """
    Flickr30K

    Reference:
    From image descriptions to visual denotations:
    New similarity metrics for semantic inference over event descriptions

    URL: https://aclanthology.org/Q14-1006/

    Dataset statistics:
    The Flickr30k dataset contains 31,000 images collected from Flickr, together with 5 reference sentences provided by human annotators.

    annotation format:
    {
        'images': list[{
            'sentids': [10, 11, 12, 13, 14],
            'imgid': 2,
            'sentences': [{
                'tokens': ['a', 'child', 'in', 'a', 'pink', 'dress'],
                'raw': 'A child in a pink dress',
                'imgid': 2 ,
                'sentid': 10
            }, ...],
            'split': 'train',
            'filename': '1000268201.jpg'
        }, ...],
        'dataset': 'flickr30k',
    }
    """
    dataset_dir = 'F30K'

    def __init__(self, root='', nlp_aug=False, verbose=True):
        super(F30K, self).__init__()
        self.dataset_dir = op.join(root, self.dataset_dir)
        self.img_dir = op.join(self.dataset_dir, 'flickr30k-images/')
        if nlp_aug:
            self.anno_path = op.join(self.dataset_dir, 'nlp_aug.json')
        else:
            self.anno_path = op.join(self.dataset_dir, 'karpathy/dataset_flickr30k.json')
        self._check_before_run()

        self.train_annos, self.test_annos, self.val_annos = self._split_anno(self.anno_path)

        self.train, self.train_id_container = self._process_anno(self.train_annos, training=True)
        self.test, self.test_id_container = self._process_anno(self.test_annos)
        self.val, self.val_id_container = self._process_anno(self.val_annos)

        if verbose:
            self.logger.info("=> F30K Images and Captions are loaded")
            self.show_dataset_info()

    def _split_anno(self, anno_path: str):
        train_annos, test_annos, val_annos = [], [], []
        annos = read_json(anno_path)['images']
        for anno in annos:
            if anno['split'] == 'train':
                train_annos.append(anno)
            elif anno['split'] == 'test':
                test_annos.append(anno)
            else:
                val_annos.append(anno)
        return train_annos, test_annos, val_annos

    def _process_anno(self, annos: List[dict], training=False):
        pid_container = set()
        if training:
            dataset = []
            for anno in annos:
                img_path = op.join(self.img_dir, anno['filename'])
                img_id = anno['imgid']
                sentences = anno['sentences'] # caption list
                for sentence in sentences:
                    assert img_id == sentence['imgid']
                    caption = sentence['raw']
                    dataset.append((img_id, -1,  img_path, caption))

            return dataset, pid_container
        else:
            dataset = {}
            img_paths = []
            captions = []
            img_ids = []
            caption_pids = []
            for anno in annos:

                img_path = op.join(self.img_dir, anno['filename'])
                img_paths.append(img_path)
                img_id = anno['imgid']
                img_ids.append(img_id)

                sentences = anno['sentences'] # caption list
                for sentence in sentences:
                    captions.append(sentence['raw'])
                    caption_pids.append(sentence['imgid'])
            dataset = {
                "image_pids": img_ids,
                "img_paths": img_paths,
                "caption_pids": caption_pids,
                "captions": captions
            }
            return dataset, pid_container

    def _check_before_run(self):
        """Check if all files are available before going deeper"""
        if not op.exists(self.dataset_dir):
            raise RuntimeError("'{}' is not available".format(self.dataset_dir))
        if not op.exists(self.img_dir):
            raise RuntimeError("'{}' is not available".format(self.img_dir))
        if not op.exists(self.anno_path):
            raise RuntimeError("'{}' is not available".format(self.anno_path))


from urllib import response
import requests, base64, subprocess, argparse, datetime

def login():
    url = 'http://127.0.0.1:5000/soap/server_sa'
    header = {'SOAPAction': 'urn:NETGEAR-ROUTER:service:DeviceConfig:1#SOAPLogin'}
    data = '<Username>admin</Username><Password>password123</Password>'

    response = requests.post(url, headers=header, data=data)
    cookie = response.cookies.get_dict()["sess_id"]
    #print(cookie)
    return cookie

def sendUpdate(loginCookie, xml):
    timeout = 10
    url = 'http://127.0.0.1:5000/soap/server_sa'
    header = {'SOAPAction': 'urn:NETGEAR-ROUTER:service:DeviceConfig:1#UpdateNewFirmware'}
    cookie = {'sess_id': loginCookie}
    data = xml

    try:
        response = requests.post(url, data=data, headers=header, cookies=cookie, timeout=timeout)
        print(response.text)
    except requests.exceptions.ConnectionError:
        print("Connection aborted :(\n")
        if (crashTest()):
            dumpCrash(xml)

    except requests.exceptions.ReadTimeout:
        dumpCrash(xml)

def craftXML(payload):
    xml = '<?xml version="1.0"?>\r\n'
    xml += '<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" SOAP-ENV:encodingStyle="http://schemas.xmlsoap.$\r\n'
    xml += '<SOAP-ENV:Body>\r\n'
    xml += 'UpdateNewFirmware\r\n'
    xml += '<NewFirmware>\r\n'
    xml += base64.b64encode(payload).decode("utf-8")
    xml += '\r\n</NewFirmware>\r\n'
    xml += '</SOAP-ENV:Body>\r\n'
    xml += '</SOAP-ENV:Envelope>\r\n'

    return xml

def crashTest():
    url = 'http://127.0.0.1:5000/soap/server_sa'
    try:
        requests.get(url)
        return False
    except:
        return True

def dumpCrash(xml):
    print("\nFOUND POTENTIAL CRASH\n")
    ts = datetime.datetime.now().strftime("%m-%d-%Y_%H:%M:%S") + ".dmp"
    print("Saving dump at ./"+ts)
    f = open(ts, "w+")
    f.write(xml)
    f.close()
    exit(1)

def readPayload(path):
    with open(path, 'rb') as f:
        return f.read()

def main(firmware_path, nofuzz, seed):
    cookie = login()

    if nofuzz:
        payload = readPayload(firmware_path)
        xml = craftXML(payload)
        sendUpdate(cookie, xml)

    else:
        while(True):
            command = 'radamsa '

            if seed:
                command += '--seed ' + "% s" % seed + ' '
                seed += 1

            payload = subprocess.check_output(command + firmware_path, shell=True)
            xml = craftXML(payload)
            sendUpdate(cookie, xml)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Fuzz the upnpd daemon using radamsa by making http requests")
    parser.add_argument("-f", "--firmware", help="Firmware in .chk format", nargs=1, required=True)
    parser.add_argument("-s", "--seed", help="Initial seed for Radamsa", nargs=1, required=False, type=int)
    parser.add_argument("--nofuzz", help="Send the given firmware without fuzzing", action="store_true")
    args = parser.parse_args()

    seed = ''
    if args.seed:
        seed = args.seed[0]

    main(args.firmware[0], args.nofuzz, seed)


'''
    Old version have a bug

class Node:
    def __init__(self, data):
        self.data = data
        self.left = None
        self.right = None
        self.level = None

    def __str__(self):
        return str(self.data)

class BST:
    def __init__(self):
        self.root = None

    def insert(self, data):
        self.root = self._insert(self.root, data)
        return self.root

    def _insert(self, root, data):
        if root is None:
            return Node(data)
        else:
            if data < root.data:
                root.left = self._insert(root.left, data)
            else:
                root.right = self._insert(root.right, data)
        return root

    def delete(self, r, data):
        self.root = BST._deleteNodeS(self.root, data)

    def _deleteNodeS(root: Node, key: int):
        if root is None: return root
        if int(key) < int(root.data):
            root.left = BST._deleteNodeS(root.left, key)
        elif int(key) > int(root.data):
            root.right = BST._deleteNodeS(root.right, key)
        else:
            if root.left is None or root.right is None:
                root = root.left if root.right is None else root.right
            else:
                temp = root.right
                while temp.left is not None:
                    temp = temp.left
                root.data = temp.data
                root.right = BST._deleteNodeS(root.right, temp.data)
        return root

    def search(self,key):
        return BST._search(self.root,key)

    def _search(root,key):
        if root is None :
            return None
        if int(root.data) < int(key):
            return BST._search(root.right,key)
        elif int(root.data) > int(key) :
            return BST._search(root.left,key)

        else:
            return root
'''
#new version fix the bug
class Node:
    def __init__(self, data):
        self.data = data
        self.left = None
        self.right = None
        self.level = None

    def __str__(self):
        return str(self.data)

class BinarySearchTree:
    def __init__(self):
        self.root = None

    def insert(self, data):
        self.root = self._insert(self.root, data)
        return self.root

    def _insert(self, root, data):
        if root is None:
            return Node(data)
        else:
            if data < root.data:
                root.left = self._insert(root.left, data)
            else:
                root.right = self._insert(root.right, data)
        return root

    def delete(self, data):
        self.root = BinarySearchTree._deleteNodeS(self.root, data)

    def _deleteNodeS(root: Node, key: int):
        if root is None: return root
        if int(key) < int(root.data):
            root.left = BinarySearchTree._deleteNodeS(root.left, key)
        elif int(key) > int(root.data):
            root.right = BinarySearchTree._deleteNodeS(root.right, key)
        else:
            if root.left is None or root.right is None:
                root = root.left if root.right is None else root.right
            else:
                temp = root.right
                while temp.left is not None:
                    temp = temp.left
                root.data = temp.data
                root.right = BinarySearchTree._deleteNodeS(root.right, temp.data)
        return root

def printTree90(node, level=0):
    if node != None:
        printTree90(node.right, level + 1)
        print('     ' * level, node)
        printTree90(node.left, level + 1)

def inOrder(root):
    if root is not None:
        # Left Subtree
        inOrder(root.left)
        # Root
        traversal.append(root.data)
        # Right Subtree
        inOrder(root.right)

tree = BinarySearchTree()
data = input("Enter Input : ").split(',')
for command in data:
    traversal = []
    inOrder(tree.root)
    command, data = command.split()
    data = int(data)
    if command == 'i':
        print(f'insert {data}')
        tree.insert(data)
        printTree90(tree.root)
    elif command == 'd':
        print(f'delete {data}')
        if tree.root is None:
            print('Error! Not Found DATA')
        else:
            if int(data) not in traversal :
                print('Error! Not Found DATA')
                printTree90(tree.root)
            else:
                tree.delete(data)
                printTree90(tree.root)


#! python3
'''
Zebra Puzzle
'''

import itertools

def immediate_right(house1, house2):
    '''
    House house1 is immediately right of house2 if house1-house2 == 1.
    '''
    return house1 - house2 == 1

def next_to(house1, house2):
    '''
    Two houses are next to each other if they differ by 1.
    '''
    return abs(house1 - house2) == 1

def zebra_puzzle():
    '''
    Return a tuple (WATER, ZEBRA) indicating their house numbers.
    '''
    houses = first, _, middle, _, _ = [1, 2, 3, 4, 5]
    orderings = list(itertools.permutations(houses))  # 1
    return next((WATER, ZEBRA)
                for (red, green, ivory, yellow, blue) in orderings
                if immediate_right(green, ivory)  # 6
                for (Englishman, Spaniard, Ukrainian, Japanese,
                     Norwegian) in orderings if Englishman is red  # 2
                if Norwegian is first  # 10
                if next_to(Norwegian, blue)  #15
                for (coffee, tea, milk, oj, WATER) in orderings
                if coffee is green  # 4
                if Ukrainian is tea  # 5
                if milk is middle  # 9
                for (OldGold, Kools, Chesterfields, LuckyStrike,
                     Parliaments) in orderings if Kools is yellow  # 8
                if LuckyStrike is oj  # 13
                if Japanese is Parliaments  # 14
                for (dog, snails, fox, horse, ZEBRA) in orderings
                if Spaniard is dog  # 3
                if OldGold is snails  # 7
                if next_to(Chesterfields, fox)  # 11
                if next_to(Kools, horse)  # 12
                )

def timed_call(fn, *args):
    '''
    Call function with args; return the time in seconds and result.
    '''
    import time
    start_time = time.clock()
    result = fn(*args)
    end_time = time.clock()
    return end_time - start_time, result

def average(numbers):
    "Return the average (arithmetic mean) of a sequence of numbers."
    return sum(numbers) / float(len(numbers))

def timed_calls(n, fn, *args):
    """Call fn(*args) repeatedly: n times if n is an int, or up to
    n seconds if n is a float; return the min, avg, and max time"""
    if isinstance(n, int):
        times = [timed_call(fn, *args)[0] for _ in range(n)]
    else:
        times = []
        total_time = 0.0
        while total_time < n:
            time = timed_call(fn, *args)[0]
            times.append(time)
            total_time += time
    return min(times), average(times), max(times)

def c(sequence):
    '''
    Generate items in sequence; keeping counts as we go.
    c.starts is the number of sequences started; c.items is number of items generated.
    '''
    c.starts += 1
    for item in sequence:
        c.items += 1
        yield item

def instrument_fn(fn, *args):
    '''
    Instrument for count how many items generator from generator.
    '''
    c.starts, c.items = 0, 0
    result = fn(*args)
    print('%s got %s with %5d iters over %7d items' % (fn.__name__, result,
                                                       c.starts, c.items))

def floor_puzzle():
    '''
    Hopper, Kay, Liskov, Perlis, and Ritchie live on
    different floors of a five-floor apartment building.
    Hopper does not live on the top floor.
    Kay does not live on the bottom floor.
    Liskov does not live on either the top or the bottom floor.
    Perlis lives on a higher floor than does Kay.
    Ritchie does not live on a floor adjacent to Liskov's.
    Liskov does not live on a floor adjacent to Kay's.
    '''
    floors = bottom, _, _, _, top = [1, 2, 3, 4, 5]
    orderings = itertools.permutations(floors)  # init
    for Hopper, Kay, Liskov, Perlis, Ritchie in orderings:
        if (Hopper is not top  # 1
                and Kay is not bottom  # 2
                and Liskov is not top and Liskov is not bottom  # 3
                and Perlis > Kay  # 4
                and abs(Ritchie - Liskov) > 1  # 5
                and abs(Liskov - Kay) > 1  # 6
            ):
            return [Hopper, Kay, Liskov, Perlis, Ritchie]

if __name__ == '__main__':
    print(timed_calls(1000, zebra_puzzle))
    print(floor_puzzle())


Pcurve = 11 # The proven prime

def modinv(a,n=Pcurve):
    lm, hm = 1,0
    low, high = a%n,n
    while low > 1:
        ratio = high/low
        nm, new = hm-lm*ratio, high-low*ratio
        lm, low, hm, high = nm, new, lm, low
    return lm % n

def ECadd(a,b):
    LamAdd = ((b[1]-a[1]) * modinv(b[0]-a[0],Pcurve)) % Pcurve
    x = (LamAdd*LamAdd-a[0]-b[0]) % Pcurve
    y = (LamAdd*(a[0]-x)-a[1]) % Pcurve
    return (x,y)

def ECdouble(a):
    Lam = ((3*a[0]*a[0]+Acurve) * modinv((2*a[1]),Pcurve)) % Pcurve
    x = (Lam*Lam-2*a[0]) % Pcurve
    y = (Lam*(a[0]-x)-a[1]) % Pcurve
    return (x,y)

def EccMultiply(GenPoint,ScalarHex):
    ScalarBin = str(bin(ScalarHex))[2:]
    Q=GenPoint
    for i in range (1, len(ScalarBin)):
        Q=ECdouble(Q)
        if ScalarBin[i] == "1":
            Q=ECadd(Q,GenPoint)
    return (Q)

print("ECC Point Addition and Multiplication - 18BCI0211")
print("1. Addition \n2. Multiplication\n")
choice = input("Choice: ")

print("y^2 = x^3 + Acurve * x + Bcurve (mod Pcurve)")

Acurve = int(input("Enter Acurve: "))
Bcurve = int(input("Enter Bcurve: "))
Pcurve = int(input("Enter Pcurve: "))

if choice == 1 or choice =="1":
    p1 = eval(input("Enter P1 (x,y):"))
    p2 = eval(input("Enter P2 (x,y):"))
    print("ECC Point Addition\n")
    print("P1 + P2 = ", ECadd(p1, p2))
if choice == 2 or choice =="2":
    p1 = eval(input("Enter P1 (x,y):"))
    n = int(input("Enter Multiplication Scalar N:"))
    print("ECC Point Multiplication\n")
    print("N * P1 = ", EccMultiply(p1, n))