import os
import random
from pathlib import Path
import yaml

import pandas as pd
import pytorch_lightning as pl
import torch
from torchaudio.transforms import AmplitudeToDB, MelSpectrogram

from desed_task.data_augm import mixup
from desed_task.utils.scaler import TorchScaler
from encoder import ManyHotEncoder
import numpy as np

from utils import (
    batched_decode_preds,
    log_sedeval_metrics,
)
from desed_task.evaluation.evaluation_measures import (
    compute_per_intersection_macro_f1,
    compute_psds_from_operating_points,
    compute_psds_from_scores,
)

import sed_scores_eval

from utils import classes_labels
from model import CRNN

class SED(pl.LightningModule):

    """Pytorch lightning module for the SED task.
    Args:
        hparams: dict, the dictionary to be used for the current experiment.
        encoder: ManyHotEncoder object, object to encode and decode labels.
        sed: torch.Module, the  model to be trained.
        opt: torch.optimizer.Optimizer object, the optimizer to be used.
        train_data: torch.utils.data.Dataset subclass object, the training data to be used.
        valid_data: torch.utils.data.Dataset subclass object, the validation data to be used.
        test_data: torch.utils.data.Dataset subclass object, the test data to be used.
        train_sampler: torch.utils.data.Sampler subclass object, the sampler to be used in the training dataloader.
        scheduler: BaseScheduler subclass object, the scheduler to be used.
                   This is used to apply ramp-up during training for example.
        fast_dev_run: bool, whether to launch a run with only one batch for each set, this is for development purpose,
                   to test the code runs.
    """

    def __init__(
        self,
        hparams,
        encoder,
        sed,
        opt=None,
        train_data=None,
        valid_data=None,
        test_data=None,
        train_sampler=None,
        scheduler=None,
        fast_dev_run=False,
        evaluation=False,
    ):
        super(SED, self).__init__()
        self.hparams.update(hparams)

        self.encoder = encoder
        self.sed = sed
        self.opt = opt
        self.train_data = train_data
        self.valid_data = valid_data
        self.test_data = test_data
        self.train_sampler = train_sampler
        self.scheduler = scheduler
        self.fast_dev_run = fast_dev_run
        self.evaluation = evaluation

        if self.fast_dev_run:
            self.num_workers = 1
        else:
            self.num_workers = self.hparams["training"]["num_workers"]

        feat_params = self.hparams["feats"]

        self.mel_spec = MelSpectrogram(
            sample_rate=feat_params["sample_rate"],
            n_fft=feat_params["n_window"],
            win_length=feat_params["n_window"],
            hop_length=feat_params["hop_length"],
            f_min=feat_params["f_min"],
            f_max=feat_params["f_max"],
            n_mels=feat_params["n_mels"],
            window_fn=torch.hamming_window,
            wkwargs={"periodic": False},
            power=1,
            center=False,
        )

        # * instantiating loss fn and scaler
        self.loss_fn = torch.nn.BCELoss()

        self.scaler = self._init_scaler()

        # * buffer for event based scores which we compute using sed-eval
        self.val_buffer_synth = {
            k: pd.DataFrame() for k in self.hparams["training"]["val_thresholds"]
        }

        self.val_buffer_test = {
            k: pd.DataFrame() for k in self.hparams["training"]["val_thresholds"]
        }

        self.val_scores_postprocessed_buffer_synth = {}

        test_n_thresholds = self.hparams["training"]["n_test_thresholds"]
        test_thresholds = np.arange(
            1 / (test_n_thresholds * 2), 1, 1 / test_n_thresholds
        )
        self.test_psds_buffer = {k: pd.DataFrame() for k in test_thresholds}
        self.decoded_05_buffer = pd.DataFrame()
        self.test_scores_raw_buffer = {}
        self.test_scores_postprocessed_buffer = {}

    _exp_dir = None

    @property
    def exp_dir(self):
        if self._exp_dir is None:
            try:
                self._exp_dir = self.logger.log_dir
                if self._exp_dir is None:
                    self._exp_dir = self.hparams["log_dir"]
            except Exception as e:
                self._exp_dir = self.hparams["log_dir"]
        return self._exp_dir

    def lr_scheduler_step(self, scheduler, optimizer_idx, metric):
        scheduler.step()

    def _init_scaler(self):
        """Scaler inizialization function. It can be either a dataset or instance scaler.

        Raises:
            NotImplementedError: in case of not Implemented scaler

        Returns:
            TorchScaler: returns the scaler
        """

        if self.hparams["scaler"]["statistic"] == "instance":
            scaler = TorchScaler(
                "instance",
                self.hparams["scaler"]["normtype"],
                self.hparams["scaler"]["dims"],
            )
            return scaler
        elif self.hparams["scaler"]["statistic"] == "dataset":
            scaler = TorchScaler(
                "dataset",
                self.hparams["scaler"]["normtype"],
                self.hparams["scaler"]["dims"],
            )
        else:
            raise NotImplementedError

        if self.hparams["scaler"]["savepath"] is not None:
            if os.path.exists(self.hparams["scaler"]["savepath"]):
                scaler = torch.load(self.hparams["scaler"]["savepath"])
                print(
                    "Loaded Scaler from previous checkpoint from {}".format(
                        self.hparams["scaler"]["savepath"]
                    )
                )
                return scaler

        self.train_loader = self.train_dataloader()
        scaler.fit(
            self.train_loader,
            transform_func=lambda x: self.take_log(self.mel_spec(x[0])),
        )

        if self.hparams["scaler"]["savepath"] is not None:
            torch.save(scaler, self.hparams["scaler"]["savepath"])
            print(
                "Saving Scaler from previous checkpoint at {}".format(
                    self.hparams["scaler"]["savepath"]
                )
            )
            return scaler

    def take_log(self, mels):
        """Apply the log transformation to mel spectrograms.
        Args:
            mels: torch.Tensor, mel spectrograms for which to apply log.

        Returns:
            Tensor: logarithmic mel spectrogram of the mel spectrogram given as input
        """

        amp_to_db = AmplitudeToDB(stype="amplitude")
        amp_to_db.amin = 1e-5  # amin= 1e-5 as in librosa
        return amp_to_db(mels).clamp(min=-50, max=80)

    def training_step(self, batch, batch_indx):
        """Apply the training for one batch (a step). Used during trainer.fit

        Args:
            batch: torch.Tensor, batch input tensor
            batch_indx: torch.Tensor, 1D tensor of indexes to know which data are present in each batch.

        Returns:
           torch.Tensor, the loss to take into account.
        """

        audio, labels, _ = batch
        features = self.mel_spec(audio)

        mixup_type = self.hparams["training"].get("mixup")
        if mixup_type is not None and 0.5 > random.random():
            features, labels = mixup(features, labels, mixup_label_type=mixup_type)

        # sed forward
        preds = self.sed(self.scaler(self.take_log(features)))

        # loss on strong labels
        loss = self.loss_fn(preds, labels)

        self.log("train/loss", loss, prog_bar=True, sync_dist=True)
        self.log(
            "train/step",
            self.scheduler["scheduler"].step_num,
            prog_bar=True,
            sync_dist=True,
        )
        self.log(
            "train/lr", self.opt.param_groups[-1]["lr"], prog_bar=True, sync_dist=True
        )

        return loss

    def validation_step(self, batch, batch_indx):
        """Apply validation to a batch (step). Used during trainer.fit

        Args:
            batch: torch.Tensor, input batch tensor
            batch_indx: torch.Tensor, 1D tensor of indexes to know which data are present in each batch.
        Returns:
        """

        audio, labels, _, filenames = batch

        features = self.mel_spec(audio)
        preds = self.sed(self.scaler(self.take_log(features)))

        loss = self.loss_fn(preds, labels)

        self.log("val/synth/loss", loss)

        filenames_synth = [
            x
            for x in filenames
            if Path(x).parent == Path(self.hparams["data"]["synth_val_folder"])
        ]

        (
            scores_raw_strong,
            scores_postprocessed_strong,
            decoded_strong,
        ) = batched_decode_preds(
            preds,
            filenames_synth,
            self.encoder,
            median_filter=self.hparams["training"]["median_window"],
            thresholds=list(self.val_buffer_synth.keys()),
        )

        self.val_scores_postprocessed_buffer_synth.update(scores_postprocessed_strong)
        for th in self.val_buffer_synth.keys():
            self.val_buffer_synth[th] = pd.concat(
                [self.val_buffer_synth[th], decoded_strong[th]], ignore_index=True
            )

        return

    def validation_epoch_end(self, outputs):
        """Function applied at the end of all the validation steps of the epoch.

        Args:
            outputs: torch.Tensor, the concatenation of everything returned by validation_step.

        Returns:
            torch.Tensor, the objective metric to be used to choose the best model from for example.
        """

        # * synth val dataset
        ground_truth = sed_scores_eval.io.read_ground_truth_events(
            self.hparams["data"]["synth_val_tsv"]
        )
        audio_durations = sed_scores_eval.io.read_audio_durations(
            self.hparams["data"]["synth_val_dur"]
        )
        if self.fast_dev_run:
            ground_truth = {
                audio_id: ground_truth[audio_id]
                for audio_id in self.val_scores_postprocessed_buffer_synth
            }
            audio_durations = {
                audio_id: audio_durations[audio_id]
                for audio_id in self.val_scores_postprocessed_buffer_synth
            }
        else:
            # * drop audios without events
            ground_truth = {
                audio_id: gt for audio_id, gt in ground_truth.items() if len(gt) > 0
            }
            audio_durations = {
                audio_id: audio_durations[audio_id] for audio_id in ground_truth.keys()
            }
        psds1_sed_scores_eval = compute_psds_from_scores(
            self.val_scores_postprocessed_buffer_synth,
            ground_truth,
            audio_durations,
            dtc_threshold=0.7,
            gtc_threshold=0.7,
            cttc_threshold=None,
            alpha_ct=0,
            alpha_st=1,
            # save_dir=os.path.join(save_dir, "", "scenario1"),
        )
        intersection_f1_macro = compute_per_intersection_macro_f1(
            self.val_buffer_synth,
            self.hparams["data"]["synth_val_tsv"],
            self.hparams["data"]["synth_val_dur"],
        )
        synth_event_macro = log_sedeval_metrics(
            self.val_buffer_synth[0.5],
            self.hparams["data"]["synth_val_tsv"],
        )[0]

        obj_metric_synth_type = self.hparams["training"].get("obj_metric_synth_type")
        if obj_metric_synth_type is None:
            synth_metric = psds1_sed_scores_eval
        elif obj_metric_synth_type == "event":
            synth_metric = synth_event_macro
        elif obj_metric_synth_type == "intersection":
            synth_metric = intersection_f1_macro
        elif obj_metric_synth_type == "psds":
            synth_metric = psds1_sed_scores_eval
        else:
            raise NotImplementedError(
                f"obj_metric_synth_type: {obj_metric_synth_type} not implemented."
            )

        obj_metric = torch.tensor(synth_metric)

        self.log("val/obj_metric", obj_metric, prog_bar=True, sync_dist=True)
        self.log(
            "val/synth/psds1_sed_scores_eval", psds1_sed_scores_eval, sync_dist=True
        )
        self.log(
            "val/synth/intersection_f1_macro", intersection_f1_macro, sync_dist=True
        )
        self.log("val/synth/event_f1_macro", synth_event_macro, sync_dist=True)

        # * free the buffers
        self.val_buffer_synth = {
            k: pd.DataFrame() for k in self.hparams["training"]["val_thresholds"]
        }
        self.val_scores_postprocessed_buffer_synth = {}

        return obj_metric

    def on_save_checkpoint(self, checkpoint):
        checkpoint["sed"] = self.sed.state_dict()
        return checkpoint

    def test_step(self, batch, batch_indx):
        """Apply Test to a batch (step), used only when (trainer.test is called)

        Args:
            batch: torch.Tensor, input batch tensor
            batch_indx: torch.Tensor, 1D tensor of indexes to know which data are present in each batch.
        Returns:
        """

        audio, labels, _, filenames = batch

        features = self.mel_spec(audio)
        preds = self.sed(self.scaler(self.take_log(features)))

        if not self.evaluation:
            loss = self.loss_fn(preds, labels)

            self.log("test/loss", loss)

        # * compute psds (Polyphonic Sound Detection Score)
        (
            scores_raw_strong,
            scores_postprocessed_strong,
            decoded_strong,
        ) = batched_decode_preds(
            preds,
            filenames,
            self.encoder,
            median_filter=self.hparams["training"]["median_window"],
            thresholds=list(self.test_psds_buffer.keys()) + [0.5],
        )

        self.test_scores_raw_buffer.update(scores_raw_strong)
        self.test_scores_postprocessed_buffer.update(scores_postprocessed_strong)
        for th in self.test_psds_buffer.keys():
            self.test_psds_buffer[th] = pd.concat(
                [self.test_psds_buffer[th], decoded_strong[th]], ignore_index=True
            )

        self.decoded_05_buffer = pd.concat(
            [self.decoded_05_buffer, decoded_strong[0.5]]
        )

    def on_test_epoch_end(self):
        save_dir = os.path.join(self.exp_dir, "metrics_test")

        # * if evaluation is True, we only save the scores
        if self.evaluation:
            save_dir_raw = os.path.join(save_dir, "_scores", "raw")
            sed_scores_eval.io.write_sed_scores(
                self.test_scores_raw_buffer, save_dir_raw
            )
            print(f"\nRaw scores for  saved in: {save_dir_raw}")

            save_dir_postprocessed = os.path.join(save_dir, "_scores", "postprocessed")
            sed_scores_eval.io.write_sed_scores(
                self.test_scores_postprocessed_buffer, save_dir_postprocessed
            )
            print(f"\nPostprocessed scores for  saved in: {save_dir_postprocessed}")
        else:
            # * calculate the metrics and save them
            ground_truth = sed_scores_eval.io.read_ground_truth_events(
                self.hparams["data"]["test_tsv"]
            )
            audio_durations = sed_scores_eval.io.read_audio_durations(
                self.hparams["data"]["test_dur"]
            )
            if self.fast_dev_run:
                ground_truth = {
                    audio_id: ground_truth[audio_id]
                    for audio_id in self.test_scores_postprocessed_buffer
                }
                audio_durations = {
                    audio_id: audio_durations[audio_id]
                    for audio_id in self.test_scores_postprocessed_buffer
                }
            else:
                # drop audios without events
                ground_truth = {
                    audio_id: gt for audio_id, gt in ground_truth.items() if len(gt) > 0
                }
                audio_durations = {
                    audio_id: audio_durations[audio_id]
                    for audio_id in ground_truth.keys()
                }
            psds1_psds_eval = compute_psds_from_operating_points(
                self.test_psds_buffer,
                self.hparams["data"]["test_tsv"],
                self.hparams["data"]["test_dur"],
                dtc_threshold=0.7,
                gtc_threshold=0.7,
                alpha_ct=0,
                alpha_st=1,
                save_dir=os.path.join(save_dir, "", "scenario1"),
            )
            psds1_sed_scores_eval = compute_psds_from_scores(
                self.test_scores_postprocessed_buffer,
                ground_truth,
                audio_durations,
                dtc_threshold=0.7,
                gtc_threshold=0.7,
                cttc_threshold=None,
                alpha_ct=0,
                alpha_st=1,
                save_dir=os.path.join(save_dir, "", "scenario1"),
            )

            psds2_psds_eval = compute_psds_from_operating_points(
                self.test_psds_buffer,
                self.hparams["data"]["test_tsv"],
                self.hparams["data"]["test_dur"],
                dtc_threshold=0.1,
                gtc_threshold=0.1,
                cttc_threshold=0.3,
                alpha_ct=0.5,
                alpha_st=1,
                save_dir=os.path.join(save_dir, "", "scenario2"),
            )
            psds2_sed_scores_eval = compute_psds_from_scores(
                self.test_scores_postprocessed_buffer,
                ground_truth,
                audio_durations,
                dtc_threshold=0.1,
                gtc_threshold=0.1,
                cttc_threshold=0.3,
                alpha_ct=0.5,
                alpha_st=1,
                save_dir=os.path.join(save_dir, "", "scenario2"),
            )

            event_macro = log_sedeval_metrics(
                self.decoded_05_buffer,
                self.hparams["data"]["test_tsv"],
                os.path.join(save_dir, ""),
            )[0]

            # synth dataset
            intersection_f1_macro = compute_per_intersection_macro_f1(
                {"0.5": self.decoded_05_buffer},
                self.hparams["data"]["test_tsv"],
                self.hparams["data"]["test_dur"],
            )

            results = {
                "test/psds1_psds_eval": psds1_psds_eval,
                "test/psds1_sed_scores_eval": psds1_sed_scores_eval,
                "test/psds2_psds_eval": psds2_psds_eval,
                "test/psds2_sed_scores_eval": psds2_sed_scores_eval,
                "test/event_f1_macro": event_macro,
                "test/intersection_f1_macro": intersection_f1_macro,
            }

        if self.logger is not None:
            self.logger.log_metrics(results)
            self.logger.log_hyperparams(self.hparams, results)

        for key in results.keys():
            self.log(key, results[key], prog_bar=True, logger=True, sync_dist=True)

    def configure_optimizers(self):
        return [self.opt], [self.scheduler]

    def train_dataloader(self):
        self.train_loader = torch.utils.data.DataLoader(
            self.train_data,
            batch_size=self.hparams["training"]["batch_size"],
            num_workers=self.num_workers,
        )
        return self.train_loader

    def val_dataloader(self):
        self.val_loader = torch.utils.data.DataLoader(
            self.valid_data,
            batch_size=self.hparams["training"]["batch_size_val"],
            num_workers=self.num_workers,
            shuffle=False,
            drop_last=False,
        )
        return self.val_loader

    def test_dataloader(self):
        self.test_loader = torch.utils.data.DataLoader(
            self.test_data,
            batch_size=self.hparams["training"]["batch_size_val"],
            num_workers=self.num_workers,
            shuffle=False,
            drop_last=False,
        )
        return self.test_loader

    def forward(self, x):
        features = self.mel_spec(x)
        features = features.unsqueeze(0)
        preds = self.sed(self.scaler(self.take_log(features)))
        return preds

if __name__ == "__main__":
    with open("params.yaml", "r") as f:
        config = yaml.safe_load(f)

    encoder = ManyHotEncoder(
        list(classes_labels.keys()),
        audio_len=config["data"]["audio_max_len"],
        frame_len=config["feats"]["n_filters"],
        frame_hop=config["feats"]["hop_length"],
        net_pooling=config["data"]["net_subsample"],
        fs=config["data"]["fs"],
    )

    sed = SED(config, encoder=encoder, sed=CRNN(**config["net"]))
    print(sed.state_dict().keys())


from __future__ import annotations
import torch
import torch.nn as nn

from .qmodules import _QModule, _QActivation
from ...qbase import QRangeSpecType, QGranularitySpecType, QHParamsInitStrategySpecType

SUPPORTED_ACTIVATION_FPMODULES = (
    nn.Identity,
    nn.ReLU,
    nn.ReLU6,
    nn.LeakyReLU,
)

class QIdentity(_QActivation, nn.Identity):

    def __init__(self,
                 qrangespec:               QRangeSpecType,
                 qgranularityspec:         QGranularitySpecType,
                 qhparamsinitstrategyspec: QHParamsInitStrategySpecType):

        super(_QModule, self).__init__()

        _QActivation.__init__(self,
                              qrangespec,
                              qgranularityspec,
                              qhparamsinitstrategyspec)

    def _register_qop(self):
        raise NotImplementedError

    def _call_qop(self, x: torch.Tensor) -> torch.Tensor:
        raise NotImplementedError

    @classmethod
    def from_fp_module(cls,
                       fpm:                      nn.Identity,
                       qrangespec:               QRangeSpecType,
                       qgranularityspec:         QGranularitySpecType,
                       qhparamsinitstrategyspec: QHParamsInitStrategySpecType) -> QIdentity:
        """Special constructor to build ``QIdentity``s from FP ``Identity``s."""

        qidentity = cls(qrangespec,
                        qgranularityspec,
                        qhparamsinitstrategyspec)

        return qidentity

class QReLU(_QActivation, nn.ReLU):

    def __init__(self,
                 qrangespec:               QRangeSpecType,
                 qgranularityspec:         QGranularitySpecType,
                 qhparamsinitstrategyspec: QHParamsInitStrategySpecType,
                 inplace:                  bool = False):

        super(_QModule, self).__init__(inplace=inplace)

        _QActivation.__init__(self,
                              qrangespec,
                              qgranularityspec,
                              qhparamsinitstrategyspec)

    def _register_qop(self):
        raise NotImplementedError

    def _call_qop(self, x: torch.Tensor) -> torch.Tensor:
        raise NotImplementedError

    @classmethod
    def from_fp_module(cls,
                       fpm:                      nn.ReLU,
                       qrangespec:               QRangeSpecType,
                       qgranularityspec:         QGranularitySpecType,
                       qhparamsinitstrategyspec: QHParamsInitStrategySpecType) -> QReLU:
        """Special constructor to build ``QReLU``s from FP ``ReLU``s."""

        qrelu = cls(qrangespec,
                    qgranularityspec,
                    qhparamsinitstrategyspec,
                    inplace=fpm.inplace)

        return qrelu

class QReLU6(_QActivation, nn.ReLU6):

    def __init__(self,
                 qrangespec:               QRangeSpecType,
                 qgranularityspec:         QGranularitySpecType,
                 qhparamsinitstrategyspec: QHParamsInitStrategySpecType,
                 inplace:                  bool = False):

        super(_QModule, self).__init__(inplace=inplace)

        _QActivation.__init__(self,
                              qrangespec,
                              qgranularityspec,
                              qhparamsinitstrategyspec)

    def _register_qop(self):
        raise NotImplementedError

    def _call_qop(self, x: torch.Tensor) -> torch.Tensor:
        raise NotImplementedError

    @classmethod
    def from_fp_module(cls,
                       fpm:                      nn.ReLU6,
                       qrangespec:               QRangeSpecType,
                       qgranularityspec:         QGranularitySpecType,
                       qhparamsinitstrategyspec: QHParamsInitStrategySpecType) -> QReLU6:
        """Special constructor to build ``QReLU6``s from FP ``ReLU6``s."""

        qrelu6 = cls(qrangespec,
                     qgranularityspec,
                     qhparamsinitstrategyspec,
                     inplace=fpm.inplace)

        return qrelu6

class QLeakyReLU(_QActivation, nn.LeakyReLU):

    def __init__(self,
                 qrangespec:               QRangeSpecType,
                 qgranularityspec:         QGranularitySpecType,
                 qhparamsinitstrategyspec: QHParamsInitStrategySpecType,
                 negative_slope:           float = 1e-2,
                 inplace:                  bool = False):

        super(_QModule, self).__init__(negative_slope=negative_slope,
                                       inplace=inplace)

        _QActivation.__init__(self,
                              qrangespec,
                              qgranularityspec,
                              qhparamsinitstrategyspec)

    def _register_qop(self):
        raise NotImplementedError

    def _call_qop(self, x: torch.Tensor) -> torch.Tensor:
        raise NotImplementedError

    @classmethod
    def from_fp_module(cls,
                       fpm:                      nn.LeakyReLU,
                       qrangespec:               QRangeSpecType,
                       qgranularityspec:         QGranularitySpecType,
                       qhparamsinitstrategyspec: QHParamsInitStrategySpecType) -> QLeakyReLU:
        """Special constructor to build ``QLeakyReLU``s from FP ``LeakyReLU``s."""

        qleakyrelu = cls(qrangespec,
                         qgranularityspec,
                         qhparamsinitstrategyspec,
                         negative_slope=fpm.negative_slope,
                         inplace=fpm.inplace)

        return qleakyrelu

NNMODULE_TO_QMODULE = {
    nn.Identity:  QIdentity,
    nn.ReLU:      QReLU,
    nn.ReLU6:     QReLU6,
    nn.LeakyReLU: QLeakyReLU,
}


import os

homeDir = os.path.expanduser("~")
print(homeDir)

# Set paths for the two input files
PATHlemma = homeDir + "/data/kilgarriff_BNC/"
PATHinfl = homeDir + "/data/agid-2016.01.19/"

# Set paths for the three outputfiles
OUTPATH = homeDir + "/Dropbox/python/engInfl/"

# input file names
lemmaFile = "lemma.num"
lemmaFilePath = PATHlemma + lemmaFile

inflFile = "infl.txt"
inflFilePath = PATHinfl + inflFile

# output file names
outFileN = OUTPATH + "bncCoreNounsSgPl.csv"
outFileA = OUTPATH + "bncCoreAdjectivesPosCompSup.csv"
outFileV = OUTPATH + "bncCoreVerbsInfPastIng3Sg.csv"

# convenience function removePrevious()
def removePrevious(fileName):
    """
    Test whether file with "fileName" already exists and, if it exists, delete it.
    """
    try:
        os.remove(fileName)
        print("old file deleted")
    except OSError:
        pass

# Empty list to store lemmata
bncCoreVocab = []

# Read in the file "lemma.num" as list with embedded lists for every lemma.
with open(lemmaFilePath, 'r', encoding='ascii') as lemmata:
    for line in lemmata:
        bncCoreVocab.append(line.strip().split(" "))
        print(line)

# We are only interested in nouns, verbs, and ajdectives
bncNouns = [x for x in bncCoreVocab if x[3] == "n"]
bncVerbs = [x for x in bncCoreVocab if x[3] == "v"]
bncAdjs = [x for x in bncCoreVocab if x[3] == "a"]

len(bncNouns)
# 3262
len(bncAdjs)
# 1124
len(bncVerbs)
# 1281

# Read in the file "infl.txt" into the list "infl", with each line as one entry
infl = []
with open(inflFilePath, 'r', encoding='ascii') as lemmas:
    for line in lemmas:
        infl.append(line.strip())

len(infl)
# 112505

# collect clear sg-pl noun instances
nounsSgPl = []
for item in bncNouns:
    target = item[2]
    targetPattern = target + " "
    for entry in infl:
        if entry.startswith(targetPattern)  and len(entry.split(" ")) == 3 and entry.split(" ")[1] == "N:" and not entry.split(" ")[2].endswith("?"):
            print(entry)
            nounsSgPl.append([entry.split(" ")[0],entry.split(" ")[2]])

len(nounsSgPl)
# 3088

# write out to csv file
removePrevious(outFileN)
with open(outFileN, mode='a', encoding='utf-8') as out:
    out.write("sg,pl\n")
    for pair in nounsSgPl:
        out.write(",".join(pair) + "\n")

# collect clear positive-comparative-superlative adjective instances
adjsPCS = []
for item in bncAdjs:
    target = item[2]
    targetPattern = target + " "
    for entry in infl:
        if entry.startswith(targetPattern)  and len(entry.split(" | ")) == 2 and len(entry.split(" | ")[0].split(" ")) == 3 and entry.split(" | ")[0].split(" ")[1] == "A:" and not entry.split(" | ")[0].split(" ")[2].endswith("?") and len(entry.split(" | ")[1].split(" ")) == 1 and not entry.split(" | ")[1].split(" ")[0].endswith("?"):
            print(entry)
            adjsPCS.append([entry.split(" ")[0],entry.split(" ")[2],entry.split(" ")[4]])

len(adjsPCS)
# 308

# write to file
removePrevious(outFileA)
with open(outFileA, mode='a', encoding='utf-8') as out:
    out.write("pos,comp,sup\n")
    for triples in adjsPCS:
        out.write(",".join(triples) + "\n")

# From the point of view of the distributional data, a differentation of preterite and perfect participle is not very promising, as the forms for the regular verbs are the same: danced/danced
# Therefore, we will only collect the data from verbs with four clear forms: inf past gerund third

len(bncVerbs)
# 1281

verbsIPGT = []
for item in bncVerbs:
    target = item[2]
    targetPattern = target + " "
    for entry in infl:
        if entry.startswith(targetPattern)  and len(entry.split(" | ")) == 3 and len(entry.split(" | ")[0].split(" ")) == 3 and entry.split(" | ")[0].split(" ")[1] == "V:" and not entry.split(" | ")[0].split(" ")[2].endswith("?") and len(entry.split(" | ")[1].split(" ")) == 1 and not entry.split(" | ")[1].split(" ")[0].endswith("?") and len(entry.split(" | ")[2].split(" ")) == 1 and not entry.split(" | ")[2].split(" ")[0].endswith("?"):
            print(entry)
            verbsIPGT.append([entry.split(" ")[0],entry.split(" ")[2],entry.split(" ")[4],entry.split(" ")[6]])

len(verbsIPGT)
# 1112

# write to file
removePrevious(outFileV)
with open(outFileV, mode='a', encoding='utf-8') as out:
    out.write("plain,past,gerund,thirdSg\n")
    for quadruples in verbsIPGT:
        out.write(",".join(quadruples) + "\n")

################################################################################
##############                                                    ##############
##############                    THE END                         ##############
##############                                                    ##############
################################################################################
# # test cases
# entry = ['AA', 'N:', 'AAs?']
# entry = "large A: larger | largest"
# entry = "other A: otherrer?, otherer<? 1 | otherrest?, otherest<? 1"
# entry = "alien A: aliener | aliennest?, alienest<? 1"
# entry  = "run V: ran | run | running | runs"
# entry = "kill V: killed | killing | kills"
#
# # test loop
# targetPattern = "run" + " "
# for entry in infl:
#     if entry.startswith(targetPattern)  and len(entry.split(" | ")) == 3 and len(entry.split(" | ")[0].split(" ")) == 3 and entry.split(" | ")[0].split(" ")[1] == "V:" and not entry.split(" | ")[0].split(" ")[2].endswith("?") and len(entry.split(" | ")[1].split(" ")) == 1 and not entry.split(" | ")[1].split(" ")[0].endswith("?") and len(entry.split(" | ")[2].split(" ")) == 1 and not entry.split(" | ")[2].split(" ")[0].endswith("?"):
#        print(entry)


import sys
array = [1, 2, 3, 4, 5]

end = 2 ** len(array)
min_abs = sys.maxsize
total = sum(array)
for i in range(end):
    one_server = 0
    for j in range(len(array)):
        if bin(i>>j)[-1]=='1':
            one_server += array[j]
    print(one_server)
    dif_abs = abs(total - 2 * one_server)
    if dif_abs < min_abs:
        min_abs = dif_abs
print(min_abs)


import random

class Deck:

    DESCRIPTIONS = {"Read Spec" : "Peeks at a goal to check if correct or not",\
        "Add Code" : "Adds a line of code",\
        "Add Code - Assignment or Initialization" : "Add an Assignment or Initialization",\
        "Add Code - Arithmetic or other Basic Functions" : "Add Arithmetic Function or another Function Call",\
        "Add Code - Print or Return" : "Add a Print or Return Statement",\
        "Add Code - Conditional or Loop" : "Add an If, While, or For Statement along with its logic",\
        "Add Code - Anything" : "Add any line of code",\
        "Modify Lines" : "Modifies code",\
        "Modify Lines - Change Constants" : "Change constants on any lines",\
        "Modify Lines - Change Variables" : "Modify a variable name to another one",\
        "Modify Lines - Change Functions" : "Change a function call or an arithmetic operator",\
        "Modify Line - Change Line" : "Modify any line",\
        "Nuke" : "Deletes up to 3 lines of code.",\
        "Dead Battery" : "Traps a player",\
        "Disconnected WiFi" : "Traps a player",\
        "Stack Overflow Down" : "Everyone has a 1/2 chance of skipping their turn after playing card",\
        "Charger" : "Removes a Dead Battery trap",\
        "Internet Cafe" : "Removes a Disconnected Wifi trap",\
        "Go to Soda" : "Removes a Dead Battery and a Disconnected Wifi trap",\
        "Code Guide" : "Makes a single player immune to Stack Overflow Down effect",\
        "Stack Overflow Restored" : "Can not be skipped. Removes all effects of ongoing Stack Overflow Down.",\
        "Run Code" : "Runs the code. After 3 run cards are played total, the game ends and winner is decided."}

    def __init__(self):
        """
        List of cards and descriptions for Deck:

        Read Spec               - Peeks at a goal to check if correct or not
        (Rethink Plan           - Changes 2 of the 3 goals to different random ones.)
        Add Line                - Adds a line of code (or n characters)
        Modify Lines            - Modifies code (or < n characters)
        Nuke                    - Deletes up to 3 lines of code.
        Dead Battery            - Traps a player
        Disconnected WiFi       - Traps a player
        Stack Overflow Down     - Traps a player (special). Everyone else has a 1/2 chance of skipping their turn after playing card.
        Charger                 - Removes a Dead Battery trap
        Internet Cafe           - Removes a Disconnected Wifi trap
        Go to Soda              - Removes a Dead Battery and a Disconnected Wifi trap
        Code Guide              - Makes a single player immune to Stack Overflow Down effect
        Stack Overflow Restored - Can not be skipped. Removes all effects of ongoing Stack Overflow Down.
        (Compile/Run)           - Runs the code. After 3? run cards are played total, the game ends and winner is decided.
        Currently 100 cards total
        """

        self.cards = ["Read Spec"] * 7 \
        + ["Add Code - Assignment or Initialization"] * 5 \
        + ["Add Code - Arithmetic or other Basic Functions"] * 5 \
        + ["Add Code - Print or Return"] * 5 \
        + ["Add Code - Conditional or Loop"] * 5 \
        + ["Add Code - Anything"] * 4 \
        + ["Modify Lines - Change Constants"] * 4 \
        + ["Modify Lines - Change Variables"] * 4 \
        + ["Modify Lines - Change Functions"] * 4 \
        + ["Modify Line - Change Line"] * 4 \
        + ["Nuke"] * 5 \
        + ["Dead Battery"] * 5 \
        + ["Disconnected WiFi"] * 5 \
        + ["Stack Overflow Down"] * 3 \
        + ["Charger"] * 5 \
        + ["Internet Cafe"] * 5 \
        + ["Go to Soda"] * 5 \
        + ["Code Guide"] * 5 \
        + ["Stack Overflow Restored"] * 3 \
        + ["Run Code"] * 12

        self.discarded = []

        random.shuffle(self.cards)

    def draw(self):
        card = self.cards.pop()
        if len(self.cards) == 0:
            self.cards = self.discarded
            self.discarded = []
            random.shuffle(self.cards)
        return card

    def discard(self, card):
        self.discarded += list(card)

    def view_discard(self):
        return self.discarded

    def info(self, card):
        return Deck.DESCRIPTIONS[card]


# In this file I have implemented text classification task with
# Multi-channel CNN architecture
# Highkights :
#       Use of Dataset object (write from step-1 ) from directory
#       Customised cleaning of each element of dataset
#       Customised Tokensization of each element of dataset
#       MultiChannel CNN Model
#
import os
import tensorflow as tf
import numpy as np
import pickle
from string import punctuation
from tensorflow.keras.preprocessing import text_dataset_from_directory
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Input, Flatten, Conv1D, Dense, Dropout, MaxPooling1D, concatenate, Embedding
from tensorflow.keras.models import Model

def datset_objbuilder(main_dir_path, debuggining_info=True):
    """

    :param main_dir_path: Path to main directory
    :return:
    """
    complete_dataset = text_dataset_from_directory(
        main_dir_path,
        class_names=['neg', 'pos'],
    )

    if debuggining_info:
        print("Accessing and Building Dataset....")
        print("First_batch_of_dataset..")
        print(list(complete_dataset.take(1).as_numpy_iterator()))  # 32 files (batch_size)
        print(complete_dataset.element_spec)
        print("Dataset Building Successful")
        for files, targets in complete_dataset.take(1).as_numpy_iterator():
            print("First File content: ")
            print(files[0])
            print("Target: ")
            print(targets[0])
            break

    return complete_dataset

main_dir_path = "./dataset/txt_sentoken"
complete_dataset = datset_objbuilder(main_dir_path)

def clean_doc(doc):
    """
    Specifies how to clean the doc and return tokens

    :param doc: doc a single string content
    :return:
    """""
    # split into tokens by white space
    tokens = doc.split()
    # remove punctuation from each token
    table = str.maketrans('', '', punctuation)
    tokens = [w.translate(table) for w in tokens]
    # remove remaining tokens that are not alphabetic
    tokens = [word for word in tokens if word.isalpha()]
    # filter out short tokens
    tokens = [word for word in tokens if len(word) > 1]
    return tokens

def word_counter_builder(clean_doc, complete_dataset, debugging_info=True):
    """
    Builds word count in the entire dataset
    :param clean_doc: Function that returns TOkens and Specifies how to clean and TOkensise the data
    :param complete_dataset: dataset_object
    :return: word_count (dictionary)
    """
    batch_counter = 0
    file_counter = 0
    word_count = dict()
    # x_dataset = complete_dataset.as_numpy_iterator()  #64 batches ka dataset_obj
    # print("----x_dataset")
    # print(len(list(x_dataset)))

    for batch_content, batch_target in complete_dataset.as_numpy_iterator():
        batch_counter += 1
        print("Batch Accessed............")
        # print("Batch_content_size: ",batch_content.shape)
        # print("Batch_Target_size: ",batch_target.shape)

        for content in batch_content:
            print("Batch_files Accessed")
            file_counter += 1
            tokens = clean_doc(str(content))
            for word in tokens:
                if word not in word_count:
                    word_count[word] = 0
                word_count[word] += 1

    if debugging_info:
        print("Total batches_accessed: ", batch_counter)
        print("Total Files Accessed: ", file_counter)
        sorted_word_count = {k: v for k, v in sorted(word_count.items(), key=lambda item: item[1], reverse=True)}
        tmp = 0
        print("First top 10 elements by occcurence")
        for element in sorted_word_count.items():
            tmp += 1
            if tmp <= 10:
                print(element)
        print("Keys in total word count: ", len(list(word_count.keys())))
    return word_count

word_count = word_counter_builder(clean_doc, complete_dataset)
print("Word_count : ", len(word_count.items()))

def word_count_slicer_Token_list_generator(word_count, num_words=20000, debugging_info=True):
    """
    :param word_count: All vocab Word count
    :param num_words: These many most occuring words will be kept from total Voccab
    :return:
    """
    sorted_word_count = {k: v for k, v in sorted(word_count.items(), key=lambda item: item[1], reverse=True)}
    tmp_counter = 0
    vocab = []
    updated_word_count = {}
    for key, value in sorted_word_count.items():
        tmp_counter += 1
        if tmp_counter <= num_words:
            vocab.append(key)
            updated_word_count[key] = value
    if debugging_info:
        print("First top 10 element by occcurence in Updated Word counter")
        tmp_1 = 0
        for element in updated_word_count.items():
            tmp_1 += 1
            if tmp_1 <= 10:
                print(element)
        print("Keys in Updated Word Count: ", len(list(updated_word_count.keys())))

    return sorted(vocab), updated_word_count

tokens = word_count_slicer_Token_list_generator(word_count)[0]
updated_word_count = word_count_slicer_Token_list_generator(word_count)[1]

def create_mappings(tokens, debuggining_info=True):
    """

    :param tokens: sorted tokens list
    :return: word_to_idx and idx_to_word
    """
    word_idx = {}
    for i, word in enumerate(tokens, start=2):  # Leave 0 for padding and 1 for unkown tokens
        word_idx[word] = i
    word_idx["<UNK>"] = 1

    idx_word = {}
    for i, word in enumerate(tokens, start=2):
        idx_word[i] = word
    idx_word[1] = "<UNK>"

    if debuggining_info:
        print("Length of Word index: ", len(word_idx.items()))  # num_words + 1 (Unknown Token / 0 padding)
        print("Length of index word mapping: ", len(idx_word.items()))  # num_words + 1 (UNK TOken/ (0--padding)
        print("Length of tokens: ", len(tokens))  # num_words
        print("length of updated word counter : ", len(updated_word_count.items()))  # num_words

    return word_idx, idx_word

word_to_idx, idx_to_word = create_mappings(tokens)

vocab_size = len(tokens) + 2  # (num_words + 1 (unknown token) + 1 (0 padding to make all equal length)

# Tokenising Content ##
print("-----------------Tokenising COntent---------------------")

def Tokenise_content(word_to_idx, complete_dataset, file_path, debugging_info=True):
    """

    :param word_to_idx: Maps word to idx
    :return: Tokenised_content
    """
    batch_counter_1 = 0
    file_counter_1 = 0
    tokenised_content = []

    for BATCH_CONTENT, BATCH_TARGET in complete_dataset.as_numpy_iterator():
        batch_counter_1 += 1
        print("Batch Accessed............")
        print(batch_counter_1)
        # print("Batch_content_size: ",batch_content.shape)
        # print("Batch_Target_size: ",batch_target.shape)
        # We can use here parallel computing to Tokenise the batches directly

        for CONTENT in BATCH_CONTENT:
            file_counter_1 += 1
            print("Files Accessed So Far")
            print(file_counter_1)
            print("Working on File")
            encoded_file_content = []
            word_counter = 0
            for word in CONTENT.split():
                word = word.decode('UTF-8')
                word_counter += 1
                if word in tokens:
                    encoded_file_content.append(word_to_idx[word])
                else:
                    encoded_file_content.append(1)
            tokenised_content.append(encoded_file_content)

            if debugging_info:
                print("-------------------------------------")
                print("Actual File Content: ")
                print(CONTENT.split())
                print("Encoded File Content: ")
                print(encoded_file_content)
                print("File Length: ", word_counter)
                print("Encoded File Length: ", len([word for word in encoded_file_content]))

    print("Saving Encoded File Content")

    file_path = file_path

    with open(file_path, "wb") as f:
        pickle.dump(tokenised_content, f)
    print("File Saved Successfully")

def file_loader_saver(file_path, task, saving_content=None):
    """
    Only for .pkl files
    :param file_path: file_path
    :param task: saving / loading
    :return:
    """
    file_path = file_path
    if task == "saving":
        if saving_content is not None:
            print("Saving Content")
            with open(file_path, 'wb') as f:
                pickle.dump(saving_content, f)

            print("File saved successfully")
            print("File Saved here: ")
            print(file_path)

    if task == 'loading':
        print("Loading File")
        with open(file_path, 'rb') as f:
            file = pickle.load(f)

        return file

file_path_tokenised_content = "./tokenised_content.pkl"

tokenised_content = file_loader_saver(file_path_tokenised_content, "loading")

# Tokenise_content(word_to_idx, complete_dataset, file_path = file_path_tokenised_content,debugging_info=False)
#
# print("Loading Tokenised File Content: ")
#
# with open(file_path_tokenised_content,'rb') as f:
#     tokenised_content = pickle.load(f)
#
# print("Loading Successful ")
# This tokenised content is of variable lengths
# This tokenised content is of shape (2000 (# of files), varibale length for each sequence)
# tokenised_content_np = np.array(tokenised_content)
# print(tokenised_content_np.shape)
print(len(tokenised_content[0]))
print(len(tokenised_content[1]))

padded_tokenised_content = pad_sequences(tokenised_content)

processed_content = np.array(padded_tokenised_content)
print(processed_content.shape)
# print(processed_content[0][])

padded_tokenised_content_file_path = "./padded_tokenised_content.pkl"
file_loader_saver(padded_tokenised_content_file_path, "saving", saving_content=processed_content)

max_doc_length = processed_content.shape[1]

print("-----------------------Working on Model------------------------------")

# define the model
def define_model(length, vocab_size):
    """

    :param length: input_length
    :param vocab_size: vocab_size
    :return:
    """
    # channel 1
    inputs1 = Input(shape=(length,))
    embedding1 = Embedding(vocab_size, 100)(inputs1)
    conv1 = Conv1D(filters=32, kernel_size=4, activation='relu')(embedding1)
    drop1 = Dropout(0.5)(conv1)
    pool1 = MaxPooling1D(pool_size=2)(drop1)
    flat1 = Flatten()(pool1)
    # channel 2
    inputs2 = Input(shape=(length,))
    embedding2 = Embedding(vocab_size, 100)(inputs2)
    conv2 = Conv1D(filters=32, kernel_size=6, activation='relu')(embedding2)
    drop2 = Dropout(0.5)(conv2)
    pool2 = MaxPooling1D(pool_size=2)(drop2)
    flat2 = Flatten()(pool2)
    # channel 3
    inputs3 = Input(shape=(length,))
    embedding3 = Embedding(vocab_size, 100)(inputs3)
    conv3 = Conv1D(filters=32, kernel_size=8, activation='relu')(embedding3)
    drop3 = Dropout(0.5)(conv3)
    pool3 = MaxPooling1D(pool_size=2)(drop3)
    flat3 = Flatten()(pool3)
    # merge
    merged = concatenate([flat1, flat2, flat3])
    # interpretation
    dense1 = Dense(10, activation='relu')(merged)
    outputs = Dense(1, activation='sigmoid')(dense1)
    model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs)
    # compile
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
    # summarize
    print(model.summary())
    # plot_model(model, show_shapes=True, to_file='multichannel.png')
    return model

model_1 = define_model(max_doc_length, vocab_size)

# def checkpoint_callback_creater(checkpoint_dir):
#     """
#
#     :param checkpoint_dir: Directory where checkpoint will be saved
#     :return:
#     """
#     # Configuring Checkpoints
#     # Directory where the checkpoints will be saved
#     checkpoint_dir = checkpoint_dir
#     # Name of the checkpoint files
#     checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt_{epoch}")
#
#     checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
#         filepath=checkpoint_prefix,
#         save_weights_only=True)
#
#     return checkpoint_callback

checkpoint_dir = "./moview_review_checkpoints"
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt_{epoch}")

checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_prefix,
    save_weights_only=True)

# checkpoint_callback = checkpoint_callback_creater(chckpoint_dir)
targets_1  = list([element[1] for element in complete_dataset.as_numpy_iterator()])
targets = []
for element in complete_dataset.as_numpy_iterator():
    # element -- > 1 batch of Inputs , Targets
    batched_targets = element[1]
    # Since Last Batch is of 16 size it'll create during transition form list to np.array
    for target in batched_targets:
        targets.append(target)

print(len(targets))
targets_np = np.array(targets)
print(targets_np.shape)
# processed_content_ds = tf.data.Dataset.from_tensor_slices(processed_content)
# targets_ds = tf.data.Dataset.from_tensor_slices(targets_np)
# processed_dataset_gen = tf.data.Dataset.zip((processed_content_ds,targets_ds))
# # Creating New processed Dataset obj
# print("-------------------------------------------------------------------------")
# print("Processed COntent: ")
# print(processed_content[0])
# print("----------------------------------------------------------------------------")
# print("Processed Dataset 1st element :")
# print(list(processed_dataset_gen.take(1).as_numpy_iterator()))
# print("----------------------------------------------------------------------------")
# print("ACtual Dataset 1st element")
# print(list(complete_dataset.take(1).as_numpy_iterator()))

#history = model_1.fit([processed_content,processed_content,processed_content],targets_np, epochs=10, callbacks=[checkpoint_callback],validation_split=0.2)

model_1.load_weights(tf.train.latest_checkpoint(checkpoint_dir))
model_1.build(input_shape=(max_doc_length,vocab_size))

def model_evaluator(model,word_to_idx, evaluation_line="Awesome",debugging_info =True):
    """

    :param evaluate_line: Line to Evaluate
    :param model: Model to Make Predictions from
    :param word_to_idx: Mapping
    :return: Prediction
    """
    evaluate_line = evaluation_line

    # Preprocessing evaluate line
    encoded_evaluation_line = []
    for word in evaluate_line.split():
        if word in tokens:
            idx = word_to_idx[word]
            encoded_evaluation_line.append(idx)
        else:
            idx = 1  # Token for unknown word
            encoded_evaluation_line.append(idx)

    encoded_evaluation_line = np.reshape(encoded_evaluation_line, (1, len(encoded_evaluation_line)))
    processed_evaluated_line = pad_sequences(encoded_evaluation_line,maxlen=max_doc_length)

    if debugging_info:
        print("Encoded Evaluation Line; ")
        print(encoded_evaluation_line)
        print("Encoded Line Shape: ")
        print(np.array(processed_evaluated_line).shape)

    pred = model_1.predict(x = [processed_evaluated_line,processed_evaluated_line,processed_evaluated_line])
    if pred < 0.5:
        print("Predicted Negative Class ")
        print(pred)
    else:
        print("Predicted Positive Class")
        print(pred)

eval_line = " in 2176 on the planet mars police taking into custody an accused murderer face the title menace . there is a lot of fighting "
# eval line from cv0005_29357 -- > Negative Class

model_evaluator(model_1,word_to_idx=word_to_idx
                ,evaluation_line=eval_line)


def convert_to_int(integer_string_with_commas):
    comma_separated_parts = integer_string_with_commas.split(",")
    for i in range(len(comma_separated_parts)):
        if len(comma_separated_parts[i]) > 3:
            return None
        if i != 0 and len(comma_separated_parts[i]) != 3:
            return None
    integer_string_without_commas = "".join(comma_separated_parts)
    try:
        return int(integer_string_without_commas)
    except ValueError:
        return None

def row_to_list(row):
    row = row.rstrip("\n")
    separated_entries = row.split("\t")
    if len(separated_entries) == 2 and "" not in separated_entries:
        return separated_entries
    return None

def preprocess(raw_data_file_path, clean_data_file_path):
    with open(raw_data_file_path, "r") as input_file:
        rows = input_file.readlines()
    with open(clean_data_file_path, "w") as output_file:
        for row in rows:
            row_as_list = row_to_list(row)
            if row_as_list is None:
                continue
            area = convert_to_int(row_as_list[0])
            price = convert_to_int(row_as_list[1])
            if area is None or price is None:
                continue
            output_file.write("{0}\t{1}\n".format(area, price))


from pygame import color, rect
from states.play_state_two import Play_stage_two
from states.state import State
import pygame, os
import game_constants
from pygame.sprite import Group, Sprite
from random import randint
from states.game_elements.elements import Enemies, Nav_player,Corona_base

class Play_stage_one(State):
    def __init__(self, game):
        super().__init__(game)

        self.player = Nav_player(game)
        self.bullet_group = Group()
        self.corona_base_group = Group()
        self.enemys = Enemies()
        self.level_finish = 0
        self.create = True

    def update(self, delta_time, actions):

        self.player.update(delta_time, actions)

        if actions['action1']:
            self.bullet_group.add(self.player.create_bullet())
            if self.create:

                self.enemys.create_virus(self.game,self.corona_base_group,15)
                self.create = False

            actions['action1'] = False

        if actions['action2']:
            actions['action2'] = False
            self.game.screen.fill(game_constants.BLACK)
            self.game.draw_text('Fase II',60,game_constants.WHITE,game_constants.WIDTH/2,200)
            self.game.draw_text('VARIANTE OMICRON',20,game_constants.WHITE,game_constants.WIDTH/2,320)
            pygame.display.flip()
            pygame.time.wait(3000)

            new_state = Play_stage_two(self.game)
            new_state.enter_state()

        for bullet in self.bullet_group.sprites():
            if bullet.rect.top <= 0:
                self.bullet_group.remove(bullet)
            else:
                bullet.update(delta_time, actions)

        for bullet in self.bullet_group.sprites():
            for virus in self.corona_base_group.sprites():
                if bullet.rect.colliderect(virus.rect):
                    self.bullet_group.remove(bullet)
                    self.corona_base_group.remove(virus)
                    self.level_finish += 1

                    if self.level_finish == 5:

                        self.game.screen.fill(game_constants.BLACK)
                        self.game.draw_text('Fase II',60,game_constants.WHITE,game_constants.WIDTH/2,200)
                        self.game.draw_text('VARIANTE ÔMICRON',20,game_constants.WHITE,game_constants.WIDTH/2,320)
                        pygame.display.flip()
                        pygame.time.wait(3000)

                        new_state = Play_stage_two(self.game)
                        new_state.enter_state()

                    break

        for virus in self.corona_base_group.sprites():
            if self.player.nav_rect.colliderect(virus.rect):
                self.game.playing = False
            virus.update(delta_time, actions)

        if self.game.playing == False:
            self.game.screen.fill(game_constants.BLACK)
            self.game.draw_text('GAME OVER',60,game_constants.WHITE,game_constants.WIDTH/2,200)
            pygame.display.flip()
            pygame.time.wait(2000)

    def render(self, surface):

        surface.fill(game_constants.BLACK)
        self.player.render(surface)
        self.bullet_group.draw(surface)
        self.corona_base_group.draw(surface)
        pygame.display.update()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#
# SPDX-License-Identifier: GPL-3.0
#
# GNU Radio Python Flow Graph
# Title: Rds Bpsk Hackrf File Source
# GNU Radio version: 3.8.1.0

from gnuradio import analog
from gnuradio import blocks
import pmt
from gnuradio import digital
from gnuradio import filter
from gnuradio.filter import firdes
from gnuradio import gr
import sys
import signal
from argparse import ArgumentParser
from gnuradio.eng_arg import eng_float, intx
from gnuradio import eng_notation
import math
import osmosdr
import time

class RDS_BPSK_HackRF_File_Source(gr.top_block):

    def __init__(self):
        gr.top_block.__init__(self, "Rds Bpsk Hackrf File Source")

        ##################################################
        # Variables
        ##################################################
        self.stereo_gain = stereo_gain = .3
        self.serial = serial = "0"
        self.rds_gain = rds_gain = .5
        self.pilot_gain = pilot_gain = .3
        self.outbuffer = outbuffer = 10
        self.notes = notes = "Replays RDS data on repeat supplied from a file. No audio is added."
        self.lower_rate = lower_rate = 380000
        self.input_gain = input_gain = .3
        self.freq = freq = 106500000
        self.fm_max_dev = fm_max_dev = 80000
        self.filepath = filepath = "/home/laptop2/FISSURE/Crafted Packets/rdsA2.bin"

        ##################################################
        # Blocks
        ##################################################
        self.osmosdr_sink_0 = osmosdr.sink(
            args="numchan=" + str(1) + " " + "hackrf=" + str(serial)
        )
        self.osmosdr_sink_0.set_time_unknown_pps(osmosdr.time_spec_t())
        self.osmosdr_sink_0.set_sample_rate(1e6)
        self.osmosdr_sink_0.set_center_freq(freq, 0)
        self.osmosdr_sink_0.set_freq_corr(0, 0)
        self.osmosdr_sink_0.set_gain(10, 0)
        self.osmosdr_sink_0.set_if_gain(30, 0)
        self.osmosdr_sink_0.set_bb_gain(20, 0)
        self.osmosdr_sink_0.set_antenna('', 0)
        self.osmosdr_sink_0.set_bandwidth(0, 0)
        self.mmse_resampler_xx_2 = filter.mmse_resampler_ff(0, 44.1/(lower_rate/1000))
        self.mmse_resampler_xx_1 = filter.mmse_resampler_ff(0, 44.1/(lower_rate/1000))
        self.mmse_resampler_xx_0 = filter.mmse_resampler_cc(0, (lower_rate/10000)/100.0)
        self.low_pass_filter_0 = filter.interp_fir_filter_fff(
            1,
            firdes.low_pass(
                1,
                lower_rate,
                2.5e3,
                .5e3,
                firdes.WIN_HAMMING,
                6.76))
        self.low_pass_filter_0.set_max_output_buffer(10)
        self.gr_unpack_k_bits_bb_0 = blocks.unpack_k_bits_bb(2)
        self.gr_unpack_k_bits_bb_0.set_max_output_buffer(10)
        self.gr_sub_xx_0 = blocks.sub_ff(1)
        self.gr_sig_source_x_0_1 = analog.sig_source_f(lower_rate, analog.GR_SIN_WAVE, 19e3, 1, 0, 0)
        self.gr_sig_source_x_0_0 = analog.sig_source_f(lower_rate, analog.GR_SIN_WAVE, 57e3, 1, 0, 0)
        self.gr_sig_source_x_0 = analog.sig_source_f(lower_rate, analog.GR_SIN_WAVE, 38e3, 1, 0, 0)
        self.gr_multiply_xx_1 = blocks.multiply_vff(1)
        self.gr_multiply_xx_0 = blocks.multiply_vff(1)
        self.gr_multiply_xx_0.set_max_output_buffer(10)
        self.gr_map_bb_1 = digital.map_bb([1,2])
        self.gr_map_bb_1.set_max_output_buffer(10)
        self.gr_map_bb_0 = digital.map_bb([-1,1])
        self.gr_map_bb_0.set_max_output_buffer(10)
        self.gr_frequency_modulator_fc_0 = analog.frequency_modulator_fc(2*math.pi*fm_max_dev/lower_rate)
        self.gr_frequency_modulator_fc_0.set_max_output_buffer(10)
        self.gr_diff_encoder_bb_0 = digital.diff_encoder_bb(2)
        self.gr_diff_encoder_bb_0.set_max_output_buffer(10)
        self.gr_char_to_float_0 = blocks.char_to_float(1, 1)
        self.gr_char_to_float_0.set_max_output_buffer(10)
        self.gr_add_xx_1 = blocks.add_vff(1)
        self.gr_add_xx_1.set_max_output_buffer(10)
        self.gr_add_xx_0 = blocks.add_vff(1)
        self.blocks_unpack_k_bits_bb_0 = blocks.unpack_k_bits_bb(8)
        self.blocks_repeat_0 = blocks.repeat(gr.sizeof_float*1, 160)
        self.blocks_null_source_0 = blocks.null_source(gr.sizeof_float*1)
        self.blocks_multiply_const_vxx_0_1 = blocks.multiply_const_ff(input_gain)
        self.blocks_multiply_const_vxx_0_0_1 = blocks.multiply_const_ff(pilot_gain)
        self.blocks_multiply_const_vxx_0_0 = blocks.multiply_const_ff(rds_gain)
        self.blocks_multiply_const_vxx_0_0.set_max_output_buffer(10)
        self.blocks_multiply_const_vxx_0 = blocks.multiply_const_ff(input_gain)
        self.blocks_file_source_0 = blocks.file_source(gr.sizeof_char*1, filepath, True, 0, 0)
        self.blocks_file_source_0.set_begin_tag(pmt.PMT_NIL)

        ##################################################
        # Connections
        ##################################################
        self.connect((self.blocks_file_source_0, 0), (self.blocks_unpack_k_bits_bb_0, 0))
        self.connect((self.blocks_multiply_const_vxx_0, 0), (self.mmse_resampler_xx_2, 0))
        self.connect((self.blocks_multiply_const_vxx_0_0, 0), (self.gr_add_xx_1, 0))
        self.connect((self.blocks_multiply_const_vxx_0_0_1, 0), (self.gr_add_xx_1, 1))
        self.connect((self.blocks_multiply_const_vxx_0_1, 0), (self.mmse_resampler_xx_1, 0))
        self.connect((self.blocks_null_source_0, 0), (self.blocks_multiply_const_vxx_0, 0))
        self.connect((self.blocks_null_source_0, 0), (self.blocks_multiply_const_vxx_0_1, 0))
        self.connect((self.blocks_repeat_0, 0), (self.low_pass_filter_0, 0))
        self.connect((self.blocks_unpack_k_bits_bb_0, 0), (self.gr_diff_encoder_bb_0, 0))
        self.connect((self.gr_add_xx_0, 0), (self.gr_add_xx_1, 3))
        self.connect((self.gr_add_xx_1, 0), (self.gr_frequency_modulator_fc_0, 0))
        self.connect((self.gr_char_to_float_0, 0), (self.blocks_repeat_0, 0))
        self.connect((self.gr_diff_encoder_bb_0, 0), (self.gr_map_bb_1, 0))
        self.connect((self.gr_frequency_modulator_fc_0, 0), (self.mmse_resampler_xx_0, 0))
        self.connect((self.gr_map_bb_0, 0), (self.gr_char_to_float_0, 0))
        self.connect((self.gr_map_bb_1, 0), (self.gr_unpack_k_bits_bb_0, 0))
        self.connect((self.gr_multiply_xx_0, 0), (self.blocks_multiply_const_vxx_0_0, 0))
        self.connect((self.gr_multiply_xx_1, 0), (self.gr_add_xx_1, 2))
        self.connect((self.gr_sig_source_x_0, 0), (self.gr_multiply_xx_1, 0))
        self.connect((self.gr_sig_source_x_0_0, 0), (self.gr_multiply_xx_0, 0))
        self.connect((self.gr_sig_source_x_0_1, 0), (self.blocks_multiply_const_vxx_0_0_1, 0))
        self.connect((self.gr_sub_xx_0, 0), (self.gr_multiply_xx_1, 1))
        self.connect((self.gr_unpack_k_bits_bb_0, 0), (self.gr_map_bb_0, 0))
        self.connect((self.low_pass_filter_0, 0), (self.gr_multiply_xx_0, 1))
        self.connect((self.mmse_resampler_xx_0, 0), (self.osmosdr_sink_0, 0))
        self.connect((self.mmse_resampler_xx_1, 0), (self.gr_add_xx_0, 1))
        self.connect((self.mmse_resampler_xx_1, 0), (self.gr_sub_xx_0, 1))
        self.connect((self.mmse_resampler_xx_2, 0), (self.gr_add_xx_0, 0))
        self.connect((self.mmse_resampler_xx_2, 0), (self.gr_sub_xx_0, 0))

    def get_stereo_gain(self):
        return self.stereo_gain

    def set_stereo_gain(self, stereo_gain):
        self.stereo_gain = stereo_gain

    def get_serial(self):
        return self.serial

    def set_serial(self, serial):
        self.serial = serial

    def get_rds_gain(self):
        return self.rds_gain

    def set_rds_gain(self, rds_gain):
        self.rds_gain = rds_gain
        self.blocks_multiply_const_vxx_0_0.set_k(self.rds_gain)

    def get_pilot_gain(self):
        return self.pilot_gain

    def set_pilot_gain(self, pilot_gain):
        self.pilot_gain = pilot_gain
        self.blocks_multiply_const_vxx_0_0_1.set_k(self.pilot_gain)

    def get_outbuffer(self):
        return self.outbuffer

    def set_outbuffer(self, outbuffer):
        self.outbuffer = outbuffer

    def get_notes(self):
        return self.notes

    def set_notes(self, notes):
        self.notes = notes

    def get_lower_rate(self):
        return self.lower_rate

    def set_lower_rate(self, lower_rate):
        self.lower_rate = lower_rate
        self.gr_frequency_modulator_fc_0.set_sensitivity(2*math.pi*self.fm_max_dev/self.lower_rate)
        self.gr_sig_source_x_0.set_sampling_freq(self.lower_rate)
        self.gr_sig_source_x_0_0.set_sampling_freq(self.lower_rate)
        self.gr_sig_source_x_0_1.set_sampling_freq(self.lower_rate)
        self.low_pass_filter_0.set_taps(firdes.low_pass(1, self.lower_rate, 2.5e3, .5e3, firdes.WIN_HAMMING, 6.76))
        self.mmse_resampler_xx_0.set_resamp_ratio((self.lower_rate/10000)/100.0)
        self.mmse_resampler_xx_1.set_resamp_ratio(44.1/(self.lower_rate/1000))
        self.mmse_resampler_xx_2.set_resamp_ratio(44.1/(self.lower_rate/1000))

    def get_input_gain(self):
        return self.input_gain

    def set_input_gain(self, input_gain):
        self.input_gain = input_gain
        self.blocks_multiply_const_vxx_0.set_k(self.input_gain)
        self.blocks_multiply_const_vxx_0_1.set_k(self.input_gain)

    def get_freq(self):
        return self.freq

    def set_freq(self, freq):
        self.freq = freq
        self.osmosdr_sink_0.set_center_freq(self.freq, 0)

    def get_fm_max_dev(self):
        return self.fm_max_dev

    def set_fm_max_dev(self, fm_max_dev):
        self.fm_max_dev = fm_max_dev
        self.gr_frequency_modulator_fc_0.set_sensitivity(2*math.pi*self.fm_max_dev/self.lower_rate)

    def get_filepath(self):
        return self.filepath

    def set_filepath(self, filepath):
        self.filepath = filepath
        self.blocks_file_source_0.open(self.filepath, True)

def main(top_block_cls=RDS_BPSK_HackRF_File_Source, options=None):
    tb = top_block_cls()

    def sig_handler(sig=None, frame=None):
        tb.stop()
        tb.wait()
        sys.exit(0)

    signal.signal(signal.SIGINT, sig_handler)
    signal.signal(signal.SIGTERM, sig_handler)

    tb.start()
    try:
        input('Press Enter to quit: ')
    except EOFError:
        pass
    tb.stop()
    tb.wait()

if __name__ == '__main__':
    main()

Flow Graph Library/Single-Stage Flow Graphs/RDS_BPSK_HackRF_File_Source.py


import requests
import time
import PyPDF2
import logging

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S'
)
logger = logging.getLogger(__name__)
API_KEY = ""

class GPTRequest:
    MODELS = ['gpt-3.5-turbo', 'text-davinci-003', 'text-babbage-001', 'text-curie-001', 'text-cushing-001', 'text-edison-002']

    def __init__(self):
        self.engine = GPTRequest.MODELS[0]
        self.all_prompt = ""
        self.all_response = ""

    def change_engine(self, index):
        if index < len(GPTRequest.MODELS):
            self.engine = GPTRequest.MODELS[index]
            logger.info(f'AI Model has switched to {self.engine}')

    def auto_reply(self, prompt: str, previous_prompt: str, previous_response: str):
        """Calls the OpenAI API endpoint, processes the output, and extracts the reply."""
        gpt_endpoint = "https://api.openai.com/v1/chat/completions"
        common_endpoint = "https://api.openai.com/v1/completions"
        headers = {
            "Content-Type": "application/json",
            "Authorization": f"Bearer {API_KEY}"
        }
        self.all_prompt += f"{prompt}\n"
        if 'gpt' in self.engine.lower():
            system_message = f"previous prompt: {previous_prompt[-2000:]}, previous response: {previous_response[-2000:]}"
            data = {
                "model": self.engine,
                "messages": [
                    {"role": "system", "content": system_message},
                    {"role": "assistant", "content": ""},
                    {"role": "user", "content": f"{prompt}"},
                ],
                "max_tokens": 500,
                "temperature": 0.5,
                "top_p": 1,
            }
            response = requests.post(gpt_endpoint, headers=headers, json=data)
            result = response.json()
            generated_text = result["choices"][0]["message"]["content"]
            self.all_response += f"{generated_text}\n"
            logger.info(f'Response: {generated_text}')
        else:
            data = {
                "model": self.engine,
                "prompt": prompt,
                "max_tokens": 400,
                "temperature": 0.5,
                "top_p": 1,
            }
            response = requests.post(common_endpoint, headers=headers, json=data)
            result = response.json()
            generated_text = result["choices"][0]["text"]
            logger.info(f'Response: {generated_text}')

class PDFReader:
    @staticmethod
    def read_pdf(file_path, max_tokens=4000):
        """限制字符有点问题，WIP"""
        with open(file_path, 'rb') as f:
            pdf_reader = PyPDF2.PdfReader(f)
            text = ""
            for page in pdf_reader.pages:
                page_text = page.extract_text()
                text += ' ' + page_text
                if len(text.split()) > max_tokens:
                    text = ' '.join(text.split()[:max_tokens])
                    break
        return text

logger.info('Starting the application')  # Log an informational message

gpt_request = GPTRequest()
file_path_pdf = "test.pdf"
scan = PDFReader()
pdf_text = scan.read_pdf(file_path=file_path_pdf)
convo_limit = 10

for _ in range(convo_limit):
    user_prompt = pdf_text
    logger.info(f'User prompt: {user_prompt}')
    gpt_request.auto_reply(prompt=user_prompt, previous_prompt=gpt_request.all_prompt, previous_response=gpt_request.all_response)
    time.sleep(20)

logger.info('Application finished')


#Note! For the code to work you need to replace all the placeholders with
#Your own details. e.g. account_sid, lat/lon, from/to phone numbers.

import requests
import os
from twilio.rest import Client
from twilio.http.http_client import TwilioHttpClient

OWM_Endpoint = "https://api.openweathermap.org/data/2.5/onecall"
api_key = os.environ.get("77bdb6ff02c171ac7d0e3678999c59a1")
account_sid = "AC5e121a66d2536267703454d8b8828d03"
auth_token = os.environ.get("c3ebf1ec95ade46906d4e160b7249eb0")

weather_params = {
    "lat": "8.905540",
    "lon": "7.187090",
    "appid": '77bdb6ff02c171ac7d0e3678999c59a1',
    "exclude": "current,minutely,daily"
}

response = requests.get(OWM_Endpoint, params=weather_params)
response.raise_for_status()
weather_data = response.json()
weather_slice = weather_data["hourly"][:12]

will_rain = False

for hour_data in weather_slice:
    condition_code = hour_data["weather"][0]["id"]
    if int(condition_code) < 700:
        will_rain = True

if will_rain:
    proxy_client = TwilioHttpClient()
    proxy_client.session.proxies = {'https': os.environ['https_proxy']}

    client = Client(account_sid, auth_token, http_client=proxy_client)
    message = client.messages \
        .create(
            body="It is going to rain today.",
            from_='+13156286894',
            to='+2349064663629'
            )
    print(message.status)


import torch.nn as nn
import torch.optim as optim
from old_models.models_util import *

# Discriminator hyperparameters
lr_d = 0.001
d_dim = 128

# Generator hyperparameters
lr_g = 0.001
z_size = 100
g_dim = 256

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Linear') != -1:
        nn.init.normal_(m.weight.data, 0.0, 0.02)
        m.weight.data = m.weight.data.float()
    elif classname.find('BatchNorm') != -1:
        nn.init.normal_(m.weight.data, 1.0, 0.02)
        nn.init.constant_(m.bias.data, 0)
        m.weight.data = m.weight.data.float()
        m.bias.data = m.bias.data.float()

class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()

        def discriminator_block(in_filters, out_filters):
            return [nn.Linear(in_filters, out_filters), nn.LeakyReLU(0.2, inplace=True), nn.Dropout(0.25)]

        self.layers = nn.Sequential(
            nn.Flatten(),
            # input layer
            *discriminator_block(keys_size * (MAX_KEYPRESSES + 1), d_dim),
            # hidden layers
            *discriminator_block(d_dim, 2 * d_dim),
            *discriminator_block(2 * d_dim, 4 * d_dim),
            # output layer
            nn.Linear(4 * d_dim, 1),
            nn.Sigmoid()
        )

    def forward(self, x):
        return self.layers(x)

class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()

        def generator_block(in_filters, out_filters, bn=True):
            block = [nn.Linear(in_filters, out_filters)]
            if bn:
                block.append(nn.BatchNorm1d(out_filters))
            block.append(nn.LeakyReLU(0.2, inplace=True))
            return block

        self.layers = nn.Sequential(
            # input layer
            *generator_block(z_size, g_dim, bn=False),
            # hidden layers
            *generator_block(g_dim, 2 * g_dim),
            #*generator_block(2 * g_dim, 4 * g_dim),
            # output layer
            #nn.Linear(4 * g_dim, keys_size * (MAX_KEYPRESSES + 1)),  # includes 0
            nn.Linear(2 * g_dim, keys_size * (MAX_KEYPRESSES + 1)),  # includes 0
            nn.Unflatten(1, (keys_size, MAX_KEYPRESSES + 1)),
            nn.Softmax(dim=2)
        )

    def forward(self, x):
        return self.layers(x)

    def generate(self):
        z = np.random.uniform(-1, 1, size=(batch_size, z_size))
        z = torch.from_numpy(z).to(DEVICE).float()
        return self(z)

D = Discriminator()
G = Generator()

D.to(DEVICE)
G.to(DEVICE)

D.float()
G.float()

D.apply(weights_init)
G.apply(weights_init)

d_optimizer = optim.Adam(D.parameters(), lr=lr_d)
g_optimizer = optim.Adam(G.parameters(), lr=lr_g)

gan_model = GANModel(D, G, "gan_hdcas_data.pt", nn.MSELoss())

def train_d(real_keys):
    d_optimizer.zero_grad()
    D_output_real = D(real_keys)
    # loss on real keys
    r_loss = gan_model.real_loss(D_output_real)
    #r_loss = torch.relu(torch.ones_like(D_output_real).to(DEVICE) - D_output_real).mean()

    with torch.no_grad():
        fake_keys = G.generate()

    # Compute the discriminator losses on fake keys
    D_output_fake = D(fake_keys)
    f_loss = gan_model.fake_loss(D_output_fake)
    #f_loss = torch.relu(torch.ones_like(D_output_fake).to(DEVICE) + D_output_fake).mean()
    # add up real and fake losses and perform backprop
    d_loss = r_loss + f_loss
    d_loss.backward()
    d_optimizer.step()
    return d_loss.item()

def train_g():
    g_optimizer.zero_grad()

    fake_keys = G.generate()
    D_fake = D(fake_keys)

    #g_loss = torch.relu(torch.ones_like(D_fake).to(DEVICE) - D_fake).mean()
    g_loss = gan_model.real_loss(D_fake)

    g_loss.backward()
    g_optimizer.step()
    return g_loss.item()

def train_model(num_epochs=40):
    D.train()
    G.train()
    for epoch in range(num_epochs):
        for batch_i, (real_keys, game_state) in enumerate(data_loader_full):
            d_loss = train_d(real_keys)
            gan_model.d_losses.append(d_loss)
            g_loss = train_g()
            gan_model.g_losses.append(g_loss)

            # Print some loss stats
            if batch_i % 400 == 0:
                # print discriminator and generator loss
                print('Epoch [{:d}/{:d}] | Batch [{:d}/{:d}] | d_loss: {:6.4f} | g_loss: {:6.4f}'.format(
                    epoch + 1, num_epochs, batch_i, len(data_loader_full), d_loss, g_loss))

def sample_generator():
    G.eval()
    x, y = next(iter(data_loader_full))
    x = x.argmax(dim=2)
    x_fake = G.generate().argmax(dim=2)
    empty = torch.tensor([0]).float()
    for i in range(len(x)):
        plotter.plot_sample(torch.cat((x[i].cpu(), empty)), y[i].cpu(), False)
        plotter.plot_sample(torch.cat((x_fake[i].cpu(), empty)), y[i].cpu(), False)


import heapq
import sys

input = sys.stdin.readline
heap = []
I = int(input())
for i in range(I):
    a = int(input())
    a = (a*-1)
    if (a == 0):
        if(len(heap) == 0):
            print(0)
        else:
            print(-1*(heapq.heappop(heap)))
    else:
        heapq.heappush(heap,a)


class Solution:
    def subarraysWithKDistinct(self, A, K) -> int:
        lens = len(A)
        count = 0

        start = 0
        end = K + start

        temp = set(A[start: end])
        while len(temp) < K:

            count += 1

        for index in range(end, lens):
            pass

        print(count)
        return count

from leetcode.data import data

data = [1, 2, 3, 2]

if __name__ == "__main__":
    import cProfile
    my = Solution()
    cProfile.run("my.subarraysWithKDistinct(data, 2)", "prof.txt")
    import pstats
    p = pstats.Stats("prof.txt")
    p.sort_stats("time").print_stats()


from flask import Flask, request, redirect, render_template
from models import db, connect_db, Pet
from forms import AddPetForm, EditPetForm

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///Work'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SQLALCHEMY_ECHO'] = True
app.config['SECRET_KEY'] = "secret"

connect_db(app)
db.create_all()

@app.route('/')
def homePage():
    pets = Pet.query.all()
    return render_template("homepage.html", pets=pets)

@app.route('/add')
def petForm():
    form = AddPetForm()
    return render_template("petform.html", form=form)

@app.route('/add', methods=["POST"])
def petAdd():
    form = AddPetForm()

    new_pet = Pet(name=form.name.data, species=form.species.data, photo_url=form.photo_url.data, age=form.age.data, notes=form.notes.data)
    db.session.add(new_pet)
    db.session.commit()

    return redirect('/')

@app.route('/<int:pet_id>')
def petEdit(pet_id):
    pet = Pet.query.get_or_404(pet_id)
    form = EditPetForm(obj=pet)

    return render_template("editform.html", form=form, pet=pet)

@app.route('/<int:pet_id>', methods=["POST"])
def petUpdate(pet_id):
    pet = Pet.query.get_or_404(pet_id)
    form = EditPetForm(obj=pet)

    pet.notes = form.notes.data
    pet.available = form.available.data
    pet.photo_url = form.photo_url.data

    db.session.commit()

    return redirect('/')


import pandas as pd
from numpy import NaN
import math
import numpy as np
import matplotlib.pyplot as plt
import time
import datetime

df1 = pd.read_csv("C:/Users/zy108/Desktop/9733_machine learning/project/data1.csv")

date_num_list = list(set(list(df1['date_num'])))
date_num_list.sort()
print(date_num_list)
min_date_num = date_num_list[0]
max_date_num = date_num_list[-1]

pseudo_index_list = list(range(int(min_date_num * 10), int(max_date_num * 10 + 1), 1))
index_list = []
for item in pseudo_index_list:
    index_list.append(float(item) / 10)

print(len(index_list))
sum_date = len(index_list)
columns = ['date_num', 'date', 'best_ask_price', 'best_ask_size', 'best_bid_price', 'best_bid_size', 'ask_total_size', 'bid_total_size',
           'ask_weighted_average_price', 'bid_weighted_average_price', 'best_mid_price', 'mid_weighted_average_price']
df = pd.DataFrame(np.arange(0, sum_date * 12).reshape(sum_date, 12), index = index_list, columns= columns)
for item in columns:
    df[item] = 0

df1.index = df1['date_num']
for i in date_num_list:
    print(i)
    df2 = df1[df1['date_num'] == i]
    date = list(df2['date'])[0]
    best_bid_list = list(df2['best_bid_price'])
    bid_size_list = list(df2['best_bid_size'])
    best_bid_price = max(best_bid_list)
    best_bid_size = 0
    for j in range(len(best_bid_list)):
        if best_bid_list[j] == best_bid_price:
            best_bid_size = best_bid_size + bid_size_list[j]

    best_ask_list = list(df2['best_ask_price'])
    ask_size_list = list(df2['best_ask_size'])
    best_ask_price = max(best_ask_list)
    best_ask_size = 0
    for j in range(len(best_ask_list)):
        if best_ask_list[j] == best_ask_price:
            best_ask_size = best_ask_size + ask_size_list[j]

    ask_total_size_list = list(df2['ask_total_size'])
    bid_total_size_list = list(df2['bid_total_size'])
    ask_total_size = sum(ask_total_size_list)
    bid_total_size = sum(bid_total_size_list)

    num_sum = 0
    ask_weighted_list = list(df2['ask_weighted_average_price'])
    for j in range(len(ask_weighted_list)):
        num_sum = num_sum + ask_weighted_list[j] * ask_total_size_list[j]

    ask_weighted_average_price = float(num_sum) / ask_total_size

    num_sum = 0
    bid_weighted_list = list(df2['bid_weighted_average_price'])
    for j in range(len(bid_weighted_list)):
        num_sum = num_sum + bid_weighted_list[j] * bid_total_size_list[j]

    bid_weighted_average_price = float(num_sum) / bid_total_size

    best_mid_price = np.mean([best_ask_price, best_bid_price])

    mid_weighted_average_price = np.mean([ask_weighted_average_price, bid_weighted_average_price])
    df.loc[i, 'date_num'] = i
    df.loc[i, 'date'] = date
    df.loc[i, 'best_ask_price'] = best_ask_price
    df.loc[i, 'best_ask_size'] = best_ask_size
    df.loc[i, 'best_bid_price'] = best_bid_price
    df.loc[i, 'best_bid_size'] = best_bid_size
    df.loc[i, 'ask_total_size'] = ask_total_size
    df.loc[i, 'bid_total_size'] = bid_total_size
    df.loc[i, 'ask_weighted_average_price'] = ask_weighted_average_price
    df.loc[i, 'bid_weighted_average_price'] = bid_weighted_average_price
    df.loc[i, 'best_mid_price'] = best_mid_price
    df.loc[i, 'mid_weighted_average_price'] = mid_weighted_average_price

df.to_csv("C:/Users/zy108/Desktop/9733_machine learning/project/data2.csv", index=False)

for i in index_list[::-1]:
    flag = df.loc[i, 'date_num']
    if flag == 0:
        df.loc[i, 'date_num'] = i
        timeStamp = i
        d = datetime.datetime.utcfromtimestamp(timeStamp)
        date = str(d.strftime("%Y-%m-%d %H:%M:%S.%f"))
        df.loc[i, 'date'] = date
        df.loc[i, 'best_ask_price'] = best_ask_price
        df.loc[i, 'best_ask_size'] = best_ask_size
        df.loc[i, 'best_bid_price'] = best_bid_price
        df.loc[i, 'best_bid_size'] = best_bid_size
        df.loc[i, 'ask_total_size'] = ask_total_size
        df.loc[i, 'bid_total_size'] = bid_total_size
        df.loc[i, 'ask_weighted_average_price'] = ask_weighted_average_price
        df.loc[i, 'bid_weighted_average_price'] = bid_weighted_average_price
        df.loc[i, 'best_mid_price'] = best_mid_price
        df.loc[i, 'mid_weighted_average_price'] = mid_weighted_average_price
    else:
        best_ask_price = df.loc[i, 'best_ask_price']
        best_ask_size = df.loc[i, 'best_ask_size']
        best_bid_price = df.loc[i, 'best_bid_price']
        best_bid_size = df.loc[i, 'best_bid_size']
        ask_total_size = df.loc[i, 'ask_total_size']
        bid_total_size = df.loc[i, 'bid_total_size']
        ask_weighted_average_price = df.loc[i, 'ask_weighted_average_price']
        bid_weighted_average_price = df.loc[i, 'bid_weighted_average_price']
        best_mid_price = df.loc[i, 'best_mid_price']
        mid_weighted_average_price = df.loc[i, 'mid_weighted_average_price']

df['0.1'] = df['best_mid_price'].shift(-1)
df['0.5'] = df['best_mid_price'].shift(-5)
df['1'] = df['best_mid_price'].shift(-10)
df['5'] = df['best_mid_price'].shift(-50)
df['10'] = df['best_mid_price'].shift(-100)
df['30'] = df['best_mid_price'].shift(-300)

df['output_0.1'] = df.apply(lambda x: 1 if x['0.1'] > x['best_mid_price'] else -1 if x['0.1'] < x['best_mid_price'] else 0, axis=1)
df['output_0.5'] = df.apply(lambda x: 1 if x['0.5'] > x['best_mid_price'] else -1 if x['0.5'] < x['best_mid_price'] else 0, axis=1)
df['output_1'] = df.apply(lambda x: 1 if x['1'] > x['best_mid_price'] else -1 if x['1'] < x['best_mid_price'] else 0, axis=1)
df['output_5'] = df.apply(lambda x: 1 if x['5'] > x['best_mid_price'] else -1 if x['5'] < x['best_mid_price'] else 0, axis=1)
df['output_10'] = df.apply(lambda x: 1 if x['10'] > x['best_mid_price'] else -1 if x['10'] < x['best_mid_price'] else 0, axis=1)
df['output_30'] = df.apply(lambda x: 1 if x['30'] > x['best_mid_price'] else -1 if x['30'] < x['best_mid_price'] else 0, axis=1)

df = df.loc[:, columns + ['output_0.1', 'output_0.5', 'output_1', 'output_5', 'output_10', 'output_30']]
df.to_csv("C:/Users/zy108/Desktop/9733_machine learning/project/data.csv", index=False)

print(df)


import soundfile as sf
import sounddevice as sd
import sys
import queue
import numpy as np
import io
import os
import json
import requests
import time

q = queue.Queue()
fname = 'new_file.wav'
sr = 44100
server_ip = '54.87.118.144'

def callback(indata, frames, time, status):
    if status:
        print(status, file=sys.stderr)
    q.put(indata.copy())

def is_silent(snd_data: np.ndarray):
    return np.abs(snd_data).max() < 10 ** -2

def record(path, sample_rate, channels=1):
    if path in os.listdir():
        os.remove(path)
    try:
        with sf.SoundFile(path, mode='x', samplerate=sample_rate, channels=channels, subtype=None) as file:
            i = 0
            silent = 0
            key = input('Press a key to start recording: ')
            time.sleep(0.5)
            with sd.InputStream(samplerate=sample_rate, device=0, channels=channels, callback=callback):
                print(f"Ask me a question: ")
                try:
                    while True:
                        to_write = q.get()
                        if not is_silent(to_write):
                            silent = 0
                        else:
                            silent += 1
                            if silent == 10 and i > 10:
                                break
                        file.write(to_write)
                        i += 1
                except RuntimeError as re:
                    print(f"{re}. If recording was stopped by the user, then this can be ignored")

    except RuntimeError:
        print('error')

record(fname, sr)
with io.open(fname, 'rb') as file:
    content = file.read()

os.remove(fname)
print('waiting for an answer...')
to_send = {'content': list(content), 'sr': sr}
to_send = json.dumps(to_send)

out = requests.post(f'http://{server_ip}:5000/audio', json=to_send)
content = json.loads(out.content)
sr = int(content['sr'])
content = np.array(content['content'])
sd.play(content, sr, blocking=True)
sf.write('ourfile.wav', content, samplerate=sr)


import imp

import sys
from unittest import result

lose_parenthesis = sys.stdin.readline().rstrip().split('-')

nums = []

for parenthesis in lose_parenthesis:
    nums.append(sum(map(int, parenthesis.split('+'))))

result = nums[0]

for i in range(1, len(nums)):
    result -= nums[i]

print(result)


# -*- coding: utf-8 -*-
"""
Created on Fri Dec 27 12:24:35 2019

@author: asus
"""

# Problem 11. Greatest product of 4 adj numbers.
# Admittedly, this one is a bit tougher for me..
# I'm certain there's a more proper way to approach this.
# NOTE NOTE NOTE!!! This is a super naive approach. If I were to redo this
# I would have one set of nested row and column indexing for-loops and would
# have 4 separate loops within to get products for k. E.g.,
# Row: [i,k+k]
# Column [i+k,j]
# Neg Diag: [i-k,j+k]
# Pos Diag: [i+k,j+k]

# My brain is tired :| Don't judge me!

import numpy as np
M = np.loadtxt('MatrixReadInForProblem11.txt',dtype='i',delimiter=' ')
MatrixOfProducts = []
Product = 1
LargestProductc = 1

for i in range(20):
    for j in range(16):
        for k in range(4):
            Product *= M[i,j+k]
        if Product > LargestProductc:
            LargestProductc = Product
        Product = 1
print("Largest Row Product:" ,LargestProductc)
MatrixOfProducts.append(LargestProductc)
Product = 1
LargestProductr = 1

for j in range(20):
    for i in range(16):
        for k in range(4):
            Product *= M[i+k,j]
        if Product > LargestProductr:
            LargestProductr = Product
        Product = 1

print("Largest Col Product: ",LargestProductr)
MatrixOfProducts.append(LargestProductr)
Product = 1
LargestProductd1 = 1

for i in range(0,17):
    for j in range(0,17):
        for k in range(4):
            Product *= M[i+k,j+k]
        if Product > LargestProductd1:
            LargestProductd1 = Product
        Product = 1

print("Largest Product neg diag: ",LargestProductd1)
MatrixOfProducts.append(LargestProductd1)
Product = 1
LargestProductd2 = 1

for i in range(3,20):
    for j in range(0,17):
        for k in range(4):
            Product *= M[i-k,j+k]
        if Product > LargestProductd2:
            LargestProductd2 = Product
        Product = 1

print("Largest Product pos diag: ",LargestProductd2)
MatrixOfProducts.append(LargestProductd2)

print("Solution to Problem 11: ",max(MatrixOfProducts))

# Problem 12
# The nth triangle number is the sum 1 + 2 + ... + n
# Find the first triangle number with over 500 divisors
# Commented out for efficiency since the code doesn't complete quickly enough.

# =============================================================================
# TriNumb = 0
# MaxDivisors = 2
# n = 1
# DivisorCount = 0
# import math
#
# while MaxDivisors < 500:
#     # for i in range(1,n+1):
#     #    TriNumb += i
#     TriNumb = n*(n+1)/2
#
#     for j in range(1,int(math.sqrt(TriNumb+1))):
#         if TriNumb%j == 0:
#             DivisorCount += 2
#
#     if DivisorCount >= 500:
#         print("Solution to Problem 12: ",TriNumb)
#         break;
#
#     # if DivisorCount > MaxDivisors:
#     #    MaxDivisors = DivisorCount
#
#     DivisorCount = 0
#     TriNumb = 0
#     n += 1
# =============================================================================

# Problem 13
# Sum of 150 digit numbers
# We're given a large "table" of digits that, I believe, need to be split to
# form 150 digit numbers that can be summed together.
# I'm a fool lol. It is 100 rows of 50 digit numbers lol. GOTCHA!

#number = np.loadtxt('NumberReadInForProblem13.txt',delimiter="\n")
#print(number)

numbers = open('NumberReadInForProblem13.txt', "r")
lines = numbers.readlines()

sum13 = 0
for i in range(len(lines)):
    sum13 += int(lines[i])

First10Digits = str(sum13)[:10]
print("Solution for Problem 13: ",First10Digits)

# Problem 14
# Collatz Problem
# This code isn't very efficient.
# Might be exceeding the 1 minute rule.
# I need to check all values under 1M
# But there may be some rule allowing certain numbers to be skipped
# =============================================================================
# LongestSeq = 10
# LongestSeqSN = 1
# for i in range(10,1000000):
#     x = i
#     SeqLen = 1
#
#     while x != 1:
#         if x%2 == 0:
#             x /= 2
#             SeqLen += 1
#
#         elif x%2 != 0:
#             x = (x*3 + 1) / 2
#             SeqLen += 2
#
#     if SeqLen > LongestSeq:
#         LongestSeq = SeqLen
#         LongestSeqSN = i
#
# print("Solution for Problem 14: ",LongestSeqSN)
# =============================================================================

# Problem 15
# Approaching it from matrix standpoint. Every bottom left and top right vertex
# can be added to equal the bottom right vertex. If I fill the first row and
# first column with 1's, then I can add and fill the whole matrix.

# Table size
t = 20

M = np.zeros((t+1,t+1))
M[0,:] = 1
M[:,0] = 1

for j in range(1,t+1):
    for i in range(1,t+1):
        M[i,j] = M[i-1,j] + M[i,j-1]

print("Solution to Problem 15: ",int(M[t,t]))