import pandas as pd
from matplotlib import pyplot as plt
import numpy as np
from dataclasses import dataclass
from tdf2lance.ms_representation import _grouped_std
from loguru import logger
from tdf2lance.ms_representation import PseudoScan
from psims.mzml.writer import MzMLWriter

import tempfile

import warnings

import lance
import pandas as pd

from tdf2lance.lance_ms import iter_precursor_frames, neighbor_denoise_frames
import streamlit as st
from matplotlib import pyplot as plt
import random
from typing import Literal
from tqdm.auto import tqdm

import time
import subprocess
from contextlib import contextmanager
from functools import partial
from tdf2lance.utils import StreamingMeanCalculator

@dataclass
class Timeit:
    run_name: str = "run"
    now: float = None
    run_time: float = 0

    def __post_init__(self):
        if self.now is None:
            self.now = time.monotonic()

    def __enter__(self):
        self.now = time.monotonic()
        return self

    def __exit__(self, *args):
        self.run_time = time.monotonic() - self.now
        logger.info(f"{self.run_name} took {self.run_time:.03f}s to run")

def plot_square(ax, xmin, xmax, ymin, ymax, color="red", **kwargs):
    # xs = [50, 90, 90, 50, 50]
    # ys = [100, 100, 130, 130, 100]
    # ax.plot(xs, ys, color="red")
    xs = [xmin, xmax, xmax, xmin, xmin]
    ys = [ymin, ymin, ymax, ymax, ymin]
    ax.plot(xs, ys, color=color, **kwargs)

def plot_squares(ax, xmins, xmaxs, ymins, ymaxs, color="red", **kwargs):
    for xmin, xmax, ymin, ymax in zip(xmins, xmaxs, ymins, ymaxs):
        plot_square(ax, xmin, xmax, ymin, ymax, color=color, **kwargs)

def _stats_per_label(arr, labels, unique_labels, counts, weights, aggregated_weights):
    arr = arr.astype(np.float64)

    arr_len = max(2, unique_labels.max() + 2)
    max_arr = np.zeros(arr_len)
    weighted_avg_arr = np.zeros(arr_len)

    np.maximum.at(max_arr, labels, arr)
    min_arr = max_arr.copy()
    np.minimum.at(min_arr, labels, arr)
    std_arr = _grouped_std(
        labels=labels, unique_labels=unique_labels, float_array=arr, counts=counts
    )
    np.add.at(weighted_avg_arr, labels, arr * weights)
    weighted_avg_arr /= aggregated_weights

    return {
        "max": max_arr,
        "min": min_arr,
        "std": std_arr,
        "weighted_avg": weighted_avg_arr,
    }

@dataclass
class CentroidedPrecursorIndex:
    intensities: np.ndarray
    mzs: np.ndarray
    mobilities: np.ndarray
    quad_low: np.ndarray
    quad_high: np.ndarray
    mobility_min: np.ndarray
    mobility_max: np.ndarray
    mobility_std: np.ndarray
    mz_min: np.ndarray
    mz_max: np.ndarray
    mz_std: np.ndarray
    rt: np.ndarray
    frame_index: np.ndarray

    def __post_init__(self):
        # Check that everything is the same length
        assert len(self.intensities) == len(self.mzs)
        assert len(self.intensities) == len(self.mobilities)
        assert len(self.intensities) == len(self.quad_low)
        assert len(self.intensities) == len(self.quad_high)
        assert len(self.intensities) == len(self.mobility_min)
        assert len(self.intensities) == len(self.mobility_max)
        assert len(self.intensities) == len(self.mobility_std)
        assert len(self.intensities) == len(self.mz_min)
        assert len(self.intensities) == len(self.mz_max)
        assert len(self.intensities) == len(self.mz_std)
        assert len(self.intensities) == len(self.rt)
        assert len(self.intensities) == len(self.frame_index)

    @classmethod
    def from_dataframe(cls, df: pd.DataFrame):
        return cls(
            df["intensities"].to_numpy(),
            df["mzs"].to_numpy(),
            df["mobilities"].to_numpy(),
            df["quad_low"].to_numpy(),
            df["quad_high"].to_numpy(),
            df["mobility_min"].to_numpy(),
            df["mobility_max"].to_numpy(),
            df["mobility_std"].to_numpy(),
            df["mz_min"].to_numpy(),
            df["mz_max"].to_numpy(),
            df["mz_std"].to_numpy(),
            df["rt"].to_numpy(),
            df["frame_index"].to_numpy(),
        )

    def dbscan_combine_traces(
        self,
        *,
        rt_scaling=10,
        mobility_scaling=0.01,
        quad_scaling=5,
        mz_scaling=0.01,
        dbscan_p=2,
        dbscan_min_samples=4,
        use_meanshift=False,
        keep_unclustered=True,
    ):
        from scipy.spatial import KDTree
        from sklearn.cluster import DBSCAN
        from sklearn.cluster import MeanShift

        logger.debug("Parameters for clustering:")
        logger.debug(f"rt_scaling = {rt_scaling}")
        logger.debug(f"mobility_scaling = {mobility_scaling}")
        logger.debug(f"quad_scaling = {quad_scaling}")
        logger.debug(f"mz_scaling = {mz_scaling}")
        logger.debug(f"dbscan_p = {dbscan_p}")
        logger.debug(f"dbscan_min_samples = {dbscan_min_samples}")
        logger.debug(f"use_meanshift = {use_meanshift}")
        logger.debug(f"keep_unclustered = {keep_unclustered}")

        # This is optimized for this specific dataset ..
        # Where cycle time is 1.7s~
        # rt_scaling = 1.75 * 4
        # mobility_scaling = 0.01
        # quad_scaling = 5
        # mz_scaling = 0.02
        # dbscan_min_samples = 2
        # dbscan_p = 0.5  # How much like manhattan distance is it?

        build_arr = np.stack(
            [
                self.mzs / mz_scaling,
                self.rt / rt_scaling,
                self.mobilities / mobility_scaling,
                self.quad_low / quad_scaling,
                self.quad_high / quad_scaling,
            ],
            axis=1,
        )

        if use_meanshift:
            ms = MeanShift(
                bandwidth=1, bin_seeding=True, cluster_all=False, min_bin_freq=5
            ).fit(build_arr)
            # Misleading but api compliant ...
            dbscan = ms

        else:
            kdtree = KDTree(
                build_arr,
                leafsize=100,
                balanced_tree=False,
                copy_data=False,
                compact_nodes=False,
            )
            d = kdtree.sparse_distance_matrix(kdtree, max_distance=2, p=dbscan_p)
            dbscan = DBSCAN(
                eps=1,
                min_samples=dbscan_min_samples,
                metric="precomputed",
                p=dbscan_p,
            ).fit(d)

        uniq_labs, pos, counts = np.unique(
            dbscan.labels_, return_index=True, return_counts=True
        )
        clustered_peaks = sum(counts[uniq_labs != -1])
        unclustered_peaks = sum(counts[uniq_labs == -1])
        logger.debug(
            f"Found {len(uniq_labs)} clusters with "
            f"{clustered_peaks} clustered peaks and "
            f" {unclustered_peaks} unclustered peaks"
        )

        uniq_labs, pos, counts = np.unique(
            dbscan.labels_, return_index=True, return_counts=True
        )

        # Counts is the number of peaks in each cluster
        arr_len = max(max(uniq_labs) + 2, 2)
        b_counts = np.zeros(arr_len)
        np.add.at(b_counts, uniq_labs, counts)
        del counts  # I am deleting it for my own safety ...
        # Since these counts do not match in position with the rest
        # of the arrays. The 'counts' start with the unclustered peaks
        # whilst the b_counts have the unclustered peaks at the end (position -1).

        # Quad lows and highs of each cluster
        b_quad_lows = np.zeros(arr_len)
        b_quad_highs = np.zeros(arr_len)
        np.add.at(b_quad_lows, dbscan.labels_, self.quad_low.astype(np.float64))
        np.add.at(b_quad_highs, dbscan.labels_, self.quad_high.astype(np.float64))
        b_quad_lows /= b_counts
        b_quad_highs /= b_counts

        # Intensities of each cluster
        b_intensities = np.zeros(arr_len)
        np.add.at(
            b_intensities,
            dbscan.labels_,
            self.intensities.astype(np.float64),
        )

        # Little report ...
        clustered_intensities = b_intensities[:-1].sum()
        unclustered_intensities = b_intensities[-1]
        tot_intensities = clustered_intensities + unclustered_intensities
        logger.debug(
            f"Found {len(b_intensities)-1} clusters with "
            f"{clustered_intensities}"
            f"({100*clustered_intensities/tot_intensities:.03f}) "
            f"clustered intensity and {unclustered_intensities} unclustered intensity"
        )

        # Now we need to average the mzs, mobilities and rts of each cluster
        # It is nice to have some distribution metrics ...

        mz_stats = _stats_per_label(
            arr=self.mzs,
            labels=dbscan.labels_,
            unique_labels=uniq_labs,
            counts=b_counts,
            weights=self.intensities,
            aggregated_weights=b_intensities,
        )

        mobility_stats = _stats_per_label(
            arr=self.mobilities,
            labels=dbscan.labels_,
            unique_labels=uniq_labs,
            counts=b_counts,
            weights=self.intensities,
            aggregated_weights=b_intensities,
        )

        rt_stats = _stats_per_label(
            arr=self.rt,
            labels=dbscan.labels_,
            unique_labels=uniq_labs,
            counts=b_counts,
            weights=self.intensities,
            aggregated_weights=b_intensities,
        )

        peak_widths = rt_stats["max"] - rt_stats["min"]
        msg = f"Detected peak width stats: mean={peak_widths.mean():.03f}s"
        msg += f" std={peak_widths.std():.03f}s"
        msg += f" median={np.median(peak_widths):.03f}s"
        logger.info(msg)

        out = {
            "intensities": b_intensities,
            "mzs": mz_stats["weighted_avg"],
            "mobilities": mobility_stats["weighted_avg"],
            "quad_low": b_quad_lows,
            "quad_high": b_quad_highs,
            "mobility_min": mobility_stats["min"],
            "mobility_max": mobility_stats["max"],
            "mobility_std": mobility_stats["std"],
            "mz_min": mz_stats["min"],
            "mz_max": mz_stats["max"],
            "mz_std": mz_stats["std"],
            "rt": rt_stats["weighted_avg"],
            "rt_start": rt_stats["min"],
            "rt_end": rt_stats["max"],
            "num_agg_peaks": b_counts,
        }
        if keep_unclustered:
            # TODO consider if this can be optimized by first allocating the full size
            # of the array including the unclustered peaks.
            unclustered_indices = np.where(dbscan.labels_ == -1)[0]
            unclustered_out = {
                "intensities": self.intensities.astype(np.float64)[unclustered_indices],
                "mzs": self.mzs.astype(np.float64)[unclustered_indices],
                "mobilities": self.mobilities.astype(np.float64)[unclustered_indices],
                "quad_low": self.quad_low.astype(np.float64)[unclustered_indices],
                "quad_high": self.quad_high.astype(np.float64)[unclustered_indices],
                "rt": self.rt.astype(np.float64)[unclustered_indices],
                "num_agg_peaks": np.ones(len(unclustered_indices)),
                "mz_std": np.zeros(len(unclustered_indices)),
                "mobility_std": np.zeros(len(unclustered_indices)),
            }
            unclustered_out["mobility_min"] = unclustered_out["mobilities"]
            unclustered_out["mobility_max"] = unclustered_out["mobilities"]
            unclustered_out["mz_min"] = unclustered_out["mzs"]
            unclustered_out["mz_max"] = unclustered_out["mzs"]
            unclustered_out["rt_start"] = unclustered_out["rt"]
            unclustered_out["rt_end"] = unclustered_out["rt"]

            if not set(out.keys()) == set(unclustered_out.keys()):
                msg = (
                    "Keys of clustered and unclustered peaks do not match: "
                    f"{set(out.keys())} vs {set(unclustered_out.keys())}: "
                    f"Diff = {set(out.keys()) ^ set(unclustered_out.keys())}"
                )
                raise ValueError(msg)
            out = {k: np.concatenate([v, unclustered_out[k]]) for k, v in out.items()}

        return out, dbscan.labels_

@dataclass
class RTCentroidedPrecursorIndex:
    intensities: np.ndarray
    mzs: np.ndarray
    mobilities: np.ndarray
    quad_low: np.ndarray
    quad_high: np.ndarray
    mobility_min: np.ndarray
    mobility_max: np.ndarray
    mobility_std: np.ndarray
    mz_min: np.ndarray
    mz_max: np.ndarray
    mz_std: np.ndarray
    rt: np.ndarray
    rt_start: np.ndarray
    rt_end: np.ndarray
    num_agg_peaks: np.ndarray

    def meanshift_cluster(
        self,
        *,
        rt_scaling: float,
        mobility_scaling: float,
        quad_scaling: float,
        min_bin_freq: int,
        bandwidth: float,
        cluster_all: bool,
        use_meanshift: bool,
        expand_dist: float,
    ):
        from scipy.spatial import KDTree
        from sklearn.cluster import DBSCAN, MeanShift

        logger.debug("Parameters for clustering:")
        logger.debug(f"rt_scaling = {rt_scaling} seconds")
        logger.debug(f"mobility_scaling = {mobility_scaling}")
        logger.debug(f"quad_scaling = {quad_scaling}")
        logger.debug(f"min_bin_freq = {min_bin_freq}")
        logger.debug(f"bandwidth = {bandwidth}")
        logger.debug(f"cluster_all = {cluster_all}")
        logger.debug(f"use_meanshift = {use_meanshift}")
        logger.debug(f"expansion_dist = {expand_dist}")

        build_arr = np.stack(
            [
                self.rt / rt_scaling,
                self.mobilities / mobility_scaling,
                self.quad_low / quad_scaling,
                self.quad_high / quad_scaling,
                # self.rt_start / (rt_scaling * 2),
                # self.rt_end / (rt_scaling * 2),
                # TODO consider adding rt starts and ends here...
            ],
            axis=1,
        )

        clustering = "MeanShift" if use_meanshift else "DBSCAN"
        logger.info(f"Running {clustering} clustering (combining elution groups)...")
        if use_meanshift:
            # TODO experiment doing this iterative increasing the bandwidth.
            generation_cluster_counts = []
            ms = MeanShift(
                bandwidth=bandwidth,
                bin_seeding=True,
                # TODO consider making cluster all false
                # and then assign within threshold to that cluster...
                cluster_all=False,
                min_bin_freq=min_bin_freq,
                max_iter=200,
            ).fit(build_arr)

            logger.info("Clustering overhanding peaks...")
            unclustered_kdt = KDTree(
                build_arr[ms.labels_ == -1],
                leafsize=100,
                balanced_tree=False,
                copy_data=False,
                compact_nodes=False,
            )
            centers_kdt = KDTree(
                ms.cluster_centers_,
                leafsize=100,
                balanced_tree=False,
                copy_data=False,
                compact_nodes=False,
            )
            tmp2 = unclustered_kdt.sparse_distance_matrix(
                centers_kdt,
                max_distance=expand_dist * bandwidth,
                p=1,
                output_type="coo_matrix",
            )
            # Cols in the coo matrix are the index in the cluster centers
            # Rows are the index in the unclustered peaks that match the former
            # cluster center.
            order = np.lexsort((tmp2.row, tmp2.data))
            unclustp_indices, center_indices, dists = (
                tmp2.row[order],
                tmp2.col[order],
                tmp2.data[order],
            )
            uniq_ucpii = np.unique(unclustp_indices, return_index=True)[1]
            logger.info(f"Adding {len(uniq_ucpii)} unclustered peaks to clusters")

            unlabelled = np.where(ms.labels_ == -1)[0]
            ms.labels_[unlabelled[unclustp_indices[uniq_ucpii]]] = center_indices[
                uniq_ucpii
            ]

            ## Second round ...
            unlabelled = np.where(ms.labels_ == -1)[0]
            generation_cluster_counts.append(ms.labels_.max())
            gen = 1
            while len(unlabelled) > min_bin_freq * 2:
                try:
                    gen += 1
                    logger.info(f"Starting round {gen} of clustering ...")
                    ms2 = MeanShift(
                        bandwidth=bandwidth,
                        bin_seeding=True,
                        cluster_all=False,
                        min_bin_freq=min_bin_freq,
                        max_iter=100,
                    ).fit(build_arr[unlabelled])
                    logger.info(
                        f"Adding round {gen} of clusters,"
                        f" {ms2.labels_.max() + 1}; {np.sum(ms2.labels_ >= 0)} points ..."
                    )
                    generation_cluster_counts.append(ms2.labels_.max())
                    keep_ms2 = ms2.labels_ >= 0

                    ms.labels_[unlabelled[keep_ms2]] = (
                        ms2.labels_[keep_ms2] + ms.labels_.max() + 1
                    )
                    unlabelled = np.where(ms.labels_ == -1)[0]
                except ValueError as e:
                    if "max() arg is an empty sequence" in str(e):
                        logger.info("No unclustered peaks left to cluster")
                        break
                    else:
                        raise e
            else:
                logger.info("No unclustered peaks left to cluster")

        else:
            logger.debug("building kdtree")
            kdtree = KDTree(
                build_arr,
                leafsize=100,
                balanced_tree=False,
                copy_data=False,
                compact_nodes=False,
            )
            logger.debug("building distance matrix")
            d = kdtree.sparse_distance_matrix(
                kdtree, max_distance=1.75 * bandwidth, p=0.5
            )

            logger.debug("running dbscan")
            dbscan = DBSCAN(
                eps=bandwidth,
                min_samples=min_bin_freq,
                metric="precomputed",
                p=0.5,
            ).fit(d)
            ms = dbscan

        uniq_labs, pos, counts = np.unique(
            ms.labels_, return_index=True, return_counts=True
        )
        clustered_peaks = sum(counts[uniq_labs != -1])
        unclustered_peaks = sum(counts[uniq_labs == -1])
        logger.debug(
            f"Found {len(uniq_labs)} clusters with {clustered_peaks} "
            f"clustered peaks and {unclustered_peaks} unclustered peaks"
        )

        # use max(uniq_labs)+1 instead of len(unique_labs)
        # Counts is the number of peaks in each cluster
        arr_len = max(max(uniq_labs) + 2, 2)
        b_counts = np.zeros(arr_len)
        np.add.at(b_counts, uniq_labs, counts)
        del counts  # I am deleting it for my own safety ...
        # Since these counts do not match in position with the rest
        # of the arrays. The 'counts' start with the unclustered peaks
        # whilst the b_counts have the unclustered peaks at the end (position -1).

        # Quad lows and highs of each cluster
        b_quad_lows = np.zeros(arr_len)
        b_quad_highs = np.zeros(arr_len)
        np.add.at(b_quad_lows, ms.labels_, self.quad_low.astype(np.float64))
        np.add.at(b_quad_highs, ms.labels_, self.quad_high.astype(np.float64))
        b_quad_lows /= b_counts
        b_quad_highs /= b_counts

        # Intensities of each cluster
        b_intensities = np.zeros(arr_len)
        np.add.at(
            b_intensities,
            ms.labels_,
            self.intensities.astype(np.float64),
        )

        # Little report ...
        clustered_intensities = b_intensities[:-1].sum()
        unclustered_intensities = b_intensities[-1]
        tot_intensities = clustered_intensities + unclustered_intensities
        logger.info(
            f"Found {len(b_intensities)-1} clusters with "
            f"{clustered_intensities}"
            f" ({100*clustered_intensities/tot_intensities:.03f}) "
            f"clustered intensity and {unclustered_intensities} unclustered intensity"
        )

        # Now we need to average the mzs, mobilities and rts of each cluster
        # It is nice to have some distribution metrics ...

        mz_lists = [None] * arr_len
        for i in range(arr_len):
            mz_lists[i] = self.mzs[ms.labels_ == i]

        itensity_lists = [None] * arr_len
        for i in range(arr_len):
            itensity_lists[i] = self.intensities[ms.labels_ == i]

        mobility_stats = _stats_per_label(
            arr=self.mobilities,
            labels=ms.labels_,
            unique_labels=uniq_labs,
            counts=b_counts,
            weights=self.intensities,
            aggregated_weights=b_intensities,
        )

        rt_stats = _stats_per_label(
            arr=self.rt,
            labels=ms.labels_,
            unique_labels=uniq_labs,
            counts=b_counts,
            weights=self.intensities,
            aggregated_weights=b_intensities,
        )

        generation = np.zeros(arr_len)
        cum_gen = 0
        for x in generation_cluster_counts:
            generation[cum_gen : cum_gen + x] = generation.max() + 1

        out = {
            "intensities": itensity_lists[:-1],
            "mzs": mz_lists[:-1],
            "mobilities": mobility_stats["weighted_avg"][:-1],
            "quad_low": b_quad_lows[:-1],
            "quad_high": b_quad_highs[:-1],
            "mobility_min": mobility_stats["min"][:-1],
            "mobility_max": mobility_stats["max"][:-1],
            "mobility_std": mobility_stats["std"][:-1],
            "rt": rt_stats["weighted_avg"][:-1],
            "rt_start": rt_stats["min"][:-1],
            "rt_end": rt_stats["max"][:-1],
            "num_agg_peaks": b_counts[:-1],
            "generation": generation[:-1],
        }
        return out, ms.labels_

def _empty_scan(rt, id):
    scan_id = f"scan={id}"
    return {
        "mz_array": np.array([220]),
        "intensity_array": np.array([220]),
        "scan_start_time": rt,
        "id": scan_id,
        "params": [
            "MS1 Spectrum",
            {"ms level": 1},
            {"total ion current": 220},
        ],
    }, scan_id

def iter_scans(pseudoscans):
    # I am binning here every 0.02 dalton ... same as comet ...
    offset = 1
    curr_rt = pseudoscans["rt"][0] - 1
    empty_spec, last_precursor_scan_id = _empty_scan(curr_rt, offset)
    offset += 1

    yield empty_spec

    for i in range(len(pseudoscans["rt"])):
        mz = pseudoscans["mzs"][i]
        mz_order = np.argsort(mz)
        mz = mz[mz_order]
        intensity = pseudoscans["intensities"][i]
        intensity = intensity[mz_order]

        rt = pseudoscans["rt"][i]
        quad_low = pseudoscans["quad_low"][i]
        quad_high = pseudoscans["quad_high"][i]
        quad_mid = (quad_low + quad_high) / 2

        if rt > curr_rt + 1:
            empty_spec, last_precursor_scan_id = _empty_scan(curr_rt, i + offset)
            yield empty_spec
            curr_rt = rt
            offset += 1
        if quad_high - quad_low > 50:
            logger.warning(
                f"Quad width is {quad_high - quad_low} at RT {rt} for mz {mz}"
            )
        yield {
            "mz_array": mz,
            "intensity_array": intensity,
            "scan_start_time": rt,
            "id": f"scan={i+offset}",
            "params": [
                "MSn Spectrum",
                {"ms level": 2},
                {"total ion current": sum(intensity)},
            ],
            "precursor_information": {
                "activation": [
                    "beam-type collisional dissociation",
                    {"collision energy": 25},
                ],
                "mz": quad_mid,
                "intensity": sum(intensity),
                "charge": 2,
                "isolation_window": [quad_low, quad_mid, quad_high],
                "scan_id": last_precursor_scan_id,
            },
        }

def pseudoscans_to_mzml(pseudoscans, target_file):
    scans = list(iter_scans(pseudoscans))
    logger.info(f"Writing {len(scans)} scans to {target_file}")

    with MzMLWriter(open(str(target_file), "wb"), close=True) as out:
        # Add default controlled vocabularies
        out.controlled_vocabularies()
        out.file_description(["MS1 spectrum", "MSn spectrum"])
        # Open the run and spectrum list sections
        with out.run(id="my_analysis"):
            with out.spectrum_list(count=len(scans)):
                for products in scans:
                    # Write MSn scans
                    out.write_spectrum(**products)

@dataclass
class NeighborDenoiseConfig:
    neighbor_n: int = 1  # 3 seems good
    dbscan_n: int = 3  # 4 seems good
    mz_tolerance: float = 0.011
    mobility_tolerance: float = 0.02
    sides: Literal["left", "right", "both"] = "both"

    @staticmethod
    def from_trial(trial):
        return NeighborDenoiseConfig(
            neighbor_n=trial.suggest_int("nd_neighbor_n", 1, 3),
            dbscan_n=trial.suggest_int("nd_dbscan_n", 1, 4),
            # TODO consider redefining this as a fraction of the
            # frame centroiding tolerance.
            mz_tolerance=trial.suggest_float("nd_mz_tolerance", 0.005, 0.015),
            mobility_tolerance=trial.suggest_float("nd_mobility_tolerance", 0.01, 0.12),
            sides="both",
        )

@dataclass
class IMSCentroidConfig:
    mz_tolerance: float = 0.013  # This seems good
    mobility_tolerance: float = 0.012
    dbscan_n: int = 3  # seems good
    # keep_unclustered: bool = True
    keep_unclustered: bool = False

    @staticmethod
    def from_trial(trial):
        return IMSCentroidConfig(
            mz_tolerance=trial.suggest_float("ic_mz_tolerance", 0.012, 0.022),
            mobility_tolerance=trial.suggest_float(
                "ic_mobility_tolerance", 0.008, 0.018
            ),
            dbscan_n=trial.suggest_int("ic_dbscan_n", 2, 5),
            # keep_unclustered=True, # This is better but is ver slow ...
            keep_unclustered=False,
        )

@dataclass
class CombineTracesConfig:
    """Configuration to combine points into traces along the RT dimension."""

    rt_scaling: float = 1.06  # seems to be similar to the peak width ...
    # Now defined as a fraction of the peak width
    mobility_scaling: float = 0.021  # 0.006-0.008 seem good 80% of c_mobility_scaling
    mz_scaling: float = 0.02  # 0.004-0.007 seem good if using the dbscan workflow.
    dbscan_min_samples: int = 2  # 2-4 seem good
    dbscan_p: float = 0.5  # 0.25-0.75 seem good
    # keep_unclustered: bool = True
    keep_unclustered: bool = False

    @staticmethod
    def from_trial(trial):
        return CombineTracesConfig(
            mz_scaling=trial.suggest_float(
                "t_mz_scaling",
                0.006,
                0.022,
            ),  # TODO reparametrize as a fraction of the mz tolerance in other steps
            # TODO implement a way to infer this parameter.
            rt_scaling=trial.suggest_float("t_rt_scaling", 0.4, 1.4),
            mobility_scaling=trial.suggest_float("t_mobility_scaling", 0.009, 0.022),
            dbscan_min_samples=2,  # 1 works a lil better, but takes forever
            # dbscan_min_samples=trial.suggest_int("t_dbscan_min_samples", 1, 3),
            dbscan_p=0.5,
            keep_unclustered=False,
            # keep_unclustered=trial.suggest_categorical(
            #     "t_keep_unclustered", [True, False]
            # ),
        )

@dataclass
class ClusterTracesConfig:
    """Configuration to cluster traces"""

    rt_scaling: float = 0.09
    # Now defined as number of cycle times for dbscan but
    # as a fraction of the peak width for meanshift.
    mobility_scaling: float = (
        0.0096  # 0.002-0.01 seem good for dbscan ... 0.01 for meanshift
    )
    min_bin_freq: int = 2  # 1-5 looks good
    binwidth: float = 1
    cluster_all: bool = False
    use_meanshift: bool = True
    expand_dists: float = 0

    @staticmethod
    def from_trial(trial):
        # use_meanshift = trial.suggest_categorical("c_use_meanshift", [True, False])
        use_meanshift = True
        if use_meanshift:
            return ClusterTracesConfig(
                rt_scaling=trial.suggest_float("c_rt_scaling", 0.01, 0.25),
                mobility_scaling=trial.suggest_float(
                    "c_mobility_scaling", 0.008, 0.022
                ),
                min_bin_freq=trial.suggest_int("c_min_bin_freq", 2, 9),
                binwidth=1,
                cluster_all=False,
                use_meanshift=True,
                expand_dists=0,
            )
        else:
            return ClusterTracesConfig(
                rt_scaling=trial.suggest_float("c_rt_scaling", 0.9, 5.1),
                mobility_scaling=trial.suggest_float(
                    "c_mobility_scaling", 0.001, 0.012
                ),
                min_bin_freq=trial.suggest_int("c_min_bin_freq", 4, 15),
                binwidth=0.6,
                cluster_all=False,
                use_meanshift=False,
            )

def neighbor_denoise(
    config: NeighborDenoiseConfig,
    ims_centroid_config: IMSCentroidConfig,
    targetfile=None,
):
    out_ds = "currfile.lance"
    ds = lance.dataset(out_ds)

    precursor_index = 5
    out = []

    start = 120
    runtime = 15 * 60

    logger.info(
        "Running neighbor denoise ... On precursor"
        f" {precursor_index} start {start} runtime {runtime}"
    )

    neighbor_denoise_partial = partial(
        neighbor_denoise_frames,
        quad_tolerance=None,
        mz_tolerance=config.mz_tolerance,
        mobility_tolerance=config.mobility_tolerance,
        sides=config.sides,
        yield_noisy=False,
    )

    def _inner():
        for f in neighbor_denoise_partial(
            iter_precursor_frames(
                ds,
                precursor_index=precursor_index,
                start_rt=start,
                end_rt=runtime + runtime,
            ),
            n=config.neighbor_n,
        ):
            if sum(f.n_peaks) == 0:
                continue

            if f.rt > (start + runtime):
                break

            # suppress EfficiencyWarning warnings
            with warnings.catch_warnings():
                warnings.filterwarnings(
                    "ignore",
                    message=(
                        "Precomputed sparse input was not sorted by row values."
                        " Use the function sklearn.neighbors.sort_graph_by_row_values"
                        " to sort the input by row values,"
                        " with warn_when_not_sorted=False"
                        " to remove this warning."
                    ),
                )

                # TODO consider making anothr round of neighbor simplification here ...
                # maybe a first round keeping unclustered and a second round not keeping
                asdasd = f.dbscan_sinplify_frame(
                    n=ims_centroid_config.dbscan_n,
                    mz_tolerance=ims_centroid_config.mz_tolerance,
                    mobility_tolerance=ims_centroid_config.mobility_tolerance,
                    quad_tolerance=5,
                    eps=1e-8,
                    dbscan_n=ims_centroid_config.dbscan_n,
                    keep_unclustered=ims_centroid_config.keep_unclustered,
                )
                yield asdasd

    for asdasd in neighbor_denoise_partial(_inner(), n=1):
        out.append(pd.DataFrame(asdasd.as_record()))

    out = pd.concat(out)
    if targetfile is None:
        targetfile = f"{precursor_index}_currfile.parquet"

    out.to_parquet(targetfile)

def plot_clustering(pseudoscan_labels, clustering_dict, pseudoscans_dict, outfile):
    fig, ax = plt.subplots(figsize=(25, 15))
    ax.scatter(
        clustering_dict["rt"][pseudoscan_labels == -1],
        clustering_dict["mobilities"][pseudoscan_labels == -1],
        c="black",
        s=5,
    )
    ax.scatter(
        clustering_dict["rt"],
        clustering_dict["mobilities"],
        c=np.concatenate([pseudoscans_dict["generation"], [-1]])[pseudoscan_labels],
        alpha=0.1,
        s=5,
    )
    fig.savefig("latest_generations.png", dpi=96)
    plt.close(fig)

    fig, ax = plt.subplots(figsize=(25, 15))

    plot_order = np.argsort(pseudoscan_labels)
    start_clusters = np.searchsorted(pseudoscan_labels[plot_order], 0)

    ax.set_xlabel("rt (s)")
    ax.set_ylabel("Mobility 1/K0")
    # Gray points for non clustered
    ax.scatter(
        clustering_dict["rt"],
        clustering_dict["mobilities"],
        c="black",
        alpha=0.2,
        s=5,
    )
    ax.scatter(
        clustering_dict["rt"][plot_order][:start_clusters],
        clustering_dict["mobilities"][plot_order][:start_clusters],
        c="gray",
        alpha=0.2,
        s=5,
    )
    fig.savefig("latest_unclustered.png", dpi=96)
    ax.scatter(
        clustering_dict["rt"][plot_order][start_clusters:],
        clustering_dict["mobilities"][plot_order][start_clusters:],
        c=pseudoscan_labels[plot_order][start_clusters:],
        alpha=0.2,
        s=5,
    )
    plot_squares(
        xmaxs=pseudoscans_dict["rt_end"],
        xmins=pseudoscans_dict["rt_start"],
        ymaxs=pseudoscans_dict["mobility_max"],
        ymins=pseudoscans_dict["mobility_min"],
        ax=ax,
        color="red",
        alpha=0.3,
    )

    logger.info(f"Saving clustering plot to {outfile}")
    fig.savefig(outfile, dpi=96)
    fig.savefig("latest.png", dpi=96)
    # plt.show()
    plt.close(fig)

def main(
    *,
    traces_config: CombineTracesConfig,
    cluster_config: ClusterTracesConfig,
    pq_file=None,
    plot=True,
):
    logger.info("Reading parquet file ...")
    # df = pd.read_parquet("5_3817_TIMS2_2col-80m_37_1_Slot1-46_1_4768.lance.parquet")
    if pq_file is None:
        logger.warning("No parquet file provided, using default")
        pq_file = "5_currfile.parquet"
    df = pd.read_parquet(pq_file)
    first_quad_low = df["quad_low"].min()
    unique_rts = np.sort(np.unique(df[df["quad_low"] == first_quad_low]["rt"]))
    cycle_time = np.median(np.diff(unique_rts))
    logger.info(f"Cycle time is {cycle_time:.03f}s")

    # Randomly pick 1000 peaks among the top 10% most intense peaks
    # and calculate the peak width distribution
    # logger.info("Calculating peak widths ...")
    sdf = df[df["quad_low"] == first_quad_low].sort_values(
        "intensities", ascending=False
    )
    hsdf = sdf.head(int(len(sdf) * 0.25))
    hsdf = hsdf.sample(min(1000, len(hsdf)))
    mzs = hsdf["mzs"].values
    intensities = hsdf["intensities"].values
    # Remove mzs that are very close to each other
    mz_order = np.argsort(mzs)
    mzs = mzs[mz_order]
    intensities = intensities[mz_order]

    mz_diffs = np.concatenate([np.array([0.0]), np.diff(mzs)])
    mzs = mzs[mz_diffs > 0.02]
    intensities = intensities[mz_diffs > 0.02]

    # Slow but easy version ...

    calc_widths = []
    for mz, inten in tqdm(zip(mzs, intensities)):
        sdf2 = sdf[(sdf["mzs"] > mz - 0.02) & (sdf["mzs"] < mz + 0.02)]
        # Keep only peaks more than 1% of the max intensity
        sdf2 = sdf2[(sdf2["intensities"] > 0.01 * inten)]
        rts = np.sort(sdf2["rt"].values)
        # keep only rts that are less than 3 cycles from each orther
        rt_diff = np.diff(rts)
        rt_diff_lt3 = rt_diff < (3 * cycle_time)

        # https://stackoverflow.com/a/38161867
        # Get start, stop index pairs for islands/seq. of 1s
        idx_pairs = np.where(np.diff(np.hstack(([False], rt_diff_lt3 == 1, [False]))))[
            0
        ].reshape(-1, 2)
        if len(idx_pairs) < 1:
            continue

        # Get the island lengths, whose argmax would give us the ID of longest island.
        # Start index of that island would be the desired output
        start_end_longest_seq = idx_pairs[np.diff(idx_pairs, axis=1).argmax(), :]
        rts = rts[start_end_longest_seq[0] : start_end_longest_seq[1] + 1]

        calc_widths.append(rts[-1] - rts[0])

    calc_widths = np.array(calc_widths)
    median_calc_width = np.median(calc_widths)
    logger.info(
        f"Peak width stats: mean={calc_widths.mean():.03f}s"
        f" std={calc_widths.std():.03f}s"
        f" median={np.median(calc_widths):.03f}s"
    )

    # mass_bins = np.arange(400, 1000, 0.5).astype(np.float64)
    # mass_shifts = 0.4
    # mass_bins += mass_shifts

    # rt_arr = np.zeros((len(unique_rts), len(mass_bins) - 1))
    # for (rt, ql), sdf in tqdm(
    #     df.groupby(["rt", "quad_low"]), desc="Calculting peak widths"
    # ):
    #     if ql != first_quad_low:
    #         continue
    #     rt_arr[unique_rts == rt, :] = np.histogram(
    #         sdf["mzs"], bins=mass_bins, weights=sdf["intensities"]
    #     )[0]

    # df_bkp = df
    # df = df.head(50000)

    logger.info("Filtering df")
    with Timeit("Filtering df"):
        df = df[(df["mzs"] > df["quad_high"] + 1) | (df["mzs"] < df["quad_low"] - 1)]

    logger.info("Clustering traces ...")
    with Timeit("Clustering traces"):
        try:
            dbscan_clustering, labels = CentroidedPrecursorIndex.from_dataframe(
                df
            ).dbscan_combine_traces(
                rt_scaling=traces_config.rt_scaling * median_calc_width,
                mobility_scaling=traces_config.mobility_scaling,
                quad_scaling=5,
                mz_scaling=traces_config.mz_scaling,
                dbscan_min_samples=traces_config.dbscan_min_samples,
                dbscan_p=traces_config.dbscan_p,
                use_meanshift=False,
                keep_unclustered=traces_config.keep_unclustered,
            )
        except IndexError as e:
            logger.error(f"Error clustering: {e}")
            return 0

    with Timeit("Meanshift clustering"):
        rtcp = RTCentroidedPrecursorIndex(**dbscan_clustering)
        try:
            pseudoscans, pseudoscan_labels = rtcp.meanshift_cluster(
                # might be better in terms of cycle time ...
                rt_scaling=cluster_config.rt_scaling * median_calc_width,
                mobility_scaling=cluster_config.mobility_scaling,
                quad_scaling=5,
                # min_bin_freq=cluster_config.min_bin_freq,
                min_bin_freq=cluster_config.min_bin_freq,
                bandwidth=cluster_config.binwidth,
                cluster_all=cluster_config.cluster_all,
                use_meanshift=True,
                expand_dist=cluster_config.expand_dists,
            )
            # Maybe have a two stage where unclustered get assigned to closest within
            # a range.
        except IndexError as e:
            logger.error(f"Error clustering: {e}")
            return 0

    if plot:
        with Timeit("Plotting clustering"):
            plot_clustering(
                pseudoscan_labels, dbscan_clustering, pseudoscans, pq_file + "_ms.png"
            )

    # with Timeit("Meanshift clustering"):
    #     rtcp = RTCentroidedPrecursorIndex(**dbscan_clustering)
    #     try:
    #         pseudoscans, pseudoscan_labels = rtcp.meanshift_cluster(
    #             # might be better in terms of cycle time ...
    #             rt_scaling=cluster_config.rt_scaling * cycle_time,
    #             mobility_scaling=cluster_config.mobility_scaling,
    #             quad_scaling=5,
    #             min_bin_freq=cluster_config.min_bin_freq,
    #             bandwidth=cluster_config.binwidth,
    #             cluster_all=cluster_config.cluster_all,
    #             use_meanshift=cluster_config.use_meanshift,
    #         )
    #     except IndexError as e:
    #         logger.error(f"Error clustering: {e}")
    #         return 0

    # with Timeit("Plotting clustering"):
    #     plot_clustering(
    #         pseudoscan_labels, dbscan_clustering, pseudoscans, pq_file + ".png"
    #     )

    # parent scans could be frame ids ...
    # or just an integerized retention time ...
    if len(pseudoscans["rt"]) == 0:
        logger.error("No pseudoscans found ...")
        return 0

    with tempfile.NamedTemporaryFile(suffix=".mzML", dir="tmp") as f:
        with tempfile.TemporaryDirectory(dir="tmp") as d:
            pseudoscans_to_mzml(pseudoscans, f.name)
            with Timeit("Running SAGE"):
                subprocess.run(
                    [  # noqa: S603, S607
                        "sage",
                        "--fasta",
                        "UP000005640_9606.fasta",
                        "-o",
                        d,
                        "--write-pin",
                        "sageconfig.json",
                        f.name,
                    ],
                    check=True,
                )

            pin_file = pd.read_csv(f"{d}/results.sage.pin", sep="\t")

    peptide_pinfile = (
        pin_file.groupby(["Label", "Peptide"])
        .agg({"ln(-poisson)": "min"})
        .reset_index()
    )  # I am using min to penalize having multiple times the same peptide
    deltascores = (
        peptide_pinfile["ln(-poisson)"][peptide_pinfile["Label"] == 1].sum()
        - peptide_pinfile["ln(-poisson)"][peptide_pinfile["Label"] == -1].sum()
    )

    return deltascores

from pathlib import Path

def _copy(self, target):
    import shutil

    assert self.is_file()
    shutil.copy(str(self), str(target))  # str() only there for Python < (3, 6)

Path.copy = _copy

def optuna_objective(trial):
    traces_config = CombineTracesConfig.from_trial(trial)
    cluster_config = ClusterTracesConfig.from_trial(trial)
    # traces_config = CombineTracesConfig()
    # cluster_config = ClusterTracesConfig()
    # cluster_config.expand_dists = trial.suggest_float("expand_dist", 5, 100)

    with tempfile.NamedTemporaryFile(dir="tmp", suffix=".parquet") as d:
        Path("5_currfile.parquet").copy(d.name)
        # neighbor_denoise(
        #     NeighborDenoiseConfig.from_trial(trial),
        #     IMSCentroidConfig.from_trial(trial),
        #     targetfile=d.name,
        # )
        out = main(
            traces_config=traces_config,
            cluster_config=cluster_config,
            pq_file=d.name,
            plot=False,
        )

    return out

if __name__ == "__main__":
    import optuna

    # if not Path("5_3817_TIMS2_2col-80m_37_1_Slot1-46_1_4768.lance.parquet").exists():

    if not Path("5_currfile.parquet").exists():
        neighbor_denoise(
            NeighborDenoiseConfig(),
            IMSCentroidConfig(),
        )
    baseline = main(
        traces_config=CombineTracesConfig(),
        cluster_config=ClusterTracesConfig(),
        plot=True,
    )
    logger.info(f"Baseline score: {baseline}")

    study = optuna.create_study(
        storage="sqlite:///db.sqlite3",  # Specify the storage URL here.
        study_name="thp1_clupmneigh_parallel5",
        load_if_exists=True,
        direction="maximize",
    )
    study.optimize(optuna_objective, n_trials=500, gc_after_trial=True, n_jobs=4)
    print(f"Best value: {study.best_value} (params: {study.best_params})")


from PIL import Image
import streamlit as st

# set title
st.title("My First Streamlit App")

# add sub header
st.subheader("this is the sub header")

# adding image
image = Image.open("1267203.jpg")
st.image(image, use_column_width=True)

# adding writing and markdown
st.write("writing a text !")
st.markdown("markdown cell ")

# adding a success
st.success("congrats this cell ran !")

# adding info , warning , error
st.info(" This is the information cell")
st.warning(" warning cell !")
st.error("error cell !")

# help
st.help(range)

# pandas data frame
import numpy as np
import pandas as pd

st.subheader('this is a dataframe of random numbers between 10,20')
dataframe = np.random.rand(10,20)
st.dataframe(dataframe)

st.text('another dataframe creation')
df = pd.DataFrame(np.random.rand(10,20),columns = ('col %d' %i for i in range(20)))
st.dataframe(df.style.highlight_max(axis = 1))

# display charts and graphs

data = pd.DataFrame(np.random.randn(20,3), columns = ['a','b','c'])

st.text('line chart')
st.line_chart(data)

st.text('area chart')
st.area_chart(data)

st.text('bar chart')
st.bar_chart(data)

# using matplot
from pyforest import *
st.set_option('deprecation.showPyplotGlobalUse', False)

st.text('pyplot module')
arr = np.random.normal(1,1,size = 100)
plt.hist(arr,bins = 20)
st.pyplot()

# using plotly
import plotly
import plotly.figure_factory as ff

x1 = np.random.randn(200) - 2
x2 = np.random.randn(200)
x3 = np.random.randn(200) - 5

st.text("plotly graph")
hist_data = [x1,x2,x3]
labels = ['group 1','group 2','group 3']
fig = ff.create_distplot(hist_data,labels,bin_size = [0.5,0.25,0.6])
st.plotly_chart(fig,use_container_width = True)

# adding buttons
st.text('button types')
if st.button('hey'):
    st.write('hello')

detail = st.radio('are you single ?',('yes','no'))
if detail == 'yes':
    st.write('eligible')
else:
    st.write('Not eigible')

ans = st.selectbox('are you married ? ',('yes','no'))
st.write('are you married ? : ',ans )

multans = st.multiselect('select year ? multi select ',(20,30,50,70))
st.write('years are : ',multans)

# adding slider
st.text('slider !')
age = st.slider('how old are you ?',0,90,10)
st.write('I am ',age )

value = st.slider('select range',0,90,(10,20))
st.write('range : ',value )

# number input
num = st.number_input('input number ')
st.write('Number : ',num)

# adding file or uploading
file = st.file_uploader("Choose a file",type = 'csv')
if file is not None:
    data = pd.read_csv(file,header = 0)
    st.write(data)
    st.success("file uploaded !")
else :
    st.markdown("upload csv file")

# color picker
color = st.color_picker('pick your color','#00f900')
st.write("color is : ",color)

# adding sidebar
st.sidebar.selectbox('choose model :' ,('model 1','model 2','model 3'))

# progress bar for tracking
st.text('progress bar 1')
import time
my_bar = st.progress(0)
for percent_complete in range(100):
    time.sleep(0.1)
    my_bar.progress(percent_complete+1)

st.text('alternative progress')
with st.spinner("please wait .........") :
    time.sleep(5)
st.balloons()
st.success("complete !")


# -*- coding: utf-8 -*-
"""
this code is part of PR #1731 at e2nIEE/pandapower
when the PR has been merged this file should be removed and its call replaced in ppqgis_import.py

https://github.com/e2nIEE/pandapower/blob/develop/pandapower/plotting/geo.py
"""

import sys
import math
import pandas as pd
from pandapower.auxiliary import soft_dependency_error

try:
    from pyproj import Transformer
    pyproj_INSTALLED = True
except ImportError:
    pyproj_INSTALLED = False

try:
    import geojson
    geojson_INSTALLED = True
except ImportError:
    geojson_INSTALLED = False

def convert_crs(net, epsg_in=4326, epsg_out=31467):
    """
    This function works for pandapowerNet and pandapipesNet. Documentation will refer to names from pandapower.
    Converts bus and line geodata in net from epsg_in to epsg_out
    if GeoDataFrame data is present convert_geodata_to_gis should be used to update geometries after crs conversion

    :param net: The pandapower network
    :type net: pandapowerNet|pandapipesNet
    :param epsg_in: current epsg projection
    :type epsg_in: int, default 4326 (= WGS84)
    :param epsg_out: epsg projection to be transformed to
    :type epsg_out: int, default 31467 (= Gauss-Krüger Zone 3)
    :return: net - the given pandapower network (no copy!)
    """
    is_pandapower = net.__class__.__name__ == 'pandapowerNet'
    if epsg_in == epsg_out:
        return

    if not pyproj_INSTALLED:
        soft_dependency_error(str(sys._getframe().f_code.co_name)+"()", "pyproj")
    transformer = Transformer.from_crs(epsg_in, epsg_out, always_xy=True)

    def _geo_node_transformer(r):
        (x, y) = transformer.transform(r.x, r.y)
        if is_pandapower:
            coords = r.coords
            if coords and not pd.isna(coords):
                coords = _geo_branch_transformer(coords)
            return pd.Series([x, y, coords], ["x", "y", "coords"])
        else:
            return pd.Series([x, y], ["x", "y"])

    def _geo_branch_transformer(r):
        return list(transformer.itransform(r))

    if is_pandapower:
        net.bus_geodata = net.bus_geodata.apply(lambda r: _geo_node_transformer(r), axis=1)
        net.line_geodata.coords = net.line_geodata.coords.apply(lambda r: _geo_branch_transformer(r))
        net.bus_geodata.attrs = {"crs": f"EPSG:{epsg_out}"}
        net.line_geodata.attrs = {"crs": f"EPSG:{epsg_out}"}
    else:
        net.junction_geodata = net.junction_geodata.apply(lambda r: _geo_node_transformer(r), axis=1)
        net.pipe_geodata.coords = net.pipe_geodata.coords.apply(lambda r: _geo_branch_transformer(r))
        net.junction_geodata.attrs = {"crs": f"EPSG:{epsg_out}"}
        net.pipe_geodata.attrs = {"crs": f"EPSG:{epsg_out}"}

def dump_to_geojson(net, nodes=False, branches=False):
    """
    This function works for pandapowerNet and pandapipesNet. Documentation will refer to names from pandapower.
    Dumps all primitive values from bus, bus_geodata, res_bus, line, line_geodata and res_line into a geojson object.
    It is recommended to only dump networks using WGS84 for GeoJSON specification compliance.

    :param net: The pandapower network
    :type net: pandapowerNet|pandapipesNet
    :param nodes: if True return contains all bus data, can be a list of bus ids that should be contained
    :type nodes: bool | list, default True
    :param branches: if True return contains all line data, can be a list of line ids that should be contained
    :type branches: bool | list, default True
    :return: geojson
    :return type: geojson.FeatureCollection
    """
    is_pandapower = net.__class__.__name__ == 'pandapowerNet'
    if is_pandapower:
        node_geodata = net.bus_geodata
        branch_geodata = net.line_geodata
    else:
        node_geodata = net.junction_geodata
        branch_geodata = net.pipe_geodata

    if not geojson_INSTALLED:
        soft_dependency_error(str(sys._getframe().f_code.co_name) + "()", "geojson")

    features = []
    # build geojson features for nodes
    if nodes:
        props = {}
        for table in (['bus', 'res_bus'] if is_pandapower else ['junction', 'res_junction']):
            if table not in net.keys():
                continue
            cols = net[table].columns
            # I use uid for the id of the feature, but it is NOT a unique identifier in the geojson structure,
            # as line and bus (pipe and junction) can have same ids.
            for uid, row in net[table].iterrows():
                prop = {
                    'pp_type': 'bus' if is_pandapower else 'junction',
                    'pp_index': uid,
                }
                for c in cols:
                    try:
                        prop[c] = float(row[c])
                        if math.isnan(prop[c]):
                            prop[c] = None
                    except (ValueError, TypeError):
                        prop[c] = str(row[c])
                if uid not in props:
                    props[uid] = {}
                props[uid].update(prop)
        if isinstance(nodes, bool):
            iterator = node_geodata.iterrows()
        else:
            iterator = node_geodata.loc[nodes].iterrows()
        for uid, row in iterator:
            if is_pandapower and row.coords is not None and not pd.isna(row.coords):
                # [(x, y), (x2, y2)] start and end of bus bar
                geom = geojson.LineString(row.coords)
            else:
                # this is just a bus with x, y
                geom = geojson.Point((row.x, row.y))
            features.append(geojson.Feature(geometry=geom, id=uid, properties=props[uid]))

    # build geojson features for branches
    if branches:
        props = {}
        for table in (['line', 'res_line'] if is_pandapower else ['pipe', 'res_pipe']):
            if table not in net.keys():
                continue
            cols = net[table].columns
            for uid, row in net[table].iterrows():
                prop = {
                    'pp_type': 'line' if is_pandapower else 'pipe',
                    'pp_index': uid,
                }
                for c in cols:
                    try:
                        prop[c] = float(row[c])
                        if math.isnan(prop[c]):
                            prop[c] = None
                    except (ValueError, TypeError):
                        prop[c] = str(row[c])
                if uid not in props:
                    props[uid] = {}
                props[uid].update(prop)

        # Iterating over pipe_geodata won't work
        # pipe_geodata only contains pipes that have inflection points!
        if isinstance(branches, bool):
            # if all iterating over pipe
            iterator = net.line_geodata.iterrows() if is_pandapower else net.pipe.iterrows()
        else:
            iterator = net.line_geodata.loc[branches].iterrows() if is_pandapower else net.pipe.loc[branches].iterrows()
        for uid, row in iterator:
            if not is_pandapower:
                coords = []
                from_coords = net.junction_geodata.loc[row.from_junction]
                to_coords = net.junction_geodata.loc[row.to_junction]
                coords.append([float(from_coords.x), float(from_coords.y)])
                if uid in net.pipe_geodata:
                    coords.append(net.pipe_geodata.loc[uid].coords)
                coords.append([float(to_coords.x), float(to_coords.y)])

            geom = geojson.LineString(row.coords if is_pandapower else coords)
            features.append(geojson.Feature(geometry=geom, id=uid, properties=props[uid]))
    # find and set crs if available
    crs_node = None
    if nodes and "crs" in node_geodata.attrs:
        crs_node = node_geodata.attrs["crs"]
    crs_branch = None
    if branches and "crs" in branch_geodata.attrs:
        crs_branch = branch_geodata.attrs["crs"]

    crs = {
        "type": "name",
        "properties": {
            "name": ""
        }
    }
    if crs_node:
        if crs_branch and crs_branch != crs_node:
            raise ValueError("Node and Branch crs mismatch")
        crs["properties"]["name"] = crs_node
    elif crs_branch:
        crs["properties"]["name"] = crs_branch
    else:
        crs = None
    if crs:
        return geojson.FeatureCollection(features, crs=crs)
    return geojson.FeatureCollection(features)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from PIL import Image

def main():
    base_img = Image.open('image/1.png')
    temp_img = Image.open('image/2.png')
    save_img = Image.new('RGBA', base_img.size, (0, 0, 0, 0))
    box = (166, 64, 320, 337)

    temp_img = temp_img.convert('RGBA')
    temp_img = temp_img.resize((box[2] - box[0], box[3] - box[1]))

    save_img.paste(temp_img, box)
    save_img.paste(base_img, (0, 0), base_img)

    save_img.save('mask.png')

if __name__ == '__main__':
    main()


import base64
import sqlite3
import json
import os
import sys
import glob
import math
import urllib.request as url_req
from urllib.error import URLError
from datetime import timedelta, datetime
from Utils import geohash, extract_app_data
from Devices.device import Device

class App(Device):
    """ This class receives data from the Mi Fit application, interprets it and stores it in the database """
    def __init__(self):
        super().__init__()
        self.support_heartrate = True
        self.support_steps = True
        self.support_acceleration = True
        self.support_sleep = True
        self.support_user = True
        self.support_position = True

    def action(self):
        """
        First, the function extracts the databases from the device, or uses a existing Android image.
        Thereafter, it obtains data from various databases and hands it to the interpreting functions.
        """
        print("Android image path (leave empty if live downloading):")
        path_to_image = input()
        if path_to_image == "":
            print("Extract Mi Fit data from Android phone")
            extractor = extract_app_data.Extractor()
            path_to_db = extractor.extract_one_file(
                    "/data/data/com.xiaomi.hm.health/databases/origin_db_*",
                    output_path="tmp")
        else:
            print("Extract Mit Fit data from " + path_to_image)
            path_to_db = os.path.join(path_to_image,
                    "data/data/com.xiaomi.hm.health/databases/origin_db_*")
            path_to_db = glob.glob(path_to_db)
            print(path_to_db)
            path_to_db = path_to_db[0]

        conn = sqlite3.connect(path_to_db)
        c = conn.cursor()
        res = c.execute("SELECT NAME,           \
                                BIRTHDAY,       \
                                AVATAR_URL,     \
                                GENDER,         \
                                HEIGHT,         \
                                WEIGHT,         \
                                CITY,           \
                                CREAT_TIME,     \
                                LAST_LOGIN_TIME \
                                FROM USER_INFOS")
        self.decode_user(res)
        res = c.execute("SELECT DATE, DATA, DATA_HR, SUMMARY FROM DATE_DATA")
        self.decode_data(res)
        res = c.execute("SELECT TIME, HR FROM HEART_RATE")
        self.decode_hr_table(res)
        res = c.execute("SELECT DATE,           \
                                COSTTIME,       \
                                ENDTIME,        \
                                LOCATION,       \
                                STATISTICS,     \
                                BULKLL,         \
                                BULKTIME,       \
                                BULKHR,         \
                                BULKPAUSE,      \
                                d.TRACKID       \
                                FROM TRACKDATA d\
                                JOIN TRACKRECORD r ON d.TRACKID = r.TRACKID")
        self.decode_track(res)

    def decode_track(self, data):
        """
        This method decodes information from the track tables.
        It stores longitude and latitude with data like heart rate
        """
        print("Process GPS and track data")
        for row in data:
            date = row[0]
            costtime = timedelta(seconds=row[1])
            endtime = datetime.fromtimestamp(row[2])
            location = row[3]
            bulkll = row[5]
            bulktime = row[6]
            bulkhr = row[7]
            bulkpause = row[8]
            track_id = row[9]

            # pause: pauses the user pressed during an activity
            pause_time_sec = 0
            pause_array = []
            if bulkpause:
                pause = bulkpause.split(';')
                for p in pause[:-1]:
                    p = p.split(',')
                    pause_time_sec += int(p[1])
                    pause_array.append(datetime.fromtimestamp(int(p[0])))
                    pause_array.append(timedelta(seconds=int(p[1])))
            pause_time = timedelta(seconds=pause_time_sec)

            # heart rate: heart rate data is stored in a string split by ;
            # every heart rate datum is stored as <second>:<heart rate>, while second is the second since activity start
            heartrate = 0
            heartrate_sec= 0
            timestamp = endtime-costtime-pause_time
            for h in bulkhr.split(';'):
                hr_sec = h.split(',')[0]
                hr = h.split(',')[1]
                if hr != '':
                    heartrate += int(hr)
                if hr_sec != '':
                    heartrate_sec += int(hr_sec)
                else:
                    heartrate_sec += 1
                self.heartrate.addData(heartrate, timestamp + timedelta(seconds=heartrate_sec))

            """ Explanation:
            start lat and lon are stored in location as string with comma.
            all lat and lon are stored in bulkll as integer.
            first lat and lon in bulkll are the same as in location, the following are delta to the previews position.
            As bulkll stores integers, it was not clear where to put the comma.
            Therefore, first the position of the comma is searched in the string from location.
            The position must be counted from the back and not the front, as seen in the following example:
            10.10 - 0.20 = 0.90 -> The position stays the same if counted from the back
            """
            timestamp = endtime-costtime-pause_time
            bulktime = bulktime.split(';')
            p_idx = 0
            lat, lon = geohash.decode(location)
            lat_position_of_comma = str(lat).find('.')
            lon_position_of_comma = str(lon).find('.')
            lat_value = int(bulkll.split(';')[0].split(',')[0])
            lat_position_of_comma -= len(str(lat_value))  # Negative number to count from the back
            lon_value = int(bulkll.split(';')[0].split(',')[1])
            lon_position_of_comma -= len(str(lon_value))  # Negative number to count from the back

            for idx, ll in enumerate(bulkll.split(';')[1:]):
                secs = int(bulktime[idx])
                timestamp += timedelta(seconds=secs)
                if len(pause_array) > p_idx and timestamp > pause_array[p_idx]:
                    timestamp += pause_array[p_idx+1]
                    p_idx += 2
                lat_value += int(ll.split(',')[0])
                lon_value += int(ll.split(',')[1])
                lat_fin = str(lat_value)[:lat_position_of_comma] + "." + str(lat_value)[lat_position_of_comma:]
                lon_fin = str(lon_value)[:lon_position_of_comma] + "." + str(lon_value)[lon_position_of_comma:]
                self.position.addData(timestamp, lat_fin, lon_fin, track_id)

    def decode_user(self, data):
        """ Interprets user data and stores it in the database """
        print("Process user data")
        for row in data:
            try:
                req = url_req.Request(row[2], headers={'User-Agent':"Magic Browser"})
                con = url_req.urlopen(req)
                main_path = os.path.dirname(os.path.abspath(sys.modules['__main__'].__file__))
                pic_name = "{}_{}".format(row[0], row[1])
                pic_path = os.path.join(main_path, "Report/static/pic/{}.jpg".format(pic_name))
                pic_file = open(pic_path, 'wb')
                pic_file.write(con.read())
                pic_file.close()
            except URLError:
                pic_name = ""
                print("Can not load profile picture, due to the following failure:")
                print(e)

            create_time = datetime.fromtimestamp(int(row[7])).strftime('%Y-%m-%d %H:%M:%S')
            last_login = datetime.fromtimestamp(int(row[8])).strftime('%Y-%m-%d %H:%M:%S')

            gender = "Male" if row[3] == 1 else "Female"

            self.user.setName(row[0])               # Name
            self.user.setBirthday(row[1])         # Birthday
            self.user.setAvatar(pic_name)           # Avatar
            self.user.setGender(gender)             # Gender
            self.user.setHeight(str(row[4]) + " cm")             # Height
            self.user.setWeight(str(round(row[5], 2)) + " kg")             # Weight
            self.user.setCity(row[6])               # City
            self.user.setCreate_Time(create_time)   # Create Time
            self.user.setLast_Login(last_login)     # Last Login

    def decode_hr_table(self, data):
        """ Interprets on demand measured heart rate and stores it in the database """
        print("Process heart rate data from heart rate table")
        for row in data:
            timestamp = datetime.fromtimestamp(row[0])
            hr = row[1]
            self.heartrate.addData(hr, timestamp)

    def decode_data(self, data):
        """ Hands several data to the corresponding decoding method """
        for row in data:
            day = datetime.strptime(row[0], '%Y-%m-%d')
            print("Process heart rate, steps and sleep from " + row[0])
            step_data = row[1]
            self.decode_data_step_data(step_data, day)

            hr_data = row[2]
            self.decode_data_hr_data(hr_data, day)

            summary_data = json.loads(row[3])
            self.decode_data_summary_data(summary_data)

    def decode_data_summary_data(self, data):
        """ Interprets daily summary data to obtain sleep and store is in the database """
        sleep_data = data["slp"]
        sleep_start = datetime.fromtimestamp(sleep_data["st"])
        sleep_end = datetime.fromtimestamp(sleep_data["ed"])
        self.sleep.addData(sleep_start, sleep_end)

    def decode_data_step_data(self, data, day):
        """ Interprets step data to obtain acceleration and steps per minute """
        data = json.loads(data)[0]["value"]
        data_bin = bytearray(data, 'UTF-8')
        data_bin = bytearray(base64.b64decode(data_bin))
        i = 0
        while i < len(data_bin)/3:
            timestamp = day+timedelta(minutes=i)
            type = str(data_bin[i*3])
            acceleleration = str(data_bin[i*3+1])
            step = str(data_bin[i*3+2])
            self.steps.addData(step, timestamp)
            self.acceleration.addData(acceleleration, timestamp)
            i += 1

    def decode_data_hr_data(self, data, day):
        """ Interprets heart rate data from the data table """
        i = 0
        while i < len(data):
            hr = int(data[i])
            timestamp = day+timedelta(minutes=i)
            self.heartrate.addData(hr, timestamp)
            i += 1


import json
import time
import serial
import os

info = {}

try:
	with open('/home/pi/aqn-ceri/i2c/result','r') as f:
		txt = f.read().split(',')
		info['co2'] = txt[0]
		info['cov'] = txt[1].split('\n')[0]
except ValueError:
	print("Error open file !")
	raise

try:
	with open('/home/pi/aqn-ceri/dht/result','r') as f:
		txt = f.read().split(',')
		info['temp'] = txt[0]
		info['humid'] = txt[1].split('\n')[0]
except ValueError:
	print("error open file ! ")
	raise

try:
	with open('/home/pi/aqn-ceri/result.json','w') as f:
		f.write(json.dumps(info))
except ValueError:
	print('Error open file !')
	raise

if not os.path.isfile('/home/pi/aqn-ceri/result.hist.json'):
	with open('/home/pi/aqn-ceri/result.hist.json', 'w+') as f:
		data = []
		data.append(info)
		f.write(json.dumps(data))

else:
	with open('/home/pi/aqn-ceri/result.hist.json', 'r') as f:
		data = json.load(f)
		data.append(info)

	with open('/home/pi/aqn-ceri/result.hist.json', 'w') as f:
		f.write(json.dumps(data))

port = "/dev/ttyACM0"

s1 = serial.Serial(port,115200)
s1.flushInput()

s1.write(json.dumps(info))

time.sleep(5)


from data import data
from utils import annotate
from models import model as m
from training import training
import torch
import torch.nn as nn
from torchvision import transforms
import numpy as np
import cv2

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

print(device)
print(torch.__version__)

# load data
annoations_filepath = "data/mpii_human_pose_v1_u12_1.mat"
images_base_filepath_local = "/Users/danny/Code/Data/SHPE/images/"
images_base_filepath_server = "/home/paperspace/Code/Data/SHPE/images/"

train_loader, val_loader = data.load_data(227, 128, images_base_filepath_server, annoations_filepath, True)

# create model and optimizer
model = m.get_model_resnet(34, device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
criterion = nn.MSELoss()

# train
training.train_single_lr(model, train_loader, val_loader, 0.001, 75, device, 'visibility_with_sigmoid_34_norm')

# evaluate
loss, _, o, l= training.evaluate(model, val_loader, nn.MSELoss(), device)

# load saved model and test it
checkpoint = torch.load('saved_models/cyclical.pt', map_location=device)
model.load_state_dict(checkpoint['model_state_dict'])
print(f"Train loss from checkpoint: {checkpoint['train_loss']}")
print(f"Val loss from checkpoint: {checkpoint['val_loss']}")

loss, _ = training.evaluate(model, val_loader, nn.MSELoss(), device)
print(f'Val loss after loading: {loss}')

itr = iter(val_loader)
x, y = itr.next()
for i in range(x.shape[0]):
    img = np.asarray(transforms.ToPILImage()(x[i].squeeze_(0)))
    coordinates = training.predict(model, x[i].unsqueeze(0).to(device))
    annotate.annotate_flat_image(coordinates.squeeze(), image=img)


from nineml.exceptions import NineMLUsageError, NineMLNameError
from nineml.visitors import BaseVisitor, BasePreAndPostVisitorWithContext
from nineml.abstraction.expressions import Expression

class ComponentRenameSymbol(BaseVisitor):

    """ Can be used for:
    StateVariables, Aliases, Ports
    """

    def __init__(self, component_class, old_symbol_name, new_symbol_name):
        super(ComponentRenameSymbol, self).__init__()
        self.old_symbol_name = old_symbol_name
        self.new_symbol_name = new_symbol_name
        self.namemap = {old_symbol_name: new_symbol_name}

        self.lhs_changes = []
        self.rhs_changes = []
        self.port_changes = []

        self.visit(component_class)
        component_class.validate()

    def note_lhs_changed(self, what):
        self.lhs_changes.append(what)

    def note_rhs_changed(self, what):
        self.rhs_changes.append(what)

    def note_port_changed(self, what):
        self.port_changes.append(what)

    def _update_dicts(self, *dicts):
        for d in dicts:
            # Can't use "pythonic" try/except because I want it to work for
            assert isinstance(d, dict)
            if self.old_symbol_name in d:
                d[self.new_symbol_name] = d.pop(self.old_symbol_name)

    def _action_port(self, port, **kwargs):  # @UnusedVariable
        if port.name == self.old_symbol_name:
            port._name = self.new_symbol_name
            self.note_port_changed(port)

    def action_componentclass(self, component_class, **kwargs):  # @UnusedVariable @IgnorePep8
        component_class._update_member_key(
            self.old_symbol_name, self.new_symbol_name)

    def action_parameter(self, parameter, **kwargs):  # @UnusedVariable
        if parameter.name == self.old_symbol_name:
            parameter._name = self.new_symbol_name
            self.note_lhs_changed(parameter)

    def action_alias(self, alias, **kwargs):  # @UnusedVariable
        if alias.lhs == self.old_symbol_name:
            self.note_lhs_changed(alias)
            alias.name_transform_inplace(self.namemap)
        elif self.old_symbol_name in alias.atoms:
            self.note_rhs_changed(alias)
            alias.name_transform_inplace(self.namemap)

    def action_constant(self, constant, **kwargs):  # @UnusedVariable
        if constant.name == self.old_symbol_name:
            self.note_lhs_changed(constant)
            constant.name_transform_inplace(self.namemap)

    def default_action(self, obj, nineml_cls, **kwargs):
        pass

class ComponentSubstituteAliases(BasePreAndPostVisitorWithContext):
    """
    Substitutes all references to aliases in expressions with their RHS,
    so that all expressions are defined interms of inputs to the (e.g. analog
    receive/reduce ports and parameters), constants and reserved identifiers.
    The only aliases that are retained are ones that map to analog output ports

    Parameters
    ----------
    component_class : ComponentClass
        The component class to expand the expressions of
    new_name : str
        The name for the expanded class
    """

    def __init__(self, component_class):
        super(ComponentSubstituteAliases, self).__init__()
        self.component_class = component_class
        # Outputs of the class for which corresponding aliases needed to be
        # retained. Set in 'action' method of component_class
        self.outputs = set()
        # Cache to the previously substituted aliases
        self.cache = {}
        self.visit(component_class)

    def substitute(self, expr):
        """
        Substitute alias symbols in expression with equivalent expression
        of inputs, constants and reserved identifiers

        Parameters
        ----------
        nineml_obj : Expression
            An expression object to substitute alias symbols in
        """
        if expr.temporary:
            raise NineMLUsageError(
                "Attempting to substitute aliases using temporary object "
                "{}({})".format(type(expr).__name__, expr))
        assert isinstance(expr, Expression)
        try:
            rhs = self.cache[expr.id]
        except KeyError:
            for sym in list(expr.rhs_symbols):
                # Substitute all alias symbols with their RHS expresssions
                if str(sym) in self.component_class.alias_names:
                    alias = self.get_alias(str(sym))
                    expr.subs(sym, self.substitute(alias))
            self.cache[expr.id] = rhs = expr.rhs
        return rhs

    def get_alias(self, name):
        alias = None
        for context in reversed(self.contexts):
            try:
                alias = context.parent.alias(name)
                break
            except (NineMLNameError, AttributeError):
                continue
        if alias is None:
            raise NineMLUsageError("Did not find alias '{}' in any "
                                     "context".format(name))
        return alias

    def remove_uneeded_aliases(self, container):
        container.remove(*(a for a in container.aliases
                           if a.name not in self.outputs))

    def action_alias(self, alias, **kwargs):  # @UnusedVariable
        return self.substitute(alias)

    def default_action(self, obj, nineml_cls, **kwargs):
        pass

    def default_post_action(self, *args, **kwargs):
        pass


# Licensed under a 3-clause BSD style license - see LICENSE.rst

import numpy as np
import pytest

from sofia_redux.toolkit.interpolate.interpolate import tabinv

def test_failure():
    x = [np.nan, np.nan, 1, 2, np.nan, 3, np.nan, np.nan]
    with pytest.raises(ValueError):
        tabinv(x, 1.5, fast=False)

    with pytest.raises(ValueError) as err:
        tabinv(np.zeros((3, 3)), 1.5)
    assert "array must have 1-dimension" in str(err.value).lower()

def test_fast():
    x = [np.nan, np.nan, 1, 2, np.nan, 3, np.nan, np.nan]
    assert tabinv(x, 1.5) == 2.5

def test_findidx():
    x = [np.nan, np.nan, 1, 2, 3, np.nan, np.nan]
    assert tabinv(x, 1.5, fast=False) == 2.5


import requests
import config
import datetime as dt
from pprint import pprint

class FlightData:
    """This class is responsible for structuring the flight data."""
    def __init__(self, max_stop_overs: int = 1, via_city: str = ""):
        self.max_stop_overs = max_stop_overs
        self.via_city = via_city
        self.tomorrow = (dt.datetime.now() + dt.timedelta(days=1)).strftime("%d/%m/%Y")
        self.six_months_from_now = (dt.datetime.now() + dt.timedelta(days=180)).strftime("%d/%m/%Y")

    def find_cheap_flight(self, city_code: str) -> dict or None:
        """Finds the cheapest possible flight from Colombo to the given city. The city's IATA code is expected as a parameter.
        Returns the departure date, arrival date, price in GBP (British Pounds), number of stop-overs and via cities in a dictionary"""
        params = {
            "fly_from": "CMB",
            "fly_to": city_code,
            "date_from": self.tomorrow,
            "date_to": self.six_months_from_now,
            "nights_in_dst_from": 7,
            "nights_in_dst_to": 28,
            "flight_type": "round",
            "curr": "GBP",
            "max_stopovers": 0
        }
        headers = {"apikey": config.KIWI_API_KEY}
        response = requests.get(url=f"{config.KIWI_ENDPOINT}/v2/search", params=params, headers=headers)

        try:  # Avoiding a error (if no flights)
            data = response.json()["data"][0]
        except IndexError:
            if self.max_stop_overs > 0:
                params["max_stopovers"] = self.max_stop_overs
                response = requests.get(url=f"{config.KIWI_ENDPOINT}/v2/search", params=params, headers=headers)

                try:
                    data = response.json()["data"][0]
                except IndexError:
                    return None
                else:
                    arrival = data["local_arrival"]
                    arrival_sepd = {"year": arrival.split("-")[0], "month": arrival.split("-")[1],
                                    "date": arrival.split("-")[2].split("T")[0]}
                    arrival_sepd = {key: int(value) for (key, value) in
                                    arrival_sepd.items()}  # Turning the values in arrival_sepd to ints
                    local_arrival = dt.datetime(year=arrival_sepd["year"], month=arrival_sepd["month"],
                                                day=arrival_sepd["date"])
                    via_cities = []
                    for i in data["route"]:
                        city = i["cityTo"]
                        if city not in via_cities and city != "Colombo":
                            via_cities.append(city)

                    data_dict = {
                        "departure": str(data["local_departure"]).split("T")[0],
                        "return": (local_arrival + dt.timedelta(days=int(data['nightsInDest']))).strftime("%Y-%m-%d"),
                        # Calculating the date of return
                        "price": int(data["price"]),
                        "stop_overs": len(via_cities),
                        "via_cities": via_cities
                    }
                    return data_dict

            else:
                return None

        else:
            arrival = data["local_arrival"]
            arrival_sepd = {"year": arrival.split("-")[0], "month": arrival.split("-")[1], "date": arrival.split("-")[2].split("T")[0]}
            arrival_sepd = {key: int(value) for (key, value) in arrival_sepd.items()}  # Turning the values in arrival_sepd to ints
            local_arrival = dt.datetime(year=arrival_sepd["year"], month=arrival_sepd["month"], day=arrival_sepd["date"])

            data_dict = {
                "departure": str(data["local_departure"]).split("T")[0],
                "return": (local_arrival + dt.timedelta(days=int(data['nightsInDest']))).strftime("%Y-%m-%d"),  # Calculating the date of return
                "price": int(data["price"]),
                "stop_overs": 0,
                "via_cities": None
            }
            return data_dict

# test = FlightData()
# test.find_cheap_flight("NYC")


from typing import List, Tuple

from allennlp.common.util import ensure_list
from allennlp.common.testing import AllenNlpTestCase

from allennlp_models.coref import WinobiasReader
from tests import FIXTURES_ROOT

class TestWinobiasReader:
    span_width = 5

    def test_read_from_file(self):
        conll_reader = WinobiasReader(max_span_width=self.span_width)
        instances = ensure_list(conll_reader.read(str(FIXTURES_ROOT / "coref" / "winobias.sample")))

        assert len(instances) == 2

        fields = instances[0].fields
        text = [x.text for x in fields["text"].tokens]
        assert text == [
            "The",
            "designer",
            "argued",
            "with",
            "the",
            "developer",
            "and",
            "slapped",
            "her",
            "in",
            "the",
            "face",
            ".",
        ]

        spans = fields["spans"].field_list
        span_starts, span_ends = zip(*[(field.span_start, field.span_end) for field in spans])

        candidate_mentions = self.check_candidate_mentions_are_well_defined(
            span_starts, span_ends, text
        )

        gold_span_labels = fields["span_labels"]
        gold_indices_with_ids = [(i, x) for i, x in enumerate(gold_span_labels.labels) if x != -1]
        gold_mentions_with_ids: List[Tuple[List[str], int]] = [
            (candidate_mentions[i], x) for i, x in gold_indices_with_ids
        ]
        assert gold_mentions_with_ids == [(["the", "developer"], 0), (["her"], 0)]

        fields = instances[1].fields
        text = [x.text for x in fields["text"].tokens]
        assert text == [
            "The",
            "salesperson",
            "sold",
            "some",
            "books",
            "to",
            "the",
            "librarian",
            "because",
            "she",
            "was",
            "trying",
            "to",
            "sell",
            "them",
            ".",
        ]

        spans = fields["spans"].field_list
        span_starts, span_ends = zip(*[(field.span_start, field.span_end) for field in spans])
        candidate_mentions = self.check_candidate_mentions_are_well_defined(
            span_starts, span_ends, text
        )

        gold_span_labels = fields["span_labels"]
        gold_indices_with_ids = [(i, x) for i, x in enumerate(gold_span_labels.labels) if x != -1]
        gold_mentions_with_ids: List[Tuple[List[str], int]] = [
            (candidate_mentions[i], x) for i, x in gold_indices_with_ids
        ]
        assert gold_mentions_with_ids == [
            (["The", "salesperson"], 0),
            (["some", "books"], 1),
            (["she"], 0),
            (["them"], 1),
        ]

    def check_candidate_mentions_are_well_defined(self, span_starts, span_ends, text):
        candidate_mentions = []
        for start, end in zip(span_starts, span_ends):
            # Spans are inclusive.
            text_span = text[start : (end + 1)]
            candidate_mentions.append(text_span)

        # Check we aren't considering zero length spans and all
        # candidate spans are less than what we specified
        assert all(self.span_width >= len(x) > 0 for x in candidate_mentions)
        return candidate_mentions


import plotly as py
import plotly.graph_objs as go
import plotly.figure_factory as ff
import pandas as pd
from sklearn.externals import joblib
import math

class Graphs:
    def __init__(self):
        # self.df = pd.read_csv("/home/manobhav/Downloads/train.csv")
        self.ddf = pd.read_csv("Dataset/master_lat_long.csv")
        self.ddf = self.ddf[self.ddf.Long >= 0]
        self.ddf.datefrom = pd.to_datetime(self.ddf.datefrom)
        self.ddf['year'] = self.ddf.datefrom.dt.year
        self.ddf['month'] = self.ddf.datefrom.dt.month
        self.mergedf = pd.read_csv("Dataset/merged_abstract_data.csv")

    def getCrimes(self):
        return self.mergedf.columns[10:].tolist()

    def getDistricts(self):
        return self.ddf.district.unique().tolist()

    def getPstations(self, district):
        return self.ddf[self.ddf.district == district].pstation.unique().tolist()

    def getDistrict(self, lat, long):
        lat = (lat - math.floor(lat)) * 10000000
        long = (long - math.floor(long)) * 10000000
        district_predictor = joblib.load('Pickle/district_predictor.pkl')
        return {"district": district_predictor.predict([[lat, long]])[0]}

    def getPstation(self, lat, long, district):
        lat = (lat - math.floor(lat)) * 10000000
        long = (long - math.floor(long)) * 10000000
        db = joblib.load('Pickle//districtb.pkl')
        a = [lat, long]
        a.extend(db.transform([district])[0])
        pstation_predictor = joblib.load('Pickle/pstation_predictor.pkl')
        return {"pstation": pstation_predictor.predict([a])[0]}

    def HeatMap(self, selval, dis_pol):

        if selval == 'Column':
            data = pd.crosstab(self.ddf.district, self.ddf.group_charge).apply(lambda c: (c / c.sum()) * 100,
                                                                               axis=0).apply(lambda c: c.round(2),
                                                                                axis=0)
        else:
            data = pd.crosstab(self.ddf.district, self.ddf.group_charge).apply(lambda r: (r / r.sum()) * 100,
                                                                               axis=1).apply(lambda r: r.round(2),
                                                                                axis=1)

        # trace = go.Heatmap(z=data.values.tolist(),
        #                    x=data.columns.tolist(),
        #                    y=data.index.tolist())
        # data = [trace]
        data = ff.create_annotated_heatmap(z=data.values.tolist(),
                                           x=data.columns.tolist(),
                                           y=data.index.tolist())
        return py.offline.plot(data, include_plotlyjs=False, output_type='div')

    def LineChart(self, time1, time2):
        data = []
        val = self.ddf.groupby([time1, time2])['group_charge'].count().tolist()
        c = 0
        print(self.ddf[time2].min(), self.ddf[time2].max())

        min = 0
        max = len(self.ddf[time2].unique())

        for i in self.ddf[time1].sort_values().unique():
            data.append(go.Scatter(
                x=list(range(min, max)),
                y=val[c * max:(c + 1) * max],
                name=i

            ))
            c += 1

        return py.offline.plot(data, include_plotlyjs=False, output_type='div')

    def ViolinPlot(self, time):
        data=[]

        for charge in self.ddf.group_charge.unique().tolist():
            trace=go.Box(y=self.ddf[self.ddf.group_charge==charge].hour,name=charge)
            data.append(trace)

        return py.offline.plot(data,include_plotlyjs=False, output_type='div')

    def ScatterPlot(self, x_val, y_val):

        if y_val == 'All':
            crime = 'Total Cognizable IPC crimes'
        else:
            crime = y_val

        trace = go.Scatter(
            x=self.mergedf[x_val],
            y=self.mergedf[crime],
            mode='markers',
            text=self.mergedf['District'],
            marker=dict(
                size=20,
                color=list(range(120, 120 + len(self.mergedf['District']) * 5, 5)),
                colorscale='Viridis'
            )
        )

        data = [trace]
        return py.offline.plot(data, include_plotlyjs=False, output_type='div')

    def ProbaPlot(self, hour, district, pstation, lat, long):

        logreg = joblib.load('Pickle/logreg.pkl')
        db = joblib.load('Pickle/districtb.pkl')
        pb = joblib.load('Pickle/policeb.pkl')

        a = [float(lat), float(long), int(hour)]
        a.extend(db.transform([district])[0])
        a.extend(pb.transform([pstation])[0])

        proba = logreg.predict_proba([a])
        trace = go.Pie(labels=logreg.classes_, values=proba[0])

        return py.offline.plot([trace], include_plotlyjs=False, output_type='div')


""" DESAFIO 050: Desenvolva um programa que leia seis números inteiros e mostre a soma apenas daqueles que forem
pares. Se o valor digitado for ímpar, desconsidere-o.
*SOMA DOS PARES*
"""
contador = 0
soma = 0
for c in range(1,7):
    n = int(input('Digite o valor {}: '.format(c)))
    if n % 2 == 0:
        contador +=1
        soma += n
print('Os números pares digitados são: {}'.format(contador))
print('A soma entre eles é de {}'.format(soma))


n = int(input())

def solve( num ):
    if int(num) > n:
        return 0
    ans = 1 if all(num.count(c) > 0 for c in "753") else 0
    for c in "753":
        ans += solve((num + c))
    return ans

print(solve("0"))


import requests
from datetime import datetime
import os
APP_ID="App_id"
API_KEY="Api_key"
today=datetime.now()
NUTRITIONIX_EXERCISE_ENDPOINT="https://trackapi.nutritionix.com/v2/natural/exercise"
HEADERS={
    'x-app-id':APP_ID,
    'x-app-key':API_KEY,
    'content-type':'application/json'
}
today_str=today.strftime("%d/%m/%Y")
time_str=today.time().strftime("%H:%M:%S")
query={
 "query":"i did tabata for 30mins and qingong for 10mins",
 "gender":"male",
 "weight_kg":"72.5",
 "height_cm":"167.64",
 "age":"33"
}
response=requests.post(NUTRITIONIX_EXERCISE_ENDPOINT,headers=HEADERS,json=query)
data=response.json()
exercise=data['exercises'][0]['user_input']
duration=int(data['exercises'][0]['duration_min'])
calories=int(data['exercises'][0]['nf_calories'])
headers={
    'Authorization':'Basic c2hha2EyMDIyOmNoaWxkb2Znb2Qx'
}
sheety_ENDPOINT="https://api.sheety.co/c06c8c81a5ea4e26088b5a1502f4e049/myWorkouts/workouts"
data_post={'workout':{
    'date':today_str,
    'time':time_str,
    'exercise':exercise,
    'duration':duration,
    'calories':130,
    'id':2
}
}
# res=requests.get(sheety_ENDPOINT)
# sheety_data=res.json()
# print(sheety_data)

respond=requests.post(url=sheety_ENDPOINT,json=data_post,headers=headers)
print(respond.text)


import pandas as pd
import os
import glob
pd.options.mode.chained_assignment = None  # default='warn'

from functions.fill_missing_glucose_values import fill_missing_glucose_values

def load_data(rootdir, test_index):

    all_files = glob.glob(rootdir + "/*.xlsx")
    all_files = sorted(all_files)
    #escolha do paciente para teste
    df_test = pd.read_excel(all_files[test_index], header=11)
    name = all_files[test_index] #guardar o nome do paciente de teste
    print("Dataset em analise -> ", name)
    #del all_files[test_index] #remover o nome do paciente de teste
    li = []
    for filename in all_files:
        df_int = pd.read_excel(filename, index_col=None, header=11)
        li.append(df_int)
    df = pd.concat(li, axis=0, ignore_index=True)
    cols = ['Index', 'Date', 'Time', 'Source', 'Excluded', 'Used in Calibration', 'ISIG Value', 'Sensor Event', 'Other', 'Raw-Type', 'Raw-Values','Carb Amount (grams)', 'Insulin Type', 'Insulin Units', 'Exercise Level', 'Sleep Start Time','Sleep Wake-up Time', 'Notes']
    #'Meal', 'Medication', 'Exercise', 'BG Reading (mg/dL)'
    df.drop(cols, axis=1, errors='ignore', inplace=True)
    df_test.drop(cols, axis=1, errors='ignore', inplace=True)

    df = fill_missing_glucose_values(df)
    df_test = fill_missing_glucose_values(df_test)

    return df, df_test


"""read the comm signal"""

def part_one(moves:list) -> int:
    ## reg starts at 1
    # noop : 1 cycle -> no effect
    # addx n : two cycles -> increment reg by n
    # prev cycle must finish before next in row can begin

    instructions = [0 if x[4:] == '' else int(x[5:]) for x in moves]

    # signal strength = cycle num * reg value
    cpu_reg = 1
    sum_signal_strength = 0
    cur_cycle = 1

    screen: list[str] =[]

    # [cycle[value], cycle2[value]]
    reg_history = [0]

    for count, instruction in enumerate(instructions):

        if instruction == 0:

            cpu_reg += 0

            signal_str = cpu_reg * cur_cycle

            reg_history.append(cpu_reg)

            if cur_cycle == 20 or cur_cycle % 40 == 20:
                sum_signal_strength += signal_str
                print(sum_signal_strength)

            cur_cycle += 1

        else:
            cur_cycle += 1

            cpu_reg += 0

            signal_str = cpu_reg * cur_cycle
            reg_history.append(cpu_reg)

            # if happens on an add instruction decrement one reg value( or index value?) back?
            if cur_cycle == 20 or cur_cycle % 40 == 20:
                signal_str = cur_cycle * reg_history[count]
                sum_signal_strength += signal_str
                print(sum_signal_strength)

            cur_cycle += 1
            cpu_reg += instruction
            reg_history.append(cpu_reg)

            if cur_cycle == 20 or cur_cycle % 40 == 20:
                signal_str = cur_cycle * cpu_reg
                sum_signal_strength += signal_str
                print(sum_signal_strength)
            # part two

    col = 0
    for cycle, val in enumerate(reg_history):

        if col == val + 1 or col == val or col == val - 1:
            screen.append('|')
        else:
            screen.append(' ')
        col += 1

        if col == 40:
            col = 0
            screen.append('\n')

    print(''.join(screen))

    return 1

if __name__ == "__main__":
    path = 'AdventOfCode2022\\Day10_input.txt'
    test_path = 'AdventOfCode2022\\Day10_test_input.txt'

    with open(path, 'rt', encoding='UTF-8') as inputfile:
        moves = [x.strip('\n') for x in inputfile.readlines()]
    # part one and two are combined
    part_one(moves)


from pfs.drp.fstar.modelInterpolation import ModelInterpolation
import lsst.utils
import lsst.utils.tests

import os

class ModelInterpolationTestCase(lsst.utils.tests.TestCase):
    def test(self):
        try:
            dataDir = lsst.utils.getPackageDir("fluxmodeldata")
        except LookupError:
            self.skipTest("fluxmodeldata not setup")

        if not os.path.exists(os.path.join(dataDir, "interpolator.pickle")):
            self.skipTest("makeRBFInterpolationModel.py has not been run")

        model = ModelInterpolation.fromFluxModelData(dataDir)
        wavelength, flux = model.interpolate(teff=7777, logg=3.333, metal=0.555, alpha=0.222)
        self.assertEqual(len(wavelength.shape), 1)
        self.assertEqual(wavelength.shape, flux.shape)

def setup_module(module):
    lsst.utils.tests.init()


def cdf(arr, val, start):
    total = float(len(arr))
    i = start
    found = False
    count = start
    while i < total and not found:
        found = arr[i] > val
        if not found:
            count += 1
        i += 1
    result = [count / total, i - 1]
    return result

def generate_cdf(arr, values):
    res = []
    start = 0
    for val in values:
        cdf_val = cdf(arr, val, start)
        res.append(cdf_val[0])
        start = cdf_val[1]
    return res


class Node:
  def __init__(self, val):
    self.val = val
    self.left = None
    self.right = None

a = Node('a')
b = Node('b')
c = Node('c')
d = Node('d')
e = Node('e')
f = Node('f')

a.left = b
a.right = c
b.left = d
b.right = e
c.right = f

#      a
#    /   \
#   b     c
#  / \     \
# d   e     f

def breadth_first_values(root):
  queue = [ root ]
  res = []

  if root == None:
    return res
  while queue:
    cur = queue.pop(0)
    res.append(cur.val)
    if cur.left != None:
      queue.append(cur.left)
    if cur.right != None:
      queue.append(cur.right)

  return res

print(breadth_first_values(a))


lista = []
for i in range(0, 5):
    x = int(input('Digite um valor: '))
    if i == 0 or x > lista[-1]:
        lista.append(x)
        print('Adicionando ao final da lista...')
    else:
        pos = 0
        while pos < len(lista):
            if x <= lista[pos]:
                lista.insert(pos, x)
                break
            pos += 1
        print(f'Adicionando na posição {pos} da lista...')
print('=-' * 30)
print(f'Valores adicionados na lista: {lista}')


import collections
class Solution(object):
    def minWindow(self, s, t):
        """
        :type s: str
        :type t: str
        :rtype: str
        """
        # letter and its times
        need = collections.Counter(t)
        missing = len(t)
        i = I = J = 0
        for j, ch in enumerate(s, 1):
            missing -= 1 if need[ch] > 0 else 0
            need[ch] -= 1
#             need is negative means this char could be removed
            if not missing:
                while i < j and need[s[i]] < 0:
                    need[s[i]] += 1
                    i += 1
                if not J or J - I > j - i:
                    J,I = j,i
        return s[I:J]
#     Since J starts from 1
S = "ADOBECODEBANC"
T = "AABC"
s = Solution()
s.minWindow(S,T)
# enumerate(object, start)
# a -= 1 if c > 0 else 0 等价于
# a -= c > 0
# Find the first window that contains all letters in t;
# Keep expanding the window to the right by 1 char at a time, reducing left side if possible.
# The best part is to make sure that THE ACTIVE WINDOW ALWAYS CONTAINS ALL LETTERS IN t.
# In this case, every time the window is expanded, only the new char need to be checked.