import os
import warnings

import numpy as np
import pyccl as ccl
import pymaster as nmt

from .covariance_builder import CovarianceFourier

class FourierGaussianNmt(CovarianceFourier):
    """Class to compute the Gaussian CellxCell covariance with NaMaster.

    This class uses the Narrow Kernel Approximation. It can also use the
    Toeplitz approximation.
    """

    cov_type = "gauss"

    def __init__(self, config):
        """Initialize the class with a config file or dictionary.

        Args:
            config (dict or str): If dict, it returns the configuration
                dictionary directly. If string, it asumes a YAML file and
                parses it.
        """
        super().__init__(config)

        # For NaMaster you need to pass the masks
        self.mask_files = self.config["tjpcov"].get("mask_file")
        self.mask_names = self.config["tjpcov"].get("mask_names")

        # Binning info is only needed if workspaces are not passed
        self.binning_info = self.config["tjpcov"].get("binning_info", None)

        # nside is needed if mask_files is a hdf5 file
        self.nside = self.config["tjpcov"].get("nside", None)

        # Read NaMaster specific options
        self.nmt_conf = self.config.get("NaMaster", {})
        for k in ["f", "w", "cw"]:
            if k not in self.nmt_conf:
                self.nmt_conf[k] = {}

        # Read cache from input file. It will update the cache passed as an
        # argument of the different methods
        self.cache = self.config.get("cache", {})

    def _compute_all_blocks(self, **kwargs):
        """Compute all the independent covariance blocks.

        Args:
            **kwargs: The arguments to pass to get_covariance_block. See its
                documentation.

        Returns:
            list: List of all the independent covariance blocks.
        """
        # We are redefining this method to improve performance by parallelizing
        # first the blocks that produce independent workspaces and covariance
        # workspaces. That way two processes will not be computing the same
        # (cov)workspace at the same time.

        ccl_tracers, tracer_Noise, tracer_Noise_coupled = self.get_tracer_info(
            return_noise_coupled=True
        )

        # Make a list of all pair of tracer combinations needed to compute the
        # independent workspaces
        trs_wsp = self.get_list_of_tracers_for_wsp()
        # Now the tracers for covariance workspaces (without trs_wsp)
        trs_cwsp = self.get_list_of_tracers_for_cov_wsp(remove_trs_wsp=True)

        # Make a list of all remaining combinations
        tracers_cov = self.get_list_of_tracers_for_cov_without_trs_wsp_cwsp()

        # Save blocks and the corresponding tracers, as comm.gather does not
        # return the blocks in the original order.
        blocks = []
        tracers_blocks = []
        print("Computing independent covariance blocks", flush=True)
        print("Computing the blocks for independent workspaces", flush=True)
        for tracer_comb1, tracer_comb2 in self._split_tasks_by_rank(trs_wsp):
            print(tracer_comb1, tracer_comb2, flush=True)
            cov = self.get_covariance_block_for_sacc(
                tracer_comb1=tracer_comb1, tracer_comb2=tracer_comb2, **kwargs
            )
            blocks.append(cov)
            tracers_blocks.append((tracer_comb1, tracer_comb2))

        if self.comm:
            self.comm.Barrier()

        print("Computing the blocks for independent covariance workspaces")
        for tracer_comb1, tracer_comb2 in self._split_tasks_by_rank(trs_cwsp):
            print(tracer_comb1, tracer_comb2)
            cov = self.get_covariance_block_for_sacc(
                tracer_comb1=tracer_comb1, tracer_comb2=tracer_comb2, **kwargs
            )
            blocks.append(cov)
            tracers_blocks.append((tracer_comb1, tracer_comb2))

        if self.comm:
            self.comm.Barrier()

        print("Computing the remaining blocks", flush=True)
        # Now loop over the remaining tracers
        for tracer_comb1, tracer_comb2 in self._split_tasks_by_rank(
            tracers_cov
        ):
            print(tracer_comb1, tracer_comb2, flush=True)
            cov = self.get_covariance_block_for_sacc(
                tracer_comb1=tracer_comb1, tracer_comb2=tracer_comb2, **kwargs
            )
            blocks.append(cov)
            tracers_blocks.append((tracer_comb1, tracer_comb2))

        return blocks, tracers_blocks

    def get_cl_for_cov(self, clab, nlab, ma, mb, w, nl_is_cp):
        """Computes the coupled Cell that goes into the covariance matrix.

        Args:
            clab (array): Fiducial Cell for the tracers a and b, used in the
                covariance estimation
            nlab (array): Coupled noise for the tracers a and b
            ma (array): Mask of the field a
            mb (array): Mask of the field b
            w (:obj:`pymaster.workspaces.NmtWorkspace`): NmtWorkspace of the
            fields a and b nl_is_cp (bool): True if nlab is coupled. False
            otherwise.

        Returns:
            array:  Coupled Cell with signal and noise
        """
        mean_mamb = np.mean(ma * mb)
        if mean_mamb == 0:
            cl_cp = np.zeros_like(nlab)
        elif nl_is_cp:
            cl_cp = (w.couple_cell(clab) + nlab) / mean_mamb
        else:
            cl_cp = (w.couple_cell(clab) + np.mean(ma) * nlab) / mean_mamb

        return cl_cp

    def get_covariance_block(
        self,
        tracer_comb1,
        tracer_comb2,
        use_coupled_noise=True,
        coupled=False,
        cache=None,
        clobber=False,
    ):
        """Compute a single covariance matrix for a given pair of C_ell.

        If outdir is set, it will save the covariance to a file called
        cov_tr1_tr2_tr3_tr4.npz. This file will be read and its output
        returned if found.

        Args:
            tracer_comb1 (list): List of the pair of tracer names of C_ell^1
            tracer_comb2 (list): List of the pair of tracer names of C_ell^2
            use_coupled_noise (bool, optional): If True, use the coupled noise.
                Note that if noise is provided via the cache arg, this will
                be used and assumed to be coupled if this option is True.
                Defaults to True.
            coupled (bool, optional): True to return the coupled Gaussian
                covariance (default False)
            cache (dict): Dictionary with the corresponding noise, masks,
                fields, workspaces and covariance workspaces. It accepts noise
                (keys: 'SN13', 'SN23', 'SN14', 'SN24'), masks (keys: 'm1',
                'm2', 'm3', 'm4'), fields (keys: 'f1', 'f2', 'f3', 'f4'),
                workspaces (keys: 'w13', 'w23', 'w14', 'w24', 'w12', 'w34'),
                the covariance workspace (key: 'cw') and a NmtBin (key:
                'bins').
            clobber (bool, optional): True to recalculate covariance and
                overwrite cached covariance (default False).  Note this does
                not erase the cache, or ignore the provided cache.

        Returns:
            array: Gaussian covariance matrix for a pair of C_ell.
        """
        if coupled:
            raise NotImplementedError(
                "Computing coupled covariance matrix not implemented yet"
            )
            fname = "covcp_{}_{}_{}_{}.npz".format(
                *tracer_comb1, *tracer_comb2
            )
        else:
            fname = "cov_{}_{}_{}_{}.npz".format(*tracer_comb1, *tracer_comb2)

        fname = os.path.join(self.io.outdir, fname)
        if os.path.isfile(fname) and not clobber:
            print(f"Loading saved covariance {fname}", flush=True)
            cov = np.load(fname)["cov"]
            return cov

        if cache is None:
            cache = {}
        cache.update(self.cache)

        if "bins" in cache:
            bins = cache["bins"]
            if (self.binning_info is not None) and (
                bins is not self.binning_info
            ):
                raise ValueError(
                    "Binning passed through cache is not the "
                    "same as the one passed during "
                    "initialization."
                )
        else:
            bins = self.binning_info

        # Get nbpw and ell arrays. Doing all this stuff because the window
        # function in the sacc file might be wrong.
        nbpw = self.get_nbpw()
        nell = self.get_nell(bins, self.nside, cache)

        ell = np.arange(nell)

        if "cosmo" in cache:
            cosmo = cache["cosmo"]
        else:
            cosmo = self.get_cosmology()

        tr = {}
        tr[1], tr[2] = tracer_comb1
        tr[3], tr[4] = tracer_comb2

        ncell = {}
        ncell[12] = self.get_tracer_comb_ncell(tracer_comb1)
        ncell[34] = self.get_tracer_comb_ncell(tracer_comb2)
        ncell[13] = self.get_tracer_comb_ncell((tr[1], tr[3]))
        ncell[24] = self.get_tracer_comb_ncell((tr[2], tr[4]))
        ncell[14] = self.get_tracer_comb_ncell((tr[1], tr[4]))
        ncell[23] = self.get_tracer_comb_ncell((tr[2], tr[3]))

        s = self.get_tracers_spin_dict(tr)

        ccl_tracers, tracer_Noise, tracer_Noise_coupled = self.get_tracer_info(
            return_noise_coupled=True
        )

        # Fiducial cl
        cl = {}
        # Noise (coupled or not)
        SN = {}
        if not use_coupled_noise:
            warnings.warn(
                "Computing the coupled noise from the uncoupled "
                "noise. This assumes the noise is white"
            )

        # Loop over the 4 different field combinations and fill the cl and
        # noise dictionaries
        for i in [13, 24, 14, 23]:
            i1, i2 = [int(j) for j in str(i)]
            key = f"cl{i}"
            if key in cache:
                cl[i] = cache[key]
            else:
                cl[i] = np.zeros((ncell[i], ell.size))
                cl[i][0] = ccl.angular_cl(
                    cosmo, ccl_tracers[tr[i1]], ccl_tracers[tr[i2]], ell
                )

            # Noise
            auto = tr[i1] == tr[i2]
            key = f"SN{i}"
            if key in cache:
                SN[i] = cache[key]
            else:
                SN[i] = np.zeros((ncell[i], ell.size))
                SN[i][0] = SN[i][-1] = np.ones_like(ell)
                if use_coupled_noise:
                    nl_cp = tracer_Noise_coupled[tr[i1]] if auto else 0
                    if nl_cp is None:
                        raise ValueError(
                            "Requested use_coupled_noise but "
                            "tracer_Noise_coupled is None for "
                            f"tracer {tr[i1]}. This could "
                            "mean that it does not have the "
                            "n_ell_coupled metadata information"
                            " in the sacc file."
                        )
                    SN[i] *= nl_cp
                else:
                    SN[i] *= tracer_Noise[tr[i1]] if auto else 0
                if s[i1] == 2:
                    SN[i][0, :2] = SN[i][-1, :2] = 0

        if (
            np.any(cl[13])
            or np.any(cl[24])
            or np.any(cl[14])
            or np.any(cl[23])
        ):
            # TODO: Modify depending on how TXPipe caches things
            # Mask, mask_names, field and workspaces dictionaries
            mn = self.get_mask_names_dict(tr)
            m = self.get_masks_dict(tr, cache)
            f = self.get_fields_dict(tr, cache, masks=m)
            w = self.get_workspaces_dict(tr, bins, cache, masks=m, fields=f)

            # TODO; Allow input options as output folder, if recompute, etc.
            if "cw" in cache:
                cw = cache["cw"]
            else:
                cw = self.get_covariance_workspace(
                    f[1],
                    f[2],
                    f[3],
                    f[4],
                    mn[1],
                    mn[2],
                    mn[3],
                    mn[4],
                    lmax=int(ell[-1]),
                    **self.nmt_conf["cw"],
                )

            cl_cov = {}
            cl_cov[13] = self.get_cl_for_cov(
                cl[13], SN[13], m[1], m[3], w[13], nl_is_cp=use_coupled_noise
            )
            cl_cov[23] = self.get_cl_for_cov(
                cl[23], SN[23], m[2], m[3], w[23], nl_is_cp=use_coupled_noise
            )
            cl_cov[14] = self.get_cl_for_cov(
                cl[14], SN[14], m[1], m[4], w[14], nl_is_cp=use_coupled_noise
            )
            cl_cov[24] = self.get_cl_for_cov(
                cl[24], SN[24], m[2], m[4], w[24], nl_is_cp=use_coupled_noise
            )

            cov = nmt.gaussian_covariance(
                cw,
                s[1],
                s[2],
                s[3],
                s[4],
                cl_cov[13],
                cl_cov[14],
                cl_cov[23],
                cl_cov[24],
                w[12],
                w[34],
                coupled,
            )
        else:
            size1 = ncell[12] * nbpw
            size2 = ncell[34] * nbpw
            cov = np.zeros((size1, size2))

        np.savez_compressed(fname, cov=cov)

        return cov

    def get_covariance_workspace(
        self, f1, f2, f3, f4, m1, m2, m3, m4, **kwargs
    ):
        """Return the covariance workspace of the fields f1, f2, f3, f4.

        Args:
            f1 (:obj:`pymaster.field.NmtField`):  Field 1
            f2 (:obj:`pymaster.field.NmtField`):  Field 2
            f3 (:obj:`pymaster.field.NmtField`):  Field 3
            f4 (:obj:`pymaster.field.NmtField`):  Field 4
            m1 (str): Mask name assotiated to the field 1
            m2 (str): Mask name assotiated to the field 2
            m3 (str): Mask name assotiated to the field 3
            m4 (str): Mask name assotiated to the field 4
            **kwargs:  Extra arguments to pass to
                pymaster.compute_coupling_coefficients. In addition, if
                recompute=True is passed, the cw will be recomputed even if
                found in the disk.

        Returns:
             :obj:`pymaster.covariance.NmtCovarianceWorkspace`: Covariance
             Workspace of the fields f1, f2, f3, f4
        """
        outdir = self.io.outdir
        spins = {
            m1: f1.fl.spin,
            m2: f2.fl.spin,
            m3: f3.fl.spin,
            m4: f4.fl.spin,
        }

        # Any other symmetry?
        combinations = [
            (m1, m2, m3, m4),
            (m2, m1, m3, m4),
            (m1, m2, m4, m3),
            (m2, m1, m4, m3),
            (m3, m4, m1, m2),
            (m4, m3, m1, m2),
            (m3, m4, m2, m1),
            (m4, m3, m2, m1),
        ]

        # Currently, outdir will be always not None. If not specified, it
        # will be the current directory. I leave this for now since we might
        # want to disable the option of writing files in the future for small
        # fast runs in the future.
        if outdir is not None:
            fnames = []
            isfiles = []
            for mn1, mn2, mn3, mn4 in combinations:
                s1, s2, s3, s4 = [spins[mi] for mi in [mn1, mn2, mn3, mn4]]
                f = f"cw{s1}{s2}{s3}{s4}__{mn1}__{mn2}__{mn3}__{mn4}.fits"
                f = os.path.join(outdir, f)
                if f not in fnames:
                    fnames.append(f)
                    isfiles.append(os.path.isfile(fnames[-1]))
        else:
            fnames = [None]
            isfiles = [None]

        cw = nmt.NmtCovarianceWorkspace()
        if "recompute" in kwargs:
            recompute = kwargs.pop("recompute")
        else:
            recompute = False
        if recompute or (not np.any(isfiles)):
            cw.compute_coupling_coefficients(f1, f2, f3, f4, **kwargs)
            if fnames[0] and not os.path.isfile(fnames[0]):
                cw.write_to(fnames[0])
            for fn, isf in zip(fnames[1:], isfiles[1:]):
                if isf and os.path.isfile(fn):
                    # Remove old covariance workspace if you have
                    # recomputed it
                    os.remove(fn)
        else:
            ix = isfiles.index(True)
            cw.read_from(fnames[ix])

        return cw

    def get_fields_dict(self, tracer_names, cache=None, masks=None, **kwargs):
        """Return a dictionary with the fields assotiated to the given tracers.

        Args:
            tracer_names (dict):  Dictionary of the tracer names of the same
                form as mask_name. It has to be given as {1: name1, 2: name2,
                3: name3, 4: name4}, where 12 and 34 are the pair of tracers
                that go into the first and second Cell you are computing the
                covariance for; i.e. <Cell^12 Cell^34>.
            cache (dict): Dictionary with cached variables. It will use the
                cached field if found. The keys must be 'f1', 'f2', 'f3' or
                'f4' and the values the corresponding NmtFields.
            masks (dict): Dictionary of the masks of the fields correlated with
                keys 1, 2, 3 or 4 and values the loaded masks.
            **kwargs: Arguments to pass to NaMaster when computing the
                field. They will override the ones passed in the configuration
                file through nmt_conf['f'].

        Returns:
            dict: Dictionary with the masks assotiated to the fields to be
            correlated.
        """
        mask_names = self.get_mask_names_dict(tracer_names)
        if masks is None:
            masks = self.get_masks_dict(tracer_names, cache)
        if cache is None:
            cache = {}
        spins = self.get_tracers_spin_dict(tracer_names)
        nmt_conf = self.nmt_conf["f"].copy()
        nmt_conf.update(kwargs)
        f = {}
        f_by_mask_name = {}
        for i in [1, 2, 3, 4]:
            key = f"f{i}"
            if key in cache:
                f[i] = cache[key]
            else:
                # We add the spin to make sure we distinguish fields of
                # different types even though they share the same mask
                k = mask_names[i] + str(spins[i])
                if k not in f_by_mask_name:
                    f_by_mask_name[k] = nmt.NmtField(
                        masks[i], None, spin=spins[i], **nmt_conf
                    )
                f[i] = f_by_mask_name[k]

        return f

    def get_list_of_tracers_for_wsp(self):
        """Return the tracers needed to compute the independent workspaces.

        Returns:
            list of str: List of tracers needed to compute the independent
            workspaces.
        """
        sacc_file = self.io.get_sacc_file()
        tracers = sacc_file.get_tracer_combinations()

        fnames = []
        tracers_out = []
        for i, trs1 in enumerate(tracers):
            s1, s2 = self.get_tracer_comb_spin(trs1)
            mn1, mn2 = [self.mask_names[tri] for tri in trs1]

            for trs2 in tracers[i:]:
                s3, s4 = self.get_tracer_comb_spin(trs2)
                mn3, mn4 = [self.mask_names[tri] for tri in trs2]

                fname1 = f"w{s1}{s2}__{mn1}__{mn2}.fits"
                fname2 = f"w{s3}{s4}__{mn3}__{mn4}.fits"

                if (fname1 in fnames) or (fname2 in fnames):
                    continue

                fnames.append(fname1)
                fnames.append(fname2)

                tracers_out.append((trs1, trs2))

        return tracers_out

    def get_list_of_tracers_for_cov_wsp(self, remove_trs_wsp=False):
        """Return the tracers to compute the independent covariance workspaces.

        Args:
            remove_trs_wsp (bool, optional): If True, remove the tracer
                combinations from used to generate the workspaces independently
                (i.e the output of get_list_of_tracers_for_wsp). Defaults to
                False.

        Returns:
            list of str: List of tracers needed to compute the independent
            covariance workspaces.
        """
        sacc_file = self.io.get_sacc_file()
        tracers = sacc_file.get_tracer_combinations()

        fnames = []
        tracers_out = []
        for i, trs1 in enumerate(tracers):
            s1, s2 = self.get_tracer_comb_spin(trs1)
            mn1, mn2 = [self.mask_names[tri] for tri in trs1]
            for trs2 in tracers[i:]:
                s3, s4 = self.get_tracer_comb_spin(trs2)
                mn3, mn4 = [self.mask_names[tri] for tri in trs2]

                fname = f"cw{s1}{s2}{s3}{s4}__{mn1}__{mn2}__{mn3}__{mn4}.fits"
                if fname not in fnames:
                    fnames.append(fname)
                    tracers_out.append((trs1, trs2))

        if remove_trs_wsp:
            trs_wsp = self.get_list_of_tracers_for_wsp()
            for trs in trs_wsp:
                tracers_out.remove(trs)

        return tracers_out

    def get_list_of_tracers_for_cov_without_trs_wsp_cwsp(self):
        """Return the remaining covariance tracers combinations.

        It will remove the tracer combinations used to compute the workspaces
        and covariance workspaces.

        Returns:
            list of str: List of independent tracers combinations.
        """
        tracers_out = self.get_list_of_tracers_for_cov()

        trs_cwsp = self.get_list_of_tracers_for_cov_wsp()
        for trs in trs_cwsp:
            tracers_out.remove(trs)

        return tracers_out

    def get_nell(self, bins=None, nside=None, cache=None):
        """Return the number of ells for the fiducial Cells.

        If the sacc file stored bandpowers are wrong. You will need to pass one
        of the other arguments.

        Args:
            bins ( pymaster.NmtBin): NmtBin instance with the desired
                binning.
            nside (int): Healpy map nside.
            cache (dict): Dictionary with cached variables. It will use the
                cached workspaces to read the bandpower windows.

        Returns:
            int: Number of ells for the fidicual Cells points; i.e. lmax or
            3*nside
        """
        # Extracting the workspace from the cache first to use it later in a
        # easy way.
        if (cache is not None) and ("workspaces" in cache):
            w = list(list(cache["workspaces"].values())[0].values())[0]
        else:
            w = None

        if isinstance(w, nmt.NmtWorkspace):
            # We don't want to read the workspace just to get the nell
            bpw = w.get_bandpower_windows()
            nell = bpw.shape[-1]
        elif bins is not None:
            nell = bins.lmax + 1
        else:
            s = self.io.get_sacc_file()
            try:
                dtype = s.get_data_types()[0]
                tracers = s.get_tracer_combinations(data_type=dtype)[0]
                ix = s.indices(data_type=dtype, tracers=tracers)
                bpw = s.get_bandpower_windows(ix)
                if bpw is None:
                    raise ValueError
                nell = bpw.nell
            except ValueError as e:
                # If the window functions are wrong. Do magic
                warnings.warn(
                    "The window functions in the sacc file are wrong:"
                )
                warnings.warn(str(e))
                if "binning/ell_max" in s.metadata:
                    warnings.warn(
                        "Trying to circunvent this error: we will use"
                        "nell = lmax + 1 as given in the metadata"
                    )
                    nell = s.metadata["binning/ell_max"] + 1
                    if nside is not None and nell > 3 * nside:
                        warnings.warn(
                            "lmax is larger than 3*nside. We will use "
                            "nell = 3*nside"
                        )
                        nell = 3 * nside
                elif nside is not None:
                    warnings.warn(
                        "Trying to circunvent this error: we will try"
                        "with nell = 3*nside"
                    )
                    nell = 3 * nside
                else:
                    raise ValueError(
                        "nside, NmtBin or NmtWorkspace instances "
                        "must be passed"
                    )
        return nell

    def get_workspace(self, f1, f2, m1, m2, bins, **kwargs):
        """Return the workspace of the fields f1, f2.

        Args:
            f1 (:obj:`pymaster.field.NmtField`):  Field 1
            f2 (:obj:`pymaster.field.NmtField`):  Field 2
            m1 (str): Mask name assotiated to the field 1
            m2 (str): Mask name assotiated to the field 2
            bins (:obj:`pymaster.bins.NmtBin`):  NmtBin instance
            mask_names (dict): Dictionary with tracer names as key and maks
                names as values.
            **kwargs:  Extra arguments to pass to w.compute_coupling_matrix.
                In addition, if recompute=True is passed, the cw will be
                recomputed even if found in the disk.

        Returns:
             :obj:`pymaster.covariance.NmtCovarianceWorkspace`: Covariance
             Workspace of the fields f1, f2, f3, f4
        """
        if not isinstance(bins, nmt.NmtBin):
            raise ValueError(
                "You must pass a NmtBin instance through the "
                "cache or at initialization"
            )

        outdir = self.io.outdir
        s1, s2 = f1.fl.spin, f2.fl.spin

        # Currently, outdir will be always not None. If not specified, it
        # will be the current directory. I leave this for now since we might
        # want to disable the option of writing files in the future for small
        # fast runs in the future.
        if outdir is not None:
            fname = os.path.join(outdir, f"w{s1}{s2}__{m1}__{m2}.fits")
            isfile = os.path.isfile(fname)

            # The workspace of m1 x m2 and m2 x m1 is the same.
            fname2 = os.path.join(outdir, f"w{s2}{s1}__{m2}__{m1}.fits")
            isfile2 = os.path.isfile(fname2)
        else:
            fname = isfile = fname2 = isfile2 = None

        w = nmt.NmtWorkspace()
        if "recompute" in kwargs:
            recompute = kwargs.pop("recompute")
        else:
            recompute = False

        if recompute or ((not isfile) and (not isfile2)):
            w.compute_coupling_matrix(f1, f2, bins, **kwargs)
            if fname and not os.path.isfile(fname):
                w.write_to(fname)
            if isfile2 and os.path.isfile(fname2):
                # Remove the other to avoid later confusions.
                os.remove(fname2)
        elif isfile:
            w.read_from(fname)
        else:
            w.read_from(fname2)

        return w

    def get_workspaces_dict(
        self, tracer_names, bins, cache=None, fields=None, masks=None, **kwargs
    ):
        """Return a dictionary with the workspaces for the given tracers.

        Args:
            tracer_names (dict):  Dictionary of the masks names assotiated to
                the fields to be correlated. It has to be given as {1: name1,
                2: name2, 3: name3, 4: name4}, where 12 and 34 are the pair of
                tracers that go into the first and second Cell you are
                computing the covariance for; i.e. <Cell^12 Cell^34>.
            bins (:obj:`pymaster.bins.NmtBin`): NmtBin instance with the
            desired binning.
            cache (dict): Dictionary with cached variables. It will use the
                cached field if found. The keys must be 'w12', 'w34', 'w13',
                'w23', 'w14' or 'w24' and the values the corresponding
                NmtWorkspaces. Alternatively, you can pass a dictionary with
                keys as (mask_name1, mask_name2).
            field (dict): Dictionary of the NmtFields of the fields correlated
                with keys 1, 2, 3 or 4 and values the NmtFields.
                masks (dict): Dictionary of the masks of the fields correlated
                with keys 1, 2, 3 or 4 and values the loaded masks.
            **kwargs: Arguments to pass to NaMaster when computing the
                workspace. They will override the ones passed in the
                configuration file through nmt_conf['w'].

        Returns:
            dict:  Dictionary with the workspaces assotiated to the different
            field combinations needed for the covariance. Its keys are 13, 23,
            14, 24, 12, 34; with values the corresponding
            :obj:`pymaster.workspaces.NmtWorkspaces`.
        """
        mask_names = self.get_mask_names_dict(tracer_names)
        if masks is None:
            masks = self.get_masks_dict(tracer_names, cache)
        if fields is None:
            fields = self.get_fields_dict(tracer_names, cache, masks=masks)
        if cache is None:
            cache = {}

        nmt_conf = self.nmt_conf["w"].copy()
        nmt_conf.update(kwargs)

        w = {}
        w_by_mask_name = {}
        # 12 and 34 must be first to avoid asigning them None if their maks do
        # not overlap
        for i in [12, 34, 13, 23, 14, 24]:
            i1, i2 = [int(j) for j in str(i)]
            # Workspace
            key = f"w{i}"
            if key in cache:
                w[i] = cache[key]
            else:
                s1, s2 = fields[i1].fl.spin, fields[i2].fl.spin
                # In this case you have to check for m1 x m2 and m2 x m1
                k = (mask_names[i1], mask_names[i2])
                sk = "".join(sorted(f"{s1}{s2}"))

                # Look for the workspaces of appropriate spin combinations
                cache_wsp = cache.get("workspaces", None)
                if cache_wsp is not None:
                    if sk in cache_wsp:
                        cache_wsp = cache_wsp[sk]
                    elif sk[::-1] in cache_wsp:
                        cache_wsp = cache_wsp[sk[::-1]]

                if sk not in w_by_mask_name:
                    w_by_mask_name_s = {}
                    w_by_mask_name[sk] = w_by_mask_name_s
                else:
                    w_by_mask_name_s = w_by_mask_name[sk]

                if k in w_by_mask_name_s:
                    w[i] = w_by_mask_name_s[k]
                elif k[::-1] in w_by_mask_name_s:
                    w[i] = w_by_mask_name_s[k[::-1]]
                elif (i not in [12, 34]) and (
                    np.mean(masks[i1] * masks[i2]) == 0
                ):
                    # w13, w23, w14, w24 are needed to couple the theoretical
                    # Cell and are not needed if the masks do not overlap.
                    # However, w12 and w34 are needed for
                    # nmt.gaussian_covariance, which will complain if they are
                    # None
                    w_by_mask_name_s[k] = None
                    w[i] = w_by_mask_name_s[k]
                elif (cache_wsp is not None) and (
                    (k in cache_wsp) or k[::-1] in cache_wsp
                ):
                    if k in cache_wsp:
                        fname = cache_wsp[k]
                    else:
                        fname = cache_wsp[k[::-1]]
                    wsp = nmt.NmtWorkspace()
                    wsp.read_from(fname)
                    w[i] = w_by_mask_name_s[k] = wsp
                else:
                    w_by_mask_name_s[k] = self.get_workspace(
                        fields[i1],
                        fields[i2],
                        mask_names[i1],
                        mask_names[i2],
                        bins,
                        **nmt_conf,
                    )
                    w[i] = w_by_mask_name_s[k]

        return w


'''
You are a professional robber planning to rob houses along a street. Each house has a certain amount of money stashed. All houses at this place are arranged in a circle. That means the first house is the neighbor of the last one. Meanwhile, adjacent houses have security system connected and it will automatically contact the police if two adjacent houses were broken into on the same night.

Given a list of non-negative integers representing the amount of money of each house, determine the maximum amount of money you can rob tonight without alerting the police.

Example 1:

Input: [2,3,2]
Output: 3
Explanation: You cannot rob house 1 (money = 2) and then rob house 3 (money = 2),
             because they are adjacent houses.
Example 2:

Input: [1,2,3,1]
Output: 4
Explanation: Rob house 1 (money = 1) and then rob house 3 (money = 3).
             Total amount you can rob = 1 + 3 = 4.
'''

class Solution(object):

    def rob(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        if not nums:
            return 0
        elif len(nums) == 1:
            return nums[0]
        elif len(nums) == 2:
            if nums[0] > nums[1]:
                return nums[0]
            else:
                return nums[1]
        else:
            op1 = nums[0] + self.robLinear(nums, 2, len(nums) - 2)
            op2 = nums[1] + self.robLinear(nums, 3, len(nums) - 1)
            op3 = nums[2] + self.robLinear(nums, 4, len(nums) - 1)

            return max([op1, op2, op3])

    def rob2(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """

        if not nums:
            return 0
        elif len(nums) == 1:
            return nums[0]
        elif len(nums) == 2:
            if nums[0] > nums[1]:
                return nums[0]
            else:
                return nums[1]
        else:
            # could be simpler. the end point is only either len(nums) - 1 or len(nums) - 2
            dp = [ [-1] * len(nums) for _ in range(len(nums))]
            op1 = nums[0] + self.helper(nums, 2, len(nums) - 2, dp)
            op2 = nums[1] + self.helper(nums, 3, len(nums) - 1, dp)
            op3 = nums[2] + self.helper(nums, 4, len(nums) - 1, dp)

            return max([op1, op2, op3])

    def helper(self, nums, start, end, dp):

        if start > end:
            return 0
        else:
            if start == end:
                return nums[start]
            else:
                if dp[start][end] >= 0:
                    return dp[start][end]

                op1 = nums[start] + self.helper(nums, start + 2, end, dp)
                op2 = nums[start + 1] + self.helper(nums, start + 3, end, dp)
                if op1 > op2:
                    dp[start][end] = op1
                else:
                    dp[start][end] = op2

                return dp[start][end]

    def robLinear(self, nums, start, end):
        '''
        Rob houses in a line
        '''
        preInclude = preExclude = 0
        for idx in range(start, end + 1):
            num = nums[idx]
            curInclude = preExclude + num
            curExclude = max(preInclude, preExclude)

            preInclude = curInclude
            preExclude = curExclude

        return max(preInclude, preExclude)

    @staticmethod
    def main():
        sol = Solution()
        nums = [226,174,214,16,218,48,153,131,128,17,157,142,88,43,37,157,43,221,191,68,206,23,225,82,54,118,111,46,80,49,245,63,25,194,72,80,143,55,209,18,55,122,65,66,177,101,63,201,172,130,103,225,142,46,86,185,62,138,212,192,125,77,223,188,99,228,90,25,193,211,84,239,119,234,85,83,123,120,131,203,219,10,82,35,120,180,249,106,37,169,225,54,103,55,166,124]
        print(sol.rob(nums))

if __name__ == "__main__":
    Solution.main()


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution:
    def detectCycle(self, head: ListNode) -> ListNode:
        # Floyd's algorithm
        # https://leetcode.com/problems/find-the-duplicate-number/solution/
        if head == None:
            return None
        p1 = head
        p2 = head

        while(True):
            p1 = p1.next
            p2 = p2.next

            # p2 moves twice nodes faster than p1.
            # That's why p2 would be reached to the end if exists.
            if (p2 == None):
                return None
            p2 = p2.next
            if (p2 == None):
                return None

            if p1 == p2:
                break

        # P2 would catch p1 in somewhere.
        # It should be the same distance to the entrance of the cycle
        # from the head and the p2 position
        p1 = head
        while (p1 != p2):
            p1 = p1.next
            p2 = p2.next

        return p1


#!/usr/bin/python
# -*- coding: UTF-8 -*-

import requests
from html.parser import HTMLParser
import http.client
import json
import os
import datetime
from datetime import datetime as dt
import threading
import queue
import time
import sys
import csv

brokers = []
IDs=[]
buffer2 = 0
class Worker(threading.Thread):
    def __init__(self, queue,num):
        threading.Thread.__init__(self)
        self.num = num
        self.queue = queue
        # self.broker = queue.get()[0]
        # self.ID = queue.get()[1]
    def run(self):
        global buffer2

        while not self.queue.empty():
            buffer = self.queue.get()
            self.broker = buffer[0]
            self.ID = buffer[1]
            get_html = open(self.ID+'/'+self.broker+'.txt',encoding = "utf-8",mode = 'w')
            conn = http.client.HTTPSConnection("www.nvesto.com",timeout=10)
            payload = "fromdate=2018-01-01&todate=2018-11-13&broker="+str(self.broker)
            headers = {
                'content-type': "application/x-www-form-urlencoded",
                'cache-control': "no-cache",
                'User-Agent':"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:48.0) Gecko/20100101 Firefox/48.0"
                }
            conn.request("POST", "/brokeranalysis/ajaxGetData/sotckType/tpe/stockId/"+self.ID, payload, headers)
            res = conn.getresponse()
            data = res.read()
            get_html.write(data.decode("utf-8"))
            get_html.close()

            # 解析
            shop_list = open(self.ID+'.csv',encoding = "utf-8",mode = 'a+',newline='')
            #print("被打開來")
            if buffer2 == 0:
                shop_list.write('Date'+','+'券商'+','+'buy'+','+'sell'+','+'buyPer'+','+'sellPer'+','+'netPer'+','+'buyWeek'+','+'sellWeek'+','+'netWeek'+','+'buyWeekPer'+','+'sellWeekPer'+','+'last_price'+','+'netWeekPer')
                buffer2 = 1
            try:

                data = data.decode("utf-8")
                res_data = json.loads(data)
                shop_list.write('\n')
                StartDate = time.strptime("2018/01/01", "%Y/%m/%d")
                EndDate = time.strptime("2018/11/13", "%Y/%m/%d")
                StartDate = datetime.date(StartDate[0], StartDate[1], StartDate[2])
                EndDate = datetime.date(EndDate[0], EndDate[1], EndDate[2])
                # EndDate =  datetime.date(EndDate[0], EndDate[1], EndDate[2])
                RangeDate = datetime.timedelta(days = 1)
                while StartDate <= EndDate:
                    z = StartDate + RangeDate
                    yearmonthday = str(z)
                    try:
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['date'])+',')
                        shop_list.write(str(self.broker)+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['buy'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['sell'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['buyPer'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['sellPer'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['netPer'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['buyWeek'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['sellWeek'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['netWeek'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['buyWeekPer'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['sellWeekPer'])+',')
                        shop_list.write(str(res_data['stockpriceData']['stockprice'][yearmonthday]['last_price'])+',')
                        shop_list.write(str(res_data['chartData']['ba'][yearmonthday]['netWeekPer'])+',')
                        shop_list.write('\n')
                    except Exception as e:
                        pass

                    StartDate = StartDate + RangeDate

                print("complete ID: "+str(self.ID)+" broker: "+str(self.broker)+" Worker %d" % (self.num))
                sys.stdout.flush()
            except Exception as e:
                print(e)
                print("out of range")
            #time.sleep(1)

# 建立佇列
my_queue = queue.Queue()

with open('123.csv', newline='') as csvfile:
  spamreader = csv.reader(csvfile)
  for row in spamreader:
    if row[0] != '':
        brokers.append(row[0])

    if row[1] != '':
        IDs.append(row[1])

for ID in IDs: # 證券代碼

    ID = str(ID)
    try:
        os.mkdir(ID) #創見目錄
    except Exception as e:
        print(e)
        print("File have exist "+ID)

    for broker in brokers: #卷商

        broker = str(broker)
        x = [broker,ID]
        my_queue.put(x)
    #print("All complete "+str(ID))

starttime = dt.now()
print("開始 "+str(starttime))
# 建立兩個 Worker
my_worker1 = Worker(my_queue, 1)
my_worker2 = Worker(my_queue, 2)
my_worker3 = Worker(my_queue, 3)
my_worker4 = Worker(my_queue, 4)
my_worker5 = Worker(my_queue, 5)
my_worker6 = Worker(my_queue, 6)
my_worker7 = Worker(my_queue, 7)
my_worker8 = Worker(my_queue, 8)

# 讓 Worker 開始處理資料
my_worker1.start()
my_worker2.start()
my_worker3.start()
my_worker4.start()
my_worker5.start()
my_worker6.start()
my_worker7.start()
my_worker8.start()

# 等待所有 Worker 結束
my_worker1.join()
my_worker2.join()
my_worker3.join()
my_worker4.join()
my_worker5.join()
my_worker6.join()
my_worker7.join()
my_worker8.join()

print("全部結束 "+str(dt.now()-starttime))


n, m = map(int, input().split())
visit = input().split()
cache = []
loadTime = 0
for page in visit:
	if page not in cache:
		loadTime += 1
		if len(cache) >= n:
			cache.pop(0)
	else:
		cache.remove(page)
	cache.append(page)
print(loadTime)
cache = list(map(int, cache))
cache.sort()
print(" ".join(list(map(str, cache))))


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="functionalstream", # Replace with your own username
    version="0.1.8",
    author="cjwcommuny",
    author_email="cjwcommuny@outlook.com",
    description="A functional interface for python iterator",
    packages=setuptools.find_packages(),
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/cjwcommuny/functionalstream",
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.6',
)


# -*- coding: utf-8 -*-
__author__ = 'alansanders'
import ctypes as ct
import os

pvcam=PVCAM = ct.WinDLL(os.path.dirname(__file__) +"/DLL/Pvcam32.dll")

import nplab.instrument.ccd.pvcam_h as pv
import numpy as np
import time
from nplab.instrument.ccd import CCD
from nplab.instrument.camera import Camera
from nplab.utils.gui import *
from nplab.ui.ui_tools import UiTools
from nplab import inherit_docstring

class PixisError(Exception):
    def __init__(self, msg):
        print(msg)
        i = pvcam.pl_error_code()
        msg = ct.create_string_buffer(20)
        pvcam.pl_error_message(i, msg)
        print('self.pvcam Error:', i, msg)
        pvcam.pl_self.pvcam_uninit()

class Pixis256E(CCD,Camera):
    def __init__(self):
        super(Pixis256E, self).__init__()
        try:
            self.pvcam = PVCAM #ct.windll.pvcam32
        except WindowsError as e:
            print('pvcam not found')
        cam_selection = ct.c_int16()
        # Initialize the self.pvcam library and open the camera #
        self.open_lib()
        self.open_cam(cam_selection)
        self._sequence_set = False
        self._current_frame = None

        self.exposure = 50
        self.timing_mode = 'timed'  # also 'trigger'
        self.cont_clears = True
        self.latest_image = None
        self.latest_raw_image = None
        self.masked_image = None

        #print 'trying to allocate0', (ct.c_int16 * 10)()

    def __del__(self):
        print('deleting')
        if self.cam_open:
            self.close_cam()
        self.close_lib()

    def read_image(self, exposure, timing='timed', mode='kinetics', new=True, end=True, *args,
                   **kwargs):
        """
        Read an image.

        :param exposure: Exposure time
        :param timing: 'timed' or 'trigger'
        :param mode: 'kinetics'
        :param new: If it is the first sequence in a set then new (True) sets up the sequence
        :param end: If it is the last sequence in a set then end (True) finishes the sequence
        :param kwargs: k_size=1

        :return:
        """
        if new:  # only setup the first time in a set of sequences
            # setup sequence
            if mode == 'kinetics':
                k_size = kwargs['k_size'] if 'k_size' in kwargs else 1
                self.setup_kinetics(exposure, k_size, timing)
        self.start_sequence()  # start acquisition
        while not self.check_readout():  # wait for image
            continue
        image = self.readout_image()  # readout
        if end:  # shutdown sequence if it is the last one
            self.finish_sequence()
        return image

    def open_lib(self):
        if not self.pvcam.pl_pvcam_init():
            raise PixisError("failed to init self.pvcam")
        else:
            print('init self.pvcam complete')

    def close_lib(self):
        self.pvcam.pl_pvcam_uninit()
        print("pvcam closed")

    def open_cam(self, cam_selection):
        cam_name = ct.create_string_buffer(20)
        self._handle = ct.c_int16()
        if not self.pvcam.pl_cam_get_name(cam_selection, cam_name):
            raise PixisError("didn't get cam name")
        if not self.pvcam.pl_cam_open(cam_name, ct.byref(self._handle), pv.OPEN_EXCLUSIVE):
            raise PixisError("camera didn't open")
        self.cam_open = True

    def close_cam(self):
        if self._sequence_set:
            self.finish_sequence()
        self.pvcam.pl_cam_close(self._handle)
        self.cam_open = False
        print('cam closed')

    def set_any_param(self, param_id, param_value):
        b_status = ct.c_bool()
        b_param = ct.c_bool()
        param_access = ct.c_uint16()
        param_id = ct.c_uint32(param_id)

        if not isinstance(param_value, ct._SimpleCData):
            raise TypeError("The parameter value must be passed as a ctypes instance, not a python value.")

        b_status = self.pvcam.pl_get_param(self._handle, param_id,
                                      pv.ATTR_AVAIL, ct.cast(ct.byref(b_param), ct.c_void_p))
        if b_param:
            b_status = self.pvcam.pl_get_param(self._handle, param_id,
                                          pv.ATTR_ACCESS,
                                          ct.cast(ct.byref(param_access), ct.c_void_p))

            if param_access.value == pv.ACC_READ_WRITE or param_access.value == pv.ACC_WRITE_ONLY:
                if not self.pvcam.pl_set_param(self._handle, param_id,
                                          ct.cast(ct.byref(param_value), ct.c_void_p)):
                    print("error: param %d (value = %d) did not get set" % (
                    param_id.value, param_value.value))
                    return False
            else:
                print("error: param %d is not writable: %s" % (param_id.value, param_access.value))
                return False
        else:
            print("error: param %d is not available" % param_id.value)
            return False
        return True

    def get_any_param(self, param_id):
        pass

    def set_full_frame(self, region):
        ser_size = ct.c_uint16()
        self.pvcam.pl_get_param(self._handle, pv.PARAM_SER_SIZE,
                           pv.ATTR_DEFAULT, ct.cast(ct.byref(ser_size), ct.c_void_p))
        par_size = ct.c_uint16()
        self.pvcam.pl_get_param(self._handle, pv.PARAM_PAR_SIZE,
                           pv.ATTR_DEFAULT, ct.cast(ct.byref(par_size), ct.c_void_p))
        region.s1, region.s2, region.p1, region.p2 = (0, ser_size.value - 1,
                                                      0, par_size.value - 1)
        region.sbin, region.pbin = (1, 1)

    def setup_kinetics(self, exposure, k_size, timing='trigger'):
        if self._sequence_set:  # uninitialise all previous sequences
            self.finish_sequence()

        params = {
            pv.PARAM_PMODE: (ct.c_uint32(pv.PMODE_KINETICS), 'pmode'),
            pv.PARAM_KIN_WIN_SIZE: (ct.c_uint16(k_size), 'kinetics window size'),
            pv.PARAM_PAR_SHIFT_TIME: (ct.c_uint32(9200), 'parallel shift time'),
            # pv.PARAM_SER_SHIFT_TIME : (ct.c_uint32(9200), 'serial shift time'),
            pv.PARAM_EXP_RES_INDEX: (ct.c_uint16(pv.EXP_RES_ONE_MICROSEC), 'exposure resolution'),
            pv.PARAM_CLEAR_CYCLES: (ct.c_uint16(1), 'clear cycles'),
            pv.PARAM_NUM_OF_STRIPS_PER_CLR: (ct.c_uint16(1), 'number of clear strips'),
            pv.PARAM_SHTR_OPEN_MODE: (ct.c_uint16(pv.OPEN_PRE_SEQUENCE), 'shutter open mode'),
            pv.PARAM_EDGE_TRIGGER: (ct.c_uint32(pv.EDGE_TRIG_POS), 'edge trigger'),
            pv.PARAM_GAIN_INDEX: (ct.c_uint16(3), 'gain index'),
            # pv.PARAM_CONT_CLEARS : (ct.c_bool(True), 'continuous clears 1'),
        }
        for p in params:
            status = self.set_any_param(p, params[p][0])
            if not status: print('problem with %s' % params[p][1])

        read_params = [
            pv.PARAM_PIX_TIME,
        ]
        for p in read_params:
            self.get_any_param(p)

        if not self.pvcam.pl_exp_init_seq():
            raise PixisError("init_seq failed!")

        exp_time = ct.c_uint32(exposure)
        size = ct.c_uint32()
        region = pv.rgn_type()
        self.set_full_frame(region)

        if timing == 'trigger':
            timing_mode = pv.TRIGGER_FIRST_MODE
        elif timing == 'timed':
            timing_mode = pv.TIMED_MODE
        if self.pvcam.pl_exp_setup_seq(self._handle, 1, 1, ct.byref(region),
                                  timing_mode, exp_time, ct.byref(size)):
            # print "frame size = %d" % size.value
            pass
        else:
            raise PixisError("experiment setup failed!")
        self._sequence_set = True
        self.exposure = exposure
        self.size = size
        self.timing = timing

        if timing == 'trigger':
            # status = self.set_any_param(pv.PARAM_CLEAR_MODE, ct.c_uint16(pv.CLEAR_PRE_SEQUENCE))
            # if not status: print 'problem with clear mode'
            status = self.set_any_param(pv.PARAM_CONT_CLEARS, ct.c_bool(self.cont_clears))
            if not status:
                print('problem with continuous cleans 2')
            # status = self.set_any_param(pv.PARAM_CLN_WHILE_EXPO, ct.c_uint16(1))
            # if not status: print 'problem with clear while exposing'

            # frame = (ct.c_uint16 * (self.size.value//2))()
            # self.pvcam.pl_exp_start_seq(self._handle, frame)
            # self._current_frame = frame

    def start_sequence(self):
        """
        Create a frame and start the kinetics exposure sequence. Call this method
        after setting up the trigger and then wait until check_kinetics confirms
        readout.
        """
        frame = (ct.c_uint16 * (self.size.value // 2))()
        self.pvcam.pl_exp_start_seq(self._handle, frame)
        self._current_frame = frame

    def check_readout(self):
        """
        Poll the readout status and return if the readout is complete or if it
        fails.
        """
        status = ct.c_int16()
        self.pvcam.pl_exp_check_status(self._handle, ct.byref(status),
                                  ct.byref(ct.c_int32()))
        if status.value == pv.READOUT_FAILED:
            raise PixisError("Data collection error")
        elif status.value == pv.READOUT_COMPLETE:
            return True
        else:
            return False

    def readout_image(self):
        """If the readout is complete a valid frame is returned."""
        return np.array(list(self._current_frame)).reshape((256, 1024))

    def finish_sequence(self):
        self.pvcam.pl_exp_finish_seq(self._handle, self._current_frame, 0)
        self.pvcam.pl_exp_uninit_seq()
        self._sequence_set = False

    metadata_property_names = ('exposure',)

    def read_background(self):
        """Acquire a new spectrum and use it as a background measurement."""
        self.background = self.read_image(self.exposure, self.timing, self.mode,
                                          new=True, end=True)
        self.update_config('background', self.background)

    def read_reference(self):
        """Acquire a new spectrum and use it as a reference."""
        self.reference = self.read_image(self.exposure, self.timing, self.mode,
                                         new=True, end=True)
        self.update_config('reference', self.reference)

    def raw_snapshot(self):
        try:
            image = self.read_image(self.exposure, timing='timed', mode='kinetics', new=False, end= True, k_size=1)
            return 1, image
        except Exception as e:
            self._logger.warn("Couldn't Capture because %s" % e)

@inherit_docstring(Pixis256E)
class Pixis256EQt(Pixis256E, QtCore.QObject):
    """Pixis256E subclass with Qt signals for GUI interaction."""

    image_taken = QtCore.Signal(np.ndarray)

    @inherit_docstring(Pixis256E.__init__)
    def __init__(self):
        super(Pixis256EQt, self).__init__()

    @inherit_docstring(Pixis256E.read_image)
    def read_image(self, exposure, timing='timed', mode='kinetics', new=True, end=True, *args,
                   **kwargs):
        image = super(Pixis256EQt, self).read_image(exposure, timing='timed', mode='kinetics',
                                                    new=True, end=True, *args, **kwargs)
        self.image_taken.emit(image)
        return image

class Pixis256EUI(QtWidgets.QWidget, UiTools):
    def __init__(self, pixis):
        if not isinstance(pixis, Pixis256EQt):
            raise TypeError('pixis is not an instance of Pixis256EQt')
        super(Pixis256EUI, self).__init__()
        self.pixis = pixis

        # self.exposure.setValidator(QtWidgets.QIntValidator())
        # self.exposure.textChanged.connect(self.check_state)
        # self.exposure.textChanged.connect(self.on_text_change)
        # self.mode.activated.connect(self.on_activated)
        # self.timing.activated.connect(self.on_activated)
        # self.cont_clears.stateChanged.connect(self.on_state_change)
        # self.take_image_button.clicked.connect(self.on_click)
        # self.take_bkgd_button.clicked.connect(self.on_click)
        # self.clear_bkgd_button.clicked.connect(self.on_click)
        # self.take_ref_button.clicked.connect(self.on_click)
        # self.clear_ref_button.clicked.connect(self.on_click)

        # self.pixis.image_taken.connect()

    def on_text_change(self, text):
        sender = super(Pixis256EUI, self).on_text_change(text)
        if sender == False:
            return
        elif sender == self.exposure:
            self.pixis.exposure = float(sender)

    def on_click(self):
        sender = self.sender()
        if sender == self.take_image_button:
            self.pixis.read_image(self.pixis.exposure, self.pixis.timing, self.pixis.mode,
                                  new=True, end=True)
        elif sender == self.take_bkgd_button:
            self.pixis.read_background()
        elif sender == self.clear_bkgd_button:
            self.pixis.clear_background()
        elif sender == self.take_ref_button:
            self.pixis.read_reference()
        elif sender == self.clear_ref_button:
            self.pixis.clear_reference()

    def on_activated(self, item):
        sender = self.sender()
        if sender == self.mode:
            self.pixis.mode = item
        elif sender == self.timing:
            self.pixis.timing = item

    def on_state_change(self, state):
        sender = self.sender()
        if sender == self.cont_clears:
            if state == QtCore.Qt.Checked:
                self.pixis.cont_clears = True
            elif state == QtCore.Qt.Unchecked:
                self.pixis.cont_clears = False

if __name__ == '__main__':
    import matplotlib.pyplot as plt
    p = Pixis256EQt()
    p.exposure = 1

    def test():
        p.setup_kinetics(p.exposure, 1)
        print(p.check_readout())
        imgs = []
        print('ready')
        shots = 3
        for i in range(shots):
            print('shot {0}'.format(i+1))
            p.start_sequence()
            time.sleep(0.1)
            while not (p.check_readout()): continue
            print("triggered")
            img = p.readout_image()
            imgs.append(img)
        p.finish_sequence()
        time.sleep(0.1)
        img = p.read_image(p.exposure, timing='timed', mode='kinetics', k_size=1)
        imgs.append(img)
        #p.close_cam()
        print('finished')

        print('plotting data')
        fig, axes = plt.subplots(shots+1, sharex=True)
        for i, ax in enumerate(axes):
            img = ax.imshow(imgs[i])
        print('done')
        plt.show()

    print("one")
#    app = get_qt_app()
#    ui = Pixis256EUI(pixis=p)
#    print "two"

#    ui.show()
 #   sys.exit(app.exec_())

    # p.read_image(p.exposure, timing='timed', mode='kinetics', new=True, end= False, k_size=1)
    # img = p.read_image(p.exposure, timing='timed', mode='kinetics', new=False, end= True, k_size=1)
    # plt.imshow(img)
    # plt.show()


# -*- coding: UTF-8 -*-
# -Cleaned and Checked on 11-23-2018 by JewBMX in Scrubs.
# Only browser checks for active domains.

#  ..#######.########.#######.##....#..######..######.########....###...########.#######.########..######.
#  .##.....#.##.....#.##......###...#.##....#.##....#.##.....#...##.##..##.....#.##......##.....#.##....##
#  .##.....#.##.....#.##......####..#.##......##......##.....#..##...##.##.....#.##......##.....#.##......
#  .##.....#.########.######..##.##.#..######.##......########.##.....#.########.######..########..######.
#  .##.....#.##.......##......##..###.......#.##......##...##..########.##.......##......##...##........##
#  .##.....#.##.......##......##...##.##....#.##....#.##....##.##.....#.##.......##......##....##.##....##
#  ..#######.##.......#######.##....#..######..######.##.....#.##.....#.##.......#######.##.....#..######.

'''
    OpenScrapers Project
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''

import re

try: from urlparse import urlparse, urljoin
except ImportError: from urllib.parse import urlparse, urljoin
try: from urllib import quote_plus
except ImportError: from urllib.parse import quote_plus

from openscrapers.modules import cfscrape
from openscrapers.modules import cleantitle
from openscrapers.modules import dom_parser
from openscrapers.modules import source_utils

class source:
	def __init__(self):
		self.priority = 1
		self.language = ['de']
		self.domains = ['iload.to']
		self.base_link = 'https://iload.to'
		self.search_link_mv = '/suche/%s/Filme'
		self.search_link_tv = '/suche/%s/Serien'
		self.scraper = cfscrape.create_scraper()

	def movie(self, imdb, title, localtitle, aliases, year):
		try:
			url = self.__search(self.search_link_mv, imdb, [localtitle] + source_utils.aliases_to_array(aliases))
			if not url and title != localtitle: url = self.__search(self.search_link_mv, imdb,
			                                                        [title] + source_utils.aliases_to_array(aliases))
			return url
		except:
			return

	def tvshow(self, imdb, tvdb, tvshowtitle, localtvshowtitle, aliases, year):
		try:
			url = self.__search(self.search_link_tv, imdb, [localtvshowtitle] + source_utils.aliases_to_array(aliases))
			if not url and tvshowtitle != localtvshowtitle: url = self.__search(self.search_link_tv, imdb, [
				tvshowtitle] + source_utils.aliases_to_array(aliases))
			return url
		except:
			return

	def episode(self, url, imdb, tvdb, title, premiered, season, episode):
		try:
			if not url:
				return
			query = urljoin(self.base_link, url)
			r = self.scraper.get(query).content
			r = dom_parser.parse_dom(r, 'td', attrs={'data-title-name': re.compile('Season %02d' % int(season))})
			r = dom_parser.parse_dom(r, 'a', req='href')[0].attrs['href']
			r = self.scraper.get(urljoin(self.base_link, r)).content
			r = dom_parser.parse_dom(r, 'td', attrs={'data-title-name': re.compile('Episode %02d' % int(episode))})
			r = dom_parser.parse_dom(r, 'a', req='href')[0].attrs['href']
			return source_utils.strip_domain(r)
		except:
			return

	def sources(self, url, hostDict, hostprDict):
		sources = []
		try:
			if not url:
				return sources
			query = urljoin(self.base_link, url)
			r = self.scraper.get(query).content
			r = dom_parser.parse_dom(r, 'div', attrs={'id': 'Module'})
			r = [(r, dom_parser.parse_dom(r, 'a', attrs={'href': re.compile('[^\'"]*xrel_search_query[^\'"]*')},
			                              req='href'))]
			r = [(i[0], i[1][0].attrs['href'] if i[1] else '') for i in r]
			rels = dom_parser.parse_dom(r[0][0], 'a', attrs={'href': re.compile('[^\'"]*ReleaseList[^\'"]*')},
			                            req='href')
			if rels and len(rels) > 1:
				r = []
				for rel in rels:
					relData = self.scraper.get(urljoin(self.base_link, rel.attrs['href'])).content
					relData = dom_parser.parse_dom(relData, 'table', attrs={'class': 'release-list'})
					relData = dom_parser.parse_dom(relData, 'tr', attrs={'class': 'row'})
					relData = [(dom_parser.parse_dom(i, 'td', attrs={'class': re.compile('[^\'"]*list-name[^\'"]*')}),
					            dom_parser.parse_dom(i, 'img', attrs={'class': 'countryflag'}, req='alt'),
					            dom_parser.parse_dom(i, 'td', attrs={'class': 'release-types'})) for i in relData]
					relData = [(i[0][0].content, i[1][0].attrs['alt'].lower(), i[2][0].content) for i in relData if
					           i[0] and i[1] and i[2]]
					relData = [(i[0], i[2]) for i in relData if i[1] == 'deutsch']
					relData = [(i[0], dom_parser.parse_dom(i[1], 'img', attrs={'class': 'release-type-stream'})) for i
					           in relData]
					relData = [i[0] for i in relData if i[1]]
					# relData = dom_parser.parse_dom(relData, 'a', req='href')[:3]
					relData = dom_parser.parse_dom(relData, 'a', req='href')
					for i in relData:
						i = self.scraper.get(urljoin(self.base_link, i.attrs['href'])).content
						i = dom_parser.parse_dom(i, 'div', attrs={'id': 'Module'})
						i = [(i, dom_parser.parse_dom(i, 'a',
						                              attrs={'href': re.compile('[^\'"]*xrel_search_query[^\'"]*')},
						                              req='href'))]
						r += [(x[0], x[1][0].attrs['href'] if x[1] else '') for x in i]
			r = [(dom_parser.parse_dom(i[0], 'div', attrs={'id': 'ModuleReleaseDownloads'}), i[1]) for i in r]
			r = [(dom_parser.parse_dom(i[0][0], 'a', attrs={'class': re.compile('.*-stream.*')}, req='href'), i[1]) for
			     i in r if len(i[0]) > 0]
			for items, rel in r:
				rel = urlparse(rel).query
				rel = urlparse.parse_qs(rel)['xrel_search_query'][0]
				quality, info = source_utils.get_release_quality(rel)
				items = [(i.attrs['href'], i.content) for i in items]
				items = [(i[0], dom_parser.parse_dom(i[1], 'img', req='src')) for i in items]
				items = [(i[0], i[1][0].attrs['src']) for i in items if i[1]]
				items = [(i[0], re.findall('.+/(.+\.\w+)\.\w+', i[1])) for i in items]
				items = [(i[0], i[1][0]) for i in items if i[1]]
				info = ' | '.join(info)
				for link, hoster in items:
					valid, hoster = source_utils.is_host_valid(hoster, hostDict)
					if not valid: continue
					sources.append({'source': hoster, 'quality': quality, 'language': 'de', 'url': link, 'info': info,
					                'direct': False, 'debridonly': False, 'checkquality': True})
			return sources
		except:
			return sources

	def resolve(self, url):
		try:
			url = self.scraper.get(urljoin(self.base_link, url), output='geturl').content
			return url if self.base_link not in url else None
		except:
			return

	def __search(self, search_link, imdb, titles):
		try:
			query = search_link % (quote_plus(cleantitle.query(titles[0])))
			query = urljoin(self.base_link, query)
			t = [cleantitle.get(i) for i in set(titles) if i]
			r = self.scraper.get(query).content
			r = dom_parser.parse_dom(r, 'div', attrs={'class': 'big-list'})
			r = dom_parser.parse_dom(r, 'table', attrs={'class': 'row'})
			r = dom_parser.parse_dom(r, 'td', attrs={'class': 'list-name'})
			r = dom_parser.parse_dom(r, 'a', req='href')
			r = [i.attrs['href'] for i in r if i and cleantitle.get(i.content) in t][0]
			url = source_utils.strip_domain(r)
			r = self.scraper.get(urljoin(self.base_link, url)).content
			r = dom_parser.parse_dom(r, 'a', attrs={'href': re.compile('.*/tt\d+.*')}, req='href')
			r = [re.findall('.+?(tt\d+).*?', i.attrs['href']) for i in r]
			r = [i[0] for i in r if i]
			return url if imdb in r else None
		except:
			return


import numpy as np

def is_pareto(costs, maximise=False):
    # source: https://stackoverflow.com/questions/32791911/fast-calculation-of-pareto-front-in-python
    """
    :param costs: An (n_points, n_costs) array
    :maximise: boolean. True for maximising, False for minimising
    :return: A (n_points, ) boolean array, indicating whether each point is Pareto efficient
    """
    is_efficient = np.ones(costs.shape[0], dtype=bool)
    for i, c in enumerate(costs):
        if is_efficient[i]:
            if maximise:
                is_efficient[is_efficient] = np.any(costs[is_efficient] >= c, axis=1)  # Remove dominated points
            else:
                is_efficient[is_efficient] = np.any(costs[is_efficient] <= c, axis=1)  # Remove dominated points
    return is_efficient

# def is_pareto(costs):
#     # source: https://stackoverflow.com/questions/32791911/fast-calculation-of-pareto-front-in-python
#     """
#     Find the pareto-efficient points
#     :param costs: An (n_points, n_costs) array
#     :return: A (n_points, ) boolean array, indicating whether each point is Pareto efficient
#     """
#     is_efficient = np.ones(costs.shape[0], dtype=bool)
#     for i, c in enumerate(costs):
#         if is_efficient[i]:
#             is_efficient[is_efficient] = np.any(costs[is_efficient] <= c, axis=1)  # Keep any point with a lower cost
#             is_efficient[i] = True  # And keep self
#     return is_efficient

# def is_pareto(scores):
#     # Source: https://pythonhealthcare.org/tag/pareto-front/
#     # Count number of items
#     population_size = scores.shape[0]
#     # Create a NumPy index for scores on the pareto front (zero indexed)
#     population_ids = np.arange(population_size)
#     # Create a starting list of items on the Pareto front
#     # All items start off as being labelled as on the Pareto front
#     pareto_front = np.ones(population_size, dtype=bool)
#     # Loop through each item. This will then be compared with all other items
#     for i in range(population_size):
#         # Loop through all other items
#         for j in range(population_size):
#             # Check if our 'i' point is dominated by out 'j' point
#             if all(scores[j] <= scores[i]) and any(scores[j] < scores[i]):
#                 # j dominates i. Label 'i' point as not on Pareto front
#                 pareto_front[i] = 0
#                 # Stop further comparisons with 'i' (no more comparisons needed)
#                 break
#     # Return ids of scenarios on pareto front
#     return population_ids[pareto_front]
# # Fairly fast for many datapoints, less fast for many costs, somewhat readable


import pandas as pd

class SRGJob:

    def __init__(self):
        self.samples = []
        self.fields = {}

    def add_sample(self, sample):
        self.samples.append(sample)

    def get_all_results(self, key_fields, test_name):
        """ Gets all the result values from each product and compiles it into
        a table. One column is the sample name and another column is the test
        value. Useful dataframe for doing statistical anaylsis

        Args:
            key_fields: the fields in the samples details to use to build the
                sample name. Used as the keys in the dictionary
            test_name: the name of the test to get the values for

        Returns:
            DataFrame: a dataframe with all the test result values for every sample
        """

        all_results = pd.DataFrame(columns=['sample', 'result'])

        for sample in self.samples:

            if test_name in sample.test_results_values:
                sample_name = sample.build_name(key_fields)

                for value in sample.test_results_values[test_name]:
                    all_results = all_results.append({'sample': sample_name,
                                                      'result': value}, ignore_index=True)

        # make sure the results are interpreted as numeric
        all_results['result'] = pd.to_numeric(all_results['result'])

        return all_results


#####
# This file contains all the functions used in the course
import numpy as np
#####

#####
# General Purpose
#####

### Horners algorithm for polynomial evaluation
def Horners(a,x):
    """ Horner's algorithm for polynomial evaluation
    a: coefficients of polynomial
    x: point at which to evaluate
    """
    n = len(a)-1
    a.reverse()
    p = a[0]
    for i in range(n):
        p = a[i+1]+p*x
    return p

### Least Squares
# Function to be fitted:
F = lambda x: [1,x,np.sin(x)**2,np.cos(x)**2]
# Function after fitting:
Fc = lambda x,c: c[0]+c[1]*x+c[2]*np.sin(x)**2+c[3]*np.cos(x)**2
def fit(F,Fc,x,y):
    """ Fit a function to data using least squares
    F: function to be fitted
    Fc: function after fitting
    x: x-values
    y: y-values
    """
    A = np.array(list(map(F,x)))
    c = np.linalg.solve(A.T@A, A.T@y)
    return Fc(x,c),c

### Print matrix nice
def print_table(A):
    """ Print matrix in nice format
    A: matrix
    """
    if type(A) != np.ndarray:
        A = np.array(A)
    m,n = A.shape
    for i in range(m):
        for j in range(n-1):
            print("{:10.4f}".format(A[i,j]),end=" ")
        print("{:10.4f}".format(A[i,-1]),end=" ")
        print()

def print_normal_equation(F,x,y,digits=4):
    """ Print the normal equation for a given function F
    F: function to be fitted
    x: x-values
    """
    A = np.array(list(map(F,x)))
    print_table(A.T@A)
    print(np.round(A.T@y,4))

### Print matrix to latex
def print_table_latex(A):
    """ Print normal equations in latex format
    A: matrix
    """
    if type(A) != np.ndarray:
        A = np.array(A)
    m,n = A.shape
    print(r"\begin{equation*}")
    print(r"\begin{array}{|"+n*"r"+"|}")
    for i in range(m):
        for j in range(n-1):
            print(A[i,j],end=" & ")
        print(A[i,-1],end="\\\\")
        print()
    print(r"\end{array}")
    print(r"\end{equation*}")

#####
# Root finding
#####
# Newton's method for finding roots
def Newton(f,df,x0, nmax, tol=1e-6,true_root=None):
    """
    Newton's method for finding roots of f(x)=0
    f: function in lambda form
    df: derivative of f in lambda form
    x0: initial guess
    nmax: maximum number of iterations
    tol: tolerance for stopping
    true_root: true root of f(x)=0
    """
    x = x0
    X = [x]
    for i in range(nmax):
        if true_root is not None:
            if np.abs(x-true_root) < tol:
                break
        fx = f(x)
        fp = df(x)
        x = x - fx/fp
        X.append(x)
    return X

def Newtonsys(FdF, X0, kmax):
    """ Newton's method for solving systems of equations
    FdF: function returning F(x) and dF(x)
    X0: initial guess
    kmax: maximum number of iterations
    """
    X = X0

    Xiterations = []
    for k in range(kmax):
        Fx, dFx = FdF(X)
        H = np.linalg.solve(dFx, Fx)
        X = X - H.flatten()
        Xiterations.append(X)
    return np.array(Xiterations)

# Bisection method for finding roots
def Bisection(f, a, b, nmax):
    """ Bisection method for finding roots of f(x)=0
    f: function in lambda form
    a: left endpoint of interval
    b: right endpoint of interval
    nmax: maximum number of iterations
    """
    if nmax < 1:
        raise ValueError("nmax must be a positive number")
    if a >= b:
        raise ValueError("a must be less than b")
    fa = f(a)
    fb = f(b)
    if fa * fb > 0:
        raise ValueError("f(a) and f(b) must have different signs")
    X = [0] * (nmax + 1)

    for n in range(nmax):
        c = (a + b) / 2  # The midpoint.
        fc = f(c)
        X[n] = c
        if fa * fc < 0:
            b = c
        else:
            a = c
            fa = fc
    c = (a + b) / 2
    X[nmax] = c
    return X

# Secant method for finding roots
def Secant(f,x0,x1,nmax):
    """ Secant method for finding roots of f(x)=0
    f: function in lambda form
    x0: initial guess
    x1: initial guess
    nmax: maximum number of iterations
    """
    a = x0
    b = x1
    fa = f(a)
    fb = f(b)
    X = [a,b]
    for _ in range(nmax):
        d = fb*(b-a)/(fb-fa)
        a = b
        fa = fb
        b = b - d
        fb = f(b)
        X.append(b)
    return X

#####
# Interpolation
#####

# Newton's interpolation using divided difference
def NewtonDD(x, y, t, n): # page 165 in E. Ward Cheney and David Kincaid, Numerical Mathematics and Computing, 7th edition
    """Newton interpolation polynomial using divided differences
    x, y: data points
    t: interpolation points
    n: number of data points
    """
    # divided difference table
    d = np.zeros((n, n))
    d[:, 0] = y
    for j in range(1, n):
        for i in range(n-j):
            d[i, j] = (d[i+1, j-1] - d[i, j-1]) / (x[i+j] - x[i])
    # evaluate polynomial
    p = d[0, 0]
    for j in range(1, n):
        pt = d[0, j]
        for k in range(j):
            pt = pt * (t - x[k])
        p = p + pt
    # coeficients
    c = d[0, :n]
    return p, c

# Cardinal polynomial for Lagrange interpolation
def CardinalPolynomial(x, i, t):
    """Lagrange cardinal polynomial
    x: list of nodes(knots)
    i: index of the node
    t: points to evaluate the polynomial as list
    """
    l = lambda k: np.prod([(k - x[j]) / (x[i] - x[j]) for j in range(len(x)) if j != i])
    return np.array([l(k) for k in t])

def InterpolerLagrangeForm(x, y, t):
    """ Calculates the values of the interpolating polynomial in Lagrange form
        x: list of nodes(knots)
        y: list of y-values
        t: points to evaluate the polynomial as list

    Returns:
        list: [P(t1) ... P(tm)]  a row with the m function values of the interpolating polynomial
    """
    cardinals = CardinalPolynomial(x,0,t)
    for i in range(1,len(x)):
        cardinals = np.vstack((cardinals,CardinalPolynomial(x,i,t)))

    cardinals = cardinals.T
    P_val = np.zeros(len(t))
    for idx in range(len(t)):
        P_val[idx] = np.sum(np.multiply(cardinals[idx],y))
    return P_val

# Second intepolation error sentence
def SecondInterpolationError(n,M,h):
    """ Second interpolation error sentence
    n: order of the polynomial
    M: fourth derivative of the function
    h: step size
    """
    return 1/(4*(n+1)) * M *h**(n+1)

# Inverse Quadratic Interpolation
def InverseQuadraticInterpolation(f,xGuess,n):
    """ Inverse Quadratic Interpolation
    f: function
    xGuess: list of 3 initial guesses
    n: number of iterations"""
    output = []
    for _ in range(n):
        x = list(map(f,xGuess))
        x3 = InterpolerLagrangeForm(x,xGuess,[0])
        xGuess = [xGuess[1],xGuess[2],x3[0]]
        output.append(x3[0])
    return output

# General Inverse Interpolation
def InverseInterpolation(f,xGuess,n):
    """
    Inverse Interpolation
    f: function
    xGuess: list of initial guesses
    n: number of iterations"""
    X = []
    for _ in range(n):
        x = list(map(f,xGuess))
        x_next = InterpolerLagrangeForm(x,xGuess,[0])
        xGuess = np.append(xGuess[1:],(x_next[0]))
        X.append(x_next[0])
    return X

#####
# numerical integration
#####

# Trapezoidal rule for known function
def Trapezoidal(f, a, b, n):
    """Calculates the integral of f on [a,b] using the trapez formula
        f: function
        a: lower bound
        b: upper bound
        n: number of steps
    """
    h = (b-a)/n
    x = np.linspace(a,b,n+1)
    y = f(x)
    return h/2*(y[0]+y[-1]+2*np.sum(y[1:-1]))

# Trapezoidal rule for unknown function (table) Would be the composite
def TrapezTable(y,a,b,n):
    """Calculates the integral of f on [a,b] using the trapez formula
        y: list of function values
        a: lower bound
        b: upper bound
        n: number of steps
    """
    h = (b-a)/n
    return h/2*(y[0]+y[-1]+2*np.sum(y[1:-1]))

# Simpson's rule for known function
def Simpson(f, a, b, n):
    """ Calculates the integral of f on [a,b] using the Simpson formula"""
    h = (b-a)/n
    x = np.linspace(a,b,n+1)
    y = f(x)
    return h/3*(y[0]+y[-1]+4*np.sum(y[1:-1:2])+2*np.sum(y[2:-1:2]))

# Simpson's rule for unknown function (table)
def SimpsonTable(y,x,a,b):
    """Calculates the integral of f on [a,b] using the Simpson formula
        y: list of function values
        x: list of nodes
        a: lower bound
        b: upper bound
    """
    n = len(x)
    h = (b-a)/n
    return h/3*(y[0]+y[-1]+4*np.sum(y[1:-1:2])+2*np.sum(y[2:-1:2]))

#####
# ODE 1st order
#####
# Taylor polynomial of order 2
def TaylorOrder2(f, df, x0, a, b, n):
    """ Calculates the Taylor polynomial of order 2 for f on [a,b] using n steps
        f: function as lambda function
        df: derivative of f as lambda function
        x0: point of expansion
        a: lower bound
        b: upper bound
        n: number of steps
    """
    h = (b-a)/n
    t = np.linspace(a,b,n+1)
    x = np.zeros(n+1)
    x[0] = x0
    for i in range(n):
        dx = f(t[i], x[i])
        ddx = df(t[i], x[i], dx)
        x[i+1] = x[i] + h * (dx + h/2 * (ddx))
    return t, x

# Taylor polynomial of order 4
def TaylorOrder4(f, df, ddf, x0, a, b, n):
    """ Calculates the Taylor polynomial of order 4 for f on [a,b] using n steps
        f: function as lambda function
        df: derivative of f as lambda function
        ddf: second derivative of f as lambda function
        x0: point of expansion
        a: lower bound
        b: upper bound
        n: number of steps
    """
    h = (b-a)/n
    t = np.linspace(a,b,n+1)
    x = np.zeros(n+1)
    x[0] = x0
    for i in range(n):
        dx = f(t[i], x[i])
        ddx = df(t[i], x[i], dx)
        dddx = ddf(t[i], x[i], dx, ddx)
        x[i+1] = x[i] + h * (dx + h/2 * (ddx + h/6 * dddx))
    return t, x

# euler method
def EulerODE(dxdt, x0, tspan, n):
    """ Calculates the solution of the ODE using the Euler method
        dxdt: rhs of the ODE as lambda function
        x0: point of expansion
        tspan: [a,b]
        n: number of steps
    """
    a, b = tspan
    h = (b-a)/n
    t = np.linspace(a,b,n+1)
    x = np.zeros(n+1)
    x[0] = x0
    for i in range(n):
        x[i+1] = x[i] + h * dxdt(t[i], x[i])
    return t, x

# Runge-kutta method of order 2
def RungeKutta2(dxdt, x0, tspan, n):
    """ Calculates the solution of the ODE using the Runge-Kutta method
        dxdt: rhs of the ODE as lambda function
        x0: point of expansion
        tspan: [a,b]
        n: number of steps
    """
    a, b = tspan
    h = (b-a)/n
    t = np.linspace(a,b,n+1)
    x = np.zeros(n+1)
    x[0] = x0
    for i in range(n):
        k1 = dxdt(t[i], x[i])
        k2 = dxdt(t[i] + h, x[i] + k1)
        x[i+1] = x[i] + h/2 * (k1 + k2)
    return t, x

# Runge-kutta method of order 4
def RungeKutta4(dxdt, x0, tspan, n):
    """ Calculates the solution of the ODE using the Runge-Kutta method
        dxdt: rhs of the ODE as lambda function
        x0: point of expansion
        tspan: [a,b]
        n: number of steps
    """
    a,b = tspan
    h = (b - a) / n
    h_half = h / 2.0 # for convenience
    h_sixth = h / 6.0 # for convenience
    t = np.linspace(a, b, n + 1)
    x = np.zeros(n + 1)
    x[0] = x0
    for i in range(n):
        K1 = dxdt(t[i], x[i])
        K2 = dxdt(t[i] + h_half, x[i] + K1*h_half)
        K3 = dxdt(t[i] + h_half, x[i] + K2*h_half)
        K4 = dxdt(t[i + 1], x[i] + h*K3)
        x[i+1] = x[i] + (K1 + 2 * (K2 + K3) + K4) * h_sixth
    return t, x

# Heun method
def HeunODE(dxdt, x0, tspan, n):
    """ Calculates the solution of the ODE using the Heun method
        dxdt: rhs of the ODE as lambda function
        x0: point of expansion
        tspan: [a,b]
        n: number of steps
    """
    a, b = tspan
    h = (b-a)/n
    h_half = h/2 # for convenience
    t = np.linspace(a,b,n+1)
    x = np.zeros(n+1)
    x[0] = x0
    for i in range(n): # Heun's method
        K1 = dxdt(t[i], x[i])
        K2 = dxdt(t[i+1], x[i] + h * K1)
        x[i+1] = x[i] + (K2 + K1) * h_half
    return t, x

#####
# System of ODE'se
#####
# Convert 2 ODE's to system of ODE's
def f2(x, y):
    """ Takes two ODE's and returns the system of ODE's
        x: ODE as lambda function
        y: ODE as lambda function
    """
    def f(t, X):
        return np.array([x(t, X[0], X[1]), y(t, X[0], X[1])])
    return f

# Runge-kutta method of order 4 for system of ODE's
def RungeKutta4System(f, tspan, xi, nsteps):
    """ Calculates the solution to the system of ODE's using the Runge-Kutta method
        f: system of ODE's (use f2)
        tspan: [a,b]
        xi: array of initial conditions
        nsteps: number of steps
    """
    a,b = tspan
    h = (b-a)/nsteps
    h_half = h/2 # for convenience
    h_sixth = h/6 # for convenience
    t = np.linspace(a,b,nsteps+1)
    X = np.zeros((nsteps+1, len(xi)))
    X[0] = xi
    for i in range(nsteps):
        k1 = f(t[i], X[i])
        k2 = f(t[i] + h_half, X[i] + h_half * k1)
        k3 = f(t[i] + h_half, X[i] + h_half * k2)
        k4 = f(t[i] + h, X[i] + h * k3)
        X[i+1] = X[i] + (k1 + 2*k2 + 2*k3 + k4) * h_sixth

    return t, X

#########
# Error
#########
def errorConvergece(n, error):
    """ Calculates the convergence rate of a method
    n: list of iterations
    error: list of errors
    returns: rate of convergence and order of convergence
    """
    conv = []
    r = []
    for i in range(len(n)-1):
        conv.append(np.log(error[i]/error[i+1])/np.log(n[i]/n[i+1]))
        r.append(error[i+1]/error[i])
    return conv, r

def errorConvergenceValues(x,x_true):
    """ Calcualates the convergence rate and order of a values
    x: list of approximations
    x_true: true value
    returns: rate of convergence and order of convergence
    """
    conv = []
    r = []
    errors = []
    for i in range(len(x)):
        errors.append(np.abs(x[i]-x_true))
    for i in range(len(x)-1):
        conv.append(np.log(errors[i]/errors[i+1])/np.log(x[i]/x[i+1]))
        r.append(errors[i+1]/errors[i])
    return conv, r

#######
# Factorization
#######

### Check SPD
def isSPD(A):
    """ Check if matrix is symmetric positive definite
    A: matrix
    """
    if type(A) != np.ndarray:
        A = np.array(A)
    if np.all(A == A.T):
        try:
            np.linalg.cholesky(A)
            print("Symmetric positive definite")
            return True
        except np.linalg.LinAlgError:
            print("Symmetric but not positive definite")
            return False
    else:
        print("Not symmetric")
        return False

### Cholesky factorization
def Cholesky(A):
    return np.linalg.cholesky(A)

def Cholesky2L(A):
    """ Gives back each step to finding L
    A: matrix
    """
    if type(A) != np.ndarray:
        A = np.array(A)
    n = len(A)
    L = np.zeros((n,n))
    L_sym = [["" for i in range(n)] for j in range(n)]
    for i in range(n):
        for j in range(i+1):
            if i == j:
                L[i,j] = np.sqrt(A[i,j] - np.sum(L[i,:j]**2))
                if np.sum(L[i,:j]**2)==0:
                    L_sym[i][j] = f"sqrt({A[i,j]})"
                else:
                    L_sym[i][j] = f"sqrt({A[i,j]} - {np.sum(L[i,:j])}^2)"
            else:
                L[i,j] = (A[i,j] - np.sum(L[i,:j]*L[j,:j]))/L[j,j]
                if np.sum(L[i,:j]*L[j,:j])==0:
                    L_sym[i][j] = f"{A[i,j]}/{L[j,j]}"
                else:
                    L_sym[i][j] = f"({A[i,j]} - {np.sum(L[i,:j]*L[j,:j])})/{L[j,j]}"
    return L, L_sym

### LU factorization
def LU(A):
    return np.linalg.lu(A)


import cv2
import numpy as np
from histogram import CourtFinder
from skimage.feature import hog
from sklearn.externals import joblib
import os

def subtract(original, deductible):
    return np.uint8((original - np.int32(deductible)).clip(min = 0))

def er_filter(image):
    kernel = np.ones((3,3), np.uint8)
    image = cv2.erode(image, kernel, iterations = 1)
    image = cv2.dilate(image, kernel, iterations = 1)
    return image

def gaussian_noise_filter(image):
    blur = cv2.GaussianBlur(image, (5,5), 0 )
    thresh = cv2.threshold(blur, 33, 255, cv2.THRESH_BINARY)
    return thresh[1]

class LK_homo_tracker:
    """ Optical Flow tracking using Lucas-Kanade method based on sample/python/lk_track.py """
    def __init__(self):
        # Config
        self.lk_params = dict(winSize=(15, 15), maxLevel=2,
                              criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))

        self.feature_params = dict(maxCorners=500, qualityLevel=0.3, minDistance=7, blockSize=7)
        self.track_len = 10
        self.detect_interval = 5
        self.tracks = []
        self.frame_idx = 0

    def track_frame(self, frame, mask):
        """
        takes a frame and track
        """
        homography = np.float32(np.matrix(np.identity(3)))
        frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

        if len(self.tracks) > 0:
            img0, img1 = self.prev_gray, frame_gray

            #
            p0 = np.float32([tr[-1] for tr in self.tracks]).reshape(-1, 1, 2)
            p1, st, err = cv2.calcOpticalFlowPyrLK(img0, img1, p0, None, **self.lk_params)
            p0r, st, err = cv2.calcOpticalFlowPyrLK(img1, img0, p1, None, **self.lk_params)
            d = abs(p0-p0r).reshape(-1, 2).max(-1)
            good = d < 1
            new_tracks = []
            homo_points_0 = []
            homo_points_1 = []
            for tr, (x0, y0), (x, y), good_flag in zip(self.tracks, p0.reshape(-1, 2), p1.reshape(-1, 2), good):
                if not good_flag:
                    continue
                tr.append((x, y))
                if len(tr) > self.track_len:
                    del tr[0]
                new_tracks.append(tr)
                homo_points_0.append((x0, y0))
                homo_points_1.append((x, y))
            self.tracks = new_tracks
            homography = cv2.findHomography(np.int32(homo_points_0), np.int32(homo_points_1), cv2.RANSAC, 5)[0]

        if self.frame_idx % self.detect_interval == 0:
            p = cv2.goodFeaturesToTrack(frame_gray, mask=mask, **self.feature_params)
            if p is not None:
                for x, y in np.float32(p).reshape(-1, 2):
                    self.tracks.append([(x, y)])

        self.frame_idx += 1
        self.prev_gray = frame_gray

        return (np.int32([tr[-1] for tr in self.tracks]).reshape(-1, 2), homography)

class HomoBG:
    """
    Background subtractor
    Uses median and homology to perform background subtraction
    window_size: integer. Size of window to keep track of for background model calculation
    """
    def __init__(self, window_size = 5):
        self.tracker = LK_homo_tracker()
        self.court_finder = CourtFinder()
        self.window_size =5
        self.init_frame()

    def run_test(self, video_path):
        video = cv2.VideoCapture(video_path)
        backgrounds_gray = []
        acounter = 0

        while True:
            acounter += 1
            ret, frame = video.read()
            if not ret:
                break
            cols, rows, *_ = frame.shape
            frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            # bgmed_gray = np.zeros((rows, cols))

            mask =  255 - self.court_finder.process_frame(frame)
            pts, homography = self.tracker.track_frame(frame, mask)
            if acounter > self.window_size:
                for i in range(-self.window_size, 0):
                    backgrounds_gray[i] = cv2.warpPerspective(backgrounds_gray[i], homography, (rows, cols))
                bgmed_gray = np.median(backgrounds_gray[-self.window_size:], axis = 0)
            backgrounds_gray.append(frame_gray)

            cv2.imshow("original", frame)
            if acounter > self.window_size:
                cv2.imshow("subtracted_-2", subtract(frame_gray, backgrounds_gray[-2]))
                cv2.imshow("subtracted_med", subtract(frame_gray, bgmed_gray))
            k = cv2.waitKey(10)
            if k == 27:
                print("Terminated by user")
                exit()

    def run(self, video_path, filter_funct = lambda x: x, startfrom = 0):
        video = cv2.VideoCapture(video_path)
        backgrounds_gray = []
        acounter = 0

        while True:
            acounter += 1

            ret, frame = video.read()
            if acounter < startfrom:
                continue
            if not ret:
                break
            cols, rows, *_ = frame.shape
            frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            # bgmed_gray = np.zeros((rows, cols))

            mask =  255 - self.court_finder.process_frame(frame)
            pts, homography = self.tracker.track_frame(frame, mask)
            if acounter - startfrom > self.window_size:
                for i in range(-self.window_size, 0):
                    backgrounds_gray[i] = cv2.warpPerspective(backgrounds_gray[i], homography, (rows, cols))
                bgmed_gray = np.median(backgrounds_gray[-self.window_size:], axis = 0)
            backgrounds_gray.append(frame_gray)

            if acounter - startfrom > self.window_size:
                yield (filter_funct(subtract(frame_gray, bgmed_gray)), frame_gray, frame)

    def run_frames(self, frames, filter_funct = lambda x: x, startfrom = 0):
        backgrounds_gray = []
        for counter, frame in enumerate(frames):
            cols, rows, *_ = frame.shape
            frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            mask = 255 - self.court_finder.process_frame(frame)
            pts, homography = self.tracker.track_frame(frame, mask)
            if counter - startfrom > self.window_size:
                for i in range(-self.window_size, 0):
                    backgrounds_gray[i] = cv2.warpPerspective(backgrounds_gray[i], homography, (rows, cols))
                bgmed_gray = np.median(backgrounds_gray[-self.window_size:], axis = 0)
            backgrounds_gray.append(frame_gray)
            if counter - startfrom > self.window_size:
                yield (filter_funct(subtract(frame_gray, bgmed_gray)), frame_gray, frame)

    def init_frame(self):
        self.backgrounds_gray = []
        self.counter = 0

    def run_frame(self, frame, filter_funct = lambda x: x):
        self.counter += 1
        cols, rows, *_ = frame.shape
        frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        mask = 255 - self.court_finder.process_frame(frame)
        pts, homography = self.tracker.track_frame(frame, mask)
        if self.counter > self.window_size:
            for i in range(-self.window_size, 0):
                self.backgrounds_gray[i] = cv2.warpPerspective(self.backgrounds_gray[i], homography, (rows, cols))
            bgmed_gray = np.median(self.backgrounds_gray[-self.window_size:], axis = 0)
            # cv2.imshow("bgmed_gray", np.uint8(bgmed_gray))
        self.backgrounds_gray.append(frame_gray)
        if self.counter > self.window_size:
            return (filter_funct(subtract(frame_gray, bgmed_gray)), frame_gray, frame)
        return None

class BallFinder:
    def __init__(self, filter_funct = er_filter, window_size = 5, svm_path = "data/svm/SVM_HN_second.pkl"):
        self.filter_funct = filter_funct
        self.bgsub = HomoBG(window_size = 5)
        self.init_frame()
        self.svm = joblib.load(svm_path)

    def init_frame(self):
        self.bgsub.init_frame()

    def run_frame(self, frame):
        res = self.bgsub.run_frame(frame, filter_funct = self.filter_funct)
        if not res:
            return None
        subbed, gray, frame = res

        subbed[0:5, :] = 0
        subbed[:, 628:632] = 0
        subbed[:, 0:7] = 0
        subbed[295:300, :] = 0
        subbed[250:300, 532:632] = 0
        cv2.imshow("subbed", subbed)

        circles = cv2.HoughCircles(subbed, cv2.HOUGH_GRADIENT, 1, 20, param1=100, param2 =10, minRadius =1, maxRadius = 7)

        if circles is not None:
            res = []
            for x,y,r in circles[0]:
                im = gray[y-25:y+25, x-25:x+25]
                if im.shape != (50, 50):
                    print("Skip_shape")
                    continue
                features = hog(im).reshape(1, -1)
                if len(features[0]) != 1296:
                    print("Skip_features", len(features[0]))
                    continue
                if self.svm.predict(features)[0]:
                    res.append((x,y,r))
            return res
        else:
            # print(circles)
            return circles

if __name__ == "__main__":
    a = HomoBG(window_size = 3)
    #a.run_test()
    startfrom = 0
    video_path = "data/beachVolleyball4.mov"
    output_dir = "output/tests/bv3"
    tt_dir = "data/training_data/bgsub/bv4"
    svm = joblib.load("data/svm/SVM_HN_second.pkl")
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)

    if not os.path.exists(tt_dir):
        os.makedirs(tt_dir)

    video = cv2.VideoCapture(video_path)

    for i in range(startfrom):
        ret, frame = video.read()
    f_video = cv2.VideoCapture(video_path)

    #Initialize frames for testing
    frames = []
    while True:
        ret, frame = f_video.read()
        if not ret or frame == None:
            break
        frames.append(frame)

    bf = BallFinder(window_size = 3)

    for frame in frames:
        circles = bf.run_frame(frame)
        if circles is not None:
            for (x,y,r) in circles:
                cv2.circle(frame, (x,y), r, (0,0,255), 2)

        cv2.imshow("draw", frame)
        k = cv2.waitKey(10)
        if k == 27:
            print("Terminated by user")
            exit()

#     # frame
#     for num, (i, gframe, frame) in enumerate(a.run_frames(frames, filter_funct = er_filter, startfrom = startfrom)):
#     # mov
#     # for num, (i, gframe, frame) in enumerate(a.run(video_path, filter_funct = er_filter, startfrom = startfrom)):
#         oframe = frame.copy()
#         #handle the edges because of homology bs
#         i[0:5, :] = 0
#         i[:, 628:632] = 0
#         i[:, 0:7] = 0
#         i[295:300, :] = 0

#         frame[0:5, :] = 0
#         frame[:, 628:632] = 0
#         frame[:, 0:7] = 0
#         frame[295:300, :] = 0

#         i[250:300, 532:632] = 0
#         frame[250:300, 532:632] = 0

#         cv2.imshow("test", i)
#         circles = cv2.HoughCircles(i, cv2.HOUGH_GRADIENT, 1, 20, param1=100, param2 =10, minRadius =1, maxRadius = 7)
#         if circles is not None:
#             print(circles)
#             for (x, y, r) in circles[0]:
#                 target_image = cv2.cvtColor(frame[y-25:y+25, x-25:x+25], cv2.COLOR_BGR2GRAY)
#                 if target_image is None or target_image.shape != (50, 50):
#                     continue
#                 if svm.predict(hog(target_image).reshape(1, -1))[0]:
#                     cv2.circle(frame, (x,y), r, (0, 0, 255), 2)
# #                cv2.circle(frame, (x,y), r, (0, 0, 255), 2)
#         cv2.imshow("draw", frame)
#         #Write outputs
#         # cv2.imwrite(os.path.join(output_dir, "{}.png".format(str(num).zfill(4))), frame)

#         #Write train/test data
#         try:
#             if circles is not None:
#                 print(circles)
#                 for (x, y, r) in circles[0]:
#                     image = oframe[y-50:y+50, x-50:x+50]
#                     image2 = oframe.copy()
#                     cv2.circle(image2, (x, y), 4, (0, 0, 255), 3)
#                     image2 = image2[y-50:y+50, x-50:x+50]
#                     # cv2.imshow("image", image2)
#                     # cv2.imwrite(os.path.join(tt_dir, "{}.png".format(str(num).zfill(4))), image)
#                     # cv2.imwrite(os.path.join(tt_dir, "{}_drawn.png".format(str(num).zfill(4))), image2)

#         except Exception as err:
#             print(err, x, y, frame.shape)

#         k = cv2.waitKey(10)
#         if k == 27:
#             print("Terminated by user")
#             exit()