#!/usr/bin/env python
from copy import deepcopy
from warnings import warn
from functools import partial
from multiprocessing import cpu_count

import numpy as np
from scipy.sparse import coo_matrix
from scipy.special import digamma, gammaln, psi
try:
    from scipy.misc import logsumexp
except ImportError:
    from scipy.special import logsumexp

from sklearn.base import BaseEstimator
import joblib
from joblib import Parallel, delayed

# TODO warn if can't import, and allow computation with slow
from schpf.hpf_numba import *
from schpf.util import minibatch_ix_generator
import schpf.loss as ls
import schpf

class HPF_Gamma(object):
    """Gamma variational distributions

    Parameters
    ----------
    vi_shape: np.ndarray
        Gamma shape parameter for the variational Gamma distributions.
        Ndarray.shape[0] must match `vi_rate`
    vi_rate: np.ndarray
        Gamma rate parameter for the variational Gamma distributions.
        Ndarray.shape[0] must match `vi_shape`

    Attributes
    ----------
    vi_shape : ndarray
    vi_rate : ndarray
    dims : ndarray
        The shape of vi_shape and vi_rate
    dtype : dtype
        dtype of both vi_shape and vi_rate
    """

    @staticmethod
    def random_gamma_factory(dims, shape_prior, rate_prior, dtype=np.float64):
        """Factory method to randomly initialize variational distributions

        Parameters
        ----------
        dims: list-like
            Numpy-style shape of the matrix of Gammas.
        shape_prior: float
            Prior for variational Gammas' shapes.  Must be greater than 0.
        rate_prior: float
            Prior for variational Gammas' rates.  Must be greater than 0.

        Returns
        -------
            A randomly initialized HPF_Gamma instance
        """
        vi_shape = np.random.uniform(0.5 * shape_prior, 1.5 * shape_prior,
                                     dims).astype(dtype)
        vi_rate  = np.random.uniform(0.5 * rate_prior, 1.5 * rate_prior,
                                     dims).astype(dtype)
        return HPF_Gamma(vi_shape,vi_rate)

    def __init__(self, vi_shape, vi_rate):
        """Initializes HPF_Gamma with variational shape and rates"""
        assert(vi_shape.shape == vi_rate.shape)
        assert(vi_shape.dtype == vi_rate.dtype)
        assert(np.all(vi_shape > 0))
        assert(np.all(vi_rate > 0))
        self.vi_shape = vi_shape
        self.vi_rate = vi_rate
        self.dtype = vi_shape.dtype

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            shape_equal = np.array_equal(self.vi_shape, other.vi_shape)
            rate_equal = np.array_equal(self.vi_rate, other.vi_rate)
            dtype_equal = self.dtype == other.dtype
            return shape_equal and rate_equal and dtype_equal
        return False

    @property
    def dims(self):
        assert self.vi_shape.shape == self.vi_rate.shape
        return self.vi_shape.shape

    @property
    def e_x(self):
        """Expected value of the random variable(s) given variational
        distribution(s)
        """
        return self.vi_shape / self.vi_rate

    @property
    def e_logx(self):
        """Expectation of the log of random variable given variational
        distribution(s)"""
        return digamma(self.vi_shape) - np.log(self.vi_rate)

    @property
    def entropy(self):
        """Entropy of variational Gammas"""
        return  self.vi_shape - np.log(self.vi_rate) \
                + gammaln(self.vi_shape) \
                + (1 - self.vi_shape) * digamma(self.vi_shape)

    def sample(self, nsamples=1):
        """Sample from variational distributions

        Parameters
        ----------
        nsamples: int (optional, default 1)
            Number of samples to take.

        Returns
        -------
        X_rep : np.ndarray
            An ndarray of samples from the variational distributions, where
            the last dimension is the number of samples `nsamples`
        """
        samples = []
        for i in range(nsamples):
            samples.append(np.random.gamma(self.vi_shape, 1/self.vi_rate).T)
        return np.stack(samples).T

    def combine(self, other, other_ixs):
        """ Combine with another HPF_Gamma

        Useful for combining variational distributions from training data with
        variational distributions from cells that were projected onto frozen
        beta and eta

        Parameters
        ----------
        other : `HPF_Gamma`
            Other HPF_Gamma to merge with
        other_ixs : list or ndarray
            Ordered indices of other in the merged HPF_Gamma. Must have len
            equal to other.shape[0]. Must have a maximum value less than
            self.dims[0] + other.shape[0]. May not have any repeat values.

        Returns
        -------
        combined_model : `HPF_Gamma`
        """
        assert other.dims[0] == len(other_ixs)
        assert len(np.unique(other_ixs)) == len(other_ixs)
        assert self.dims[0] + other.dims[0] > np.max(other_ixs)

        new_dims = [self.dims[0]+other.dims[0], *self.dims[1:]]
        self_ixs = np.setdiff1d(np.arange(new_dims[0]),
                other_ixs)

        new_vi_shape = np.empty(new_dims, dtype=self.dtype)
        new_vi_shape[self_ixs] = self.vi_shape
        new_vi_shape[other_ixs] = other.vi_shape

        new_vi_rate = np.empty(new_dims, dtype=self.dtype)
        new_vi_rate[self_ixs] = self.vi_rate
        new_vi_rate[other_ixs] = other.vi_rate

        return HPF_Gamma(new_vi_shape, new_vi_rate)

class scHPF(BaseEstimator):
    """scHPF as described in Levitin et al., Molecular Systems Biology 2019

    Parameters
    ----------
    nfactors: int
        Number of factors (K)
    a: float, (optional, default 0.3)
        Hyperparameter a
    ap: float (optional, default 1.0)
        Hyperparameter a'
    bp: float (optional, default None)
        Hyperparameter b'. Set empirically from observed data if not
        given.
    c: float, (optional, default 0.3)
        Hyperparameter c
    cp: float (optional, default 1.0)
        Hyperparameter c'
    dp: float (optional, default None)
        Hyperparameter d'. Set empirically from observed data if not
        given.
    min_iter: int (optional, default 30):
        Minimum number of interations for training.
    max_iter: int (optional, default 1000):
        Maximum number of interations for training.
    check_freq: int (optional, default 10)
        Number of training iterations between calculating loss.
    epsilon: float (optional, default 0.001)
        Percent change of loss for convergence.
    better_than_n_ago: int (optional, default 5)
        Stop condition if loss is getting worse.  Stops training if loss
        is worse than `better_than_n_ago`*`check_freq` training steps
        ago and getting worse.
    xi: HPF_Gamma (optional, default None)
        Variational distributions for xi
    theta: HPF_Gamma (optional, default None)
        Variational distributions for theta
    eta: HPF_Gamma (optional, default None)
        Variational distributions for eta
    beta: HPF_Gamma (optional, default None)
        Variational distributions for beta
    verbose: bool (optional, default True)
            Print messages at each check_freq
    """
    def __init__(
            self,
            nfactors,
            a=0.3,
            ap=1,
            bp=None,
            c=0.3,
            cp=1,
            dp=None,
            min_iter=30,
            max_iter=1000,
            check_freq=10,
            epsilon=0.001,
            better_than_n_ago=5,
            dtype=np.float64,
            xi=None,
            theta=None,
            eta=None,
            beta=None,
            loss=[],
            verbose=True,
            ):
        """Initialize HPF instance"""
        self.version = schpf.__version__
        self.nfactors = nfactors
        self.a = a
        self.ap = ap
        self.bp = bp
        self.c = c
        self.cp = cp
        self.dp = dp
        self.min_iter = min_iter
        self.max_iter = max_iter
        self.check_freq = check_freq
        self.epsilon = epsilon
        self.better_than_n_ago = better_than_n_ago
        self.dtype = dtype
        self.verbose = verbose

        self.xi = xi
        self.eta = eta
        self.theta = theta
        self.beta = beta

        self.loss = []

    @property
    def a(self):
        try:
            return self._a
        except AttributeError:
            msg = 'Automatically using a=0.3. If you are loading a model'
            msg += ' generated with scHPF version < 0.5 and set a custom value'
            msg += ' for a, you must manually reset it and re-save the model.'
            warn(msg, RuntimeWarning)
            return 0.3

    @a.setter
    def a(self, val):
        if val == -2:
            if self.nfactors is None:
                raise ValueError('Can only set a as a function of nfactors when'
                        ' nfactors is not None')
            else:
                self._a = 1/np.sqrt(self.nfactors)
        else:
            assert val > 0
            self._a = val

    @property
    def c(self):
        try:
            return self._c
        except AttributeError:
            msg = 'Automatically using c=0.3. If you are loading a model '
            msg += ' generated with scHPF version < 0.5 and set a custom value'
            msg += ' for c, you must manually reset it and re-save the model.'
            warn(msg, RuntimeWarning)
            return 0.3

    @c.setter
    def c(self, val):
        if val == -2:
            if self.nfactors is None:
                raise ValueError('Can only set a as a function of nfactors when'
                        ' nfactors is not None')
            else:
                self._c = 1/np.sqrt(self.nfactors)
        else:
            assert val > 0
            self._c = val

    @property
    def ngenes(self):
        return self.eta.dims[0] if self.eta is not None else None

    @property
    def ncells(self):
        return self.xi.dims[0] if self.xi is not None else None

    def cell_score(self, xi=None, theta=None):
        """Get cell score from xi and theta

        Parameters
        ----------
        xi : HPF_Gamma, (optional, default self.xi)
            varitional distributions for xi
        theta : HPF_Gamma, (optional, default self.theta)
            varitional distributions for theta

        Returns
        -------
        cell_score : ndarray
            ncell x nfactor array of cell scores
        """
        xi = self.xi if xi is None else xi
        theta = self.theta if theta is None else theta
        return self._score(xi, theta)

    def gene_score(self, eta=None, beta=None):
        """Get cell score from eta and beta

        Parameters
        ----------
        eta : HPF_Gamma, (optional, default self.eta)
            varitional distributions for eta
        beta : HPF_Gamma, (optional, default self.beta)
            varitional distributions for beta

        Returns
        -------
        gene_score : ndarray
            ngene x nfactor array of cell scores
        """
        eta = self.eta if eta is None else eta
        beta = self.beta if beta is None else beta
        return self._score(eta, beta)

    def pois_llh_pointwise(self, X, theta=None, beta=None):
        """Poisson log-likelihood (for each nonzero data)

        Attempt to use numba/cffi/gsl, use numpy otherwise

        Parameters
        ----------
        X: coo_matrix
            Data to compute Poisson log likelihood of. Assumed to be nonzero.
        theta : HPF_Gamma, optional
            If given, use for theta instead of self.theta
        beta : HPF_Gamma, optional
            If given, use for beta instead of self.beta

        Returns
        -------
        llh: ndarray
        """
        theta = self.theta if theta is None else theta
        beta = self.beta if beta is None else beta
        return ls.pois_llh_pointwise(X=X, theta=theta, beta=beta)

    def cellmean_negative_pois_llh(self, X, theta=None, beta=None):
        """Convenience method for mean negative llh of nonzero entries,
           averaged by cell

        """
        theta = self.theta if theta is None else theta
        assert(theta.vi_shape.shape[0] == X.shape[0])
        beta = self.beta if beta is None else beta
        llh_pointwise= self.pois_llh_pointwise(X=X, theta=theta, beta=beta)

        llh_csr = coo_matrix((-llh_pointwise, (X.row,X.col)), shape=X.shape).tocsr()
        sums = llh_csr.sum(axis=1).A1
        counts = np.diff(llh_csr.indptr)
        averages = sums/counts

        assert(averages.shape[0] == theta.vi_shape.shape[0])
        return averages

    def mean_negative_pois_llh(self, X, theta=None, beta=None, **kwargs):
        """Convenience method for mean negative llh of nonzero entries

        """
        theta = self.theta if theta is None else theta
        beta = self.beta if beta is None else beta
        return ls.mean_negative_pois_llh(X=X, theta=theta, beta=beta)

    def fit(self, X, **kwargs):
        """Fit an scHPF model

        Parameters
        ----------
        X: coo_matrix
            Data to fit
        loss_function : function, optional (Default: None)
            loss function to use for fit. set to negative poisson likelihood
            of X if not given
        """
        (bp, dp, xi, eta, theta, beta, loss) = self._fit(
                X, **kwargs)
        self.bp = bp
        self.dp = dp
        self.xi = xi
        self.eta = eta
        self.theta = theta
        self.beta = beta
        self.loss = loss
        return self

    def project(self, X, recalc_bp=False, replace=False, min_iter=2, max_iter=50,
            check_freq=2, **kwargs):
        """Project new cells into latent space

        Gene distributions (beta and eta) are fixed.

        Parameters
        ----------
        X: coo_matrix
            Data to project.  Should have self.ngenes columns
        recalc_bp : bool, optional (Default: False)
            Recalculated value of empirical hyperparameter bp. Do not do this
            for withheld text data.
        replace: bool, optional (Default: False)
            Replace theta and xi with projected values in self. Note that
            loss will not be updated
        min_iter: int, (Default: 2)
            Replaces self.min_iter if not None. Few iterations are needed
            because beta and eta are fixed.
        max_iter: int, (Default: 10)
            Replaces self.max_iter if not None. Few iterations are needed
            because beta and eta are fixed.
        check_freq: int, optional (Default: 2)
            Number of training iterations between calculating loss.

        Returns
        -------
        result : scHPF or ndarray
            If replace=`False`, an  scHPF object with variational
            distributions theta and xi (for the new cells in `X`) and the
            same variational distributions as self for gene distributions
            beta and eta. If replace=`True`, then the loss for the projection
            (xi and theta will be updated in self but not returned). In both
            cases, bp will only be updated for the new data if self.bp==None or
            recalc_bp=`True`.

        """
        if replace and recalc_bp:
            msg = 'Cannot replace `bp` with recalculated value'
            raise ValueError(msg)

        model = self if replace else deepcopy(self)
        if recalc_bp: model.bp = None
        (bp, _, xi, _, theta, _, loss) = model._fit(X,
                min_iter=min_iter, max_iter=max_iter, check_freq=check_freq,
                freeze_genes=True, **kwargs)
        if replace:
            self.xi = xi
            self.theta = theta
            return loss
        else:
            model.bp = bp
            model.xi = xi
            model.theta = theta
            model.loss = loss
            return model

    def _score(self, capacity, loading):
        """Get the hierarchically normalized loadings which we call the cell
        or gene score in the scHPF paper

        Parameters
        ----------
        capacity : HPF_Gamma
            xi or eta
        loading : HPF_Gamma
            theta or beta

        Returns
        -------
        score : ndarray
        """
        assert(loading.dims[0] == capacity.dims[0])
        return loading.e_x * capacity.e_x[:,None]

    def _fit(self, X, freeze_genes=False, reinit=True, loss_function=None,
            min_iter=None, max_iter=None, epsilon=None, check_freq=None,
            single_process=False, checkstep_function=None, verbose=None,
            batchsize=None, beta_theta_simultaneous=False,
            loss_smoothing=1):
        """Combined internal fit/transform function

        Parameters
        ----------
        X: coo_matrix
            Data to fit
        freeze_genes: bool, (optional, default False)
            Should we update gene variational distributions eta and beta
        reinit: bool, (optional, default True)
            Randomly initialize variational distributions even if they
            already exist. Superseded by freeze_genes. Does not affect
            self.bp and self.dp which will only be set empirically if they
            are None
        loss_function : function, (optional, default None)
            Function to use for loss, which is assumed to be nonzero and
            decrease with improvement. Must accept hyperparameters a, ap,
            bp, c, cp, and dp and the variational distributions for xi, eta,
            theta, and beta even if only some of these values are used.
            Should have an internal reference to any data used (_fit will
            not pass it any data). If `loss_function` is not given or equal
            to None, the mean negative log likelihood of nonzero values in
            training data `X` is used.
        min_iter: int (optional, default None)
            Replaces self.min_iter if given.  Useful when projecting
            new data onto an existing scHPF model.
        max_iter: int (optional, default None)
            Replaces self.max_iter if given.  Useful when projecting
            new data onto an existing scHPF model.
        epsilon: float (optional, default None)
            Replaces self.epsilon if given. Percent change of loss for
            convergence.
        check_freq : int, optional (Default: None)
            Replaces self.check_freq if given.  Useful when projecting
            new data onto an existing scHPF model.
        single_process : bool, optional (Default: False)
            Use single-threaded versions of updates
        checkstep_function : function  (optional, default None)
            A function that takes arguments bp, dp, xi, eta, theta, beta,
            and t and, if given, is called at check_interval. Intended use
            is to check additional stats during training, potentially with
            hardcoded data, but is unrestricted.  Use at own risk.
        verbose: bool (optional, default None)
            If not None, overrides self.verbose
        batchsize: int, optional (Default 0)
            number of cells per batch. When 0, all cells are used
        beta_theta_simultaneous: bool, option (Default False)
            Should updates for beta and theta be computed simultaneously.
            If False, beta is updated first, and theta is updated using
            that beta
        loss_smoothing: int, optional (Default: 1)
            Smooth loss up to `loss_smoothing` check frequencies ago. 1 results
            in no smoothing. Intended to be used with batching when assessing
            convergence based on training loss, where a good value might be
            int(ncells/n_batches)

        Returns
        -------
        bp: float
            Empirically set value for bp
        dp: float
            Empirically set value for dp. Unchanged if freeze_genes.
        xi: HPF_Gamma
            Learned variational distributions for xi
        eta: HPF_Gamma
            Learned variational distributions for eta. Unchanged if
            freeze_genes.
        theta: HPF_Gamma
            Learned variational distributions for theta
        beta: HPF_Gamma
            Learned variational distributions for beta. Unchanged if
            freeze_genes.
        loss : list
            loss at each checkstep
        """
        assert loss_smoothing > 0

        # local (convenience) vars for model
        nfactors, (ncells, ngenes) = self.nfactors, X.shape
        a, ap, c, cp = self.a, self.ap, self.c, self.cp

        # get empirically set hyperparameters and variational distributions
        bp, dp, xi, eta, theta, beta = self._setup(X, freeze_genes, reinit)

        # Make first updates for hierarchical shape prior
        # (vi_shape is constant, but want to update full distribution)
        xi.vi_shape[:] = ap + nfactors * a
        if not freeze_genes:
            eta.vi_shape[:] = cp + nfactors * c

        # setup loss function as mean negative llh of nonzero training data
        # if the loss function is not given
        if loss_function is None:
            loss_function = ls.loss_function_for_data(
                    ls.mean_negative_pois_llh, X)

        # setup batch_ix iterator
        if batchsize is not None and batchsize > 1 and batchsize <= ncells:
            batched = True
            batch_ix_generator = minibatch_ix_generator(ncells, batchsize)
        else:
            batched = False
            batch_ix_generator = None

        ## init
        loss, unsmoothed_loss, pct_change = [], [], []
        # check variable overrides
        min_iter = self.min_iter if min_iter is None else min_iter
        max_iter = self.max_iter if max_iter is None else max_iter
        epsilon = self.epsilon if epsilon is None else epsilon
        check_freq = self.check_freq if check_freq is None else check_freq
        verbose = self.verbose if verbose is None else verbose
        for t in range(max_iter):
            # setup batching
            if batch_ix_generator is None:
                batch_ix = np.arange(X.shape[0])
                batchsize = ncells
                X_batch = X
            else:
                batch_ix = next(batch_ix_generator)
                X_batch = X.tocsr()[batch_ix,:].tocoo()

            if t==0 and reinit: #randomize phi for first iteration
                random_phi = np.random.dirichlet( np.ones(nfactors),
                        X_batch.data.shape[0])
                Xphi_data = X_batch.data[:,None] * random_phi
            else:
                if single_process:
                    Xphi_data = compute_Xphi_data_numpy(X_batch, theta, beta,
                            theta_ix=batch_ix)
                else:
                    Xphi_data = compute_Xphi_data(
                            X_batch.data, X_batch.row, X_batch.col,
                            theta.vi_shape[batch_ix], theta.vi_rate[batch_ix],
                            beta.vi_shape, beta.vi_rate)

            if beta_theta_simultaneous:
                # calculate gene updates but don't assign yet
                if not freeze_genes:
                    bvs = compute_loading_shape_update(Xphi_data,
                            X_batch.col, ngenes, c)
                    bvr = compute_loading_rate_update(eta.vi_shape,
                            eta.vi_rate, theta.vi_shape[batch_ix],
                            theta.vi_rate[batch_ix])
                # cell updates
                theta.vi_shape[batch_ix] = compute_loading_shape_update(
                        Xphi_data, X_batch.row, batchsize, a)
                theta.vi_rate[batch_ix] = compute_loading_rate_update(
                        xi.vi_shape[batch_ix], xi.vi_rate[batch_ix],
                        beta.vi_shape, beta.vi_rate)
                xi.vi_rate[batch_ix] = bp + theta.e_x[batch_ix].sum(1)
                # make gene updates
                if not freeze_genes:
                    beta.vi_shape = bvs
                    beta.vi_rate = bvr
                    eta.vi_rate = dp + beta.e_x.sum(1)

            else:
                if batched:
                    # cell updates, must do first for batching
                    theta.vi_shape[batch_ix] = compute_loading_shape_update(
                            Xphi_data, X_batch.row, batchsize, a)
                    theta.vi_rate[batch_ix] = compute_loading_rate_update(
                            xi.vi_shape[batch_ix], xi.vi_rate[batch_ix],
                            beta.vi_shape, beta.vi_rate)
                    xi.vi_rate[batch_ix] = bp + theta.e_x[batch_ix].sum(1)

                if not freeze_genes:
                    #gene updates
                    beta.vi_shape = compute_loading_shape_update(Xphi_data,
                            X_batch.col, ngenes, c)
                    beta.vi_rate = compute_loading_rate_update(eta.vi_shape,
                            eta.vi_rate, theta.vi_shape[batch_ix],
                            theta.vi_rate[batch_ix])
                    eta.vi_rate = dp + beta.e_x.sum(1)

                if not batched:
                    # cell updates, doing after gene updates when not batched
                    # for legacy consistency
                    theta.vi_shape[batch_ix] = compute_loading_shape_update(
                            Xphi_data, X_batch.row, batchsize, a)
                    theta.vi_rate[batch_ix] = compute_loading_rate_update(
                            xi.vi_shape[batch_ix], xi.vi_rate[batch_ix],
                            beta.vi_shape, beta.vi_rate)
                    xi.vi_rate[batch_ix] = bp + theta.e_x[batch_ix].sum(1)

            # record llh/percent change and check for convergence
            if t % check_freq == 0:

                # chech llh
                # vX = validation_data if validation_data is not None else X
                try :
                    curr = loss_function(
                                a=a, ap=ap, bp=bp, c=c, cp=cp, dp=dp,
                                xi=xi, eta=eta, theta=theta, beta=beta)
                    unsmoothed_loss.append(curr)
                    if len(unsmoothed_loss) > loss_smoothing:
                        unsmoothed_loss = unsmoothed_loss[1:]
                    # normally this is just curr as loss_smoothing=1 by default
                    loss.append(np.mean(unsmoothed_loss))
                except NameError as e:
                    print('Invalid loss function')
                    raise e

                # calculate percent change
                try:
                    curr, prev = loss[-1], loss[-2]
                    pct_change.append(100 * (curr - prev) / np.abs(prev))
                except IndexError:
                    pct_change.append(100)
                if verbose:
                    msg = '[Iter. {0: >4}]  loss:{1:.6f}  pct:{2:.9f}'.format(
                            t, curr, pct_change[-1])
                    print(msg)
                if checkstep_function is not None:
                    checkstep_function(bp=bp, dp=dp, xi=xi, eta=eta, theta=theta,
                            beta=beta, t=t)

                # check convergence
                if len(loss) > 3 and t >= min_iter:
                    # convergence conditions (all must be met)
                    current_small = np.abs(pct_change[-1]) < self.epsilon
                    prev_small = np.abs(pct_change[-2]) < self.epsilon
                    not_inflection = not (
                            (np.abs(loss[-3]) < np.abs(prev)) \
                            and (np.abs(prev) > np.abs(curr)))
                    converged = current_small and prev_small and not_inflection
                    if converged:
                        if verbose:
                            print('converged')
                        break

                    # getting worse, and has been for better_than_n_ago checks
                    # (don't waste time on a bad run)
                    if len(loss) > self.better_than_n_ago \
                            and self.better_than_n_ago:
                        nprev = loss[-self.better_than_n_ago] \
                                if len(loss)>self.better_than_n_ago else loss[0]
                        worse_than_n_ago = np.abs(nprev) < np.abs(curr)
                        getting_worse = np.abs(prev) < np.abs(curr)
                        if worse_than_n_ago and getting_worse:
                            if verbose:
                                print('getting worse break')
                            break

            # TODO message or warning or something
            if t >= self.max_iter:
                break

        return (bp, dp, xi, eta, theta, beta, loss)

    def _setup(self, X, freeze_genes=False, reinit=True, clip=True):
        """Setup variational distributions

        Parameters
        ----------
        X: coo_matrix
            Data to fit
        freeze_genes: bool, optional (Default: False)
            Should we update gene variational distributions eta and beta
        reinit: bool, optional (Default: True)
            Randomly initialize variational distributions even if they
            already exist. Superseded by freeze_genes. Does not affect
            self.bp and self.dp (which will only be set empirically if
            they are None)
        clip : bool, optional (Default: True)
            If empirically calculating dp and bp > 1000 * dp, clip dp to
            bp / 1000.

        Returns
        -------
        bp : float
        dp : float
        xi : HPF_Gamma
        eta : HPF_Gamma
        theta : HPF_Gamma
        beta : HPF_Gamma

        """
        # locals for convenience
        nfactors, (ncells, ngenes) = self.nfactors, X.shape
        a, ap, c, cp = self.a, self.ap, self.c, self.cp
        bp, dp = self.bp, self.dp

        xi, eta, theta, beta = (self.xi, self.eta, self.theta, self.beta)

        # empirically set bp and dp
        bp, dp = self._get_empirical_hypers(X, freeze_genes, clip)

        if reinit or (xi is None):
            xi = HPF_Gamma.random_gamma_factory((ncells,), ap, bp,
                    dtype=self.dtype)
        if reinit or (theta is None):
            theta = HPF_Gamma.random_gamma_factory((ncells,nfactors), a, bp,
                    dtype=self.dtype)

        # Check if variational distributions for genes exist, create if not
        # Error if freeze_genes and eta and beta don't exists
        if freeze_genes:
            if eta is None or beta is None:
                msg = 'To fit with frozen gene variational distributions ' \
                    + '(`freeze_genes`==True), eta and beta must be set to ' \
                    + 'valid HPF_Gamma instances.'
                raise ValueError(msg)
        else:
            if reinit or (eta is None):
                eta = HPF_Gamma.random_gamma_factory((ngenes,), cp, dp,
                        dtype=self.dtype)
            if reinit or (beta is None):
                beta = HPF_Gamma.random_gamma_factory((ngenes,nfactors),
                        c, dp, dtype=self.dtype)

        return (bp, dp, xi, eta, theta, beta)

    def _get_empirical_hypers(self, X, freeze_genes=False, clip=True):
        """Get empirical values for bp, dp

        Parameters
        ----------
        X : coo_matrix
            Data to fit

        Returns
        -------
        bp : float
        dp : float
        """
        bp, dp = self.bp, self.dp
        # empirically set bp and dp
        def mean_var_ratio(X, axis):
            axis_sum = X.sum(axis=axis)
            return np.mean(axis_sum) / np.var(axis_sum)
        if bp is None:
            bp = self.ap * mean_var_ratio(X, axis=1)
        if dp is None: # dp first in case of error
            if freeze_genes:
                msg = 'dp is None and cannot be set'
                msg += ' when freeze_genes is True.'
                raise ValueError(msg)
            else:
                dp = self.cp *  mean_var_ratio(X, axis=0)
                if clip and bp > 1000 * dp:
                    old_val = dp
                    dp = bp / 1000
                    print('Clipping dp: was {} now {}'.format(old_val, dp))

        return bp, dp

    def _initialize(self, X, freeze_genes=False):
        """Shortcut to setup random distributions & set variables
        """
        bp, dp, xi, eta, theta, beta = self._setup(X, freeze_genes,
                reinit=True)
        self.bp = bp
        self.dp = dp
        self.xi = xi
        self.eta = eta
        self.theta = theta
        self.beta = beta

def load_model(file_name):
    """Load a model from a joblib file

    Parameters
    ----------
    file_name : str
        Joblib file containing a saved scHPF model

    Returns
    -------
    model : scHPF
        The scHPF model in the file
    """
    return joblib.load(file_name)

def save_model(model, file_name):
    """Save model to (joblib) file

    Serialize scHPF model as a joblib file.  Joblib is simillar to pickle,
    but preferable for objects with many numpy arrays

    Parameters
    ----------
    model : scHPF
        The scHPF model object to save
    file_name : str
        Name of file to save model to
    """
    joblib.dump(model, file_name)

def combine_across_cells(x, y, y_ixs):
    """Combine theta & xi from two scHPF instance with the same beta & eta

    Intended to be used combining variational distributions for local
    variables (theta,xi) from training data with variational distributions
    for local variables from validation or other data that was projected
    onto the same global variational distributions (beta,eta)

    If `x.bp` != `y.bp`, returned model `xy.bp` is set to None. All other
    attributes (except for the merged xi and eta) are inherited from `x`.

    Parameters
    ----------
    x : `scHPF`
    y : `scHPF`
        The scHPF instance whose rows in the output should be at the
        corresponding indices `y_ixs`
    y_ixs : ndarray
        Row indices of `y` in the returned distributions. Must be 1-d and
        have same number of rows as `y`, have no repeats, and have no index
        greater than or equal to x.ncells + y.ncells.

    Returns
    -------
    ab : `scHPF`

    """
    assert x.dp == y.dp
    assert x.eta == y.eta
    assert x.beta == y.beta

    xy = deepcopy(x)
    if y.bp != x.bp:
        xy.bp = None
    xy.xi = x.xi.combine(y.xi, y_ixs)
    xy.theta = x.theta.combine(y.theta, y_ixs)
    return xy

def run_trials(X, nfactors,
        ntrials=5,
        min_iter=30,
        max_iter=1000,
        check_freq=10,
        epsilon=0.001,
        better_than_n_ago=5,
        dtype=np.float64,
        verbose=True,
        vcells = None,
        vX = None,
        loss_function=None,
        model_kwargs = {},
        return_all = False,
        reproject = False,
        reproject_kwargs = {},
        batchsize=0,
        beta_theta_simultaneous=False,
        loss_smoothing=1
        ):
    """
    Train with multiple random initializations, selecting model with best loss

    As scHPF uses non-convex optimization, it benefits from training with
    multiple random initializations to avoid local minima.

    Parameters
    ----------
    X: coo_matrix
        Data to fit
    nfactors: int
        Number of factors (K)
    ntrials : int,  optional (Default 5)
        Number of random initializations for training
    min_iter: int, optional (Default 30)
        Minimum number of interations for training.
    max_iter: int, optional (Default 1000):
        Maximum number of interations for training.
    check_freq: int, optional (Default 10)
        Number of training iterations between calculating loss.
    epsilon: float, optional (Default 0.001)
        Percent change of loss for convergence.
    better_than_n_ago: int, optional (Default 5)
        Stop condition if loss is getting worse.  Stops training if loss
        is worse than `better_than_n_ago`*`check_freq` training steps
        ago and getting worse.
    dtype : datatype, optional (Default np.float64)
        np.float64 or np.float32
    verbose: bool, optional (Default True)
        verbose
    vcells : coo_matrix, optional (Default None)
        cells to use in a validation loss function
    vX : coo_matrix, optional (Default None)
        nonzero entries from the cells in vX
    loss_function : function, optional (Default None)
        A loss function to asses convergence that accepts data, model
        variational parameters, and model hyperparameters.  Note this is
        distinct from the `loss_function` argument in scHPF._fit (called by
        scHPF.fit and scHPF.project), which assumes a fixed reference to data
        is included in the function and *does not* accept data as an argument.
    model_kwargs: dict, optional (Default {})
        dictionary of additional keyword arguments for model
        initialization
    return_all: bool, optional (Default False)
        return all models
    reproject: bool, optional (Default False)
        Reproject the data onto the frozen gene variables before calculating
        loss. The reprojected loss will be added to the end of loss as a
        sublist. Note that this reprojection will *not* use the `loss_function`
        argument, and instead use the default provided log likelihood
    reproject_kwargs: dict, optional (Default {'replace':True})
        Only used if `reproject` is True. Keyword args for scHPF.project.
        'replace':True cannot be changed, and will be overwritten if given
    batchsize: int, optional (Defualt 0)
        Number of cells to use per training round. All cells used if 0.
    loss_smoothing: int, optional (Default: 1)
        Smooth loss up to `loss_smoothing` check frequencies ago. 1 results in
        no smoothing. Intended to be used with batching when assessing
        convergence based on training loss, where a good value might be
        int(ncells/n_batches)

    Returns
    -------
    best_model: scHPF
        The model with the best loss facter `ntrials` random initializations
        and training runs
    rejected_models: list, optional
        Rejected models, ordered by decreasing loss . Only returned if
        return_all is True
    """
    ncells, ngenes = X.shape
    if ngenes >= 20000:
        msg = 'WARNING: you are running scHPF with {} genes,'.format(ngenes)
        msg += ' which is more than the ~20k protein coding genes in the'
        msg += ' human genome. We suggest running scHPF on protein-coding'
        msg += ' genes only.'
        print(msg)

    # get the loss function for any data
    if loss_function is None:
        loss_function = partial(ls.mean_negative_pois_llh,
                single_process=False)

    # check data we're using for loss
    if vcells is not None:
        assert X.shape[1] == vcells.shape[1]
    if vX is not None:
        assert vX.shape == X.shape
    else:
        vX = X
    # setup loss fnc w/data (will be overridden if vcells is not None)
    data_loss_function = ls.loss_function_for_data(loss_function, vX)
    # setup smoothed_loss if using batches

    # run trials
    best_loss, best_model, best_t = np.finfo(np.float64).max, None, None
    models, losses = [], [] # only used if return_all
    for t in range(ntrials):
        # make a new model
        model = scHPF(nfactors=nfactors,
                    min_iter=min_iter, max_iter=max_iter,
                    check_freq=check_freq, epsilon=epsilon,
                    better_than_n_ago=better_than_n_ago,
                    verbose=verbose, dtype=dtype,
                    **model_kwargs
                    )

        # override the loss function data if we have vcells
        # (must be redone for each new model)
        if vcells is not None:
            proj_kwargs = dict(reinit=False,
                               min_iter=1,
                               max_iter=min(10, check_freq),
                               check_freq= check_freq+1,
                               verbose=False
                               )
            data_loss_function = ls.projection_loss_function(
                    loss_function, vcells, nfactors,
                    proj_kwargs=proj_kwargs)
            def checkstep_function(**kwargs):
                loss = ls.loss_function_for_data(loss_function, X)
                print('\ttrain:', '{0:.6f}'.format(loss(**kwargs)))
        else:
            checkstep_function = None

        # fit the model
        model.fit(X, loss_function=data_loss_function,
                  checkstep_function=checkstep_function,
                  batchsize=batchsize, loss_smoothing=loss_smoothing,
                  beta_theta_simultaneous=beta_theta_simultaneous)
        if reproject:
            print('Reprojecting data...')
            reproject_kwargs['replace'] = True
            reproject_kwargs['reinit'] = False
            proj_loss = model.project(X, **reproject_kwargs)
            model.loss.append(proj_loss)
            loss = proj_loss[-1]
        else:
            loss = model.loss[-1]

        if loss < best_loss:
            best_model = model
            best_loss = loss
            best_t = t
            if verbose:
                print('New best!'.format(t))
        if return_all:
            models.append(model)
            losses.append(loss)
        if verbose:
            print('Trial {0} loss: {1:.6f}'.format(t, loss))
            print('Best loss: {0:.6f} (trial {1})'.format(best_loss, best_t))

    if return_all:
        return_order = np.argsort(losses)
        ordered_models = [models[i] for i in return_order]
        assert ordered_models[0] == best_model
        return best_model, ordered_models[1:]
    else:
        return best_model

# TODO deal with verbosity
def run_trials_pool(X, nfactors,
        ntrials=5,
        njobs=0,
        max_threads=None,
        min_iter=30,
        max_iter=1000,
        check_freq=10,
        epsilon=0.001,
        better_than_n_ago=5,
        dtype=np.float64,
        verbose=True,
        vcells = None,
        vX = None,
        loss_function=None,
        model_kwargs = {},
        return_all = False,
        reproject = False,
        reproject_kwargs = {},
        batchsize=0,
        beta_theta_simultaneous=False,
        loss_smoothing=1
        ):
    """
    Train with multiple random initializations, selecting model with best loss.
    Parallelization is done at the trial level rather than within computations

    As scHPF uses non-convex optimization, it benefits from training with
    multiple random initializations to avoid local minima.

    Parameters
    ----------
    X: coo_matrix
        Data to fit
    nfactors: int or list of ints
        Number of factors (K), may be a list for multiple k
    ntrials : int,  optional (Default 5)
        Number of random initializations for training
    njobs : int, optional (Default 0)
        Maximum number of threads in the threadpool.  0 will use all available.
    min_iter: int, optional (Default 30)
        Minimum number of interations for training.
    max_iter: int, optional (Default 1000):
        Maximum number of interations for training.
    check_freq: int, optional (Default 10)
        Number of training iterations between calculating loss.
    epsilon: float, optional (Default 0.001)
        Percent change of loss for convergence.
    better_than_n_ago: int, optional (Default 5)
        Stop condition if loss is getting worse.  Stops training if loss
        is worse than `better_than_n_ago`*`check_freq` training steps
        ago and getting worse.
    dtype : datatype, optional (Default np.float64)
        np.float64 or np.float32
    verbose: bool, optional (Default True)
        verbose
    vcells : coo_matrix, optional (Default None)
        cells to use in a validation loss function
    vX : coo_matrix, optional (Default None)
        nonzero entries from the cells in vX
    loss_function : function, optional (Default None)
        A loss function that accepts data, model variational parameters,
        and model hyperparameters.  Note this is distinct from the
        `loss_function` argument in scHPF._fit (called by scHPF.fit and
        scHPF.project), which assumes a fixed reference to data is included
        in the function and *does not* accept data as an argument.
    model_kwargs: dict, optional (Default {})
        dictionary of additional keyword arguments for model
        initialization
    return_all: bool, optional (Default False)
        return all models
    reproject: bool, optional (Default False)
        Reproject the data onto the frozen gene variables before calculating
        loss. The reprojected loss will be added to the end of loss as a
        sublist. Note that this reprojection will *not* use the `loss_function`
        argument, and instead use the default provided log likelihood
    reproject_kwargs: dict, optional (Default {'replace':True})
        Only used if `reproject` is True. Keyword args for scHPF.project.
        'replace':True cannot be changed, and will be overwritten if given
    batchsize: int, optional (Defualt 0)
            Number of cells to use per training round. All cells used if 0.
    loss_smoothing: int, optional (Default: 1)
        Smooth loss up to `loss_smoothing` check frequencies ago. 1 results in
        no smoothing. Intended to be used with batching when assessing
        convergence based on training loss, where a good value might be
        int(ncells/n_batches)

    Returns
    -------
    best_models: list(scHPF)
        The model with the best loss facter `ntrials` random initializations
        and training runs for each value in nfactors
    rejected_models: list(list(scHPF)), optional
        Rejected models, ordered by corresponding nfactors and then by
        decreasing loss . Only returned if return_all is True
    """
    ngenes = X.shape[1]
    if ngenes >= 20000:
        msg = 'WARNING: you are running scHPF with {} genes,'.format(ngenes)
        msg += ' which is more than the ~20k protein coding genes in the'
        msg += ' human genome. We suggest running scHPF on protein-coding'
        msg += ' genes only.'
        print(msg)

    # get the loss function for any data
    if loss_function is None:
        loss_function = partial(ls.mean_negative_pois_llh,
                single_process=True)

    # check data we're using for loss
    if vcells is not None:
        assert X.shape[1] == vcells.shape[1]
    if vX is not None:
        assert vX.shape == X.shape
    else:
        vX = X
    # setup loss fnc w/data (will be overridden if vcells is not None)
    data_loss_function = ls.loss_function_for_data(loss_function, vX)

    # only need to create once because will be copied to processes
    # override the loss function data if we have vcells
    # (must be redone for each new model)
    if vcells is not None:
        proj_kwargs = dict(reinit=False,
                            min_iter=1,
                            max_iter=min(10, check_freq),
                            check_freq= check_freq+1,
                            verbose=False
                            )
        data_loss_function = ls.projection_loss_function(
                loss_function, vcells, nfactors,
                proj_kwargs=proj_kwargs)

    # function to fit model
    def fit_model(nfactors):
        model = scHPF(nfactors=nfactors,
                    min_iter=min_iter, max_iter=max_iter,
                    check_freq=check_freq, epsilon=epsilon,
                    better_than_n_ago=better_than_n_ago,
                    verbose=False, dtype=dtype,
                    **model_kwargs
                    )
        # fit the model
        model.fit(X, loss_function=data_loss_function,
                  checkstep_function=None, single_process=True,
                  batchsize=batchsize, loss_smoothing=loss_smoothing)
        if reproject:
            # print('Reprojecting data...')
            reproject_kwargs['replace'] = True
            proj_loss = model.project(X, loss_function=data_loss_function,
                    **reproject_kwargs)
            model.loss.append(proj_loss)
        return model

    # get nfactors for every trial
    if isinstance(nfactors, int):
        nfactors = [nfactors]
    trial_nfactors = [t for trial_set in [[K]*ntrials for K in nfactors] \
            for t in trial_set]

    # set max processes if not given
    if njobs == 0: njobs = min(cpu_count(), len(trial_nfactors))

    # training
    with Parallel(n_jobs=njobs, verbose=10) as pool: # make the pool
        candidates = pool( delayed(fit_model)(K)  for K in trial_nfactors)

    # get the best model for every K
    ordered_best, ordered_reject = [], []
    for i,K in enumerate(nfactors):
        my_candidates = candidates[i*ntrials : (i+1)*ntrials]
        loss = [m.loss[-1][-1] if reproject else m.loss[-1] for m in
                my_candidates]
        # print(list(zip([m.nfactors for m in my_candidates],loss)))
        best_ix = np.argmin(loss)
        ordered_best.append(my_candidates[best_ix])
        ordered_reject.append([my_candidates[i] for i in np.argsort(loss)[1:]])
    if return_all:
        return ordered_best, ordered_reject
    else:
        return ordered_best


# Ödev : sepet aşamasındaki kullanıcılara ürün önerisinde bulunmak

# Kullanacağımız veri seti : Online Retail II veri seti

# İstenen : Karar kurallarını 2010-2011 Germany müşterileri üzerinden yapacağız.

# İstenen : id1 : 21987 , id2 : 23235 , id3 : 22747

import pandas as pd
pd.set_option("display.max_columns", None)
pd.set_option("display.width", 500)
pd.set_option("display.expand_frame_repr", False)
from mlxtend.frequent_patterns import apriori, association_rules

# Veriyi Tanımlama :

data = pd.read_excel("week-3/online_retail_II.xlsx", sheet_name="Year 2010-2011")
df = data.copy()

########################################################################################################################

# Görev 1: Veri ön işleme

def outlier_thresholds(dataframe, variable):
    quartile1 = dataframe[variable].quantile(0.01)
    quartile3 = dataframe[variable].quantile(0.99)
    interquantile_range = quartile3 - quartile1
    up_limit = quartile3 + 1.5 * interquantile_range
    low_limit = quartile1 - 1.5 * interquantile_range
    return low_limit, up_limit

def replace_with_thresholds(dataframe, variable):
    low_limit, up_limit = outlier_thresholds(dataframe, variable)
    dataframe.loc[(dataframe[variable] < low_limit), variable] = low_limit
    dataframe.loc[(dataframe[variable] > up_limit), variable] = up_limit

def retail_data_prep(dataframe):
    dataframe.dropna(inplace=True)
    dataframe = dataframe[~dataframe["Invoice"].str.contains("C", na=False)]
    dataframe = dataframe[~dataframe["StockCode"].str.contains("POST", na=False)]
    dataframe = dataframe[dataframe["Quantity"] > 0]
    dataframe = dataframe[dataframe["Price"] > 0]
    replace_with_thresholds(dataframe, "Quantity")
    replace_with_thresholds(dataframe, "Price")
    return dataframe

df = retail_data_prep(df)

#####################################################################################

# Görev 2 : Germany müşterileri üzerinden birliktelik kuralları üretiniz.

# BURADA SADECE GERMANY İLE İLGİLENECEĞİZ.

df = df[df["Country"] == "Germany"]

# Verinin gelmesini istediğimiz durum:

# Description   NINE DRAWER OFFICE TIDY   SET 2 TEA TOWELS I LOVE LONDON    SPACEBOY BABY GIFT SET
# Invoice
# 536370                              0                                 1                       0
# 536852                              1                                 0                       1
# 536974                              0                                 0                       0
# 537065                              1                                 0                       0
# 537463                              0                                 0                       1

# Bunu da yapmak için aşağıdaki scripti yazıyoruz.

def create_invoice_product_format( df, id = False):
    if id :
        return df.groupby(["Invoice","StockCode"])["Quantity"].sum().unstack().fillna(0).applymap( lambda x: 1 if x > 0 else 0)
    else:
        return df.groupby(["Invoice","Description"])["Quantity"].sum().unstack().fillna(0).applymap( lambda x: 1 if x > 0 else 0)

# fonksiyonu df üzerinde uyguluyoruz ve istediğimiz invoice-product formatında kullanabiliyoruz.

ge_inv_pro_df = create_invoice_product_format(df, id = True)

# Eğer stockcode ile ürünü gösterirsek bu kod ile ürünün ismine ulaşabiliriz.

def check_id(dataframe, stock_code):
    product_name = dataframe[dataframe["StockCode"] == stock_code][["Description"]].values[0].tolist()
    print(product_name)

######################################################################

# Görev 3 : ID'leri verilen ürünlerin isimleri nelerdir?

print(check_id(df, 21987))
#['PACK OF 6 SKULL PAPER CUPS']
print(check_id(df, 23235))
#['STORAGE TIN VINTAGE LEAF']
print(check_id(df, 22747))
#["POPPY'S PLAYHOUSE BATHROOM"]

#####################################################################################

# Görev 4 : Sepetteki kullanıcılar için ürün önerisi yapınız.

#Birliktelik kuralı

frequent_itemsets = apriori(ge_inv_pro_df, min_support=0.01, use_colnames=True)
frequent_itemsets.sort_values("support", ascending=False).head(50)

rules = association_rules(frequent_itemsets, metric="support", min_threshold=0.01)
rules.sort_values("support", ascending=False).head()

rules.sort_values("lift", ascending=False).head(500)

#Ürün önerme

def arl_recommender(rules_df, product_id, rec_count=1):

    sorted_rules = rules_df.sort_values("lift", ascending=False)
    recommendation_list = []

    for i, product in sorted_rules["antecedents"].items():
        for j in list(product):
            if j == product_id:
                recommendation_list.append(list(sorted_rules.iloc[i]["consequents"]))
    recommendation_list = list({item for item_list in recommendation_list for item in item_list})

    return recommendation_list[:rec_count]

check_id(df,21987)
print(arl_recommender(rules, 21987, 3))
print("*******************************************")
check_id(df,23235)
print(arl_recommender(rules, 23235, 3))
print("*******************************************")
check_id(df,22747)
print(arl_recommender(rules, 22747, 3))
print("*******************************************")

#####################################################################################

# Görev 5 : Önerilen ürünlerin isimleri nelerdir?

# ['PACK OF 6 SKULL PAPER CUPS'] ' önerilen ürünler -------> ['BLUE POLKADOT PLATE '] ,['SET OF 60 PANTRY DESIGN CAKE CASES '] ve ['SPACEBOY BIRTHDAY CARD']

# ['STORAGE TIN VINTAGE LEAF'] '   önerilen ürünler -------> ['RED RETROSPOT MINI CASES'] , ['REGENCY CAKESTAND 3 TIER'] ve ['PLASTERS IN TIN SPACEBOY']

# ["POPPY'S PLAYHOUSE BATHROOM"] '   önerilen ürünler -------> ['JUMBO SHOPPER VINTAGE RED PAISLEY'] , ['PLASTERS IN TIN SPACEBOY'] ve ['PLASTERS IN TIN WOODLAND ANIMALS']


import os
import numpy as np
import PIL.Image as Image
import torch
from torch.utils import data
import pdb
import random

def rotated_rect_with_max_area(w, h, angle):
    """
  Given a rectangle of size wxh that has been rotated by 'angle' (in
  radians), computes the width and height of the largest possible
  axis-aligned rectangle (maximal area) within the rotated rectangle.
  """
    if w <= 0 or h <= 0:
        return 0, 0

    width_is_longer = w >= h
    side_long, side_short = (w, h) if width_is_longer else (h, w)

    # since the solutions for angle, -angle and 180-angle are all the same,
    # if suffices to look at the first quadrant and the absolute values of sin,cos:
    sin_a, cos_a = abs(np.sin(angle)), abs(np.cos(angle))
    if side_short <= 2. * sin_a * cos_a * side_long or abs(sin_a - cos_a) < 1e-10:
        # half constrained case: two crop corners touch the longer side,
        #   the other two corners are on the mid-line parallel to the longer line
        x = 0.5 * side_short
        wr, hr = (x / sin_a, x / cos_a) if width_is_longer else (x / cos_a, x / sin_a)
    else:
        # fully constrained case: crop touches all 4 sides
        cos_2a = cos_a * cos_a - sin_a * sin_a
        wr, hr = (w * cos_a - h * sin_a) / cos_2a, (h * cos_a - w * sin_a) / cos_2a

    return wr, hr

class BaseFolder(data.Dataset):
    def __init__(self, root, crop=None, rotate=None, flip=False,
                 mean=None, std=None):
        super(BaseFolder, self).__init__()
        self.mean, self.std = mean, std
        self.flip = flip
        self.rotate = rotate
        self.crop = crop
        img_dir = os.path.join(root, 'images')
        gt_dir = os.path.join(root, 'masks')
        names = ['.'.join(name.split('.')[:-1]) for name in os.listdir(gt_dir)]
        self.img_filenames = [os.path.join(img_dir, name+'.jpg') for name in names]
        self.gt_filenames = [os.path.join(gt_dir, name+'.png') for name in names]
        self.names = names

    def random_crop(self, *images):
        images = list(images)
        sz = [img.size for img in images]
        sz = set(sz)
        assert(len(sz)==1)
        w, h = sz.pop()
        th, tw = int(self.crop*h), int(self.crop*w)
        if w == tw and h == th:
            return 0, 0, h, w
        i = random.randint(0, h - th)
        j = random.randint(0, w - tw)
        results = [img.crop((j, i, j + tw, i + th)) for img in images]
        return tuple(results)

    def random_flip(self, *images):
        if self.flip and random.randint(0, 1):
            images = list(images)
            results = [img.transpose(Image.FLIP_LEFT_RIGHT) for img in images]
            return tuple(results)
        else:
            return images

    def random_rotate(self, *images):
        images = list(images)
        sz = [img.size for img in images]
        sz = set(sz)
        assert(len(sz)==1)
        w, h = sz.pop()
        degree = random.randint(-1*self.rotate, self.rotate)
        images_r = [img.rotate(degree, expand=1) for img in images]
        w_b, h_b = images_r[0].size
        w_r, h_r = rotated_rect_with_max_area(w, h, np.radians(degree))
        ws = (w_b - w_r) / 2
        ws = max(ws, 0)
        hs = (h_b - h_r) / 2
        hs = max(hs, 0)
        we = ws + w_r
        he = hs + h_r
        we = min(we, w_b)
        he = min(he, h_b)
        results = [img.crop((ws, hs, we, he)) for img in images_r]
        return tuple(results)

    def __len__(self):
        return len(self.names)

class PriorFolder(BaseFolder):
    def __init__(self, root, prior_dir, size=256, crop=None, rotate=None, flip=False,
                 mean=None, std=None):
        super(PriorFolder, self).__init__(root, crop=crop, rotate=rotate, flip=flip, mean=mean, std=std)
        self.size = size
        self.pr_filenames = [os.path.join(prior_dir, name+'.png') for name in self.names]

    def __getitem__(self, index):
        # load image
        img_file = self.img_filenames[index]
        img = Image.open(img_file)
        gt_file = self.gt_filenames[index]
        name = self.names[index]
        gt = Image.open(gt_file)
        WW, HH = gt.size
        img = img.resize((WW, HH))
        pr_file = self.pr_filenames[index]
        pr = Image.open(pr_file)
        pr = pr.resize((WW, HH))
        if self.crop is not None:
            img, gt, pr = self.random_crop(img, gt, pr)
        if self.rotate is not None:
            img, gt, pr = self.random_rotate(img, gt, pr)
        if self.flip:
            img, gt, pr = self.random_flip(img, gt, pr)
        img, gt, pr = [_img.resize((self.size, self.size)) for _img in [img, gt, pr]]
        img = np.array(img, dtype=np.float64) / 255.0
        gt = np.array(gt, dtype=np.uint8)
        gt[gt != 0] = 1
        pr = np.array(pr, dtype=np.float64) / 255.0
        if len(img.shape) < 3:
            img = np.stack((img, img, img), 2)
        if img.shape[2] > 3:
            img = img[:, :, :3]
        if len(gt.shape) > 2:
            gt = gt[:, :, 0]
        if self.mean is not None:
            img -= self.mean
        if self.std is not None:
            img /= self.std
        img = img.transpose(2, 0, 1)
        img = torch.from_numpy(img).float()
        gt = torch.from_numpy(gt).float()
        pr = torch.from_numpy(pr).float()
        return img, gt, pr, name, WW, HH

class Folder(BaseFolder):
    def __init__(self, root, scales=[256], crop=None, rotate=None, flip=False,
                 mean=None, std=None):
        super(Folder, self).__init__(root, crop=crop, rotate=rotate, flip=flip, mean=mean, std=std)
        self.scales = scales

    def __getitem__(self, index):
        # load image
        img_file = self.img_filenames[index]
        img = Image.open(img_file)
        gt_file = self.gt_filenames[index]
        name = self.names[index]
        gt = Image.open(gt_file)
        WW, HH = gt.size
        img = img.resize((WW, HH))
        if self.crop is not None:
            img, gt = self.random_crop(img, gt)
        if self.rotate is not None:
            img, gt = self.random_rotate(img, gt)
        if self.flip:
            img, gt = self.random_flip(img, gt)
        max_size = max(self.scales)
        img = img.resize((max_size, max_size))
        gts = [gt.resize((s, s)) for s in self.scales]

        img = np.array(img, dtype=np.float64) / 255.0
        gts = [np.array(gt, dtype=np.uint8) for gt in gts]
        for gt in gts: gt[gt != 0] = 1
        if len(img.shape) < 3:
            img = np.stack((img, img, img), 2)
        if img.shape[2] > 3:
            img = img[:, :, :3]
        for i, gt in enumerate(gts):
            if len(gt.shape) > 2:
                gts[i] = gt[:, :, 0]
        if self.mean is not None:
            img -= self.mean
        if self.std is not None:
            img /= self.std
        img = img.transpose(2, 0, 1)
        img = torch.from_numpy(img).float()
        gts = [torch.from_numpy(gt).float() for gt in gts]
        return img, gts, name

def collate_more(data):
    images, gts, name = zip(*data)
    gts = list(map(list, zip(*gts)))

    images = torch.stack(images, 0)
    gts = [torch.stack(gt, 0) for gt in gts]

    return images, gts, name

if __name__ == "__main__":
    import matplotlib.pyplot as plt
    from datetime import datetime
    random.seed(datetime.now())
    sb = Folder('/home/zeng/data/datasets/saliency_Dataset/ECSSD',
                     crop=None, rotate=10, flip=True)
    img, gt, _ = sb.__getitem__(random.randint(0, 1000))
    img = img.numpy().transpose((1, 2, 0))
    plt.imshow(img)
    plt.show()
    plt.imshow(gt[0])
    plt.show()

    # sb = PriorFolder('/home/zeng/data/datasets/saliency_Dataset/ECSSD',
    #                  '/home/zeng/data/datasets/saliency_Dataset/results/ECSSD-Sal/SRM',
    #                  crop=None, rotate=None, flip=True, size=256)
    # img, gt, pr, _, _, _ = sb.__getitem__(0)
    # img = img.numpy().transpose((1, 2, 0))
    # plt.imshow(img)
    # plt.show()
    # plt.imshow(gt)
    # plt.show()
    # plt.imshow(pr)
    # plt.show()


from os.path import *
from shutil import move
import time

def get_download_path():
    """Returns the default downloads path for linux or windows"""
    if os.name == 'nt':
        import winreg
        sub_key = r'SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders'
        downloads_guid = '{374DE290-123F-4565-9164-39C4925E467B}'
        with winreg.OpenKey(winreg.HKEY_CURRENT_USER, sub_key) as key:
            location = winreg.QueryValueEx(key, downloads_guid)[0]
        return location
    else:
        return os.path.join(os.path.expanduser('~'), 'downloads')
def newest_file(path, extension=None):
    download_wait(path)
    files = os.listdir(path)
    if extension:
        paths = [os.path.join(path, basename)
                 for basename in files if splitext(basename)[1] == extension]
    else:
        paths = [os.path.join(path, basename)
                 for basename in files]
    if not paths:
        raise FileNotFoundError
    return max(paths, key=os.path.getctime)

def download_wait(path_to_downloads):
    seconds = 0
    dl_wait = True
    while dl_wait and seconds < 20:
        time.sleep(0.5)
        dl_wait = False
        for fname in os.listdir(path_to_downloads):
            if fname.endswith('.crdownload'):
                dl_wait = True
        seconds += 0.5
    return seconds

def move_newest_downloaded_file(
        self, filename, destination_dir=None, check_name=False):
    name, extension = splitext(filename)
    download_wait(get_download_path())
    new_file = newest_file(get_download_path(), extension)
    new_file_base, extension = splitext(basename(new_file))
    destination = join(destination_dir, filename)
    os.makedirs(destination_dir, exist_ok=True)
    if check_name is False:
        move(
            join(
                get_download_path(),
                new_file_base + extension), destination)
    else:
        move(
            join(
                get_download_path(),
                name + extension), destination)

    return destination


from django.urls import reverse
from django.shortcuts import redirect, render
from django.db.models import Q
from django.http import HttpResponseRedirect, HttpRequest
from django.conf import settings
from modules.serializers import OrderProducts

from modules.exceptions import *
from .reports import generate_report
from .forms import *
from .models import *

def get_shop_params():
    categories = ProductCategory.objects.all()
    params = {
        "categories": categories,
    }
    return params

@server_error_decorator
@is_staff_decorator
def index(request):
    with open(settings.PATH_LOG / f"api_view_logs.log", encoding="utf-8") as file:
        api_logs = file.read()
    with open(settings.PATH_LOG / f"orders_logs.log", encoding="utf-8") as file:
        order_logs = file.read()
    with open(settings.PATH_LOG / f"server_logs.log", encoding="utf-8") as file:
        server_logs = file.read()
    with open(settings.PATH_LOG / f"logs.log", encoding="utf-8") as file:
        logs = file.read()

    params = {
        "api_logs": api_logs,
        "order_logs": order_logs,
        "server_logs": server_logs,
        "logs": logs,
    }

    form = SearchForm()
    if "query" in request.GET:
        form = SearchForm(request.GET)
        if form.is_valid():
            cd = form.cleaned_data
            prod_results = Product.objects.filter(
                Q(name__icontains=cd["query"]) | Q(vendor__icontains=cd["query"])
            )
            prod_total_results = prod_results.count()

            news_results = News.objects.filter(
                Q(title__icontains=cd["query"]) | Q(news__icontains=cd["query"])
            )
            news_total_results = news_results.count()

            orders_results = Order.objects.filter(Q(id__icontains=cd["query"]))
            orders_total_results = orders_results.count()

            users_results = User.objects.filter(
                Q(username__icontains=cd["query"])
                | Q(first_name__icontains=cd["query"])
                | Q(last_name__icontains=cd["query"])
                | Q(email__icontains=cd["query"])
                | Q(id__icontains=cd["query"])
            )
            users_total_results = users_results.count()

            search_params = {
                "prod_results": prod_results,
                "prod_total_results": prod_total_results,
                "news_results": news_results,
                "news_total_results": news_total_results,
                "orders_results": orders_results,
                "orders_total_results": orders_total_results,
                "users_results": users_results,
                "users_total_results": users_total_results,
            }

            context = {
                **params,
                **search_params,
                "form": form,
                "cd": cd,
            }
            return render(request, "crm/index.html", context)

    context = {**params, "form": form}

    if str(request.GET.get("name")) != "None":
        context["name"] = str(request.GET.get("name"))
    return render(request, "crm/index.html", context)

@server_error_decorator
@is_staff_decorator
def news_edit(request, news_id):
    try:
        new_item = News.objects.get(pk=news_id)
    except News.DoesNotExist:
        message = "Такой новости не существует."
        return render(request, "dashboard/404.html", {"message": message})

    if request.method == "POST":
        form = NewsEdit(request.POST, request.FILES, instance=new_item)
        if form.is_valid():
            news_form = form.save(commit=False)
            news_form.pub_date = datetime.now()
            news_form.save()
            return HttpResponseRedirect(reverse("crm:news_list"))
    else:
        form = NewsEdit(instance=new_item)
    return render(
        request,
        "crm/news_edit.html",
        {
            "form": form,
            "new_item": new_item,
        },
    )

@server_error_decorator
@is_staff_decorator
def products_list(request):
    products = Product.objects.all().order_by("id")
    categories = ProductCategory.objects.all().order_by("id")
    filter_del = str(request.GET.get("deleted"))
    restore = str(request.GET.get("restored"))

    params = {
        "categories": categories,
    }

    form = SearchForm()
    if "query" in request.GET:
        form = SearchForm(request.GET)
        if form.is_valid():
            cd = form.cleaned_data
            products = Product.objects.filter(
                Q(name__icontains=cd["query"]) | Q(vendor__icontains=cd["query"])
            )

            context = {
                **params,
                "products": products,
                "form": form,
                "cd": cd,
            }
            return render(request, "crm/products_list.html", context)

    context = {
        **params,
        "products": products,
        "form": form,
    }

    if filter_del != "None":
        if isinstance(int(filter_del), int):
            product_to_delete = Product.objects.get(id=int(filter_del))
            product_to_delete.is_active = False
            product_to_delete.save()
            return render(request, "crm/products_list.html", context)

    if restore != "None":
        if isinstance(int(restore), int):
            product_to_restore = Product.objects.get(id=int(restore))
            product_to_restore.is_active = True
            product_to_restore.save()
            return render(request, "crm/products_list.html", context)

    return render(request, "crm/products_list.html", context)

@server_error_decorator
@is_staff_decorator
def product_edit(request, product_id):
    try:
        product_item = Product.objects.get(pk=product_id)
    except Product.DoesNotExist:
        message = "Такого продукта нет."
        return render(request, "dashboard/404.html", {"message": message})
    if request.method == "POST":
        form = ProductEdit(request.POST, request.FILES, instance=product_item)
        if form.is_valid():
            product_form = form.save(commit=False)
            product_form.save()
            return HttpResponseRedirect(reverse("crm:products_list"))
    else:
        form = ProductEdit(instance=product_item)
    return render(
        request,
        "crm/product_edit.html",
        {
            "form": form,
            "product_item": product_item,
        },
    )

@server_error_decorator
@is_staff_decorator
def create_news(request):
    profile = Profile.objects.get(client_id=request.user)
    assert isinstance(request, HttpRequest)
    form = NewsEdit(request.POST, request.FILES)
    if request.method == "POST":
        if form.is_valid():
            new_f = form.save(commit=False)
            new_f.pub_date = datetime.now()
            new_f.save()
            return redirect("crm:news_list")
        else:
            form = NewsEdit()
    return render(
        request,
        "crm/create_news.html",
        {
            "form": form,
            "profile": profile,
        },
    )

@server_error_decorator
@is_staff_decorator
def create_product(request):
    assert isinstance(request, HttpRequest)
    form = ProductEdit(request.POST, request.FILES)
    if request.method == "POST":
        if form.is_valid():
            new_f = form.save(commit=False)
            new_f.save()
            return redirect("crm:products_list")
        else:
            form = ProductEdit()
    return render(
        request,
        "crm/create_product.html",
        {
            "form": form,
        },
    )

@server_error_decorator
@is_staff_decorator
def news_list(request):
    news = News.objects.all().order_by("id")
    filter_del = str(request.GET.get("deleted"))
    restore = str(request.GET.get("restored"))

    form = SearchForm()
    if "query" in request.GET:
        form = SearchForm(request.GET)
        if form.is_valid():
            cd = form.cleaned_data
            news = News.objects.filter(
                Q(title__icontains=cd["query"]) | Q(news__icontains=cd["query"])
            )

            context = {
                # "profile": profile,
                "news": news,
                "form": form,
                "cd": cd,
            }
            return render(request, "crm/news_list.html", context)

    context = {
        "news": news,
        "form": form,
    }
    if filter_del != "None":
        if isinstance(int(filter_del), int):
            new_to_delete = News.objects.get(id=int(filter_del))
            new_to_delete.is_active = False
            new_to_delete.save()
            return render(request, "crm/news_list.html", context)

    if restore != "None":
        if isinstance(int(restore), int):
            new_to_restore = News.objects.get(id=int(restore))
            new_to_restore.is_active = True
            new_to_restore.save()
            return render(request, "crm/news_list.html", context)

    return render(request, "crm/news_list.html", context)

@server_error_decorator
@is_staff_decorator
def user_orders(request):
    users = User.objects.all()
    profiles = Profile.objects.all()
    orders_list = Order.objects.all().order_by("-id")
    # profile = Profile.objects.get(client_id=request.user)
    status1 = OrderStatus.objects.get(status_name="В сборке")
    status2 = OrderStatus.objects.get(status_name="В доставке")
    status3 = OrderStatus.objects.get(status_name="Доставлен")
    status4 = OrderStatus.objects.get(status_name="Отменен")

    statuses = {
        "status1": status1,
        "status2": status2,
        "status3": status3,
        "status4": status4,
    }

    form = SearchForm()
    if "query" in request.GET:
        form = SearchForm(request.GET)
        if form.is_valid():
            cd = form.cleaned_data

            users = User.objects.filter(
                Q(username__icontains=cd["query"])
                | Q(first_name__icontains=cd["query"])
                | Q(last_name__icontains=cd["query"])
                | Q(email__icontains=cd["query"])
                | Q(id__icontains=cd["query"])
            )

            search_params = {
                "users": users,
            }

            context = {
                "users": users,
                "profiles": profiles,
                # "profile": profile,
                "orders_list": orders_list,
                **statuses,
                **search_params,
                "form": form,
                "cd": cd,
            }
            return render(request, "crm/orders.html", context)

    context = {
        "users": users,
        "profiles": profiles,
        # "profile": profile,
        "orders_list": orders_list,
        **statuses,
        "form": form,
    }

    return render(request, "crm/orders.html", context)

@server_error_decorator
@is_staff_decorator
def user_order_edit(request, order_id):
    try:
        order = Order.objects.get(pk=order_id)
    except Order.DoesNotExist:
        message = "Такого заказа не существует."
        return render(request, "dashboard/404.html", {"message": message})

    if request.method == "POST":
        form = OrderEdit(request.POST, instance=order)
        if form.is_valid():
            order_form = form.save(commit=False)
            order_form.save()

            return HttpResponseRedirect(reverse("crm:orders"))
    else:
        form = OrderEdit(instance=order)

    serializer = OrderProducts(order, many=False)
    keys = serializer.data["product_list"].keys()
    get_keys = []
    amount = []
    for key in keys:
        get_keys.append(int(key))
        amount.append(int(serializer.data["product_list"].get(key)))
    order_products = Product.objects.filter(id__in=get_keys)

    profile = Profile.objects.get(client_id=request.user)
    title = order
    total = float(0)
    leng = int(0)
    for product in order_products:
        total += product.get_sale() * amount[leng]
        leng += 1
    context = {
        "form": form,
        "keys": keys,
        "amount": amount,
        "order_products": order_products,
        "title": title,
        "profile": profile,
        "order": order,
        "total": total,
    }
    return render(request, "crm/order_edit.html", context)

@server_error_decorator
@is_superuser_decorator
def user_permissions(request):
    profile = Profile.objects.get(client_id=request.user)
    users = User.objects.all().order_by("-is_staff")
    context = {
        "profile": profile,
        "users": users,
    }
    return render(request, "crm/user_permissions.html", context)

@server_error_decorator
@is_superuser_decorator
def make_user_client(user_id):
    target_user = User.objects.get(id=user_id)
    try:
        target_user.is_staff = False
        target_user.is_superuser = False
        target_user.save()
        return redirect("crm:permissions")
    except target_user.username == "igor":
        return redirect("crm:permissions")

@server_error_decorator
@is_superuser_decorator
def make_user_manager(user_id):
    target_user = User.objects.get(id=user_id)
    try:
        target_user.is_staff = True
        target_user.is_superuser = False
        target_user.save()
        return redirect("crm:permissions")
    except target_user.username == "igor":
        return redirect("crm:permissions")

@server_error_decorator
@is_superuser_decorator
def make_user_superuser(user_id):
    target_user = User.objects.get(id=user_id)
    try:
        target_user.is_staff = True
        target_user.is_superuser = True
        target_user.save()
        return redirect("crm:permissions")
    except target_user.username == "igor":
        return redirect("crm:permissions")

@server_error_decorator
def product_detail(request, product_id):
    reviews = Review.objects.order_by('-pub_date')
    try:
        product = Product.objects.get(pk=product_id, is_active=True)
    except Product.DoesNotExist:
        message = 'Такого товара не существует.'
        return render(request, 'dashboard/404.html', {'message': message})
    title = str(product.name)

    params = {
        'product': product,
        'title': title,
        'reviews': reviews,
    }

    context = {**get_shop_params(),
               **params}
    return render(request, 'crm/product_detail.html', context)

@server_error_decorator
def news_detail(request, news_id):
    try:
        new = News.objects.get(pk=news_id, is_active=True)
    except News.DoesNotExist:
        message = 'Такой новости не существует.'
        return render(request, 'dashboard/404.html', {'message': message})

    context = {
        'current_new': new,
    }
    if request.user.is_active:
        profile = Profile.objects.get(client_id=request.user)
        context['profile'] = profile
    return render(request, 'crm/news_detail.html', context)

@server_error_decorator
@is_staff_decorator
def support_list(request):
    get_r = str(request.GET.get("get_r"))
    read = str(request.GET.get("read"))
    cq = str(request.GET.get("current_question"))

    sup_list = Support.objects.all().order_by("-id")

    if get_r != "None":
        if isinstance(int(get_r), int):
            sup_list = Support.objects.filter(checked=get_r).order_by("-id")

    try:
        current_question = Support.objects.get(id=cq)
    except:
        current_question = sup_list.first()

    context = {
        "sup_list": sup_list,
        "current_question": current_question,
    }

    if read != "None":
        if isinstance(int(read), int):
            current_question.checked = True
            current_question.save()
            return render(request, "crm/support_list.html", context)
    return render(request, "crm/support_list.html", context)

@server_error_decorator
@is_staff_decorator
def templates(request):
    tid = str(request.GET.get("tid"))
    temp_list = Templates.objects.filter()
    context = {"temp_list": temp_list}
    if tid and tid.isnumeric():
        template = Templates.objects.get(id=int(tid))
        context["template"] = template
    return render(request, "crm/templates_list.html", context)

@server_error_decorator
@is_staff_decorator
def reports(request):
    if request.method == "POST":
        form = OrderPeriodForm(request.POST)
        if form.is_valid():
            orders = Order.objects.all().order_by("-date_created")
            generate_report(orders)
    else:
        form = OrderPeriodForm()

    context = {
        "form": form,
    }
    return render(request, "crm/reports.html", context)


import os.path
import yaml
import sys
import traceback
import luigi
import luigi.local_target
import pieshell
import traceback
import math
import time
import requests
import poltergust_luigi_utils # Add GCS luigi opener

DB_URL = os.environ.get("DB_URL")

def make_environment(envpath, environment, log):
    _ = pieshell.env(exports=dict(pieshell.env._exports))
    if not os.path.exists(envpath):
        envdir = os.path.dirname(envpath)
        if not os.path.exists(envdir):
            os.makedirs(envdir)
        +_.virtualenv(envpath, **environment.get("virtualenv", {}))
    +_.bashsource(envpath + "/bin/activate")
    for dep in environment["dependencies"]:
        for line in _.pip.install(dep):
            log(line)

def iter_to_pcnt(x, a=0.5):
    return 100 * a * math.log(1 + x) / (1 + a*math.log(1 + x))

class Logging(object):
    def on_failure(self, exception):
        traceback_string = traceback.format_exc()
        msg = "Runtime error:\n%s" % traceback_string
        self._log(msg)
        self.close_log()
        return msg

    def on_success(self):
        self._log("DONE")
        self.close_log()

    def close_log(self):
        with self.logfile().open("w") as f:
            f.write("\n".join(self.msgs) + "\n")

    def _log(self, msg):
        if not hasattr(self, "msgs"):
            self.msgs = []
        print(msg)
        self.msgs.append(msg)

    def log(self, msg):
        self._log(msg)
        self.set_status_message("\n".join(self.msgs))
        self.set_progress_percentage(iter_to_pcnt(len(self.msgs)))

class MakeEnvironment(Logging, luigi.Task):
    path = luigi.Parameter()
    hostname = luigi.Parameter()

    def run(self):
        with luigi.contrib.opener.OpenerTarget(self.path).open("r") as f:
            environment = yaml.load(f, Loader=yaml.SafeLoader)
        make_environment(self.envdir().path, environment, self.log)
        with self.output().open("w") as f:
            f.write("DONE")

    def envdir(self):
        return luigi.local_target.LocalTarget(
            os.path.join("/tmp/environments", self.path.replace("://", "/").lstrip("/")))

    def logfile(self):
        return luigi.contrib.opener.OpenerTarget("%s.%s.log.txt" % (self.path, self.hostname))

    def output(self):
        return luigi.local_target.LocalTarget(
            os.path.join("/tmp/environments", self.path.replace("://", "/").lstrip("/"), "done"))

class RunTask(Logging, luigi.Task):
    path = luigi.Parameter()
    hostname = luigi.Parameter()
    retry_on_error = luigi.Parameter(default=False)

    @property
    def scheduler(self):
        return self.set_progress_percentage.__self__._scheduler

    @property
    def scheduler_url(self):
        return self.scheduler._url

    def run(self):
        src = '%s.config.yaml' % (self.path,)
        fs = self.output().fs

        try:
            cfg = luigi.contrib.opener.OpenerTarget('%s.config.yaml' % (self.path,))
            try:
                with cfg.open("r") as f:
                    task = yaml.load(f, Loader=yaml.SafeLoader)
            except:
                if not cfg.fs.exists(cfg.path):
                    # The task has already been marked as done by another worker.
                    return
                raise

            with luigi.contrib.opener.OpenerTarget(task["environment"]).open("r") as f:
                environment = yaml.load(f, Loader=yaml.SafeLoader)

            env = MakeEnvironment(path=task["environment"], hostname=self.hostname)
            yield env
            envpath = env.envdir().path

            _ = pieshell.env(envpath, interactive=True)
            +_.bashsource(envpath + "/bin/activate")

            _._exports.update(environment.get("variables", {}))
            _._exports.update(task.get("variables", {}))

            command = task.get("command", None)

            task_args = dict(task.get("task", {}))
            task_name = task_args.pop("name", None)
            task_args["scheduler-url"] = self.scheduler_url
            task_args["retcode-already-running"] = "10"
            task_args["retcode-missing-data"] = "20"
            task_args["retcode-not-run"] = "25"
            task_args["retcode-task-failed"] = "30"
            task_args["retcode-scheduling-error"] = "35"
            task_args["retcode-unhandled-exception"] = "40"

            if command is None:
                command = "luigi(task_name, **task_args)"

            scope = pieshell.environ.EnvScope(env=_)
            scope["task_name"] = task_name
            scope["task_args"] = task_args

            # Rerun the task until it is actually being run (or is done!)
            # We need to do this, since luigi might exit because all
            # dependencies of a task are already being run by other
            # workers, and our task can't even start...
            while True:
                try:
                    for line in eval(command, scope):
                        self.log(line)
                except pieshell.PipelineFailed as e:
                    self.log(str(e))
                    if e.pipeline.exit_code <= 25:
                        time.sleep(5)
                        continue
                    raise
                break

        except Exception as e:
            self.on_failure(e)
            if self.retry_on_error:
                raise
            dst = '%s.error.yaml' % (self.path,)
            try:
                fs.move(src, dst)
            except:
                # Might already have been moved by another node...
                pass
        else:
            self.on_success()
            dst = '%s.done.yaml' % (self.path,)
            try:
                fs.move(src, dst)
            except:
                # Might already have been moved by another node...
                pass

        if DB_URL is not None:
            r = requests.get(DB_URL, params={"pipeline_url": self.path})
            if r.status_code != 200:
                print("Unable to update status", r.status_code, r.text)

    def logfile(self):
        return luigi.contrib.opener.OpenerTarget("%s.%s.log.txt" % (self.path, self.hostname))

    def output(self):
         return luigi.contrib.opener.OpenerTarget(
             '%s.done.yaml' % (self.path,))

class RunTasks(luigi.Task):
    path = luigi.Parameter()
    hostname = luigi.Parameter()

    def requires(self):
        return [RunTask(path=path.replace(".config.yaml", ""), hostname=self.hostname)
                for path in self.output().fs.list_wildcard('%s/*.config.yaml' % (self.path,))]

    def run(self):
        pass

    def output(self):
        # This target should never actually be created...
        return luigi.contrib.opener.OpenerTarget('%s/done' % (self.path,))

# luigi --module emerald_algorithms_evaluation.luigi Pipeline --param-evaluation_name=test-luigi-redhog-1


def main():
    a, b, k = map(int, input().split())
    a, b = min(a, b), max(a, b)

    lst = []
    for i in range(1, a+1):
        if a % i == 0:
            lst.append(i)
    lst.sort(reverse=True)
    cnt = 0
    for num in lst:
        if b % num == 0:
            cnt += 1
            if cnt == k:
                print(num)
                break

if __name__ == '__main__':
    main()