# Author: Alexander Fabisch  -- <afabisch@informatik.uni-bremen.de>
# Author: Christopher Moody <chrisemoody@gmail.com>
# Author: Nick Travers <nickt@squareup.com>
# License: BSD 3 clause (C) 2014

# This is the exact and Barnes-Hut t-SNE implementation. There are other
# modifications of the algorithm:
# * Fast Optimization for t-SNE:
#   https://cseweb.ucsd.edu/~lvdmaaten/workshops/nips2010/papers/vandermaaten.pdf

from numbers import Integral, Real
from time import time

import numpy as np
from scipy import linalg
from scipy.sparse import csr_matrix, issparse
from scipy.spatial.distance import pdist, squareform

from ..base import (
    BaseEstimator,
    ClassNamePrefixFeaturesOutMixin,
    TransformerMixin,
    _fit_context,
)
from ..decomposition import PCA
from ..metrics.pairwise import _VALID_METRICS, pairwise_distances
from ..neighbors import NearestNeighbors
from ..utils import check_random_state
from ..utils._openmp_helpers import _openmp_effective_n_threads
from ..utils._param_validation import Interval, StrOptions, validate_params
from ..utils.validation import _num_samples, check_non_negative

# mypy error: Module 'sklearn.manifold' has no attribute '_utils'
# mypy error: Module 'sklearn.manifold' has no attribute '_barnes_hut_tsne'
from . import _barnes_hut_tsne, _utils  # type: ignore

MACHINE_EPSILON = np.finfo(np.double).eps

def _joint_probabilities(distances, desired_perplexity, verbose):
    """Compute joint probabilities p_ij from distances.

    Parameters
    ----------
    distances : ndarray of shape (n_samples * (n_samples-1) / 2,)
        Distances of samples are stored as condensed matrices, i.e.
        we omit the diagonal and duplicate entries and store everything
        in a one-dimensional array.

    desired_perplexity : float
        Desired perplexity of the joint probability distributions.

    verbose : int
        Verbosity level.

    Returns
    -------
    P : ndarray of shape (n_samples * (n_samples-1) / 2,)
        Condensed joint probability matrix.
    """
    # Compute conditional probabilities such that they approximately match
    # the desired perplexity
    distances = distances.astype(np.float32, copy=False)
    conditional_P = _utils._binary_search_perplexity(
        distances, desired_perplexity, verbose
    )
    P = conditional_P + conditional_P.T
    sum_P = np.maximum(np.sum(P), MACHINE_EPSILON)
    P = np.maximum(squareform(P) / sum_P, MACHINE_EPSILON)
    return P

def _joint_probabilities_nn(distances, desired_perplexity, verbose):
    """Compute joint probabilities p_ij from distances using just nearest
    neighbors.

    This method is approximately equal to _joint_probabilities. The latter
    is O(N), but limiting the joint probability to nearest neighbors improves
    this substantially to O(uN).

    Parameters
    ----------
    distances : sparse matrix of shape (n_samples, n_samples)
        Distances of samples to its n_neighbors nearest neighbors. All other
        distances are left to zero (and are not materialized in memory).
        Matrix should be of CSR format.

    desired_perplexity : float
        Desired perplexity of the joint probability distributions.

    verbose : int
        Verbosity level.

    Returns
    -------
    P : sparse matrix of shape (n_samples, n_samples)
        Condensed joint probability matrix with only nearest neighbors. Matrix
        will be of CSR format.
    """
    t0 = time()
    # Compute conditional probabilities such that they approximately match
    # the desired perplexity
    distances.sort_indices()
    n_samples = distances.shape[0]
    distances_data = distances.data.reshape(n_samples, -1)
    distances_data = distances_data.astype(np.float32, copy=False)
    conditional_P = _utils._binary_search_perplexity(
        distances_data, desired_perplexity, verbose
    )
    assert np.all(np.isfinite(conditional_P)), "All probabilities should be finite"

    # Symmetrize the joint probability distribution using sparse operations
    P = csr_matrix(
        (conditional_P.ravel(), distances.indices, distances.indptr),
        shape=(n_samples, n_samples),
    )
    P = P + P.T

    # Normalize the joint probability distribution
    sum_P = np.maximum(P.sum(), MACHINE_EPSILON)
    P /= sum_P

    assert np.all(np.abs(P.data) <= 1.0)
    if verbose >= 2:
        duration = time() - t0
        print("[t-SNE] Computed conditional probabilities in {:.3f}s".format(duration))
    return P

def _kl_divergence(
    params,
    P,
    degrees_of_freedom,
    n_samples,
    n_components,
    skip_num_points=0,
    compute_error=True,
):
    """t-SNE objective function: gradient of the KL divergence
    of p_ijs and q_ijs and the absolute error.

    Parameters
    ----------
    params : ndarray of shape (n_params,)
        Unraveled embedding.

    P : ndarray of shape (n_samples * (n_samples-1) / 2,)
        Condensed joint probability matrix.

    degrees_of_freedom : int
        Degrees of freedom of the Student's-t distribution.

    n_samples : int
        Number of samples.

    n_components : int
        Dimension of the embedded space.

    skip_num_points : int, default=0
        This does not compute the gradient for points with indices below
        `skip_num_points`. This is useful when computing transforms of new
        data where you'd like to keep the old data fixed.

    compute_error: bool, default=True
        If False, the kl_divergence is not computed and returns NaN.

    Returns
    -------
    kl_divergence : float
        Kullback-Leibler divergence of p_ij and q_ij.

    grad : ndarray of shape (n_params,)
        Unraveled gradient of the Kullback-Leibler divergence with respect to
        the embedding.
    """
    X_embedded = params.reshape(n_samples, n_components)

    # Q is a heavy-tailed distribution: Student's t-distribution
    dist = pdist(X_embedded, "sqeuclidean")
    dist /= degrees_of_freedom
    dist += 1.0
    dist **= (degrees_of_freedom + 1.0) / -2.0
    Q = np.maximum(dist / (2.0 * np.sum(dist)), MACHINE_EPSILON)

    # Optimization trick below: np.dot(x, y) is faster than
    # np.sum(x * y) because it calls BLAS

    # Objective: C (Kullback-Leibler divergence of P and Q)
    if compute_error:
        kl_divergence = 2.0 * np.dot(P, np.log(np.maximum(P, MACHINE_EPSILON) / Q))
    else:
        kl_divergence = np.nan

    # Gradient: dC/dY
    # pdist always returns double precision distances. Thus we need to take
    grad = np.ndarray((n_samples, n_components), dtype=params.dtype)
    PQd = squareform((P - Q) * dist)
    for i in range(skip_num_points, n_samples):
        grad[i] = np.dot(np.ravel(PQd[i], order="K"), X_embedded[i] - X_embedded)
    grad = grad.ravel()
    c = 2.0 * (degrees_of_freedom + 1.0) / degrees_of_freedom
    grad *= c

    return kl_divergence, grad

def _kl_divergence_bh(
    params,
    P,
    degrees_of_freedom,
    n_samples,
    n_components,
    angle=0.5,
    skip_num_points=0,
    verbose=False,
    compute_error=True,
    num_threads=1,
):
    """t-SNE objective function: KL divergence of p_ijs and q_ijs.

    Uses Barnes-Hut tree methods to calculate the gradient that
    runs in O(NlogN) instead of O(N^2).

    Parameters
    ----------
    params : ndarray of shape (n_params,)
        Unraveled embedding.

    P : sparse matrix of shape (n_samples, n_sample)
        Sparse approximate joint probability matrix, computed only for the
        k nearest-neighbors and symmetrized. Matrix should be of CSR format.

    degrees_of_freedom : int
        Degrees of freedom of the Student's-t distribution.

    n_samples : int
        Number of samples.

    n_components : int
        Dimension of the embedded space.

    angle : float, default=0.5
        This is the trade-off between speed and accuracy for Barnes-Hut T-SNE.
        'angle' is the angular size (referred to as theta in [3]) of a distant
        node as measured from a point. If this size is below 'angle' then it is
        used as a summary node of all points contained within it.
        This method is not very sensitive to changes in this parameter
        in the range of 0.2 - 0.8. Angle less than 0.2 has quickly increasing
        computation time and angle greater 0.8 has quickly increasing error.

    skip_num_points : int, default=0
        This does not compute the gradient for points with indices below
        `skip_num_points`. This is useful when computing transforms of new
        data where you'd like to keep the old data fixed.

    verbose : int, default=False
        Verbosity level.

    compute_error: bool, default=True
        If False, the kl_divergence is not computed and returns NaN.

    num_threads : int, default=1
        Number of threads used to compute the gradient. This is set here to
        avoid calling _openmp_effective_n_threads for each gradient step.

    Returns
    -------
    kl_divergence : float
        Kullback-Leibler divergence of p_ij and q_ij.

    grad : ndarray of shape (n_params,)
        Unraveled gradient of the Kullback-Leibler divergence with respect to
        the embedding.
    """
    params = params.astype(np.float32, copy=False)
    X_embedded = params.reshape(n_samples, n_components)

    val_P = P.data.astype(np.float32, copy=False)
    neighbors = P.indices.astype(np.int64, copy=False)
    indptr = P.indptr.astype(np.int64, copy=False)

    grad = np.zeros(X_embedded.shape, dtype=np.float32)
    error = _barnes_hut_tsne.gradient(
        val_P,
        X_embedded,
        neighbors,
        indptr,
        grad,
        angle,
        n_components,
        verbose,
        dof=degrees_of_freedom,
        compute_error=compute_error,
        num_threads=num_threads,
    )
    c = 2.0 * (degrees_of_freedom + 1.0) / degrees_of_freedom
    grad = grad.ravel()
    grad *= c

    return error, grad

def _gradient_descent(
    objective,
    p0,
    it,
    n_iter,
    n_iter_check=1,
    n_iter_without_progress=300,
    momentum=0.8,
    learning_rate=200.0,
    min_gain=0.01,
    min_grad_norm=1e-7,
    verbose=0,
    args=None,
    kwargs=None,
):
    """Batch gradient descent with momentum and individual gains.

    Parameters
    ----------
    objective : callable
        Should return a tuple of cost and gradient for a given parameter
        vector. When expensive to compute, the cost can optionally
        be None and can be computed every n_iter_check steps using
        the objective_error function.

    p0 : array-like of shape (n_params,)
        Initial parameter vector.

    it : int
        Current number of iterations (this function will be called more than
        once during the optimization).

    n_iter : int
        Maximum number of gradient descent iterations.

    n_iter_check : int, default=1
        Number of iterations before evaluating the global error. If the error
        is sufficiently low, we abort the optimization.

    n_iter_without_progress : int, default=300
        Maximum number of iterations without progress before we abort the
        optimization.

    momentum : float within (0.0, 1.0), default=0.8
        The momentum generates a weight for previous gradients that decays
        exponentially.

    learning_rate : float, default=200.0
        The learning rate for t-SNE is usually in the range [10.0, 1000.0]. If
        the learning rate is too high, the data may look like a 'ball' with any
        point approximately equidistant from its nearest neighbours. If the
        learning rate is too low, most points may look compressed in a dense
        cloud with few outliers.

    min_gain : float, default=0.01
        Minimum individual gain for each parameter.

    min_grad_norm : float, default=1e-7
        If the gradient norm is below this threshold, the optimization will
        be aborted.

    verbose : int, default=0
        Verbosity level.

    args : sequence, default=None
        Arguments to pass to objective function.

    kwargs : dict, default=None
        Keyword arguments to pass to objective function.

    Returns
    -------
    p : ndarray of shape (n_params,)
        Optimum parameters.

    error : float
        Optimum.

    i : int
        Last iteration.
    """
    if args is None:
        args = []
    if kwargs is None:
        kwargs = {}

    p = p0.copy().ravel()
    update = np.zeros_like(p)
    gains = np.ones_like(p)
    error = np.finfo(float).max
    best_error = np.finfo(float).max
    best_iter = i = it

    tic = time()
    for i in range(it, n_iter):
        check_convergence = (i + 1) % n_iter_check == 0
        # only compute the error when needed
        kwargs["compute_error"] = check_convergence or i == n_iter - 1

        error, grad = objective(p, *args, **kwargs)

        inc = update * grad < 0.0
        dec = np.invert(inc)
        gains[inc] += 0.2
        gains[dec] *= 0.8
        np.clip(gains, min_gain, np.inf, out=gains)
        grad *= gains
        update = momentum * update - learning_rate * grad
        p += update

        if check_convergence:
            toc = time()
            duration = toc - tic
            tic = toc
            grad_norm = linalg.norm(grad)

            if verbose >= 2:
                print(
                    "[t-SNE] Iteration %d: error = %.7f,"
                    " gradient norm = %.7f"
                    " (%s iterations in %0.3fs)"
                    % (i + 1, error, grad_norm, n_iter_check, duration)
                )

            if error < best_error:
                best_error = error
                best_iter = i
            elif i - best_iter > n_iter_without_progress:
                if verbose >= 2:
                    print(
                        "[t-SNE] Iteration %d: did not make any progress "
                        "during the last %d episodes. Finished."
                        % (i + 1, n_iter_without_progress)
                    )
                break
            if grad_norm <= min_grad_norm:
                if verbose >= 2:
                    print(
                        "[t-SNE] Iteration %d: gradient norm %f. Finished."
                        % (i + 1, grad_norm)
                    )
                break

    return p, error, i

@validate_params(
    {
        "X": ["array-like", "sparse matrix"],
        "X_embedded": ["array-like", "sparse matrix"],
        "n_neighbors": [Interval(Integral, 1, None, closed="left")],
        "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable],
    },
    prefer_skip_nested_validation=True,
)
def trustworthiness(X, X_embedded, *, n_neighbors=5, metric="euclidean"):
    r"""Indicate to what extent the local structure is retained.

    The trustworthiness is within [0, 1]. It is defined as

    .. math::

        T(k) = 1 - \frac{2}{nk (2n - 3k - 1)} \sum^n_{i=1}
            \sum_{j \in \mathcal{N}_{i}^{k}} \max(0, (r(i, j) - k))

    where for each sample i, :math:`\mathcal{N}_{i}^{k}` are its k nearest
    neighbors in the output space, and every sample j is its :math:`r(i, j)`-th
    nearest neighbor in the input space. In other words, any unexpected nearest
    neighbors in the output space are penalised in proportion to their rank in
    the input space.

    Parameters
    ----------
    X : {array-like, sparse matrix} of shape (n_samples, n_features) or \
        (n_samples, n_samples)
        If the metric is 'precomputed' X must be a square distance
        matrix. Otherwise it contains a sample per row.

    X_embedded : {array-like, sparse matrix} of shape (n_samples, n_components)
        Embedding of the training data in low-dimensional space.

    n_neighbors : int, default=5
        The number of neighbors that will be considered. Should be fewer than
        `n_samples / 2` to ensure the trustworthiness to lies within [0, 1], as
        mentioned in [1]_. An error will be raised otherwise.

    metric : str or callable, default='euclidean'
        Which metric to use for computing pairwise distances between samples
        from the original input space. If metric is 'precomputed', X must be a
        matrix of pairwise distances or squared distances. Otherwise, for a list
        of available metrics, see the documentation of argument metric in
        `sklearn.pairwise.pairwise_distances` and metrics listed in
        `sklearn.metrics.pairwise.PAIRWISE_DISTANCE_FUNCTIONS`. Note that the
        "cosine" metric uses :func:`~sklearn.metrics.pairwise.cosine_distances`.

        .. versionadded:: 0.20

    Returns
    -------
    trustworthiness : float
        Trustworthiness of the low-dimensional embedding.

    References
    ----------
    .. [1] Jarkko Venna and Samuel Kaski. 2001. Neighborhood
           Preservation in Nonlinear Projection Methods: An Experimental Study.
           In Proceedings of the International Conference on Artificial Neural Networks
           (ICANN '01). Springer-Verlag, Berlin, Heidelberg, 485-491.

    .. [2] Laurens van der Maaten. Learning a Parametric Embedding by Preserving
           Local Structure. Proceedings of the Twelfth International Conference on
           Artificial Intelligence and Statistics, PMLR 5:384-391, 2009.
    """
    n_samples = _num_samples(X)
    if n_neighbors >= n_samples / 2:
        raise ValueError(
            f"n_neighbors ({n_neighbors}) should be less than n_samples / 2"
            f" ({n_samples / 2})"
        )
    dist_X = pairwise_distances(X, metric=metric)
    if metric == "precomputed":
        dist_X = dist_X.copy()
    # we set the diagonal to np.inf to exclude the points themselves from
    # their own neighborhood
    np.fill_diagonal(dist_X, np.inf)
    ind_X = np.argsort(dist_X, axis=1)
    # `ind_X[i]` is the index of sorted distances between i and other samples
    ind_X_embedded = (
        NearestNeighbors(n_neighbors=n_neighbors)
        .fit(X_embedded)
        .kneighbors(return_distance=False)
    )

    # We build an inverted index of neighbors in the input space: For sample i,
    # we define `inverted_index[i]` as the inverted index of sorted distances:
    # inverted_index[i][ind_X[i]] = np.arange(1, n_sample + 1)
    inverted_index = np.zeros((n_samples, n_samples), dtype=int)
    ordered_indices = np.arange(n_samples + 1)
    inverted_index[ordered_indices[:-1, np.newaxis], ind_X] = ordered_indices[1:]
    ranks = (
        inverted_index[ordered_indices[:-1, np.newaxis], ind_X_embedded] - n_neighbors
    )
    t = np.sum(ranks[ranks > 0])
    t = 1.0 - t * (
        2.0 / (n_samples * n_neighbors * (2.0 * n_samples - 3.0 * n_neighbors - 1.0))
    )
    return t

class TSNE(ClassNamePrefixFeaturesOutMixin, TransformerMixin, BaseEstimator):
    """T-distributed Stochastic Neighbor Embedding.

    t-SNE [1] is a tool to visualize high-dimensional data. It converts
    similarities between data points to joint probabilities and tries
    to minimize the Kullback-Leibler divergence between the joint
    probabilities of the low-dimensional embedding and the
    high-dimensional data. t-SNE has a cost function that is not convex,
    i.e. with different initializations we can get different results.

    It is highly recommended to use another dimensionality reduction
    method (e.g. PCA for dense data or TruncatedSVD for sparse data)
    to reduce the number of dimensions to a reasonable amount (e.g. 50)
    if the number of features is very high. This will suppress some
    noise and speed up the computation of pairwise distances between
    samples. For more tips see Laurens van der Maaten's FAQ [2].

    Read more in the :ref:`User Guide <t_sne>`.

    Parameters
    ----------
    n_components : int, default=2
        Dimension of the embedded space.

    perplexity : float, default=30.0
        The perplexity is related to the number of nearest neighbors that
        is used in other manifold learning algorithms. Larger datasets
        usually require a larger perplexity. Consider selecting a value
        between 5 and 50. Different values can result in significantly
        different results. The perplexity must be less than the number
        of samples.

    early_exaggeration : float, default=12.0
        Controls how tight natural clusters in the original space are in
        the embedded space and how much space will be between them. For
        larger values, the space between natural clusters will be larger
        in the embedded space. Again, the choice of this parameter is not
        very critical. If the cost function increases during initial
        optimization, the early exaggeration factor or the learning rate
        might be too high.

    learning_rate : float or "auto", default="auto"
        The learning rate for t-SNE is usually in the range [10.0, 1000.0]. If
        the learning rate is too high, the data may look like a 'ball' with any
        point approximately equidistant from its nearest neighbours. If the
        learning rate is too low, most points may look compressed in a dense
        cloud with few outliers. If the cost function gets stuck in a bad local
        minimum increasing the learning rate may help.
        Note that many other t-SNE implementations (bhtsne, FIt-SNE, openTSNE,
        etc.) use a definition of learning_rate that is 4 times smaller than
        ours. So our learning_rate=200 corresponds to learning_rate=800 in
        those other implementations. The 'auto' option sets the learning_rate
        to `max(N / early_exaggeration / 4, 50)` where N is the sample size,
        following [4] and [5].

        .. versionchanged:: 1.2
           The default value changed to `"auto"`.

    n_iter : int, default=1000
        Maximum number of iterations for the optimization. Should be at
        least 250.

    n_iter_without_progress : int, default=300
        Maximum number of iterations without progress before we abort the
        optimization, used after 250 initial iterations with early
        exaggeration. Note that progress is only checked every 50 iterations so
        this value is rounded to the next multiple of 50.

        .. versionadded:: 0.17
           parameter *n_iter_without_progress* to control stopping criteria.

    min_grad_norm : float, default=1e-7
        If the gradient norm is below this threshold, the optimization will
        be stopped.

    metric : str or callable, default='euclidean'
        The metric to use when calculating distance between instances in a
        feature array. If metric is a string, it must be one of the options
        allowed by scipy.spatial.distance.pdist for its metric parameter, or
        a metric listed in pairwise.PAIRWISE_DISTANCE_FUNCTIONS.
        If metric is "precomputed", X is assumed to be a distance matrix.
        Alternatively, if metric is a callable function, it is called on each
        pair of instances (rows) and the resulting value recorded. The callable
        should take two arrays from X as input and return a value indicating
        the distance between them. The default is "euclidean" which is
        interpreted as squared euclidean distance.

    metric_params : dict, default=None
        Additional keyword arguments for the metric function.

        .. versionadded:: 1.1

    init : {"random", "pca"} or ndarray of shape (n_samples, n_components), \
            default="pca"
        Initialization of embedding.
        PCA initialization cannot be used with precomputed distances and is
        usually more globally stable than random initialization.

        .. versionchanged:: 1.2
           The default value changed to `"pca"`.

    verbose : int, default=0
        Verbosity level.

    random_state : int, RandomState instance or None, default=None
        Determines the random number generator. Pass an int for reproducible
        results across multiple function calls. Note that different
        initializations might result in different local minima of the cost
        function. See :term:`Glossary <random_state>`.

    method : {'barnes_hut', 'exact'}, default='barnes_hut'
        By default the gradient calculation algorithm uses Barnes-Hut
        approximation running in O(NlogN) time. method='exact'
        will run on the slower, but exact, algorithm in O(N^2) time. The
        exact algorithm should be used when nearest-neighbor errors need
        to be better than 3%. However, the exact method cannot scale to
        millions of examples.

        .. versionadded:: 0.17
           Approximate optimization *method* via the Barnes-Hut.

    angle : float, default=0.5
        Only used if method='barnes_hut'
        This is the trade-off between speed and accuracy for Barnes-Hut T-SNE.
        'angle' is the angular size (referred to as theta in [3]) of a distant
        node as measured from a point. If this size is below 'angle' then it is
        used as a summary node of all points contained within it.
        This method is not very sensitive to changes in this parameter
        in the range of 0.2 - 0.8. Angle less than 0.2 has quickly increasing
        computation time and angle greater 0.8 has quickly increasing error.

    n_jobs : int, default=None
        The number of parallel jobs to run for neighbors search. This parameter
        has no impact when ``metric="precomputed"`` or
        (``metric="euclidean"`` and ``method="exact"``).
        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
        ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
        for more details.

        .. versionadded:: 0.22

    Attributes
    ----------
    embedding_ : array-like of shape (n_samples, n_components)
        Stores the embedding vectors.

    kl_divergence_ : float
        Kullback-Leibler divergence after optimization.

    n_features_in_ : int
        Number of features seen during :term:`fit`.

        .. versionadded:: 0.24

    feature_names_in_ : ndarray of shape (`n_features_in_`,)
        Names of features seen during :term:`fit`. Defined only when `X`
        has feature names that are all strings.

        .. versionadded:: 1.0

    learning_rate_ : float
        Effective learning rate.

        .. versionadded:: 1.2

    n_iter_ : int
        Number of iterations run.

    See Also
    --------
    sklearn.decomposition.PCA : Principal component analysis that is a linear
        dimensionality reduction method.
    sklearn.decomposition.KernelPCA : Non-linear dimensionality reduction using
        kernels and PCA.
    MDS : Manifold learning using multidimensional scaling.
    Isomap : Manifold learning based on Isometric Mapping.
    LocallyLinearEmbedding : Manifold learning using Locally Linear Embedding.
    SpectralEmbedding : Spectral embedding for non-linear dimensionality.

    References
    ----------

    [1] van der Maaten, L.J.P.; Hinton, G.E. Visualizing High-Dimensional Data
        Using t-SNE. Journal of Machine Learning Research 9:2579-2605, 2008.

    [2] van der Maaten, L.J.P. t-Distributed Stochastic Neighbor Embedding
        https://lvdmaaten.github.io/tsne/

    [3] L.J.P. van der Maaten. Accelerating t-SNE using Tree-Based Algorithms.
        Journal of Machine Learning Research 15(Oct):3221-3245, 2014.
        https://lvdmaaten.github.io/publications/papers/JMLR_2014.pdf

    [4] Belkina, A. C., Ciccolella, C. O., Anno, R., Halpert, R., Spidlen, J.,
        & Snyder-Cappione, J. E. (2019). Automated optimized parameters for
        T-distributed stochastic neighbor embedding improve visualization
        and analysis of large datasets. Nature Communications, 10(1), 1-12.

    [5] Kobak, D., & Berens, P. (2019). The art of using t-SNE for single-cell
        transcriptomics. Nature Communications, 10(1), 1-14.

    Examples
    --------
    >>> import numpy as np
    >>> from sklearn.manifold import TSNE
    >>> X = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]])
    >>> X_embedded = TSNE(n_components=2, learning_rate='auto',
    ...                   init='random', perplexity=3).fit_transform(X)
    >>> X_embedded.shape
    (4, 2)
    """

    _parameter_constraints: dict = {
        "n_components": [Interval(Integral, 1, None, closed="left")],
        "perplexity": [Interval(Real, 0, None, closed="neither")],
        "early_exaggeration": [Interval(Real, 1, None, closed="left")],
        "learning_rate": [
            StrOptions({"auto"}),
            Interval(Real, 0, None, closed="neither"),
        ],
        "n_iter": [Interval(Integral, 250, None, closed="left")],
        "n_iter_without_progress": [Interval(Integral, -1, None, closed="left")],
        "min_grad_norm": [Interval(Real, 0, None, closed="left")],
        "metric": [StrOptions(set(_VALID_METRICS) | {"precomputed"}), callable],
        "metric_params": [dict, None],
        "init": [
            StrOptions({"pca", "random"}),
            np.ndarray,
        ],
        "verbose": ["verbose"],
        "random_state": ["random_state"],
        "method": [StrOptions({"barnes_hut", "exact"})],
        "angle": [Interval(Real, 0, 1, closed="both")],
        "n_jobs": [None, Integral],
    }

    # Control the number of exploration iterations with early_exaggeration on
    _EXPLORATION_N_ITER = 250

    # Control the number of iterations between progress checks
    _N_ITER_CHECK = 50

    def __init__(
        self,
        n_components=2,
        *,
        perplexity=30.0,
        early_exaggeration=12.0,
        learning_rate="auto",
        n_iter=1000,
        n_iter_without_progress=300,
        min_grad_norm=1e-7,
        metric="euclidean",
        metric_params=None,
        init="pca",
        verbose=0,
        random_state=None,
        method="barnes_hut",
        angle=0.5,
        n_jobs=None,
    ):
        self.n_components = n_components
        self.perplexity = perplexity
        self.early_exaggeration = early_exaggeration
        self.learning_rate = learning_rate
        self.n_iter = n_iter
        self.n_iter_without_progress = n_iter_without_progress
        self.min_grad_norm = min_grad_norm
        self.metric = metric
        self.metric_params = metric_params
        self.init = init
        self.verbose = verbose
        self.random_state = random_state
        self.method = method
        self.angle = angle
        self.n_jobs = n_jobs

    def _check_params_vs_input(self, X):
        if self.perplexity >= X.shape[0]:
            raise ValueError("perplexity must be less than n_samples")

    def _fit(self, X, skip_num_points=0):
        """Private function to fit the model using X as training data."""

        if isinstance(self.init, str) and self.init == "pca" and issparse(X):
            raise TypeError(
                "PCA initialization is currently not supported "
                "with the sparse input matrix. Use "
                'init="random" instead.'
            )

        if self.learning_rate == "auto":
            # See issue #18018
            self.learning_rate_ = X.shape[0] / self.early_exaggeration / 4
            self.learning_rate_ = np.maximum(self.learning_rate_, 50)
        else:
            self.learning_rate_ = self.learning_rate

        if self.method == "barnes_hut":
            X = self._validate_data(
                X,
                accept_sparse=["csr"],
                ensure_min_samples=2,
                dtype=[np.float32, np.float64],
            )
        else:
            X = self._validate_data(
                X, accept_sparse=["csr", "csc", "coo"], dtype=[np.float32, np.float64]
            )
        if self.metric == "precomputed":
            if isinstance(self.init, str) and self.init == "pca":
                raise ValueError(
                    'The parameter init="pca" cannot be used with metric="precomputed".'
                )
            if X.shape[0] != X.shape[1]:
                raise ValueError("X should be a square distance matrix")

            check_non_negative(
                X,
                (
                    "TSNE.fit(). With metric='precomputed', X "
                    "should contain positive distances."
                ),
            )

            if self.method == "exact" and issparse(X):
                raise TypeError(
                    'TSNE with method="exact" does not accept sparse '
                    'precomputed distance matrix. Use method="barnes_hut" '
                    "or provide the dense distance matrix."
                )

        if self.method == "barnes_hut" and self.n_components > 3:
            raise ValueError(
                "'n_components' should be inferior to 4 for the "
                "barnes_hut algorithm as it relies on "
                "quad-tree or oct-tree."
            )
        random_state = check_random_state(self.random_state)

        n_samples = X.shape[0]

        neighbors_nn = None
        if self.method == "exact":
            # Retrieve the distance matrix, either using the precomputed one or
            # computing it.
            if self.metric == "precomputed":
                distances = X
            else:
                if self.verbose:
                    print("[t-SNE] Computing pairwise distances...")

                if self.metric == "euclidean":
                    # Euclidean is squared here, rather than using **= 2,
                    # because euclidean_distances already calculates
                    # squared distances, and returns np.sqrt(dist) for
                    # squared=False.
                    # Also, Euclidean is slower for n_jobs>1, so don't set here
                    distances = pairwise_distances(X, metric=self.metric, squared=True)
                else:
                    metric_params_ = self.metric_params or {}
                    distances = pairwise_distances(
                        X, metric=self.metric, n_jobs=self.n_jobs, **metric_params_
                    )

            if np.any(distances < 0):
                raise ValueError(
                    "All distances should be positive, the metric given is not correct"
                )

            if self.metric != "euclidean":
                distances **= 2

            # compute the joint probability distribution for the input space
            P = _joint_probabilities(distances, self.perplexity, self.verbose)
            assert np.all(np.isfinite(P)), "All probabilities should be finite"
            assert np.all(P >= 0), "All probabilities should be non-negative"
            assert np.all(
                P <= 1
            ), "All probabilities should be less or then equal to one"

        else:
            # Compute the number of nearest neighbors to find.
            # LvdM uses 3 * perplexity as the number of neighbors.
            # In the event that we have very small # of points
            # set the neighbors to n - 1.
            n_neighbors = min(n_samples - 1, int(3.0 * self.perplexity + 1))

            if self.verbose:
                print("[t-SNE] Computing {} nearest neighbors...".format(n_neighbors))

            # Find the nearest neighbors for every point
            knn = NearestNeighbors(
                algorithm="auto",
                n_jobs=self.n_jobs,
                n_neighbors=n_neighbors,
                metric=self.metric,
                metric_params=self.metric_params,
            )
            t0 = time()
            knn.fit(X)
            duration = time() - t0
            if self.verbose:
                print(
                    "[t-SNE] Indexed {} samples in {:.3f}s...".format(
                        n_samples, duration
                    )
                )

            t0 = time()
            distances_nn = knn.kneighbors_graph(mode="distance")
            duration = time() - t0
            if self.verbose:
                print(
                    "[t-SNE] Computed neighbors for {} samples in {:.3f}s...".format(
                        n_samples, duration
                    )
                )

            # Free the memory used by the ball_tree
            del knn

            # knn return the euclidean distance but we need it squared
            # to be consistent with the 'exact' method. Note that the
            # the method was derived using the euclidean method as in the
            # input space. Not sure of the implication of using a different
            # metric.
            distances_nn.data **= 2

            # compute the joint probability distribution for the input space
            P = _joint_probabilities_nn(distances_nn, self.perplexity, self.verbose)

        if isinstance(self.init, np.ndarray):
            X_embedded = self.init
        elif self.init == "pca":
            pca = PCA(
                n_components=self.n_components,
                svd_solver="randomized",
                random_state=random_state,
            )
            # Always output a numpy array, no matter what is configured globally
            pca.set_output(transform="default")
            X_embedded = pca.fit_transform(X).astype(np.float32, copy=False)
            # PCA is rescaled so that PC1 has standard deviation 1e-4 which is
            # the default value for random initialization. See issue #18018.
            X_embedded = X_embedded / np.std(X_embedded[:, 0]) * 1e-4
        elif self.init == "random":
            # The embedding is initialized with iid samples from Gaussians with
            # standard deviation 1e-4.
            X_embedded = 1e-4 * random_state.standard_normal(
                size=(n_samples, self.n_components)
            ).astype(np.float32)

        # Degrees of freedom of the Student's t-distribution. The suggestion
        # degrees_of_freedom = n_components - 1 comes from
        # "Learning a Parametric Embedding by Preserving Local Structure"
        # Laurens van der Maaten, 2009.
        degrees_of_freedom = max(self.n_components - 1, 1)

        return self._tsne(
            P,
            degrees_of_freedom,
            n_samples,
            X_embedded=X_embedded,
            neighbors=neighbors_nn,
            skip_num_points=skip_num_points,
        )

    def _tsne(
        self,
        P,
        degrees_of_freedom,
        n_samples,
        X_embedded,
        neighbors=None,
        skip_num_points=0,
    ):
        """Runs t-SNE."""
        # t-SNE minimizes the Kullback-Leiber divergence of the Gaussians P
        # and the Student's t-distributions Q. The optimization algorithm that
        # we use is batch gradient descent with two stages:
        # * initial optimization with early exaggeration and momentum at 0.5
        # * final optimization with momentum at 0.8
        params = X_embedded.ravel()

        opt_args = {
            "it": 0,
            "n_iter_check": self._N_ITER_CHECK,
            "min_grad_norm": self.min_grad_norm,
            "learning_rate": self.learning_rate_,
            "verbose": self.verbose,
            "kwargs": dict(skip_num_points=skip_num_points),
            "args": [P, degrees_of_freedom, n_samples, self.n_components],
            "n_iter_without_progress": self._EXPLORATION_N_ITER,
            "n_iter": self._EXPLORATION_N_ITER,
            "momentum": 0.5,
        }
        if self.method == "barnes_hut":
            obj_func = _kl_divergence_bh
            opt_args["kwargs"]["angle"] = self.angle
            # Repeat verbose argument for _kl_divergence_bh
            opt_args["kwargs"]["verbose"] = self.verbose
            # Get the number of threads for gradient computation here to
            # avoid recomputing it at each iteration.
            opt_args["kwargs"]["num_threads"] = _openmp_effective_n_threads()
        else:
            obj_func = _kl_divergence

        # Learning schedule (part 1): do 250 iteration with lower momentum but
        # higher learning rate controlled via the early exaggeration parameter
        P *= self.early_exaggeration
        params, kl_divergence, it = _gradient_descent(obj_func, params, **opt_args)
        if self.verbose:
            print(
                "[t-SNE] KL divergence after %d iterations with early exaggeration: %f"
                % (it + 1, kl_divergence)
            )

        # Learning schedule (part 2): disable early exaggeration and finish
        # optimization with a higher momentum at 0.8
        P /= self.early_exaggeration
        remaining = self.n_iter - self._EXPLORATION_N_ITER
        if it < self._EXPLORATION_N_ITER or remaining > 0:
            opt_args["n_iter"] = self.n_iter
            opt_args["it"] = it + 1
            opt_args["momentum"] = 0.8
            opt_args["n_iter_without_progress"] = self.n_iter_without_progress
            params, kl_divergence, it = _gradient_descent(obj_func, params, **opt_args)

        # Save the final number of iterations
        self.n_iter_ = it

        if self.verbose:
            print(
                "[t-SNE] KL divergence after %d iterations: %f"
                % (it + 1, kl_divergence)
            )

        X_embedded = params.reshape(n_samples, self.n_components)
        self.kl_divergence_ = kl_divergence

        return X_embedded

    @_fit_context(
        # TSNE.metric is not validated yet
        prefer_skip_nested_validation=False
    )
    def fit_transform(self, X, y=None):
        """Fit X into an embedded space and return that transformed output.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape (n_samples, n_features) or \
            (n_samples, n_samples)
            If the metric is 'precomputed' X must be a square distance
            matrix. Otherwise it contains a sample per row. If the method
            is 'exact', X may be a sparse matrix of type 'csr', 'csc'
            or 'coo'. If the method is 'barnes_hut' and the metric is
            'precomputed', X may be a precomputed sparse graph.

        y : None
            Ignored.

        Returns
        -------
        X_new : ndarray of shape (n_samples, n_components)
            Embedding of the training data in low-dimensional space.
        """
        self._check_params_vs_input(X)
        embedding = self._fit(X)
        self.embedding_ = embedding
        return self.embedding_

    @_fit_context(
        # TSNE.metric is not validated yet
        prefer_skip_nested_validation=False
    )
    def fit(self, X, y=None):
        """Fit X into an embedded space.

        Parameters
        ----------
        X : {array-like, sparse matrix} of shape (n_samples, n_features) or \
            (n_samples, n_samples)
            If the metric is 'precomputed' X must be a square distance
            matrix. Otherwise it contains a sample per row. If the method
            is 'exact', X may be a sparse matrix of type 'csr', 'csc'
            or 'coo'. If the method is 'barnes_hut' and the metric is
            'precomputed', X may be a precomputed sparse graph.

        y : None
            Ignored.

        Returns
        -------
        self : object
            Fitted estimator.
        """
        self.fit_transform(X)
        return self

    @property
    def _n_features_out(self):
        """Number of transformed output features."""
        return self.embedding_.shape[1]

    def _more_tags(self):
        return {"pairwise": self.metric == "precomputed"}


#
#  Project: MXCuBE
#  https://github.com/mxcube.
#
#  This file is part of MXCuBE software.
#
#  MXCuBE is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  MXCuBE is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with MXCuBE.  If not, see <http://www.gnu.org/licenses/>.

import pickle

__credits__ = ["MXCuBE collaboration"]
__license__ = "LGPLv3+"

class Property:
    def __init__(self, property_name, default_value=None):
        self.name = property_name

        self.type = "undefined"
        self.value = None
        self.old_value = None
        self.hidden = False
        self._editor = None
        self.comment = ""

        if default_value is None:
            self.default_value = None
        else:
            self.set_default_value(default_value, True)

    def __getstate__(self):
        dict = self.__dict__.copy()  # copy the dict since we change it
        if hasattr(self, "_editor"):
            del dict["_editor"]  # remove ref. to editor
        return dict

    def __setstate__(self, dict):
        self.__dict__.update(dict)

        if not "hidden" in dict:
            self.hidden = False

        self._editor = None

    def get_name(self):
        return self.name

    def get_type(self):
        return self.type

    def get_user_value(self):
        return self.get_value()

    def get_value(self):
        return self.value

    def get_default_value(self):
        return self.default_value

    def set_value(self, property_value):
        self.value = property_value

    def set_default_value(self, value, set_as_value=False):
        saved_value = self.value
        saved_old_value = self.old_value

        try:
            self.set_value(value)
        except ValueError:
            raise ValueError(
                "cannot set default value to %s : incompatible types." % str(value)
            )

        self.default_value = self.value

        if not set_as_value:
            self.value = saved_value
            self.old_value = saved_old_value

    def set_comment(self, comment):
        self.comment = comment

    def get_comment(self):
        return self.comment

    def as_dict(self):
        return {"type": self.type, "value": self.value}

    def set_from_dict(self, property_dict):
        self.type = property_dict["type"]
        self.value = property_dict["value"]

class StringProperty(Property):
    def __init__(self, property_name, default_value=""):
        Property.__init__(self, property_name, default_value)

        self.type = "string"

    def set_value(self, property_value):
        self.old_value = self.value
        self.value = str(property_value)

class IntegerProperty(Property):
    def __init__(self, property_name, default_value=None):
        Property.__init__(self, property_name, default_value)

        self.type = "integer"

    def set_value(self, property_value):
        try:
            new_value = int(property_value)
        except ValueError:
            raise ValueError("%s is not a valid integer value." % repr(property_value))

        self.old_value = self.value
        self.value = new_value

class BooleanProperty(Property):
    def __init__(self, property_name, default_value=False):
        Property.__init__(self, property_name, default_value)

        self.type = "boolean"

    def set_value(self, property_value):
        self.old_value = self.value

        if property_value:
            self.value = 1
        else:
            self.value = 0

class ComboProperty(Property):
    def __init__(self, property_name, choices=[], default_value=None):
        self.set_choices(choices)

        Property.__init__(self, property_name, default_value)
        self.type = "combo"

    def add_choice(self, choice):
        self.choices.append(str(choice))

    def set_choices(self, choices):
        try:
            self.choices = list(choices)
        except BaseException:
            raise ValueError("%s cannot be converted into a list" % repr(choices))

    def get_choices(self):
        return self.choices

    def set_value(self, property_value):
        str_value = str(property_value)

        for choice in self.choices:
            if str_value == choice:
                self.old_value = self.value
                self.value = str_value
                return
        raise ValueError(
            "%s is not a valid choice for combo" % repr(str(property_value))
        )

class FloatProperty(Property):
    def __init__(self, property_name, default_value=None):
        Property.__init__(self, property_name, default_value)

        self.type = "float"

    def set_value(self, property_value):
        try:
            new_value = float(property_value)
        except ValueError:
            raise ValueError("%s is not a valid float value" % repr(property_value))

        self.old_value = self.value
        self.value = new_value

class FileProperty(Property):
    def __init__(self, property_name, filter="All (*.*)", default_value=None):
        Property.__init__(self, property_name, default_value or "")

        self.type = "file"
        self.filter = filter

    def set_value(self, property_value):
        import os.path

        new_value = os.path.abspath(str(property_value))

        self.old_value = self.value
        self.value = new_value

    def get_filter(self):
        return self.filter

class ColorProperty(Property):
    def __init__(self, property_name, default_value=None):
        Property.__init__(self, property_name, default_value)

        self.type = "color"

    def set_value(self, property_value):
        self.old_value = self.value
        self.value = property_value

class FormatStringProperty(Property):
    def __init__(self, property_name, default_value=None):
        self.format_string = None
        self.format_string_length = 0

        Property.__init__(self, property_name, default_value)

        self.type = "formatString"

    def set_value(self, property_value):
        self.old_value = self.value
        self.value = str(property_value)

        if property_value.startswith("+"):
            prefix = "+"
            property_value = property_value[1:]
        elif property_value.startswith(" "):
            prefix = ""
            property_value = property_value[1:]
        else:
            prefix = ""

        parts = property_value.split(".")

        if len(parts) == 2:
            self.format_string = (
                "%" + prefix + str(len(parts[0])) + "." + str(len(parts[1])) + "f"
            )
        elif len(parts) == 1:
            self.format_string = "%" + prefix + str(len(parts[0])) + ".0f"
        else:
            raise ValueError

        self.format_string_length = sum(map(len, parts)) + 1

    def get_user_value(self):
        return self.value

    def get_value(self):
        return self.format_string

    def get_format_length(self):
        return self.format_string_length

class PropertyBag:
    def __init__(self):
        self.properties = {}

    def add_property(
        self,
        property_name,
        property_type,
        arg1=None,
        arg2=None,
        comment="",
        hidden=False,
    ):
        if property_type == "string":
            if arg1 is None:
                arg1 = ""
            new_property = StringProperty(property_name, arg1)
        elif property_type == "integer":
            new_property = IntegerProperty(property_name, arg1)
        elif property_type == "combo":
            if list(arg1):
                new_property = ComboProperty(property_name, arg1, arg2)
            else:
                new_property = ComboProperty(property_name, default_value=arg1)
        elif property_type == "boolean":
            new_property = BooleanProperty(property_name, arg1)
        elif property_type == "float":
            new_property = FloatProperty(property_name, arg1)
        elif property_type == "file":
            new_property = FileProperty(property_name, arg1, arg2)
        elif property_type == "color":
            new_property = ColorProperty(property_name, default_value=arg1)
        elif property_type == "formatString":
            new_property = FormatStringProperty(property_name, default_value=arg1)
        else:
            new_property = Property(property_name, arg1)

        new_property.comment = comment
        new_property.hidden = hidden

        self.properties[property_name] = new_property

        self.update_editor()

    def update_editor(self):
        for prop_name, prop in self.properties.items():
            if prop._editor is not None:
                # the properties are being edited,
                # refresh property editor
                editor = prop._editor()

                if editor is not None:
                    editor.set_property_bag(self)
                break

    def del_property(self, property_name):
        prop_editor = None
        for prop_name, prop in self.properties.items():
            prop_editor = prop._editor
            break

        try:
            del self.properties[property_name]
        except KeyError:
            pass
        else:
            try:
                editor = prop_editor()
                if editor is not None:
                    editor.set_property_bag(self)
            except TypeError:
                pass

    def get_property(self, property_name):
        try:
            return self.properties[property_name]
        except KeyError:
            return Property("")

    def hide_property(self, property_name):
        prop = self.properties.get(property_name, None)
        if prop is not None and not prop.hidden:
            prop.hidden = True
            if prop._editor is not None:
                editor = prop._editor()
                if editor is not None:
                    editor.set_property_bag(self)

    def show_property(self, property_name):
        prop = self.properties.get(property_name, None)
        if prop is not None and prop.hidden:
            prop.hidden = False
            if prop._editor is not None:
                editor = prop._editor()
                if editor is not None:
                    editor.set_property_bag(self)

    def __repr__(self):
        return repr(pickle.dumps(self))

    def __str__(self):
        return "<PropertyBag instance>"

    def __iter__(self):
        keys = sorted(self.properties.keys())

        for key in keys:
            yield self.properties[key]

    def __len__(self):
        return len(self.properties)

    def __getitem__(self, property_key):
        item = self.properties.get(property_key)
        return None if item is None else item.get_value()

    def __setitem__(self, property_name, property):
        self.properties[property_name] = property

    def is_empty(self):
        return len(self.properties) == 0

    def has_property(self, name):
        return name in self.properties


print("Hello! This program converts kilometers into miles.")

while True:
    Km = int(input("Enter the number of kilometers: "))
    Miles = (Km * 1.6)
    print(str(Km) + " kilometers is " + str(Miles) + " miles.")
    additional = input("Would you like to do another conversion? ")
    if additional.lower() != "y" and additional.lower() != "yes":
        print("Thank you for using this converter.")
        break


from socket import *
import time, curses, random

host = "localhost"
port = 21567
addr = (host,port)
buf = 1024

newSck = socket(AF_INET,SOCK_STREAM)
#newSck.connect(('localhost',2727))

#init curses module
#stdscr = curses.initscr()
#curses.cbreak()
#stdscr.keypad(1)

#simulate misclassifications
errArr = [1,0,1,1,0,1,1,1,1,1]   #find out code for replicating ones

while(1):
	#get keypress
	ch = stdscr.getch()

	#time to aqcuire and analyze signal
	#time.sleep(1.5)

	#error code
	bool = random.choice(errArr)
	if bool == 0:
		if ch == 258:
			ch = 259
		else:
			ch == 258

	if ch == ord('q'):  #quit
		break
	elif ch == 258:     #down key
	  	decision = 0
	elif ch == 259:     #up key
		decision = 1

	data = "%s" % (decision)
	newSck.sendto(data,addr)
	#newSck.recv(100)

#return terminal to normal behavior
curses.nocbreak()
stdscr.keypad(0)
curses.endwin()

newSck.close()


import json
import ast
from main.constants import *

def load_features(
	input_file,
	load_mapping=False,
	key_val_separator=KEY_VAL_SEPARATOR
	):
	"""
	input_file: input_file_path
		each line is item in feature list
	load_mapping: whether or not to load (feature, value) mapping
	key_val_separator: separator between feature and value if load_mapping

	output:
		list / dictionary of features
	"""
	if load_mapping:
		features = dict()
		try:
			with open(input_file, "r") as file:
				lines = file.readlines()
				for line in lines:
					[feature_name, feature_value] = line.strip('\n').split(key_val_separator)
					if feature_name not in features:
						features[feature_name] = list()
					features[feature_name].append(feature_value)

		except IOError:
			print("{} doesn't exists".format(input_file))

	else:
		features = list()
		try:
			with open(input_file, "r") as file:
				lines = file.readlines()
				for line in lines:
					features.append(line)

		except IOError:
			print("{} doesn't exists".format(input_file))

	return features

def load_feature_vec_to_int(input_file):
	"""
	input_file: input_file_path
		each line is json object contains feature_id and feature_dict

	output:
		mapping from feature_dict to feature_id
	"""
	feature_vector_to_int = dict()
	try:
		with open(input_file, "r") as file:
			lines = file.readlines()
			for line in lines:
				line = json.load(line.strip('\n'))
				feature_dict = line[FEATURE_DICT]
				feature_id = line[FEATURE_ID]
				feature_vector_to_int[frozenset(feature_dict.items())] = feature_id

	except IOError:
		print("{} doesn't exists".format(input_file))

	return feature_vector_to_int

def load_mapping(
	input_file,
	value_is_set,
	eval_set={0,1},
	col_separator=COL_SEPARATOR
	):
	"""
	input_file: input_file_path
		each line is (key, value) pair separated by col_separator
	value_is_set: whether or not value is set
	eval_set: for each line of (key, value), indices to calculate
	col_separator: separator between key and values

	output:
		mapping from key to value in dictionary
	"""
	mapping = dict()
	try:
		with open(input_file, "r") as file:
			lines = file.readlines()
			for line in lines:
				[key, val] = line.strip('\n').split(col_separator)

				if 0 in eval_set:
					key = ast.literal_eval(key)

				if 1 in eval_set:
					val = ast.literal_eval(val)

				if value_is_set:
					if key not in mapping:
						mapping[key] = list()
					mapping[key].append(val)

				else:
					mapping[key] = val

	except IOError:
		print("{} doesn't exists".format(input_file))

	return mapping

def load_reviews(
	input_file,
	eval_set={0,1,2,3},
	col_separator=COL_SEPARATOR
	):
	"""
	input_file: input_file_path
		each line is col_separateor separated review [int_user_id, restaurant, rating, timestamp]
	eval_set: for each review, indices to calculate
	col_separator: separator between key and values

	output:
		reviews: list of reviews [int_user_id, restaurant, rating, timestamp]
	"""
	reviews = list()
	try:
		with open(input_file, "r") as file:
			lines = file.readlines()
			for line in lines:
				[user, restaurant, rating, timestamp] = line.strip('\n').split(col_separator)
				if 0 in eval_set:
					user = ast.literal_eval(user)

				if 1 in eval_set:
					restaurant = ast.literal_eval(restaurant)

				if 2 in eval_set:
					rating = ast.literal_eval(rating)

				if 3 in eval_set:
					timestamp = ast.literal_eval(timestamp)

				reviews.append([user, restaurant, rating, timestamp])

	except IOError:
		print("{} doesn't exists".format(input_file))

	return reviews


#!/usr/bin/env python
# coding: utf-8

# # Tutorial 1
#
# ## Question 1
# A time `t` in seconds can be converted to days, hours, minutes and seconds using integer division:
#
# 1. Divide `t` by 60; set `t` to the quotient and call the remainder `s`.
# 1. Divide `t` by 60; set `t` to the quotient and call the remainder `m`.
# 1. Divide `t` by 24; set `d` to the quotient and call the remainder `h`.
#
# Calculate the number of days, hours, minutes and seconds in one million seconds. Display the result as follows:
# ```
# 1000000 seconds is xx days H:M:S
# ```

# In[1]:

# It should be reasonably easy to follow the instructions in order to produce the correct code to perform the calculation.
# The only tricky part is to maintain the original value of `t` in a new variable `t_orig` so that we can include it in the final print statement.

t = 1000000
t_orig = t
s = t % 60
t = t // 60
m = t % 60
t = t // 60
h = t % 24
d = t // 24

print(t_orig, " seconds is ", d, " days ", h, ":", m, ":", s, sep="")

# ## Question 2
#
# Iodine-131 is a radioisotope with a half-life $t_\mathrm{half} = 8.02$ days.
#
# 1. Use the equation $r = \left(\frac{1}{2}\right) ^ {1/t_\mathrm{half}}$ to calculate the daily decay rate (i.e. the proportion of a sample of Iodine 131 that decays in one day).
# 1. A sample of radioactive material containing Iodine-131 has an activity of $1.67 \times 10^{12}$ Becquerels. Write a program which calculates the number of days it takes for the sample to reach an activity of $10000$ Becquerels.
# 1. Print the activity of the sample once every 7 days.

# In[2]:

# This is very similar to the bacterial growth example in the notes.
# You shouldn't need to know much about radioactive decay to answer this.
# However, this is the kind of foundational knowledge that I would expect all Natural Scientists to be aware of,
# even if you don't specialise in physics.

t_half = 8.02
r = 0.5 ** (1/t_half)
print("Daily decay rate:", r)

activity = 1.67e12

i = 0
while activity > 10000:

    activity = activity * r

    i += 1 # or i = i + 1

    if i % 7 == 0:
        print(activity)

print("Number of days:", i)

# ## Question 3
#
# We can calculate how many digits there are in a number by repeatedly dividing by 10 until the number is less than one.
#
# 1. Write peudo-code for this calculation.
# 1. Write a Python program to implement it, displaying the result in the format `1234 has 4 digits.`
# 1. Test your program with a variety of numbers.
# 1. Extend your program so that it works for negative numbers.
#

#
#
# 1. Pseudocode.
#
# This is just an example. It is most important that it makes sense to you. Notice that each line corresponds to one line of Python.
# ```
# set value of x
# set y to value of x
# set i = 0
# repeat until x less than 1:
#     divide x 10
#     increase i by 1
# display value of y and i
# ```

# In[3]:

# 2.

x = 1234
y = x
i = 0
while x > 1:
    x = x / 10
    i += 1
print(y, "has", i, "digits")

#3.

# Change the value of x in the code above and test that the result is correct.
# It's a good idea to test 'edge cases' such as 1000, 99999

#4.

# The simplest way to do this is to multiply x by -1 if it is negative.
# This is quite a clever 'trick' that allows us to re-use our existing code!

x = -1234.
y = x
if x < 0:
    x = x * -1

i = 0
while x > 1:
    x = x / 10
    i += 1
print(y, "has", i, "digits")

# ## Question 4 (The Collatz Problem)
#
# Given a positive integer `n`, the Collatz Operation is defined as follows:
#
#  - If the number is even, divide it by two
#  - If the number is odd, triple it and add one
#
# Repeatedly applying the Collatz Operation results in a sequence of integers which eventually reaches the number 1.
#
# > Write a program which generates the Collatz Sequence for a given positive integer $n$.
#
# For example, if $n = 5$ then your program should produce the following:
#
# ```
# 5
# 16
# 8
# 4
# 2
# 1
# ```
#
# Given a positive integer $n$, define the Collatz Number to be the length of the Collatz Sequence for $n$. For example, the Collatz number of 5 is 6.
#
# > Write a program which calculates and prints the Collatz number of a given number $n$. For example, if $n=5$ your program should print `The Collatz number of 5 is 6`.
#
# > Write a program which finds the smallest number $n$ whose Collatz Number is greater than 200.
#

# In[4]:

# Despite a completely different problem, we can use a very similar pattern that
# we used in the previous two questions.

n = 5
m = n
i = 1
while n > 1:
    if n % 2 == 0:
        n = n // 2
    else:
        n = 3*n + 1
    i += 1
print("The Collatz Number of", m, "is", i)

# In[5]:

# The second part is tricky because we need to repeat the previous
# code for sequential numbers. But we already know how to do this:
# using while loops. Thus, we have a while loop within a while loop.
# The outer one terminates when we reach a collatz number greater than 200.

# Keeping track of the various variables is difficult and it might take some
# perseverence to get this right. Using a function to encapsulate the inner loop
# might make it easier - but we haven't studied functions yet.

n = 1
i = 1
while i <= 200: # while i is less than *or equal to* 200
    i = 1
    m = n
    while m > 1:
        if m % 2 == 0:
            m = m // 2
        else:
            m = 3 * m + 1
        i += 1

    print(n, i)
    n = n + 1

print("The Collatz Number of ", n - 1, "is", i) #note that we have to subract 1 from n here

# In[ ]:


#__________________________________________________________________________
#
# (C) dGB Beheer B.V.; (LICENSE) http://opendtect.org/OpendTect_license.txt
# Author:        O. Ibrahim
# Date:          Sept 2021
#
# _________________________________________________________________________
# various tools machine learning using PyTorch platform
#
import os, json, pickle, joblib
import numpy as np
import dgbpy.keystr as dgbkeys
import dgbpy.hdf5 as dgbhdf5
import odpy.hdf5 as odhdf5

try:
  import torch
  import torch.nn as nn
  from torch.utils.data import DataLoader
  import dgbpy.torch_classes as tc
  device = torch.device('cpu')
except ModuleNotFoundError:
  device = None
  pass

def hasTorch():
  try:
    import torch
  except ModuleNotFoundError:
    return False
  return True

platform = (dgbkeys.torchplfnm, 'PyTorch')

withtensorboard = dgbkeys.getDefaultTensorBoard()

default_transforms = []

defbatchstr = 'defaultbatchsz'

torch_dict = {
    dgbkeys.decimkeystr: False,
    'nbchunk': 10,
    'epochs': 15,
    'patience': 10,
    'epochdrop': 5,
    'split': 0.2,
    'nbfold': 5,
    'criterion': nn.CrossEntropyLoss() if hasTorch() else None,
    'batch': 8,
    'learnrate': 1e-4,
    'type': None,
    'prefercpu': None,
    'scale': dgbkeys.globalstdtypestr,
    'transform':default_transforms,
    'withtensorboard': withtensorboard,
    'tofp16': True,
}

def getMLPlatform():
  return platform[0]

cudacores = [ '1', '2', '4', '8', '16', '32', '48', '64', '96', '128', '144', '192', '256', \
              '288',  '384',  '448',  '480',  '512',  '576',  '640',  '768', \
              '896',  '960',  '1024', '1152', '1280', '1344', '1408', '1536', \
              '1664', '1792', '1920', '2048', '2176', '2304', '2432', '2496', \
              '2560', '2688', '2816', '2880', '2944', '3072', '3584', '3840', \
              '4352', '4608', '4992', '5120' ]

def can_use_gpu():
  if torch.cuda.is_available():
    return True
  return False

def set_compute_device(prefercpu):
  global device
  if not prefercpu:
    prefercpu = not can_use_gpu()
  device = torch.device('cuda:0' if not prefercpu else 'cpu')

def get_torch_infos():
  ret = {
    'hastorchgpu': can_use_gpu(),
    dgbkeys.prefercpustr: not can_use_gpu(),
    'batchsizes': cudacores,
    defbatchstr: torch_dict['batch']
  }
  return json.dumps( ret )

def getParams(
    nntype=torch_dict['type'],
    dodec = torch_dict[dgbkeys.decimkeystr],
    nbchunk = torch_dict['nbchunk'],
    learnrate=torch_dict['learnrate'],
    epochdrop = torch_dict['epochdrop'],
    epochs=torch_dict['epochs'],
    patience=torch_dict['patience'],
    batch=torch_dict['batch'],
    prefercpu = torch_dict['prefercpu'],
    validation_split=torch_dict['split'],
    nbfold=torch_dict['nbfold'],
    scale=torch_dict['scale'],
    transform=torch_dict['transform'],
    withtensorboard=torch_dict['withtensorboard'],
    tofp16=torch_dict['tofp16']):
  ret = {
    dgbkeys.decimkeystr: dodec,
    'type': nntype,
    'nbchunk': nbchunk,
    'learnrate': learnrate,
    'epochdrop': epochdrop,
    'epochs': epochs,
    'patience': patience,
    'split':validation_split,
    'nbfold':nbfold,
    'batch': batch,
    'scale': scale,
    'transform': transform,
    'withtensorboard': withtensorboard,
    'tofp16': tofp16,
  }
  if prefercpu == None:
    prefercpu = not can_use_gpu()
  ret.update({'prefercpu': prefercpu})
  if not dodec:
    ret['nbchunk'] = 1
  return ret

def getDefaultModel(setup,type=torch_dict['type']):
  isclassification = setup[dgbhdf5.classdictstr]
  inp_shape = setup[dgbkeys.inpshapedictstr]
  attribs = dgbhdf5.getNrAttribs(setup)
  model_shape = get_model_shape(inp_shape, attribs, True)
  if isclassification:
    nroutputs = len(setup[dgbkeys.classesdictstr])
  else:
    nroutputs = dgbhdf5.getNrOutputs( setup )
  if len(tc.TorchUserModel.mlmodels) < 1:
    tc.TorchUserModel.mlmodels = tc.TorchUserModel.findModels()
  if type==None:
      type = getModelsByInfo( setup )
  if tc.TorchUserModel.findName(type):
    return tc.TorchUserModel.findName(type).model(model_shape, nroutputs, attribs)
  return None

def getModelsByType( learntype, classification, ndim ):
    predtype = tc.DataPredType.Continuous
    outtype = tc.OutputType.Pixel
    dimtype = tc.DimType(ndim)
    if dgbhdf5.isImg2Img(learntype):
        outtype = tc.OutputType.Image
    if classification or dgbhdf5.isSeisClass( learntype ):
            predtype = tc.DataPredType.Classification
    return tc.TorchUserModel.getNamesByType(pred_type=predtype, out_type=outtype, dim_type=dimtype)

def getModelsByInfo( infos ):
    shape = infos[dgbkeys.inpshapedictstr]
    if isinstance(shape,int):
        ndim = 1
    else:
        ndim = len(shape) - shape.count(1)
    modelstypes = getModelsByType( infos[dgbkeys.learntypedictstr],
                                   infos[dgbhdf5.classdictstr],
                                   ndim )
    if len(modelstypes) < 1:
        return None
    return modelstypes[0]

def get_model_shape( shape, nrattribs, attribfirst=True ):
  ret = ()
  if attribfirst:
    ret += (nrattribs,)
  if isinstance( shape, int ):
    ret += (shape,)
    if not attribfirst:
      ret += (nrattribs,)
    return ret
  else:
    for i in shape:
      if i > 1:
        ret += (i,)
  if attribfirst:
    if len(ret) == 1:
      ret += (1,)
  else:
    if len(ret) == 0:
      ret += (1,)
  if not attribfirst:
    ret += (nrattribs,)
  return ret

def getModelDims( model_shape, data_format ):
  if data_format == 'channels_first':
    ret = model_shape[1:]
  else:
    ret = model_shape[:-1]
  if len(ret) == 1 and ret[0] == 1:
    return 0
  return len(ret)

savetypes = ( 'onnx', 'joblib', 'pickle' )
defsavetype = savetypes[0]

def load( modelfnm ):
  model = None
  h5file = odhdf5.openFile( modelfnm, 'r' )
  modelgrp = h5file['model']
  savetype = odhdf5.getText( modelgrp, 'type' )

  modeltype = odhdf5.getText(h5file, 'type')
  if modeltype=='Sequential' or modeltype=='Net':
    savetype = savetypes[1]
  if savetype == savetypes[0]:
    modfnm = odhdf5.getText( modelgrp, 'path' )
    modfnm = dgbhdf5.translateFnm( modfnm, modelfnm )
    from dgbpy.torch_classes import OnnxTorchModel
    model = OnnxTorchModel(str(modfnm))
  elif savetype == savetypes[1]:
    modfnm = odhdf5.getText( modelgrp, 'path' )
    modfnm = dgbhdf5.translateFnm( modfnm, modelfnm )
    model = joblib.load( modfnm )
  elif savetype == savetypes[2]:
    modeldata = modelgrp['object']
    model = pickle.loads( modeldata[:].tostring() )
  h5file.close()
  return model

def onnx_from_torch(model, infos):
  attribs = dgbhdf5.getNrAttribs(infos)
  isclassification = infos[dgbhdf5.classdictstr]
  model_shape = get_model_shape(infos[dgbkeys.inpshapedictstr], attribs, True)
  dims = getModelDims(model_shape, True)
  if isclassification:
    nroutputs = len(infos[dgbkeys.classesdictstr])
    predtype = tc.DataPredType.Classification
  else:
    predtype = tc.DataPredType.Continuous
    nroutputs = dgbhdf5.getNrOutputs( infos )
  input_size = torch_dict['batch']
  if model.__class__.__name__ == 'UNet' or 'UNet_VGG19':
    input_size = 1
  if dims  == 3:
    dummy_input = torch.randn(input_size, model_shape[0], model_shape[1], model_shape[2], model_shape[3])
  elif dims == 2:
    dummy_input = torch.randn(input_size, model_shape[0], model_shape[1], model_shape[2])
  elif dims == 1 or dims == 0:
    dummy_input = torch.randn(input_size, model_shape[0], model_shape[1])
  if model.__class__.__name__ == 'Sequential':
    from dgbpy.mlmodel_torch_dGB import ResNet18
    model_instance = ResNet18(nroutputs, dims, attribs)
  elif model.__class__.__name__ == 'Net':
    from dgbpy.torch_classes import Net
    model_instance = Net(nroutputs, dims, attribs)
  elif model.__class__.__name__ == "dGBLeNet":
    from dgbpy.torch_classes import dGBLeNet
    model_instance = dGBLeNet(model_shape, nroutputs, dims, attribs, predtype)
  elif model.__class__.__name__ == 'UNet':
    from dgbpy.torch_classes import UNet
    model_instance = UNet(out_channels=nroutputs, dim=dims, in_channels=attribs, n_blocks=model.n_blocks)
  elif model.__class__.__name__ == 'UNet_VGG19':
    from dgbpy.torch_classes import UNet_VGG19
    model_instance = UNet_VGG19(model_shape, nroutputs, attribs)
  elif model.__class__.__name__ == 'GraphModule':
    model_instance = torch.jit.trace(model.cpu(), dummy_input)
  model_instance.load_state_dict(model.state_dict())
  return model_instance, dummy_input

def save( model, outfnm, infos, save_type=defsavetype ):
  h5file = odhdf5.openFile( outfnm, 'w' )
  odhdf5.setAttr( h5file, 'backend', 'PyTorch' )
  odhdf5.setAttr( h5file, 'torch_version', torch.__version__ )
  odhdf5.setAttr( h5file, 'type', model.__class__.__name__ )
  odhdf5.setAttr( h5file, 'model_config', json.dumps((str(model)) ))
  modelgrp = h5file.create_group( 'model' )
  odhdf5.setAttr( modelgrp, 'type', save_type )
  if model.__class__.__name__ == 'Sequential' or model.__class__.__name__ == "Net":
    save_type = savetypes[1]
  if save_type == savetypes[0]:
    joutfnm = os.path.splitext( outfnm )[0] + '.onnx'
    retmodel, dummies = onnx_from_torch(model, infos)
    input_name = ['input']
    dynamic_axes = {'input': {0: 'batch_size'}}
    torch.onnx.export(retmodel, dummies, joutfnm, input_names=input_name, dynamic_axes=dynamic_axes)
    odhdf5.setAttr( modelgrp, 'path', joutfnm )
  elif save_type == savetypes[1]:
    joutfnm = os.path.splitext( outfnm )[0] + '.joblib'
    joblib.dump( model.state_dict(), joutfnm )
    odhdf5.setAttr( modelgrp, 'path', joutfnm )
  elif save_type == savetypes[2]:
    exported_modelstr = pickle.dumps(model.state_dict())
    exported_model = np.frombuffer( exported_modelstr, dtype='S1', count=len(exported_modelstr) )
    modelgrp.create_dataset('object',data=exported_model)
  h5file.close()

def train(model, imgdp, params, cbfn=None, logdir=None, silent=False):
    from dgbpy.torch_classes import Trainer, AdaptiveLR
    trainloader, testloader = DataGenerator(imgdp,batchsize=params['batch'],scaler=params['scale'],transform=params['transform'])
    criterion = torch_dict['criterion']
    if imgdp[dgbkeys.infodictstr][dgbkeys.classdictstr]==False:
      criterion = nn.MSELoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=params['learnrate'])
    if params['epochdrop'] < 1:
      scheduler = None
    else:
      scheduler = AdaptiveLR(optimizer, params['learnrate'], params['epochdrop'])
    tensorboard = None
    if logdir != None and params['withtensorboard']:
      from torch.utils.tensorboard import SummaryWriter
      tensorboard = SummaryWriter(log_dir=logdir)
    set_compute_device(params['prefercpu'])
    trainer = Trainer(
        model=model,
        criterion=criterion,
        optimizer=optimizer,
        scheduler=scheduler,
        device = device,
        training_DataLoader=trainloader,
        validation_DataLoader=testloader,
        tensorboard=tensorboard,
        epochs=params['epochs'],
        earlystopping = params['patience'],
        imgdp=imgdp,
        silent = silent,
        tofp16=params['tofp16']
    )
    model = trainer.fit(cbs = cbfn)
    return model

def transfer(model):
  """
    Transfer learning utility function for fine-tuning a Torch model.

    This function takes a Torch model and prepares it for transfer learning by selectively
    setting layers to be trainable. The layers to be made trainable are determined as follows:

    1. All layers before the first Conv1D, Conv2D, or Conv3D layer (or a Sequential containing such layers)
       are set to trainable.

    2. All layers after the last Conv1D, Conv2D, Conv3D, or Dense layer (or a Sequential containing
       such layers) are set to trainable.

    3. All layers between the first and last Conv1D, Conv2D, Conv3D, or Dense layer (or a Sequential
        containing such layers) are set to non-trainable.
  """
  def getTrainableLayers(model):
    first_lyr = None
    last_lyr = None

    for name, layer in model.named_modules():
        if isinstance(layer, (nn.Conv1d, nn.Conv2d, nn.Conv3d)):
            if first_lyr is None:
                first_lyr = name
            last_lyr = name
        elif isinstance(layer, nn.Linear) and first_lyr:
          last_lyr = name
    return first_lyr, last_lyr

  from dgbpy.torch_classes import OnnxTorchModel
  if isinstance(model, OnnxTorchModel):
    model = model.convert_to_torch()

  first_conv_layer, last_layer = getTrainableLayers(model)
  found_first, found_last = False, False

  for name, param in model.named_parameters():
    if first_conv_layer and name.startswith(first_conv_layer):
      param.requires_grad = True
      found_first = True
    elif last_layer and name.startswith(last_layer):
      param.requires_grad = True
      found_last = True
    elif found_first and not found_last:
      param.requires_grad = False
    else:
      param.requires_grad = True
  return model

def apply( model, info, samples, scaler, isclassification, withpred, withprobs, withconfidence, doprobabilities ):
  attribs = dgbhdf5.getNrAttribs(info)
  model_shape = get_model_shape(info[dgbkeys.inpshapedictstr], attribs, True)
  ndims = getModelDims(model_shape, 'channels_first')
  sampleDataset = tc.DatasetApply(samples, info, isclassification, 1, ndims=ndims)
  ret = {}
  res = None
  try:
    img2img = info[dgbkeys.seisimgtoimgtypestr]
    img2img = True
  except KeyError:
    img2img = False

  if info[dgbkeys.learntypedictstr] == dgbkeys.seisclasstypestr or \
      info[dgbkeys.learntypedictstr] == dgbkeys.loglogtypestr:
      drop_last = True
  else:
    drop_last = False
  batch_size = torch_dict['batch']
  dataloader = getDataLoader(sampleDataset, batch_size=batch_size, drop_last=drop_last)
  if isclassification:
    nroutputs = len(info[dgbkeys.classesdictstr])
    predtype = tc.DataPredType.Classification
  else:
    nroutputs = dgbhdf5.getNrOutputs(info)
    predtype = tc.DataPredType.Continuous

  if model.__class__.__name__ == "OrderedDict":
    from dgbpy.mlmodel_torch_dGB import ResNet18
    from dgbpy.torch_classes import dGBLeNet, Net
    seis_log_arch = [ResNet18, dGBLeNet, Net]
    if info[dgbkeys.learntypedictstr] == dgbkeys.seisclasstypestr or \
      info[dgbkeys.learntypedictstr] == dgbkeys.loglogtypestr or info[dgbkeys.learntypedictstr] == dgbkeys.seisproptypestr:
      for arch in seis_log_arch:
        try:
          if arch.__name__ == 'dGBLeNet': dfdm = arch(nroutputs, dim=ndims, nrattribs=attribs, predtype=predtype)
          else: dfdm = arch(nroutputs, dim=ndims, nrattribs=attribs)
          dfdm.load_state_dict(model)
        except RuntimeError: pass
    elif info[dgbkeys.learntypedictstr] == dgbkeys.seisimgtoimgtypestr:
      from dgbpy.torch_classes import UNet
      if isclassification:
        dfdm = UNet(out_channels=nroutputs, n_blocks=1, dim=ndims)
      else:
        dfdm = UNet(out_channels=1,  n_blocks=1, dim=ndims)
      dfdm.load_state_dict(model)
  elif model.__class__.__name__ == 'OnnxTorchModel':
    dfdm = model

  predictions = []
  predictions_prob = []
  dfdm.eval()
  for input in dataloader:
      with torch.no_grad():
        out = dfdm(input)
        pred = out.detach().numpy()
        pred_prob = pred.copy()
        if isclassification:
          pred = np.argmax(pred, axis=1)
        for _ in pred:
          predictions.append(_)
        for _prob in pred_prob:
          predictions_prob.append(_prob)

  if withpred:
    if isclassification:
      if not (doprobabilities or withconfidence):
        try:
          res = np.array(predictions_prob)
          res = np.transpose(np.array(predictions))
        except AttributeError:
          pass

  if isclassification and (doprobabilities or withconfidence or withpred):
    if len(ret)<1:
      allprobs = (np.array(predictions)).transpose()
    else:
      allprobs = ret[dgbkeys.preddictstr]
    indices = None
    if withconfidence or not img2img or (img2img and nroutputs>2):
      N = 2
      if img2img:
        indices = np.argpartition(allprobs,-N,axis=1)[:,-N:]
      else:
        indices = np.argpartition(allprobs,-N,axis=0)[-N:]
    if withpred and isinstance( indices, np.ndarray ):
      if img2img:
        ret.update({dgbkeys.preddictstr: indices[:,-1:]})
      else:
        ret.update({dgbkeys.preddictstr: indices[-1:]})
    if doprobabilities and len(withprobs) > 0:
      res = np.copy(allprobs[withprobs])
      ret.update({dgbkeys.probadictstr: res})
    if withconfidence:
      N = 2
      predictions_prob = np.array(predictions_prob)
      x = predictions_prob.shape[0]
      indices = np.argpartition(predictions_prob.transpose(),-N,axis=0)[-N:].transpose()
      sortedprobs = predictions_prob.transpose()[indices.ravel(),np.tile(np.arange(x),N)].reshape(N,x)
      res = np.diff(sortedprobs,axis=0)
      ret.update({dgbkeys.confdictstr: res})

  if withpred:
    res = np.transpose(np.array(predictions))
    ret.update({dgbkeys.preddictstr: res})

  if doprobabilities:
    res = np.transpose( np.array(predictions_prob) )
    ret.update({dgbkeys.probadictstr: res})
  if info[dgbkeys.learntypedictstr] == dgbkeys.seisimgtoimgtypestr and withpred:
    if ndims==3:
      if isclassification:
        ret[dgbkeys.preddictstr] = ret[dgbkeys.preddictstr].transpose(3, 2, 1, 0)
      else:
        ret[dgbkeys.preddictstr] = ret[dgbkeys.preddictstr].transpose(4, 3, 2, 1, 0)
    elif ndims==2:
      if isclassification:
        ret[dgbkeys.preddictstr] = ret[dgbkeys.preddictstr].transpose(2, 1, 0)
      else:
        ret[dgbkeys.preddictstr] = ret[dgbkeys.preddictstr].transpose(3, 2, 1, 0)
  return ret

def getDataLoader(dataset, batch_size=torch_dict['batch'], drop_last=False):
    dataloader = DataLoader(dataset=dataset, batch_size=batch_size, shuffle=False, drop_last=drop_last)
    return dataloader

class ChunkedDataLoader(DataLoader):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def set_chunk(self, ichunk):
        return self.dataset.set_chunk(ichunk)

    def set_fold(self, ichunk, ifold):
        return self.dataset.set_fold(ichunk, ifold)

    def set_transform_seed(self):
        self.dataset.set_transform_seed()

    def get_batchsize(self):
        return self.batch_size

    def __iter__(self):
        for batch in super().__iter__():
            yield batch

def getDataLoaders(traindataset, testdataset, batchsize=torch_dict['batch']):
    trainloader = ChunkedDataLoader(dataset=traindataset, batch_size=batchsize, shuffle=False, drop_last=True)
    testloader= ChunkedDataLoader(dataset=testdataset, batch_size=batchsize, shuffle=False, drop_last=True)
    return trainloader, testloader

def getDatasetPars(imgdp, _forvalid):
    info = imgdp[dgbkeys.infodictstr]
    if _forvalid:
      x_data = imgdp[dgbkeys.xvaliddictstr]
      y_data = imgdp[dgbkeys.yvaliddictstr]
    else:
      x_data = imgdp[dgbkeys.xtraindictstr]
      y_data = imgdp[dgbkeys.ytraindictstr]
    inp_ch = x_data.shape[1]
    attribs = dgbhdf5.getNrAttribs(info)
    model_shape = get_model_shape(info[dgbkeys.inpshapedictstr], attribs, True)
    ndims = getModelDims(model_shape, True)
    return x_data, y_data, info, inp_ch, ndims

def DataGenerator(imgdp, batchsize, scaler=None, transform=list()):
    from dgbpy.torch_classes import TrainDatasetClass, TestDatasetClass
    train_dataset = TrainDatasetClass(imgdp, scaler, transform=transform)
    test_dataset = TestDatasetClass(imgdp, scaler)

    trainloader, testloader = getDataLoaders(train_dataset, test_dataset, batchsize)
    return trainloader, testloader


# Faça um programa para o cálculo de uma folha de pagamento, sabendo que os descontos são:

# a. X% IR - depende do salário bruto
# b. 3% Sindicato
# c. 11% FGTS - não é descontado (é a empresa que deposita).
# d. 10% INSS

# O Salário Líquido corresponde ao Salário Bruto menos os descontos. O programa deverá pedir ao usuário o valor da sua hora e a quantidade de horas trabalhadas no mês.
#
# Desconto do IR:
# Salário Bruto até 900 (inclusive) - isento
# Salário Bruto até 1500 (inclusive) - desconto de 5%
# Salário Bruto até 2500 (inclusive) - desconto de 10%
# Salário Bruto acima de 2500 - desconto de 20%
#
# Imprima na tela as informações, dispostas conforme o exemplo.
# No exemplo o valor da hora é 5 e a quantidade de hora é 220.
print(f"\033[33m========================== CALCULA SALÁRIO ==========================\033[0m")

valor_hora = float(input("Valor da hora trabalhada: "))
horas_mes = float(input("Total de horas trabalhadas no mês: "))

# cálculo salário bruto
salario_b = valor_hora * horas_mes

# descontos que não variam o porcentual
inss = salario_b * 0.1
sindicato = salario_b * 0.03
fgts = salario_b * 0.11

# função para aplicar o desconto do IR
def descontoIR(salario):
    if salario > 2500:
        ir = salario * 0.2
    elif salario > 1500:
        ir = salario * 0.1
    elif salario > 900:
        ir = salario * 0.05
    elif salario <= 900 and salario > 0:
        ir = 0
    else:
        return print("Valor IR Inválido!")
    return ir

# cálculo salário líquido
total_descontos = inss + sindicato + descontoIR(salario_b)
salario_liquido = salario_b - total_descontos

# Impressão resultado
print(f'Salário Bruto: R${salario_b}\n (-) IR: R${descontoIR(salario_b)}\n (-) INSS: R${inss}\n (-) Sindicato: R${sindicato}\n FGTS: R${fgts}\n Total Descontos: R${total_descontos}\n Salário Líquido: R${salario_liquido}')

print(f"\033[33m========================== =============== ==========================\033[0m")

02 - Estrutura de Decisão/12_calcula-salario-liquido.py


import wx
from wx.lib.mixins.listctrl import ColumnSorterMixin

import kenshisaves as ks

show_characters = set([])
ignore_characters = set([])
additional_character_attrs = {}

class StatsPanel(wx.Panel, ColumnSorterMixin):
	def __init__(self, parent, num_cat):
		wx.Panel.__init__(self, parent, wx.ID_ANY)

		self.my_category = ks.categories[num_cat]
		cols = ["Name"] + self.my_category["attributes"] + ["Comment"]
		self.ctrl_attributes_spreadsheet = wx.ListCtrl(self, style = wx.LC_REPORT)

		for i, s in enumerate(cols):
			self.ctrl_attributes_spreadsheet.InsertColumn(i + 1, s)

		self.n_columns = len(cols)
		ColumnSorterMixin.__init__(self, self.n_columns)

		self.itemDataMap = {}
		self.column_min_width = []
		for i in range(self.ctrl_attributes_spreadsheet.GetColumnCount()):
			self.ctrl_attributes_spreadsheet.SetColumnWidth(i, wx.LIST_AUTOSIZE_USEHEADER)
			self.column_min_width.append(self.ctrl_attributes_spreadsheet.GetColumnWidth(i))

		self.main_sizer = wx.BoxSizer(wx.VERTICAL)
		self.main_sizer.Add(self.ctrl_attributes_spreadsheet, proportion = 1, flag = wx.EXPAND|wx.ALL, border = 10)
		self.SetSizer(self.main_sizer)

	def GetListCtrl(self):
		return self.ctrl_attributes_spreadsheet

	def Update(self):
		global additional_character_attrs
		con = self.ctrl_attributes_spreadsheet
		con.DeleteAllItems()
		self.itemDataMap = {}

		n_item = 0
		for character, attributes in ks.attribs.items():
			if show_characters and not character in show_characters: continue
			if character in ignore_characters: continue
			n_item = con.InsertItem(n_item, character)

			self.itemDataMap[n_item] = [character]
			for n_column, attr in enumerate(self.my_category["attributes"]):
				val = attributes[attr]
				con.SetItem(n_item, n_column+1, "%.02f\t" % val)
				self.itemDataMap[n_item].append(val)

			if character in additional_character_attrs.keys():
				if "color" in additional_character_attrs[character].keys():
					con.SetItemBackgroundColour(n_item, wx.Colour(additional_character_attrs[character]["color"]))
				if "comment" in additional_character_attrs[character].keys():
					con.SetItem(n_item, len(self.my_category["attributes"])+1, additional_character_attrs[character]["comment"])
					self.itemDataMap[n_item].append(additional_character_attrs[character]["comment"])

			con.SetItemData(n_item, n_item)
			n_item += 1
		con.SetColumnWidth(0, wx.LIST_AUTOSIZE)

class MainWindow(wx.Frame):

	def __init__(self, parent, id, title):

		wx.Frame.__init__(self, parent, wx.ID_ANY, title, size=(500, 500), style=wx.DEFAULT_FRAME_STYLE|wx.NO_FULL_REPAINT_ON_RESIZE)

		self.ReadSettings()
		self.ctrl_url = wx.TextCtrl(self, value = self.wildcard)
		self.bt_scan = wx.Button(self, label = "Update")
		self.bt_scan.Bind(wx.EVT_BUTTON, self.OnUpdate)

		self.nb = wx.Notebook(self)

		self.tabs = []
		for i, cat in enumerate(ks.categories):
			self.tabs.append(StatsPanel(self.nb, i))
			self.nb.AddPage(self.tabs[-1], cat["name"])

		self.url_sizer = wx.BoxSizer(wx.HORIZONTAL)
		self.url_sizer.Add(self.ctrl_url, proportion = 1, flag = wx.EXPAND|wx.ALL, border = 10)
		self.url_sizer.Add(self.bt_scan, flags = wx.SizerFlags().Right().Border(wx.ALL, 10))

		self.main_sizer = wx.BoxSizer(wx.VERTICAL)
		self.main_sizer.Add(self.url_sizer, flags = wx.SizerFlags().Expand())
		self.main_sizer.Add(self.nb, proportion = 1, flag = wx.EXPAND|wx.ALL, border = 10)

		self.SetSizer(self.main_sizer)

	def ReadSettings(self):
		global show_characters, ignore_characters, additional_character_attrs
		with open("settings.txt") as f:
			for line in f:
				if line.startswith("#"): continue
				if line.startswith("wildcard"):
					self.wildcard = line.split("=")[1].strip()
				elif line.startswith("show"):
					show_characters = set([x.strip() for x in line.split("=")[1].split(";")])
				elif line.startswith("ignore"):
					ignore_characters = set([x.strip() for x in line.split("=")[1].split(";")])
				elif line.startswith("@"):
					s = line[1:]
					char_names, char_attrs = [x.strip() for x in s.split("=")]
					char_names = [x.strip() for x in char_names.split(";")]
					char_attrs = [x.strip() for x in char_attrs.split(";")]
					for char_name in char_names:
						additional_character_attrs[char_name] = {}
						for ca in char_attrs:
							k, v = ca.split(":")
							additional_character_attrs[char_name][k.strip()] = v.strip()

	def OnUpdate(self, evt):
		global ignore_characters, show_characters
		self.ReadSettings()
		the_path = self.ctrl_url.GetValue()
		ks.parse(the_path)

		#counting total and displayed characters
		#(can't check with set inclusion operation, because names may repeat)
		n_chars_total = 0
		n_chars_ignored = 0
		chars = list(ks.attribs.keys())
		for c in chars:
			if show_characters and not c in show_characters: continue
			n_chars_total += 1
			if c in ignore_characters: n_chars_ignored += 1
		self.SetTitle(f"Total {n_chars_total} characters, {n_chars_total - n_chars_ignored} displayed, {n_chars_ignored} ignored")
		for tab in self.tabs:
			tab.Update()

app = wx.App()
frame = MainWindow(None, -1, "Kenshi Saves Parser")
frame.Show(1)
app.MainLoop()


N, M = map(int, input().split())
P = [int(input()) for _ in range(M)]

price = 0  # 최소 가격
max_profit = 0  # 최대 수익

for p in sorted(set(P), reverse=True):
    profit = (
        len([x for x in P if x >= p]) * p
        if N >= len([x for x in P if x >= p])
        else N * p
    )
    if profit >= max_profit:
        price = p
        max_profit = profit

print(price, max_profit)


from openerp.osv import fields, osv

to_19_fr = (u'zéro', 'un', 'deux', 'trois', 'quatre', 'cinq', 'six',
          'sept', 'huit', 'neuf', 'dix', 'onze', 'douze', 'treize',
          'quatorze', 'quinze', 'seize', 'dix-sept', 'dix-huit', 'dix-neuf')
tens_fr = ('vingt', 'trente', 'quarante', 'Cinquante', 'Soixante', 'Soixante-dix', 'Quatre-vingts', 'Quatre-vingt Dix')
denom_fr = ('',
          'Mille', 'Millions', 'Milliards', 'Billions', 'Quadrillions',
          'Quintillion', 'Sextillion', 'Septillion', 'Octillion', 'Nonillion',
          'Décillion', 'Undecillion', 'Duodecillion', 'Tredecillion', 'Quattuordecillion',
          'Sexdecillion', 'Septendecillion', 'Octodecillion', 'Icosillion', 'Vigintillion')

def _convert_nn_fr(val):
    """ convert a value < 100 to French
    """
    if val < 20:
        return to_19_fr[val]
    for (dcap, dval) in ((k, 20 + (10 * v)) for (v, k) in enumerate(tens_fr)):
        if dval + 10 > val:
            if val % 10:
                if dval == 70 or dval == 90:
                    return tens_fr[dval / 10 - 3] + '-' + to_19_fr[val % 10 + 10]
                else:
                    return dcap + '-' + to_19_fr[val % 10]
            return dcap

def _convert_nnn_fr(val):
    """ convert a value < 1000 to french

        special cased because it is the level that kicks
        off the < 100 special case.  The rest are more general.  This also allows you to
        get strings in the form of 'forty-five hundred' if called directly.
    """
    word = ''
    (mod, rem) = (val % 100, val // 100)
    if rem > 0:
        if rem == 1:
            word = 'Cent'
        else:
            word = to_19_fr[rem] + ' Cent'
        if mod > 0:
            word += ' '
    if mod > 0:
        word += _convert_nn_fr(mod)
    return word

def french_number(val):
    if val < 100:
        return _convert_nn_fr(val)
    if val < 1000:
        return _convert_nnn_fr(val)
    for (didx, dval) in ((v - 1, 1000 ** v) for v in range(len(denom_fr))):
        if dval > val:
            mod = 1000 ** didx
            l = val // mod
            r = val - (l * mod)
            if l == 1:
                ret = denom_fr[didx]
            else:
                ret = _convert_nnn_fr(l) + ' ' + denom_fr[didx]
            if r > 0:
                ret = ret + ', ' + french_number(r)
            return ret

def amount_to_text_fr(number, currency):
    number = '%.2f' % number
    units_name = currency
    list = str(number).split('.')
    start_word = french_number(abs(int(list[0])))
    end_word = french_number(int(list[1]))
    cents_number = int(list[1])
    cents_name = (cents_number > 1) and ' Centimes' or ' Centime'
    final_result = start_word + ' ' + units_name + ' ' + end_word + ' ' + cents_name
    return final_result

class account_invoice(osv.osv):
    _inherit = "account.invoice"

    def _amount_in_words(self, cr, uid, ids, field_name, arg, context=None):
        res = {}
        for inv in self.browse(cr, uid, ids, context=context):
            res[inv.id] = {
                'amount_words': '0.0',
                            }
            res[inv.id] = amount_to_text_fr(inv.amount_total, inv.currency_id.symbol)
        return res

    _columns = {
        'amount_to_text': fields.function(_amount_in_words, string='In Words', type="char", store=True, help="The amount in words"),
    }
account_invoice()

class sale_order(osv.osv):
    _inherit = "sale.order"

    def _amount_in_words(self, cr, uid, ids, field_name, arg, context=None):
        res = {}
        for order in self.browse(cr, uid, ids, context=context):
            res[order.id] = {
                'amount_words': '0.0',
                            }
            res[order.id] = amount_to_text_fr(order.amount_total, order.pricelist_id.currency_id.symbol)
        return res

    _columns = {
        'amount_to_text': fields.function(_amount_in_words, string='In Words', type="char", store=True, help="The amount in words"),
    }
sale_order()


t=int(input())

for T in range(t):
    x,y=map(int,input().split())
    z=abs(x-y)
    if x>y :
        if z%2==0:
            print(1)
        else:
            print(2)
    elif x<y:
        if z%2!=0:
            print(1)
        elif z%4==0:
            print(3)
        else:
            print(2)
    elif x==y:
        print(0)


from OCRhandling import EasyOCR,TesseractOCR
from PDFhandling import PdfHandler
from IOhandling import Outputhandler
from Imghandling import ImageHandler

class PdfToFile:
    def __init__(self, inputpdffile, hojas):
        self.pdffile = inputpdffile
        self.numhojas = hojas
        self.imagehandler = ImageHandler()
        self.OCR = TesseractOCR()
        self.PDF = PdfHandler(self.pdffile)
        self.File = Outputhandler()

    def process(self):
        for hoja in range(0, self.numhojas):
            imagen_hoja = self.PDF.get_image_from_page(hoja)
            imagen_hoja_tratada = self.imagehandler.treat_image(imagen_hoja)
            self.OCR.get_text_from_image(imagen_hoja_tratada)
            self.File.write(self.OCR)
            print('Hoja' + str(hoja))

prueba = PdfToFile('./input/A02103.pdf',1)
prueba.process()


def pascal_walk(n):
    """
    Print pascal walk of S <= 500 positions
    such that we sum to n.
    """
    # Each row of the Pascal's triangle sums to 2^r, where r >= 0 is the row.
    # To sum to 1 <= n <= 10^9, we'll need at most 29 rows since sum of all
    # these rows = 2^30 - 1 > 10^9

    # Strategy: get binary representation of n - 29, and sum rows corresponding to
    # a set bit. To 'skip' a row, just traverse down edge of 1's. Then, at the end
    # we will traverse down edge of 1's to reach n.

    # To handle cases n <= 29, we will just traverse down edge of 1's.
    if n <= 29:
        for r in range(n):
            print("{:d} {:d}".format(r+1, 1))
        return

    n_less = n - 29
    r = 1
    k = 1
    sum = 0
    while n_less > 0:
        bit = n_less & 1
        if bit == 0:
            print("{:d} {:d}".format(r, k))
            sum += 1
        else:
            forward = k == 1
            print("{:d} {:d}".format(r, k))
            for i in range(r - 1):
                if forward:
                    k += 1
                else:
                    k -= 1
                print("{:d} {:d}".format(r, k))
            sum += 1 << r - 1

        r += 1
        if k > 1:
            k += 1
        n_less >>= 1

    while sum < n:
        print("{:d} {:d}".format(r, k))
        sum += 1
        r += 1
        if k > 1:
            k += 1

def main():
    num_cases = int(input())
    for i in range(num_cases):
        n = int(input())
        print("Case #{:d}:".format(i+1))
        pascal_walk(n)

if __name__ == "__main__":
    main()