import json
import os
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import GridSearchCV, train_test_split
import mlflow
from util import (
    DataPrepUtil, initialize_column_transformer, initialize_pipeline, plot_roc_curve,
    plot_learning_curve, config, record_hyperparameters
)

if __name__ == "__main__":
    # Data cleaning step
    data_prep_util = DataPrepUtil()
    X_train, y_train = data_prep_util.load_train_data(config.get('paths').get('train_data'))
    X_train, X_test, y_train, y_test = train_test_split(
        X_train, y_train, test_size=config.get('test_size'), random_state=config.get('random_state'),
        stratify=y_train
    )

    # Hyperparameter tuning, metric and parameter logging
    mlflow.set_experiment(config.get('mlflow').get('exp_name'))
    RUN_NAME = os.path.basename(__file__).split('.')[0]
    with mlflow.start_run(run_name=RUN_NAME):
        print('Starting run {}...'.format(RUN_NAME))
        mlflow.log_param('test_size', config.get('test_size'))
        pipeline = initialize_pipeline(
            initialize_column_transformer(scale_values=True),
            KNeighborsClassifier()
        )
        param_grid = {
            'model__n_neighbors': config.get('hyperparams').get('knn').get('n_neighbors'),
            'model__weights': config.get('hyperparams').get('knn').get('weights')
        }
        gs_clf = GridSearchCV(
            pipeline, param_grid=param_grid,
            scoring=config.get('scoring'), cv=config.get('k'), verbose=1
        )
        gs_clf.fit(X_train, y_train)
        mlflow.log_params(gs_clf.best_params_)
        mlflow.log_metrics({
                '_'.join([config.get('scoring'), 'train']): gs_clf.score(X_train, y_train),
                '_'.join([config.get('scoring'), 'test']): gs_clf.score(X_test, y_test)
        })

        print('Plotting ROC curve...')
        img_fn = 'roc_{}.png'.format(RUN_NAME)
        plot_roc_curve(gs_clf, X_test, y_test, img_fn)

        # print('Plotting learning curve...')
        # img_fn = 'learning_curve_{}.png'.format(RUN_NAME)
        # plot_learning_curve(gs_clf, X_train, y_train, img_fn)

        print('Recording hyperparameters used...')
        txt_fn = 'hyperparams_{}.txt'.format(RUN_NAME)
        record_hyperparameters(txt_fn, config.get('hyperparams').get('knn'))


import numpy as np
import matplotlib.pyplot as plt # For verbose
import scipy.optimize
import sys          # For error handling
from copy import copy

#######################################
### 1: Frequency sampling functions ###
#######################################

# 1.0: dithering
def drawDithering(m,bounds = None):
    '''Draws m samples a <= x < b, with bounds=(a,b) (default: (0,2*pi)).'''
    if bounds is None:
        (lowb,highb) = (0,2*np.pi)
    else:
        (lowb,highb) = bounds
    return np.random.uniform(low=lowb,high=highb,size=m)

# 1.1: frequency sampling functions
# 1.1.1: gaussian sampling
def drawFrequencies_Gaussian(d,m,Sigma = None):
    '''draws frequencies according to some sampling pattern'''
    if Sigma is None:
        Sigma = np.identity(d)
    Om = np.random.multivariate_normal(np.zeros(d), np.linalg.inv(Sigma), m).T # inverse of sigma
    return Om

# 1.1.2: folded gaussian sampling
def drawFrequencies_FoldedGaussian(d,m,Sigma = None):
    '''draws frequencies according to some sampling pattern
    omega = R*Sigma^{-1/2}*phi, for R from folded Gaussian with variance 1, phi uniform'''
    if Sigma is None:
        Sigma = np.identity(d)
    R = np.abs(np.random.randn(m)) # folded standard normal distribution radii
    phi = np.random.randn(d,m)
    phi = phi / np.linalg.norm(phi,axis=0) # normalize -> randomly sampled from unit sphere
    SigFact = np.linalg.inv(np.linalg.cholesky(Sigma))

    Om = SigFact@phi*R

    return Om

# 1.1.3: adapted radius sampling
def sampleFromPDF(pdf,x,nsamples=1):
    '''x is a vector (the support of the pdf), pdf is the values of pdf eval at x'''
    # Note that this can be more general than just the adapted radius distribution

    pdf = pdf/np.sum(pdf) # ensure pdf is normalized

    cdf = np.cumsum(pdf)

    # necessary?
    cdf[-1] = 1.

    sampleCdf = np.random.uniform(0,1,nsamples)

    sampleX = np.interp(sampleCdf, cdf, x)

    return sampleX

def pdfAdaptedRadius(r,KMeans=False):
    '''up to a constant'''
    if KMeans:
        return r*np.exp(-(r**2)/2)  # Dont take the gradient according to sigma into account
    else:
        return np.sqrt(r**2 + (r**4)/4)*np.exp(-(r**2)/2)

def drawFrequencies_AdaptedRadius(d,m,Sigma = None,KMeans=False):
    '''draws frequencies according to some sampling pattern
    omega = R*Sigma^{-1/2}*phi, for R from adapted with variance 1, phi uniform'''
    if Sigma is None:
        Sigma = np.identity(d)

    # Sample the radii
    r = np.linspace(0,5,2001)
    R = sampleFromPDF(pdfAdaptedRadius(r,KMeans),r,nsamples=m)

    phi = np.random.randn(d,m)
    phi = phi / np.linalg.norm(phi,axis=0) # normalize -> randomly sampled from unit sphere
    SigFact = np.linalg.inv(np.linalg.cholesky(Sigma))

    Om = SigFact@phi*R

    return Om

def pdf_diffOfGaussians(r,GMM_upper=None,GMM_lower=None):
    """Here, GMM is given in terms of SD and not variance"""
    if isinstance(GMM_upper,tuple):
        (weights_upper,sigmas_upper) = GMM_upper
    elif GMM_upper is None:
        weights_upper = np.array([]) # Empty array
    else:
        (weights_upper,sigmas_upper) = (np.array([1.]),np.array([GMM_upper]))

    if isinstance(GMM_lower,tuple):
        (weights_lower,sigmas_lower) = GMM_lower
    elif GMM_lower is None:
        weights_lower = np.array([])
    else:
        (weights_lower,sigmas_lower) = (np.array([1.]),np.array([GMM_lower]))

    res = np.zeros(r.shape)
    # Add
    for k in range(weights_upper.size):
        res += weights_upper[k]*np.exp(-0.5*(r**2)/(sigmas_upper[k]**2))
    # Substract
    for k in range(weights_lower.size):
        res -= weights_lower[k]*np.exp(-0.5*(r**2)/(sigmas_lower[k]**2))

    # Ensure pdf is positive
    pdf_is_negative = res < 0
    if any(pdf_is_negative):
        print(res[:5])
        # Print a warning if the negative pdf values are significant (not due to rounding errors)
        tol = 1e-8
        if np.max(np.abs(res[np.where(pdf_is_negative)[0]])) > tol:
            print("WARNING: negative pdf values detected and replaced by zero, check the validity of your input")
        # Correct the negative values
        res[np.where(pdf_is_negative)[0]] = 0.

    return res

def drawFrequencies_diffOfGaussians(d,m,GMM_upper,GMM_lower=None,verbose=0):
    '''draws frequencies according to some sampling pattern
    omega = R*Sigma^{-1/2}*phi, TODO, phi uniform'''

    # reasonable sampling
    n_Rs = 1001
    if isinstance(GMM_upper,tuple):
        R_max = 4*np.max(GMM_upper[1]) # GMM_upper is (weights, cov)-type tuple
    else:
        R_max = 4*GMM_upper
    r = np.linspace(0,R_max,n_Rs)

    if verbose > 0:
        plt.plot(r,pdf_diffOfGaussians(r,GMM_upper,GMM_lower))
        plt.xlabel('frequency norm r')
        plt.ylabel('pdf(r)')
        plt.show()

    # sample from the diff of gaussians pdf
    R = sampleFromPDF(pdf_diffOfGaussians(r,GMM_upper,GMM_lower),r,nsamples=m)

    phi = np.random.randn(d,m)
    phi = phi / np.linalg.norm(phi,axis=0) # normalize -> randomly sampled from unit sphere

    Om = phi*R

    return Om

# General function for convenience
def drawFrequencies(drawType,d,m,Sigma = None,nb_cat_per_dim=None):
    """Draw the 'frequencies' or projection matrix Omega for sketching.

    Arguments:
        - drawType: a string indicating the sampling pattern (Lambda) to use, one of the following:
            -- "gaussian"       or "G"  : Gaussian sampling > Lambda = N(0,Sigma^{-1})
            -- "foldedGaussian" or "FG" : Folded Gaussian sampling (i.e., the radius is Gaussian)
            -- "adaptedRadius"  or "AR" : Adapted Radius heuristic
        - d: int, dimension of the data to sketch
        - m: int, number of 'frequencies' to draw (the target sketch dimension)
        - Sigma: is either:
            -- (d,d)-numpy array, the covariance of the data (note that we typically use Sigma^{-1} in the frequency domain).
            -- a tuple (w,cov) describing a scale mixture of Gaussians where,
                -- w:  (K,)-numpy array, the weights of the scale mixture
                -- cov: (K,d,d)-numpy array, the K different covariances in the mixture
            -- None: same as Sigma = identity matrix (belongs to (d,d)-numpy array case)
                 If Sigma is None (default), we assume that data was normalized s.t. Sigma = identity.
        - nb_cat_per_dim: (d,)-array of ints, the number of categories per dimension for integer data,
                    if its i-th entry = 0 (resp. > 0), dimension i is assumed to be continuous (resp. int.).
                    By default all entries are assumed to be continuous. Frequencies for int data is drawn as follows:
                    1. Chose one dimension among the categorical ones, set Omega along all others to zero
                    2. For the chosen dimension with C categories, we draw its component omega ~ U({0,...,C-1}) * 2*pi/C

    Returns:
        - Omega: (d,m)-numpy array containing the 'frequency' projection matrix
    """
    # Parse drawType input
    if drawType.lower() in ["drawfrequencies_gaussian","gaussian","g"]:
        drawFunc = drawFrequencies_Gaussian
    elif drawType.lower() in ["drawfrequencies_foldedgaussian","foldedgaussian","folded_gaussian","fg"]:
        drawFunc = drawFrequencies_FoldedGaussian
    elif drawType.lower() in ["drawfrequencies_adapted","adaptedradius","adapted_radius","ar"]:
        drawFunc = drawFrequencies_AdaptedRadius
    elif drawType.lower() in ["drawfrequencies_adapted_kmeans","adaptedradius_kmeans","adapted_radius_kmeans","arkm","ar-km"]:
        drawFunc = lambda _a,_b,_c : drawFrequencies_AdaptedRadius(_a,_b,_c,KMeans=True)
    else:
        raise ValueError("drawType not recognized")

    # Handle no input
    if Sigma is None:
        Sigma = np.identity(d)

    # Handle
    if isinstance(Sigma,np.ndarray):
        Omega = drawFunc(d,m,Sigma)

    # Handle mixture-type input
    elif isinstance(Sigma,tuple):
        (w,cov) = Sigma # unpack
        K = w.size
        # Assign the frequencies to the mixture components
        assignations = np.random.choice(K,m,p=w)
        Omega = np.zeros((d,m))
        for k in range(K):
            active_index = (assignations == k)
            if any(active_index):
                Omega[:,np.where(active_index)[0]] = drawFunc(d,active_index.sum(),cov[k])

    elif (isinstance(Sigma,float) or isinstance(Sigma,int)) and Sigma > 0:
        Omega = drawFunc(d,m,Sigma*np.eye(d))

    else:
        raise ValueError("Sigma not recognized")

    # If needed, overwrite the integer entries
    if nb_cat_per_dim is not None:
        intg_index = np.nonzero(nb_cat_per_dim)[0]
        d_intg = np.size(intg_index)

        Omega_intg = np.zeros((d_intg,m))
        for intgdim_localindex,intg_globalindex in enumerate(intg_index):
            C = nb_cat_per_dim[intg_globalindex]
            Omega_intg[intgdim_localindex,:] = (2*np.pi/C)*np.random.randint(0,C,(1,m))
        # Mask
        masks_pool = np.eye(d_intg)
        masks = masks_pool[np.random.choice(d_intg,m)].T
        Omega_intg = Omega_intg*masks

        Omega[intg_index] = Omega_intg

    return Omega

# The following funtions allows to estimate Sigma (some utils first, main function is "estimate_Sigma")
# Optimization problem to fit a GMM curve to the data
def _fun_grad_fit_sigmas(p,R,z):
    """
    Function and gradient to solve the optimization problem
        min_{w,sigs2} sum_{i = 1}^n ( z[i] - sum_{k=1}^K w[k]*exp(-R[i]^2*sig2[k]/2) )^2
    Arguments:
        - p, a (2K,) numpy array obtained by stacking
            - w : (K,) numpy array
            - sigs2 : (K,) numpy array
        - R: (n,) numpy array, data to fit (x label)
        - z: (n,) numpy array, data to fit (y label)
    Returns:
        - The function evaluation
        - The gradient
    """

    K = p.size//2
    w = p[:K]
    sigs2 = p[K:]
    n = R.size
    # Naive implementation, TODO better?
    fun = 0
    grad = np.zeros(2*K)
    for i in range(n):
        fun += (z[i] - w@np.exp(-(sigs2*R[i]**2)/2.))**2
        grad[:K] += (z[i] - w@np.exp(-(sigs2*R[i]**2)/2.)) * (- np.exp(-(sigs2*R[i]**2)/2.)) # grad of w
        grad[K:] += (z[i] - w@np.exp(-(sigs2*R[i]**2)/2.)) * (- w * np.exp(-(sigs2*R[i]**2)/2.)) * (-0.5*R[i]**2) # grad of sigma2
    return (fun,grad)

# For normalization in the optimization problem
def _callback_fit_sigmas(p):
    K = p.size//2
    p[:K] /= np.sum(p[:K])

def estimate_Sigma_from_sketch(z,Phi,K=1,c=20,mode='max',sigma2_bar=None,weights_bar=None,should_plot=False):
    # Parse
    if mode == 'max':
        mode_criterion = np.argmax
    elif mode == 'min':
        mode_criterion = np.argmin
    else:
        raise ValueError("Unrecocgnized mode ({})".format(mode))

    # sort the freqs by norm
    Rs = np.linalg.norm(Phi.Omega,axis=0)
    i_sort = np.argsort(Rs)
    Rs = Rs[i_sort]
    z_sort = z[i_sort]/Phi.c_norm # sort and normalize individual entries to 1

    # Initialization
    # number of freqs per box
    s = Phi.m//c
    # init point
    if sigma2_bar is None:
        sigma2_bar = np.random.uniform(0.3,1.6,K)
    if weights_bar is None:
        weights_bar = np.ones(K)/K

    # find the indices of the max of each block
    jqs = np.empty(c)
    for ic in range(c):
        j_max = mode_criterion(np.abs(z_sort)[ic*s:(ic+1)*s]) + ic*s
        jqs[ic] = j_max
    jqs = jqs.astype(int)
    R_tofit = Rs[jqs]
    z_tofit = np.abs(z_sort)[jqs]

    # Set up the fitting opt. problem
    f = lambda p: _fun_grad_fit_sigmas(p,R_tofit,z_tofit) # cost

    p0 = np.zeros(2*K) # initial point
    p0[:K] = weights_bar # w
    p0[K:] = sigma2_bar

    # Bounds of the optimization problem
    bounds = []
    for k in range(K): bounds.append([1e-5,1]) # bounds for the weigths
    for k in range(K): bounds.append([5e-4*sigma2_bar[k],2e3*sigma2_bar[k]]) # bounds for the sigmas -> cant cange too much

    # Solve the sigma^2 optimization problem
    sol = scipy.optimize.minimize(f, p0,jac = True, bounds = bounds,callback=_callback_fit_sigmas)
    p = sol.x
    weights_bar = np.array(p[:K])/np.sum(p[:K])
    sigma2_bar = np.array(p[K:])

    # Plot if required
    if should_plot:
        plt.figure(figsize=(10,5))
        rfit = np.linspace(0,Rs.max(),100)
        zfit = np.zeros(rfit.shape)
        for k in range(K):
            zfit += weights_bar[k]*np.exp(-(sigma2_bar[k]*rfit**2)/2.)
        plt.plot(Rs,np.abs(z_sort),'.')
        plt.plot(R_tofit,z_tofit,'.')
        plt.plot(rfit,zfit)
        plt.xlabel('R')
        plt.ylabel('|z|')
        plt.show()

    return sigma2_bar

def estimate_Sigma(dataset,m0,K=None,c=20,n0=None,drawFreq_type = "AR",nIterations=5,mode='max',verbose=0):
    """Automatically estimates the "Sigma" parameter(s) (the scale of data clusters) for generating the sketch operator.

    We assume here that Sigma = sigma2_bar * identity matrix.
    To estimate sigma2_bar, lightweight sketches of size m0 are generated from (a small subset of) the dataset
    with candidate values for sigma2_bar. Then, sigma2_bar is updated by fitting a Gaussian
    to the absolute values of the obtained sketch. Cfr. https://arxiv.org/pdf/1606.02838.pdf, sec 3.3.3.

    Arguments:
        - dataset: (n,d) numpy array, the dataset X: n examples in dimension d
        - m0: int, number of candidate 'frequencies' to draw (can be typically smaller than m).
        - K:  int (default 1), number of scales to fit (if > 1 we fit a scale mixture)
        - c:  int (default 20), number of 'boxes' (i.e. number of maxima of sketch absolute values to fit)
        - n0: int or None, if given, n0 samples from the dataset are subsampled to be used for Sigma estimation
        - drawType: a string indicating the sampling pattern (Lambda) to use in the pre-sketches, either:
            -- "gaussian"       or "G"  : Gaussian sampling > Lambda = N(0,Sigma^{-1})
            -- "foldedGaussian" or "FG" : Folded Gaussian sampling (i.e., the radius is Gaussian)
            -- "adaptedRadius"  or "AR" : Adapted Radius heuristic
        - nIterations: int (default 5), the maximum number of iteration (typically stable after 2 iterations)
        - mode: 'max' (default) or 'min', describe which sketch entries per block to fit
        - verbose: 0,1 or 2, amount of information to print (default: 0, no info printed). Useful for debugging.

    Returns: If K = 1:
                - Sigma: (d,d)-numpy array, the (diagonal) estimated covariance of the clusters in the dataset;
             If K > 1: a tuple (w,Sigma) representing the scale mixture model, where:
                - w:     (K,)-numpy array, the weigths of the scale mixture (sum to 1)
                - Sigma: (K,d,d)-numpy array, the dxd covariances in the scale mixture
    """

    return_format_is_matrix = K is None
    K = 1 if K is None else K

    (n,d) = dataset.shape
    # X is the subsampled dataset containing only n0 examples
    if n0 is not None and n0<n:
        X = dataset[np.random.choice(n,n0,replace=False)]
    else:
        X = dataset

    # Check if we dont overfit the empirical Fourier measurements
    if (m0 < (K * 2)*c):
        print("WARNING: overfitting regime detected for frequency sampling fitting")

    # Initialization
    #maxNorm = np.max(np.linalg.norm(X,axis=1))
    sigma2_bar = np.random.uniform(0.3,1.6,K)
    weights_bar = np.ones(K)/K

    # Actual algorithm
    for i in range(nIterations):
        # Draw frequencies according to current estimate
        sigma2_bar_matrix = np.outer(sigma2_bar,np.eye(d)).reshape(K,d,d)  # covariances in (K,d,d) format
        Omega0 = drawFrequencies(drawFreq_type,d,m0,Sigma = (weights_bar,sigma2_bar_matrix))

        # Compute unnormalized complex exponential sketch
        Phi0 = SimpleFeatureMap("ComplexExponential",Omega0)
        z0 = computeSketch(X,Phi0)

        should_plot = verbose > 1 or (verbose > 0 and i >= nIterations - 1)
        sigma2_bar = estimate_Sigma_from_sketch(z0,Phi0,K,c,mode,sigma2_bar,weights_bar,should_plot)

    if return_format_is_matrix:
        Sigma = sigma2_bar[0]*np.eye(d)
    else:
        sigma2_bar_matrix = np.outer(sigma2_bar,np.eye(d)).reshape(K,d,d)  # covariances in (K,d,d) format
        Sigma = (weights_bar,sigma2_bar_matrix)

    return Sigma

#######################################
###    2: Feature map functions     ###
#######################################

# 2.1: Common sketch nonlinearities and derivatives
def _complexExponential(t,T=2*np.pi):
    return np.exp(1j*(2*np.pi)*t/T)
def _complexExponential_grad(t,T=2*np.pi):
    return ((1j*2*np.pi)/T)*np.exp(1j*(2*np.pi)*t/T)

def _universalQuantization(t,Delta=np.pi,centering=True):
    if centering:
        return ( (t // Delta) % 2 )*2-1 # // stands for "int division
    else:
        return ( (t // Delta) % 2 ) # centering=false => quantization is between 0 and +1

def _universalQuantization_complex(t,Delta=np.pi,centering=True):
    return _universalQuantization(t-Delta/2,Delta=Delta,centering=centering) + 1j*_universalQuantization(t-Delta,Delta=Delta,centering=centering)

def _sawtoothWave(t,T=2*np.pi,centering=True):
    if centering:
        return ( t % T )/T*2-1
    else:
        return ( t % T )/T # centering=false => output is between 0 and +1

def _sawtoothWave_complex(t,T=2*np.pi,centering=True):
    return _sawtoothWave(t-T/4,T=T,centering=centering) + 1j*_sawtoothWave(t-T/2,T=T,centering=centering)

def _triangleWave(t,T=2*np.pi):
    return (2*(t % T)/T ) - (4*(t % T)/T - 2)*( (t // T) % 2 ) - 1

def _fourierSeriesEvaluate(t,coefficients,T=2*np.pi):
    """T = period
    coefficients = F_{-K}, ... , F_{-1}, F_{0}, F_{1}, ... F_{+K}"""
    K = (coefficients.shape[0]-1)/2
    ks = np.arange(-K,K+1)
    # Pre-alloc
    ft = np.zeros(t.shape) + 0j
    for i in range(2*int(K)+1):
        ft += coefficients[i]*np.exp(1j*(2*np.pi)*ks[i]*t/T)
    return ft

# dict of nonlinearities and their gradient returned as a tuple
_dico_nonlinearities = {
    "complexexponential":(_complexExponential,_complexExponential_grad),
    "universalquantization":(_universalQuantization,None),
    "universalquantization_complex":(_universalQuantization_complex,None),
    "sawtooth":(_sawtoothWave,None),
    "sawtooth_complex":(_sawtoothWave_complex,None),
    "cosine": (lambda x: np.cos(x),lambda x: -np.sin(x))
 }

# 2.2 FeatureMap objects
# Abstract feature map class
class FeatureMap:
    """Template for a generic Feature Map. Useful to check if an object is an instance of FeatureMap."""
    def __init__(self):
        pass
    def __call__(self):
        raise NotImplementedError("The way to compute the feature map is not specified.")
    def grad(self):
        raise NotImplementedError("The way to compute the gradient of the feature map is not specified.")

# TODO find a better name
class SimpleFeatureMap(FeatureMap):
    """Feature map the type Phi(x) = c_norm*f(Omega^T*x + xi)."""
    def __init__(self, f, Omega, xi = None, c_norm = 1.):
        """
        - f can be one of the following:
            -- a string for one of the predefined feature maps:
                -- "complexExponential"
                -- "universalQuantization"
                -- "cosine"
            -- a callable function
            -- a tuple of function (specify the derivative too)

        """
        # 1) extract the feature map
        self.name = None
        if isinstance(f, str):
            try:
                (self.f,self.f_grad) = _dico_nonlinearities[f.lower()]
                self.name = f # Keep the feature function name in memory so that we know we have a specific fct
            except KeyError:
                raise NotImplementedError("The provided feature map name f is not implemented.")
        elif callable(f):
            (self.f,self.f_grad) = (f,None)
        elif (isinstance(f,tuple)) and (len(f) == 2) and (callable(f[0]) and callable(f[1])):
            (self.f,self.f_grad) = f
        else:
            raise ValueError("The provided feature map f does not match any of the supported types.")

        # 2) extract Omega the projection matrix TODO allow callable Omega for fast transform
        if (isinstance(Omega,np.ndarray) and Omega.ndim == 2):
            self.Omega = Omega
            (self.d,self.m) = Omega.shape
        else:
            raise ValueError("The provided projection matrix Omega should be a (d,m) numpy array.")
        # 3) extract the dithering
        if xi is None:
            self.xi = np.zeros(self.m)
        else:
            self.xi = xi
        # 4) extract the normalization constant
        if isinstance(c_norm, str):
            if c_norm.lower() in ['unit','normalized']:
                self.c_norm = 1./np.sqrt(self.m)
            else:
                raise NotImplementedError("The provided c_norm name is not implemented.")
        else:
            self.c_norm = c_norm

    # magic operator to be able to call the FeatureMap object as a function
    def __call__(self,x):
        return self.c_norm*self.f(np.matmul(self.Omega.T,x.T).T + self.xi) # Evaluate the feature map at x

    def grad(self,x):
        """Gradient (Jacobian matrix) of Phi, as a (d,m)-numpy array"""
        return self.c_norm*self.f_grad(np.matmul(self.Omega.T,x.T).T + self.xi)*self.Omega

#######################################
### 3: Actual sketching functions   ###
#######################################

#################################
# 3.1 GENERAL SKETCHING ROUTINE #
#################################
def computeSketch(dataset, featureMap, datasetWeights = None, batch_size = None):
    """
    Computes the sketch of a dataset given a generic feature map.

    More precisely, evaluates
        z = sum_{x_i in X} w_i * Phi(x_i)
    where X is the dataset, Phi is the sketch feature map, w_i are weights assigned to the samples (typically 1/n).

    Arguments:
        - dataset        : (n,d) numpy array, the dataset X: n examples in dimension d
        - featureMap     : the feature map Phi, given as one of the following:
            -- a function, z_x_i = featureMap(x_i), where x_i and z_x_i are (n,)- and (m,)-numpy arrays, respectively
            -- a FeatureMap instance (e.g., constructed as featureMap = SimpleFeatureMap("complexExponential",Omega) )
        - datasetWeights : (n,) numpy array, optional weigths w_i in the sketch (default: None, corresponds to w_i = 1/n)

    Returns:
        - sketch : (m,) numpy array, the sketch as defined above
    """
    # TODOs:
    # - add possibility to specify classes and return one sketch per class
    # - defensive programming
    # - efficient implementation, take advantage of parallelism

    (n,d) = dataset.shape # number of samples, dimension

    # Determine the sketch dimension and sanity check: the dataset is nonempty and the map works
    if isinstance(featureMap,FeatureMap): # featureMap is the argument, FeatureMap is the class
        m = featureMap.m
    else:
        try:
            m = featureMap(dataset[0]).shape[0]
        except:
            raise ValueError("Unexpected error while calling the sketch feature map:", sys.exc_info()[0])

    sketch = np.zeros(m)

    # Split the batches
    if batch_size is None:
        batch_size = int(1e6/m) # Rough heuristic, best batch size will vary on different machines
    nb_batches = int(np.ceil(n/batch_size))

    if datasetWeights is None:
        for b in range(nb_batches):
            sketch = sketch + featureMap(dataset[b*batch_size:(b+1)*batch_size]).sum(axis=0)
        sketch /= n
    else:
        sketch = datasetWeights@featureMap(X)
    return sketch

#################################
# 3.2 PRIVATE SKETCHING METHODS #
#################################
def sensisitivty_sketch(featureMap,n = 1,DPdef = 'UDP',sensitivity_type = 1):
    """
    Computes the sensitity of a provided sketching function.

    The noisy sketch operator A(X) is given by
        A(X) := (1/n)*[sum_{x_i in X} featureMap(x_i)] + w
    where w is Laplacian or Gaussian noise.

    Arguments:
        - featureMap, the sketch the sketch featureMap (Phi), provided as either:
            -- a FeatureMap object with a known sensitivity (i.e., complex exponential or universal quantization periodic map)
            -- (m,featureMapName,c_normalization): tuple (deprectated, only useful for code not supporting FeatureMap objects),
                that should contain:
                -- m: int, the sketch dimension
                -- featureMapName: string, name of sketch feature function f, values supported:
                    -- 'complexExponential' (f(t) = exp(i*t))
                    -- 'universalQuantization_complex' (f(t) = sign(exp(i*t)))
                -- c_normalization: real, the constant before the sketch feature function (e.g., 1. (default), 1./sqrt(m),...)
        - n: int, number of sketch contributions being averaged (default = 1, useful to add noise on n independently)
        - DPdef: string, name of the Differential Privacy variant considered, i.e. the neighbouring relation ~:
            -- 'remove', 'add', 'remove/add', 'UDP' or 'standard': D~D' iff D' = D U {x'} (or vice versa)
            -- 'replace', 'BDP': D~D' iff D' = D \ {x} U {x'} (or vice versa)
        - sensitivity_type: int, 1 (default) for L1 sensitivity, 2 for L2 sensitivity.

    Returns: a positive real, the L1 or L2 sensitivity of the sketching operator defined above.

    Cfr: Differentially Private Compressive K-means, https://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=8682829.
    """

    # TODO include real cases (cosine, real universal quantization)

    # The sensitivity is of the type: c_feat*c_
    if isinstance(featureMap,FeatureMap):
        m = featureMap.m
        featureMapName = featureMap.name
        c_normalization = featureMap.c_norm
    elif (isinstance(featureMap,tuple)) and (len(featureMap) == 3):
        (m,featureMapName,c_normalization) = featureMap
    else:
        raise ValueError('The featureMap argument does not match one of the supported formats.')

    # Sensitivity is given by S = c_featureMap * c_sensitivity_type * c_DPdef, check all three conditions (ughh)
    if featureMapName.lower() == 'complexexponential':
        if sensitivity_type == 1:
            if DPdef.lower() in ['remove','add','remove/add','standard','udp']:
                return m*np.sqrt(2)*(c_normalization/n)
            elif DPdef.lower() in ['replace','bdp']:
                return 2*m*np.sqrt(2)*(c_normalization/n)
        elif sensitivity_type == 2:
            if DPdef.lower() in ['remove','add','remove/add','standard','udp']:
                return np.sqrt(m)*(c_normalization/n)
            elif DPdef.lower() in ['replace','bdp']:
                return np.sqrt(m)*np.sqrt(2)*(c_normalization/n)
    elif featureMapName.lower() == 'universalquantization_complex': # Assuming normalized in [-1,+1], TODO check real/complex case?
        if sensitivity_type == 1:
            if DPdef.lower() in ['remove','add','remove/add','standard','udp']:
                return m*2*(c_normalization/n)
            elif DPdef.lower() in ['replace','bdp']:
                return 2*m*2*(c_normalization/n)
        elif sensitivity_type == 2:
            if DPdef.lower() in ['remove','add','remove/add','standard','udp']:
                return np.sqrt(m)*np.sqrt(2)*(c_normalization/n)
            elif DPdef.lower() in ['replace','bdp']:
                return np.sqrt(2)*np.sqrt(m)*np.sqrt(2)*(c_normalization/n)
    print(sensitivity_type)
    raise Exception('You provided ({},{});\nThe sensitivity for this (feature map,DP definition) combination is not implemented.'.format(featureMapName.lower(),DPdef.lower()))
    return None

def computeSketch_DP(dataset, featureMap, epsilon, delta = 0,DPdef = 'UDP',useImproveGaussMechanism=True,budget_split_num = None):
    """
    Computes the Differentially Private sketch of a dataset given a generic feature map.

    More precisely, evaluates the DP sketching mechanism:
        z = ( sum_{x_i in X} Phi(x_i) + w_num )/( |X| + w_den )
    where X is the dataset, Phi is the sketch feature map, w_num and w_den are Laplacian or Gaussian random noise.

    Arguments:
        - dataset        : (n,d) numpy array, the dataset X: n examples in dimension d
        - featureMap, the sketch the sketch featureMap (Phi), provided as either:
            -- a FeatureMap object with a known sensitivity (i.e., complex exponential or universal quantization periodic map)
            -- (featureMap(x_i),m,featureMapName,c_normalization): tuple (deprectated, only useful for old code),
                that should contain:
                -- featMap: a function, z_x_i = featMap(x_i), where x_i and z_x_i are (n,)- and (m,)-numpy arrays, respectively
                -- m: int, the sketch dimension
                -- featureMapName: string, name of sketch feature function f, values supported:
                    -- 'complexExponential' (f(t) = exp(i*t))
                    -- 'universalQuantization' (f(t) = sign(exp(i*t)))
                -- c_normalization: real, the constant before the sketch feature function (e.g., 1. (default), 1./sqrt(m),...)
        - epsilon: real > 0, the privacy parameter epsilon
        - delta:  real >= 0, the privacy parameter delta in approximate DP; if delta=0 (default), we have "pure" DP.
        - DPdef: string, name of the Differential Privacy variant considered, i.e. the neighbouring relation ~:
            -- 'remove', 'add', 'remove/add', 'UDP' or 'standard' (default): D~D' iff D' = D U {x'} (or vice versa)
            -- 'replace', 'BDP': D~D' iff D' = D \ {x} U {x'} (or vice versa)
        - useImproveGaussMechanism: bool, if True (default) use the improved Gaussian mechanism[1] rather than usual bounds[2].
        - budget_split_num: 0 < real < 1, fraction of epsilon budget to allocate to the numerator (ignored in BDP).
                            By default, we assign a fraction of (2*m)/(2*m+1) on the numerator.

    Returns:
        - sketch : (m,) numpy array, the differentially private sketch as defined above
    """

    # Extract dataset size
    (n,d) = dataset.shape

    # Compute the nonprivate, usual sketch
    if isinstance(featureMap,FeatureMap):
        z_clean = computeSketch(dataset, featureMap)
    elif (isinstance(featureMap,tuple)) and (callable(featureMap[0])):
        featMap = featureMap[0]
        featureMap = featureMap[1:]
        z_clean = computeSketch(dataset, featMap)

    if epsilon == np.inf: # Non-private
        return z_clean

    useBDP = DPdef.lower() in ['replace','bdp'] # otherwise assume UDP, TODO DEFENSIVE

    # We will need the sketch size
    m = z_clean.size

    # Split privacy budget
    if useBDP: # Then no noise on the denom
        budget_split_num = 1.
    elif budget_split_num is None:
        budget_split_num = (2*m)/(2*m + 1)
    # TODO defensive programming to block budget split > 1?
    epsilon_num = budget_split_num*epsilon

    # Compute numerator noise
    if delta > 0:
        # Gaussian mechanism
        S = sensisitivty_sketch(featureMap,DPdef = DPdef,sensitivity_type = 2) # L2

        if useImproveGaussMechanism: # Use the sharpened bounds
            from .third_party import calibrateAnalyticGaussianMechanism
            sigma = calibrateAnalyticGaussianMechanism(epsilon_num, delta, S)
        else: # use usual bounds
            if epsilon >= 1: raise Exception('WARNING: with epsilon >= 1 the sigma bound doesn\'t hold! Privacy is NOT ensured!')
            sigma = np.sqrt(2*np.log(1.25/delta))*S/epsilon_num
        noise_num = np.random.normal(scale = sigma, size=m) + 1j*np.random.normal(scale = sigma, size=m) # TODO real
    else:
        # Laplacian mechanism
        S = sensisitivty_sketch(featureMap,DPdef = DPdef,sensitivity_type = 1) # L1
        beta = S/epsilon_num # L1 sensitivity/espilon
        noise_num = np.random.laplace(scale = beta, size=m) + 1j*np.random.laplace(scale = beta, size=m)

    # Add denominator noise if needed
    if useBDP: # Then no noise on the denom
        return z_clean + (noise_num/n)
    else:
        num = (z_clean*n) + noise_num
        beta_den = 1/(epsilon - epsilon_num) # rest of the privacy budget
        den = n + np.random.laplace(scale = beta_den)
        return num/den

## Useful: compute the sketch of a GMM
def fourierSketchOfGaussian(mu,Sigma,Omega,xi=None,scst=None):
    res = np.exp(1j*(mu@Omega) -np.einsum('ij,ij->i', np.dot(Omega.T, Sigma), Omega.T)/2.)
    if xi is not None:
        res = res*np.exp(1j*xi)
    if scst is not None: # Sketch constant, eg 1/sqrt(m)
        res = scst*res
    return res

def fourierSketchOfGMM(GMM,featureMap):
    '''Returns the complex exponential sketch of a Gaussian Mixture Model

    Parameters
    ----------
    GMM: (weigths,means,covariances) tuple, the Gaussian Mixture Model, with
        - weigths:     (K,)-numpy array containing the weigthing factors of the Gaussians
        - means:       (K,d)-numpy array containing the means of the Gaussians
        - covariances: (K,d,d)-numpy array containing the covariance matrices of the Gaussians
    featureMap: the sketch the sketch featureMap (Phi), provided as either:
        - a SimpleFeatureMap object (i.e., complex exponential or universal quantization periodic map)
        - (Omega,xi): tuple with the (d,m) Fourier projection matrix and the (m,) dither (see above)

    Returns
    -------
    z: (m,)-numpy array containing the sketch of the provided GMM
    '''
    # Parse GMM input
    (w,mus,Sigmas) = GMM
    K = w.size

    # Parse featureMap input
    if isinstance(featureMap,SimpleFeatureMap):
        Omega = featureMap.Omega
        xi = featureMap.xi
        d = featureMap.d
        m = featureMap.m
        scst = featureMap.c_norm # Sketch normalization constant, e.g. 1/sqrt(m)
    elif isinstance(featureMap,tuple):
        (Omega,xi) = featureMap
        (d,m) = Omega.shape
        scst = 1. # This type of argument passing does't support different normalizations
    else:
        raise ValueError('The featureMap argument does not match one of the supported formats.')

    z = 1j*np.zeros(m)
    for k in range(K):
        z += fourierSketchOfGaussian(mus[k],Sigmas[k],Omega,xi,scst)
    return z

def fourierSketchOfBox(box,featureMap,nb_cat_per_dim=None, dimensions_to_consider=None):
    '''Returns the complex exponential sketch of the indicator function on a parallellipiped (box).
    For dimensions that flagged as integer, considers the indicator on a set of integers instead.

    Parameters
    ----------
    box: (d,2)-numpy array, the boundaries of the box (x in R^d is in the box iff box[i,0] <= x_i <= box[i,1])
    featureMap: the sketch the sketch featureMap (Phi), provided as either:
        - a SimpleFeatureMap object (is assumed to use the complex exponential map)
        - (Omega,xi): tuple with the (d,m) Fourier projection matrix and the (m,) dither (see above)

    Additional Parameters
    ---------------------
    nb_cat_per_dim: (d,)-array of ints, the number of categories per dimension for integer data,
                    if its i-th entry = 0 (resp. > 0), dimension i is assumed to be continuous (resp. int.).
                    By default all entries are assumed to be continuous.
    dimensions_to_consider: array of ints (between 0 and d-1), [0,1,...d-1] by default.
                    The box is restricted to the prescribed dimensions.
                    This is helpful to solve problems on a subsets of all dimensions.

    Returns
    -------
    z: (m,)-numpy array containing the sketch of the indicator function on the provided box
    '''
    ## Parse input
    # Parse box input
    (d,_) = box.shape
    c_box = (box[:,1]+box[:,0])/2 # Center of the box in each dimension
    l_box = (box[:,1]-box[:,0])/2 # Length (well, half of the length) of the box in each dimension

    # Parse featureMap input
    if isinstance(featureMap,SimpleFeatureMap):
        Omega = featureMap.Omega
        xi = featureMap.xi
        d = featureMap.d
        m = featureMap.m
        scst = featureMap.c_norm # Sketch normalization constant, e.g. 1/sqrt(m)
    elif isinstance(featureMap,tuple):
        (Omega,xi) = featureMap
        (d,m) = Omega.shape
        scst = 1. # This type of argument passing does't support different normalizations
    else:
        raise ValueError('The featureMap argument does not match one of the supported formats.')

    # Parse nb_cat_per_dim
    if nb_cat_per_dim is None:
        nb_cat_per_dim = np.zeros(d)

    # Parse dimensions to consider
    if dimensions_to_consider is None:
        dimensions_to_consider = np.arange(d)

     ## Compute sketch
    z = scst*np.exp(1j*xi)
    for i in dimensions_to_consider:
        mask_valid = np.abs(Omega[i]) > 1e-15
        # CHECK IF INTEGER OR CONTINUOUS
        if nb_cat_per_dim[i] > 0:
            low_int  = box[i,0]
            high_int = box[i,1] + 1 # python counting convention
            C = high_int - low_int
            newTerm = np.ones(m) + 1j*np.zeros(m)
            newTerm[mask_valid] = (1/C)*(np.exp(1j*Omega[i,mask_valid]*high_int)-np.exp(1j*Omega[i,mask_valid]*low_int))/(np.exp(1j*Omega[i,mask_valid])-1)
            z *= newTerm
        else:
            # If continuous
            # To avoid divide by zero error, use that lim x->0 sin(a*x)/x = a
            sincTerm = np.zeros(m)

            sincTerm[mask_valid] = np.sin(Omega[i,mask_valid]*l_box[i])/Omega[i,mask_valid]
            sincTerm[~mask_valid] = l_box[i]

            z *= 2*np.exp(1j*Omega[i]*c_box[i])*sincTerm
    return z

### TODOS FOR SKETCHING.PY

# Short-term:
#  - Add support of private sketching for the real variants of the considered maps
#  - Add the square nonlinearity, for sketching for PCA for example

# Long-term:
# - Fast sketch computation


def part_one():
    increased = 0
    last_number = 170
    with open('Input.txt') as file:
        for line in file:
            tmp_number = int(line)
            if tmp_number > last_number:
                increased += 1
            last_number = tmp_number
    print("It increases: ", increased, "times.")

part_one()

def part_two():
    with open("Input.txt") as file:
        content = [int(line) for line in file]
        # A + B + C < B + C +D if A < D --> Only compare first and fourth item.
        zipped = zip(content, content[3:])
        result = sum(x < y for x, y in zipped)
        print("The Packages increases: ", result, "times.")

part_two()


import os
import json
import time
from collections import OrderedDict
import pandas as pd

with open("songs.json", encoding="utf-8", newline="") as plik:
    dane = json.load(plik)

parent_dir = "C:/Users/grzeg/PycharmProjects/pythonProject1/"
directory = "piosenki"

path = os.path.join(parent_dir, directory)
os.mkdir(path)

artistList = []
print(dane)
for i in range(len(dane)):
    artistList.append(dane[i]['artist'])
print(len(artistList))

res = list(OrderedDict.fromkeys(artistList))
print(len(res))

root_path = "C:/Users/grzeg/PycharmProjects/pythonProject1/piosenki/"
for x in res:
    pathArtist = os.path.join(root_path, x)
    print(pathArtist)
    os.mkdir(pathArtist)

for y in range(len(dane)):
    artist = ((dane[y]['artist'])) + "/"
    title = ((dane[y]['title']))
    lyrics = ((dane[y]['lyrics']))

    _dir = os.path.join(root_path, artist)

    completeName = os.path.join(_dir, title + ".txt")
    print(completeName)
    file1 = open(completeName, "w")
    file1.write(lyrics)
    file1.close()


file1=open(r'myfile.txt')
v,c,u,l=0,0,0,0
str1=file1.read()
for i in str1:
    if(i>='a' and i<='z'):
        l+=1
    elif(i>='A' and i<='Z'):
        u+=1
for j in str1:
    j=j.lower()
    if(j=='a' or j=='e' or j=='i' or j=='o' or j=='u'):
        v+=1
    else:
        if j.isalpha():
            c+=1

print("lowercase count:",l)
print("uppercase count:",u)
print("vowels count:",v)
print("consonants count:",c)


import torch
import torch.nn as nn

class ServerSolver:
    def __init__(self,
                 do_selection=True,           # whether to do client selection, if false selection_size is ignored
                 estimation_interval=1,       # [tau] time interval to estimate the importance weights
                 selection_size=30            # [K] size of selected clients for each cluster
                 ):
        self.do_selection = do_selection
        self.estimation_interval = estimation_interval
        self.selection_size = selection_size

    def to_json_dict(self):
        param_dict = {
            'do_selection': self.do_selection,
            'estimation_interval': self.estimation_interval,
            'selection_size': self.selection_size
        }
        return param_dict

class BaseClientSolver:
    def __init__(self):
        self.batch_size = -1
        self.optimizer_type = None
        self.criterion = None
        self.local_epoch = -1
        self.lr = -1.
        self.lr_scheduler_type = None
        self.lr_step = -1
        self.lr_gamma = -1.
        self.classification = False
        self.num_classes = -1
        self.reg_weight = -1.            # [lambda] regularization coefficient
        self.count_smoother = -1.        # [sigma] smoother for zero data match

    def to_json_dict(self):
        param_dict = {
            'batch_size': self.batch_size,
            'optimizer_type': self.optimizer_type.__name__,
            'criterion': type(self.criterion).__name__,
            'local_epoch': self.local_epoch,
            'lr': self.lr,
            'lr_scheduler_type': self.lr_scheduler_type.__name__,
            'lr_step': self.lr_step,
            'lr_gamma': self.lr_gamma,
            'classification': self.classification,
            'num_classes': self.num_classes,
            'reg_weight': self.reg_weight,
            'count_smoother': self.count_smoother
        }
        return param_dict

class MnistClientSolver(BaseClientSolver):
    def __init__(self):
        super(MnistClientSolver, self).__init__()
        self.batch_size = 10
        self.optimizer_type = torch.optim.Adam
        self.criterion = nn.CrossEntropyLoss()
        self.local_epoch = 10
        self.lr = 1e-4
        self.lr_scheduler_type = torch.optim.lr_scheduler.StepLR
        self.lr_step = 99999999
        self.lr_gamma = 1.0
        self.classification = True
        self.num_classes = 10
        self.reg_weight = 0.1
        self.count_smoother = 0.0001

class LettersCNNClientSolver(BaseClientSolver):
    def __init__(self):
        super(LettersCNNClientSolver, self).__init__()
        self.batch_size = 5
        self.optimizer_type = torch.optim.Adam
        self.criterion = nn.CrossEntropyLoss()
        self.local_epoch = 5
        self.lr = 1e-5
        self.lr_scheduler_type = torch.optim.lr_scheduler.StepLR
        self.lr_step = 99999999
        self.lr_gamma = 1.0
        self.classification = True
        self.num_classes = 26
        self.reg_weight = 0.1
        self.count_smoother = 0.0001

class LRClientSolver(BaseClientSolver):
    def __init__(self):
        super(LRClientSolver, self).__init__()
        self.batch_size = 10
        self.optimizer_type = torch.optim.Adam
        self.criterion = nn.MSELoss()
        self.local_epoch = 10
        self.lr = 5e-3
        self.lr_scheduler_type = torch.optim.lr_scheduler.StepLR
        self.lr_step = 99999999
        self.lr_gamma = 1.0
        self.classification = False
        self.reg_weight = 1.0
        self.count_smoother = 0.0001

class Cifar10CNNClientSolver(BaseClientSolver):
    def __init__(self):
        super(Cifar10CNNClientSolver, self).__init__()
        self.batch_size = 64
        self.optimizer_type = torch.optim.Adam
        self.criterion = nn.CrossEntropyLoss()
        self.estimate_criterion = nn.CrossEntropyLoss(reduction='none')
        self.local_epoch = 10
        self.lr = 5e-4
        self.lr_scheduler_type = torch.optim.lr_scheduler.StepLR
        self.lr_step = 99999999
        self.lr_gamma = 1.0
        self.classification = True
        self.num_classes = 10
        self.reg_weight = 0.01
        self.count_smoother = 0.0001


# Create your models here.
from django.db import models
from datetime import date

# Create your models here.
from jsonfield import JSONField
from django.contrib.auth.models import User

today = date.today()

DENIALS= (
    ("NOT SATISFACTORY", "NOT SATISFACTORY"),
    ("WORKER DENIED", "WORKER DENIED")
)

SLOTS = (
    ("MORNING", "MORNING"),
    ("EVENING", "EVENING")
)

STAT = (
    ("UPCOMING", "UPCOMING"),
    ("ONGOING", "ONGOING"),
    ("COMPLETED", "COMPLETED")
)

TRANS_TYPE = (

    ("DEBIT", "DEBIT"),
    ("CREDIT", "CREDIT"),
)
PURPOSE = (
    ("HIRING", "HIRING"),
    ("REFUND", "REFUND"),
    ("ADD MONEY", "ADD MONEY"),
    ("REFERRAL", "REFERRAL"),
    ("NEW USER", "NEW USER"),
    ("TRANSFER", "TRANSFER"),
)

RATING = (
    (1, 1),
    (2, 2),
    (3, 3),
    (4, 4),
    (5, 5),
)

REPORT_TYPE = (
    ("DUTY DENIAL", "DUTY DENIAL"),
    ("MISBEHAVIOUR", "MISBEHAVIOUR"),
    ("TIME DELAY", "TIME DELAY"),
    ("NOT RESPONDING", "NOT RESPONDING"),
    ("DEMANDING MORE WAGES", "DEMANDING MORE WAGES"),
    ("COMMITMENT ISSUE", "COMMITMENT ISSUE"),
    ("MESSED UP WORK", "MESSED UP WORK"),
    ("NON CONSENTUAL WORK", "NON CONSENTUAL WORK"),
)

ACTIONS = (
    ("COMPLETE REFUND", "COMPLETE REFUND"),
    ("PARTIAL REFUND", "PARTIAL REFUND"),
    ("REPLACEMENT", "REPLACEMENT"),
    ("NO ACTION REQUIRED", "NO ACTION REQUIRED"),
)

STATUS = (
    ("DENIED", "DENIED"),
    ("UNDER REVIEW", "UNDER REVIEW"),
    ("COMPLETED", "COMPLETED"),
)

CITY = (
    ("Bhopal", "Bhopal"),
    ("Pune", "Pune"),
    ("Indore", "Indore"),
    ("Bangalore", "Bangalore"),
)

# Create your models here.
class workers(models.Model):

    workerid = models.CharField(max_length=20, unique=True)
    agreeNo = models.BigIntegerField(null=True, unique=True)
    name = models.CharField(max_length=200)
    img =  models.ImageField(upload_to='prof')
    skills = JSONField(null=True)
    exp = JSONField(default=int, null=True)
    dateBirth = models.DateField(auto_now=False)
    lang = JSONField(null=True)
    wages = models.FloatField(null=True)
    avgWork = models.FloatField(null=True)
    distance = models.FloatField()
    offDay = models.CharField(max_length=20, default='Sunday')
    idType = models.CharField(max_length=20, default='Aadhar Card')
    idValue = models.CharField(max_length=50, default='000000000')
    active = models.BooleanField(default=False)
    gend = models.CharField(max_length=10)
    contact = models.BigIntegerField()
    strtime = models.TimeField(auto_now=False, auto_now_add=False)
    endtime = models.TimeField(auto_now=False, auto_now_add=False)
    add = models.TextField()
    city = models.CharField(max_length=100)
    coor = JSONField(default=float)
    ageid = models.CharField(max_length=20)
    verified = models.BooleanField(default=True, max_length=1)
    updated_at = models.DateTimeField(auto_now=True)
    services = models.CharField(max_length=100, null=True)
    wagestype = models.CharField(max_length=50, null=True)

    def __str__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.workerid

class appUser(models.Model):
    userID = models.AutoField(primary_key=True, auto_created=True, unique=True, verbose_name="UserID")
    username = models.CharField(max_length = 100, blank=True, serialize=True, unique=True)
    name = models.CharField(max_length=200, blank=True, serialize=True)
    contact = models.BigIntegerField(max_length=200, unique=True, default="9090000000", blank=True, serialize=True)
    profile = models.ImageField(verbose_name= "Profile Image", upload_to= "profile", null=True, blank=True, serialize=True)
    email = models.CharField(max_length=200, unique=False, blank=True, serialize=True)
    created_on = models.DateTimeField(auto_now_add=True)

    def __int__(self):
        return self.userID

class transaction(models.Model):
    userT = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
    amount = models.FloatField(null=False, default=0.0)
    orderID = models.CharField(max_length=100, unique=True)
    transID = models.CharField(max_length=200, unique=True)
    transType = models.CharField(max_length=20, choices=TRANS_TYPE)
    purpose = models.CharField(max_length=30, choices=PURPOSE)
    timestamp = models.DateTimeField(auto_now=True)

    def __str__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.transID

class hired(models.Model):
    hiredID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="HireID")
    userH = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
    workerIDH = models.ForeignKey(workers, default= "BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerID", to_field="workerid")
    orderid = models.ForeignKey(transaction, on_delete=models.CASCADE, verbose_name="orderID", to_field="orderID")
    date = models.DateField(default=today.strftime("%Y-%m-%d"))
    slot = models.CharField(choices=SLOTS, max_length=20)
    status = models.CharField(choices=STAT, max_length=20, default="UPCOMING")
    extension = models.IntegerField(max_length=2, default=1)
    updated_at = models.DateTimeField(auto_now=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.hiredID

class workRating(models.Model):
    ratingID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="RateID")
    userR = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
    workerIDR = models.ForeignKey(workers, default= "BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerID", to_field="workerid")
    rat_1 = models.IntegerField(null=True, choices=RATING)
    rat_2 = models.IntegerField(null=True, choices=RATING)
    rat_3 = models.IntegerField(null=True, choices=RATING)
    rat_4 = models.IntegerField(null=True, choices=RATING)
    comment = models.TextField(null=True)
    hiredOn = models.DateField(null=False, auto_now=True)
    updated_at = models.DateTimeField(auto_now=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.ratingID

class saved(models.Model):
    savedID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="SaveID")
    userS = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
    workerIDS = models.ForeignKey(workers, default= "BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerID", to_field="workerid")
    updated_at = models.DateTimeField(auto_now=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.savedID

class filterCache(models.Model):
    CacheID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="CacheID")
    userF = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="userID")
    location = JSONField(null=True)
    wages = models.IntegerField(null=False)
    jobs = JSONField(null=False)
    city = models.TextField(default='Bhopal', choices=CITY)
    add = models.TextField(null=True, max_length=100)
    updated_at = models.DateTimeField(auto_now=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.CacheID

class reportWorker(models.Model):
     reportID =models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="ReportID")
     userRe = models.ForeignKey(appUser, default='shiva12', verbose_name="User", on_delete=models.SET_DEFAULT, to_field="username")
     workerIDW = models.ForeignKey(workers, default= "BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerID", to_field="workerid")
     reportType = models.CharField(max_length = 200, choices=REPORT_TYPE)
     description = models.TextField(editable=True)
     actionNeed = models.CharField(max_length=200, choices=ACTIONS)
     status = models.CharField(max_length=100, choices= STATUS)
     updated_at = models.DateTimeField(auto_now=True)

     def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.reportID

class dutyDenials(models.Model):
    denyID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="denialID")
    worker = models.ForeignKey(workers, default="BHO1001", on_delete=models.SET_DEFAULT, verbose_name="workerid", to_field="workerid")
    user = models.ForeignKey(appUser, default="shiva12", on_delete=models.SET_DEFAULT, verbose_name="username", to_field="username")
    transid = models.ForeignKey(transaction, on_delete=models.CASCADE, verbose_name="transID", to_field="transID")
    reason = models.CharField(max_length = 100, null=True, choices=DENIALS)
    updated_at = models.DateTimeField(auto_now=True, auto_created=True)

    def __int__(self): # __str__ for Python 3, __unicode__ for Python 2
        return self.denyID

class workerCalls(models.Model):
    callID = models.IntegerField(primary_key=True, auto_created=True, default=0, unique=True, verbose_name="callID")
    workerid = models.ForeignKey(workers, default="BHO1001", on_delete=models.SET_DEFAULT, verbose_name="WorkerID", to_field="workerid")
    userid = models.ForeignKey(appUser, default="BHO1001", on_delete=models.SET_DEFAULT, verbose_name="UserID", to_field="username")
    callSid = models.CharField(max_length=100, null=False, default="123456")
    record = models.URLField(max_length=200, verbose_name="CallRecord", null=True)
    date = models.DateField(auto_now=True, auto_created=True)
    transid = models.CharField(max_length=100, default="HIR00001", null=False)
    updated_at = models.DateTimeField(auto_now=True, auto_created=True)


from hashlib import md5
from typing import List

def get_file_hash(absolute_path:str) -> str:
    with open(absolute_path, 'rb') as f:
        hash_obj = md5()
        hash_obj.update(f.read())
        return hash_obj.hexdigest()

def get_list_hash(data:List[str], encoding:str = 'latin1') -> str:
    hash_obj = md5()
    for chunk in sorted(data):
        hash_obj.update(chunk.encode(encoding))
    return hash_obj.hexdigest()


# -*- coding: utf-8 -*-

from toolium.pageobjects.page_object import PageObject
from selenium.common.exceptions import NoSuchElementException

from toolium.pageelements import *
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.select import Select
import selenium.webdriver.support.expected_conditions as WAITCON
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from toolium.pageelements import InputText, Button
import random
import string
import time
from selenium.webdriver.common.by import By

class TabsPageObject(PageObject):
    """
    :Class represents web page elements for Login page.
    """

    def init_page_elements(self):

        """ Open page elements

        :returns: this page object instance
        """
        self.genre = Button(By.XPATH, '//*[@text="Songs by genre"]', wait=True)
        self.rock = Button(By.XPATH, '//*[@text="Rock songs"]', wait=True)
        self.song = Button(By.XPATH, '//*[@text="Silent Partner"]', wait=True)

    def SelectGenre(self):
        """ Select genre group

        :returns: this page object instance
        """
        try:
            self.logger.info("\n=====================================Genre======================================")
            self.logger.debug("Opening genres")
            self.logger.debug("Checking if the button is present : %s", self.genre)
            self.logger.debug("\n-----------------------------------------------------------------------------------")

            self.genre.wait_until_visible()
            self.genre.click()
            time.sleep(2)
            return True
        except NoSuchElementException:
            self.logger.debug("error", "Element {} does not exist".format(self.genre))
            return None

    def Rock(self):
        """ Select Rock music genre

        :returns: this page object instance
        """
        try:
            self.logger.info("\n===============================Selecting Rock music from genre=================================")
            self.logger.debug("Choose a genre")
            self.logger.debug("Checking if the pageElement is present : %s", self.rock)
            self.logger.debug("\n-----------------------------------------------------------------------------------")

            self.rock.wait_until_visible()
            self.rock.click()
            time.sleep(2)
            return True
        except NoSuchElementException:
            self.logger.debug("error", "Element {} does not exist".format(self.rock))
            return None

    def SelectedSong(self):
        """ Select play selected song

        :returns: this page object instance
        """
        try:
            self.logger.info("\n===============================Playing awakening=================================")
            self.logger.debug("Playing song.")
            self.logger.debug("Checking if the pageElement is present : %s", self.song)
            self.logger.debug("\n-----------------------------------------------------------------------------------")

            self.song.wait_until_visible()
            self.song.click()
            time.sleep(10)
            return True
        except NoSuchElementException:
            self.logger.debug("error", "Element {} does not exist".format(self.song))
            return None


# !/usr/bin/env python3
# coding: utf-8

from src.minority_game import  MinorityGameWithRandomChoice
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
if __name__ == "__main__":
    # set agent num
    agent_num = 201
    # set round number from 10000 to 100000
    rounds = range(10000,110000,10000)

    mean_list = []
    stdd_list = []
    round_num = 100000

    # Run the game for different round number
    game = MinorityGameWithRandomChoice(agent_num, round_num)
    mean_list,stdd_list = game.run_game()

    # print result
    print(mean_list)
    print(stdd_list)
    # plot mean
    fig = plt.figure(figsize=(8, 4))
    plt.plot(rounds,mean_list)
    plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.2)
    fig.suptitle('Mean Value')
    plt.xlabel('Iteration number')
    plt.ylabel('Average numbers of winners')
    fig.savefig('ex3-1-mean.jpg')

    # plot SD
    fig2 = plt.figure(figsize=(8, 4))
    plt.plot(rounds,stdd_list)
    plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.2)
    fig2.suptitle('Standard Deviation')
    plt.xlabel('Iteration number')
    plt.ylabel('Average numbers of winners')
    fig2.savefig('ex3-1-std.jpg')


import spacy
import pandas as pd

# Load the spaCy model
nlp = spacy.load('en_core_web_sm')
nlp.max_length = 4000000

# This function filters out the stop words and the punctuation. It also applies Lemmatization
def filter_and_lemmatize(document):
    filtered_tokens = []
    for token in document:
        if not token.is_stop and not token.is_punct and token.text != '\n':
            filtered_tokens.append(token)
    lemmas = [token.lemma_ for token in filtered_tokens]
    return lemmas

# Filter tokens based on their POS
def filter_pos_tags(document):
    allowed_postags = ['NOUN', 'ADJ', 'ADV']
    return [tok.text for tok in document if tok.pos_ in allowed_postags]

# Function to perform NER on a list of tokens
def perform_ner(tokens):
    text = ' '.join(tokens)  # Convert list of tokens to a string
    doc = nlp(text)
    entities = [(ent.text, ent.label_) for ent in doc.ents]
    return entities

# Read the JSON file into a pandas DataFrame
combined_file_path = 'material/combined.json'
print('Reading JSON file...')
df = pd.read_json(combined_file_path)

# Process the text and save the results
print('Processing texts...')
doc = nlp(df['text'][0])
data = []
for token in doc:
    data.append([token.text, token.lemma_, token.pos_, token.tag_, token.dep_, token.shape_, token.is_alpha, token.is_stop])

# Create the pandas DataFrame for token details
tokens_details = pd.DataFrame(data, columns=['Text', 'Lemma', 'coarse POS', 'fine POS', 'Dependency', 'Shape', 'is Alpha', 'is Stop'])

# Apply spaCy model to all documents
print('Applying spaCy model to all documents...')
df['spacy_document'] = df['text'].apply(nlp)

# Process files: Filter and lemmatize, POS tag, NER
print('Processing files...')
df['clean_text'] = df['spacy_document'].apply(filter_and_lemmatize)
print(df)
df['pos_filtered_text'] = df['spacy_document'].apply(filter_pos_tags)
df['ner'] = df['pos_filtered_text'].apply(perform_ner)

# Print the DataFrame
print('DataFrame:')
with pd.option_context('display.max_rows', None, 'display.max_columns', None):
    print(df)

# Save the DataFrame to a new JSON file
output_file_path = 'material/data.json'
df['spacy_document'] = df['spacy_document'].apply(lambda doc: [{'text': token.text,
                                                                'lemma': token.lemma_,
                                                                'pos': token.pos_} for token in doc])

df.to_json(output_file_path, orient='records')

print('DataFrame saved to:', output_file_path)


# 实现2路归并排序算法

def merge(leftSource = [], rightSource = []):
    lenLeft = len(leftSource)
    lenRight = len(rightSource)
    # 定义存储合并后的list数组
    merge_one = [0] * (lenLeft + lenRight)
    mi = 0 # 定义合并list的下标
    li = 0 # 定义左边数组的下标
    ri = 0 # 定义右边数组的下标
    while li < lenLeft and ri < lenRight:
        # 左右和右边依次取第一个数字比较，小的放合并数组中
        if leftSource[li] <= rightSource[ri]:
           merge_one[mi] = leftSource[li]
           li = li + 1
        else:
            merge_one[mi] = rightSource[ri]
            ri = ri + 1
        mi = mi + 1
    if  li < lenLeft: # 判断左边数组是否已经取值完毕
        for i in range(li, lenLeft):
            merge_one[mi] = leftSource[i]
            mi = mi + 1
    else: # 判断右边数组是否已经取值完毕
        for j in range(ri, lenRight):
            merge_one[mi] = rightSource[j]
            mi = mi + 1
    return merge_one

def merge_sort(to_sort = []):
    if len(to_sort) == 1:
        return to_sort
    mid = len(to_sort) // 2
    left = to_sort[:mid]
    right = to_sort[mid:]

    left = merge_sort(left)
    right = merge_sort(right)

    merged = merge(left, right)

    return merged

if __name__ == '__main__':
    print(merge_sort([10,20,9,50,30,99,3]))


'''
Problem
-------------------
Given an array nums of n integers and an integer target, find three integers in nums such that the sum is closest to target. Return the sum of the three integers. You may assume that each input would have exactly one solution.

Constraints
-------------------
3 <= nums.length <= 10^3\
-10^3 <= nums[i] <= 10^3\
-10^4 <= target <= 10^4\

Thinking
-------------------

'''
from typing import List

def threeSumClosest(nums: List[int], target: int) -> int:
    nums.sort()
    closest_sum = 2**31-1
    for i in range(len(nums)):
        j, k = i+1, len(nums)-1
        while j < k:
            curr_sum = nums[i] + nums[j] + nums[k]
            if curr_sum == target:
                return curr_sum
            if abs(curr_sum-target) < abs(closest_sum-target):
                closest_sum = curr_sum
            if curr_sum < target:
                j = j+1
            else:
                k = k-1
    return closest_sum

if __name__ == "__main__":
    print(threeSumClosest([-1, 2, 1, -4], 1))


from datetime import datetime

class AERONETConstants:
    def __init__(self):

        self.DATEREF = datetime(1970, 1, 1, 0, 0, 0)
        self.ThuillierMethodDefault = 'interp'

        #CSV Original File

        #NCFile
        self.dim_time = 'TimeIndex'
        self.dim_wl = 'Central_wavelenghts'


from django.conf.urls import patterns, include, url

from django.contrib import admin
admin.autodiscover()

from issues.views import *

urlpatterns = patterns('',
    # Examples:
    url(r'^$', home, name='home'),
    url(r'^issues/$', issues_list, name='issue_list'),
    url(r'^issues/(\d+)/', issue_articles_list),
    url(r'^weblog/', include('weblog.urls.entries')),
    url(r'^weblog/categories/', include('weblog.urls.categories')),

    url(r'^admin/', include(admin.site.urls)),
)