import time
import re
import inspect

import numpy as np
import pandas as pd
from scipy import stats, optimize, interpolate, signal, linalg
import patsy as ps
import statsmodels.api as sm
import sklearn.metrics as met
import sklearn.linear_model as lm
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin


def validpar(func, x):
    '''
    Avoid invalid parameters for a function.
    
    Parameters
    ----------
    func : function
        Which cannot accept invalid parameters.
    x : dict
        Including pairs of parameter name and value.
    
    Returns
    -------
    op : dict
        Valid parameters for the function.
    '''
    op = dict(i for i in x.items() if i[0] in inspect.signature(func).parameters.keys())
    return(op)


def strnum(x, func_reduce=max):
    '''
    Convert string to number, only keep numerical part (0-9 or .), otherwise return np.nan.
    
    Parameters
    ----------
    x : string
        To convert to numbers.
    func_reduce : function
        Function to reduce multiple numbers.
    
    Returns
    -------
    op : float
    '''
    if isinstance(x, str):
        numL = re.findall(r'\d[\.\d]+', x)
        if numL:
            return(func_reduce([float(i) for i in numL]))
        else:
            return(np.nan)
    else:
        return(x)

    
def plyargs(func, argL, argname, f_concat=list, argcon={}, **kwargs):
    '''
    Apply function on a list of arguments.
    
    Parameters
    ----------
    func : function
        To apply arguments.
    argL : list (of lists)
        Iterated argument values.
    argname : list
        Argument names, same length as argL.
    f_concat : function
        To concat iterated outputs of the function.
    argcon : dict
        arguments of f_con
    Additional keyword arguments will be passed as keywords to the function.
    
    Returns
    -------
    op : list-like
        iterated outputs of the function.
    '''
    op = f_concat(map(lambda arg: func(**dict(zip(*[argname, arg])), **kwargs), zip(*argL)), **argcon)
    return(op)


def apply_df(df, func, axis=0, n_jobs=1, **kwargs):
    '''
    Apply function on each row/column of dataframe and return a Series.
    
    Parameters
    ----------
    df : DataFrame
        To apply function.
    func : function
        Function with input (index, Series) to apply to each column/row.
    axis :  {0, 1}, default 0
        - 0 or 'index': apply function along rows.
        - 1 or 'columns': apply function along columns.
    n_jobs : int, optional (default=1)
        The number of jobs to run in parallel.
    Additional keyword arguments will be passed as keywords to the function.
    
    Returns
    -------
    S : list-like
        iterated outputs of f.
    '''
    if axis == 0:
        pairs = df.iteritems()
        index = df.columns
    else:
        pairs = df.iterrows()
        index = df.index
    S = pd.Series([func(i_val) for i, i_val in pairs], index, **kwargs)
    return(S)

    
def cross_join(left, right, combine_index=True):
    '''
    Cross outer join of two DataFrames along column.
    
    Parameters
    ----------
    left : DataFrame
    right : DataFrame
    combine_index : bool
        Whether combine index name of two DataFrames
    
    Returns
    -------
    DataFrame
        
    '''
    out = pd.merge(left.assign(_key=1), right.assign(_key=1), on="_key").drop("_key", axis=1)
    if combine_index:
        out.index = ["{}_{}".format(i, j) for i in left.index for j in right.index]
    return(out)
    
    
def collinearvif(df):
    '''
    Compute variance inflation factor (VIF) of a DataFrame.
    
    Parameters
    ----------
    df : DataFrame
    
    Returns
    -------
    op : Series
        VIF values of each column.
    '''
    op = pd.Series(np.diag(np.linalg.inv(df.corr())), df.columns, name="CollinearVIF")
    return(op)


def summary(df, n=5, pct=[0.1, 0.5, 0.9]):
    '''
    Summarize DataFrame along columns for data type, sample size, numerical statistics and frequency
    
    Parameters
    ----------
    df : DataFrame
        To summary.
    n : int
        The number of foremost frequent categories of frequency table.
    pct : list
        Percentiles for numerical statistics.

    Returns
    -------
    op : DataFrame
        summary of DataFrame along columns.
    '''
    
    def freq(s):
        op = pd.value_counts(s)
        op = pd.concat([pd.Series(op.index[:n]).rename(lambda x: "FreqCat{}".format(x+1)), 
                        pd.Series(op.values[:n]).rename(lambda x: "FreqVal{}".format(x+1)).T, 
                        pd.Series(op.iloc[n:].sum(), index=["Freq_Others"])])
        return(op)
    op = pd.concat([df.dtypes.rename("Type"), 
                    df.notnull().sum().rename("N"), 
                    df.describe(pct).iloc[1:].T, 
                    df.apply(freq).T], axis=1).loc[df.columns]
    return(op)


## Clean Data

def CLtimenum(s):
    '''Convert datetime variable to numeric variable.'''
    op = pd.DatetimeIndex(s)
    op = pd.get_dummies(pd.DataFrame(dict(zip(*[["{}__{}".format(s.name, i) for i in ["Month", "Year"]], [op.month, op.year]])), index=s.index).astype("O"))
    return(op)


def CLnormal(df):
    '''
    Normalize the distribution of numeric DataFrame to standard normal distribution by rank.
    
    See also
    --------
    CLscale
    '''
    op = pd.DataFrame(stats.norm.ppf((df.rank(pct = True) - 0.5/df.shape[0]).fillna(0.5)), index=df.index, columns=df.columns)
    return(op)


def CLscale(df):
    '''
    Standardize the distribution of numeric DataFrame to mean 0 and standard deviation 1.
    
    See also
    --------
    CLnormal
    '''
    op = ((df - df.mean())/df.std()).fillna(0)
    return(op)


def CLsparse_cat(s, sp=0.01):
    '''
    Clean a object Series with sparse values converted to "others".
    '''
    sfreq = pd.value_counts(s) > sp*s.shape[0]
    op = s.where(s.isin(sfreq.index[sfreq]), "others")
    return(op)


def CLsparse(s):
    '''
    Check the percentage of non-most-frequent and non-null values in a Series.
    '''
    op = 1 - s.isnull().mean()
    if op > 0:
        op -= pd.value_counts(s).iloc[0]/s.shape[0]
    return(op)


def CLdata(df, sp=0, cor=1, f_norm=CLscale, formula=None, **kwargs):
    '''
    Clean a pandas DataFrame according to dtype (numeric or object)
    
    Parameters
    ----------
    df : DataFrame
        data to clean.
    sp : float
        between [0, 1), threshold of ratio of unique and missing values to delete variables.
    cor : float
        between (0, 1], threshold of absolute correlation to remove later collinear variables.